usearch 2.15.1 → 2.15.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/binding.gyp +1 -1
- package/c/CMakeLists.txt +10 -3
- package/cpp/README.md +1 -1
- package/cpp/bench.cpp +2 -2
- package/include/usearch/index.hpp +27 -13
- package/include/usearch/index_dense.hpp +45 -88
- package/include/usearch/index_plugins.hpp +128 -40
- package/javascript/usearch.test.js +15 -13
- package/package.json +1 -1
- package/prebuilds/darwin-arm64+x64/usearch.node +0 -0
- package/prebuilds/linux-arm64/usearch.node +0 -0
- package/prebuilds/linux-x64/usearch.node +0 -0
- package/simsimd/CODE_OF_CONDUCT.md +128 -0
- package/simsimd/CONTRIBUTING.md +18 -8
- package/simsimd/Cargo.lock +1 -1
- package/simsimd/Cargo.toml +1 -1
- package/simsimd/MANIFEST.in +2 -0
- package/simsimd/README.md +446 -158
- package/simsimd/build.rs +11 -1
- package/simsimd/c/lib.c +172 -40
- package/simsimd/include/simsimd/binary.h +11 -11
- package/simsimd/include/simsimd/curved.h +804 -0
- package/simsimd/include/simsimd/dot.h +333 -334
- package/simsimd/include/simsimd/geospatial.h +9 -3
- package/simsimd/include/simsimd/mesh.h +69 -0
- package/simsimd/include/simsimd/probability.h +178 -150
- package/simsimd/include/simsimd/simsimd.h +581 -107
- package/simsimd/include/simsimd/sparse.h +807 -0
- package/simsimd/include/simsimd/spatial.h +737 -502
- package/simsimd/include/simsimd/types.h +145 -75
- package/simsimd/javascript/simsimd.ts +4 -4
- package/simsimd/package.json +1 -1
- package/simsimd/rust/lib.rs +165 -0
|
@@ -5,8 +5,8 @@
|
|
|
5
5
|
* @date March 14, 2023
|
|
6
6
|
*
|
|
7
7
|
* Contains:
|
|
8
|
-
* - L2 (Euclidean) squared distance
|
|
9
|
-
* - Cosine (Angular) similarity
|
|
8
|
+
* - L2 (Euclidean) regular and squared distance
|
|
9
|
+
* - Cosine (Angular) distance - @b not similarity!
|
|
10
10
|
*
|
|
11
11
|
* For datatypes:
|
|
12
12
|
* - 64-bit IEEE floating point numbers
|
|
@@ -14,6 +14,7 @@
|
|
|
14
14
|
* - 16-bit IEEE floating point numbers
|
|
15
15
|
* - 16-bit brain floating point numbers
|
|
16
16
|
* - 8-bit signed integral numbers
|
|
17
|
+
* - 4-bit signed integral numbers
|
|
17
18
|
*
|
|
18
19
|
* For hardware architectures:
|
|
19
20
|
* - Arm (NEON, SVE)
|
|
@@ -27,7 +28,7 @@
|
|
|
27
28
|
|
|
28
29
|
#include "types.h"
|
|
29
30
|
|
|
30
|
-
#include "dot.h" // `
|
|
31
|
+
#include "dot.h" // `_simsimd_reduce_f32x8_haswell`
|
|
31
32
|
|
|
32
33
|
#ifdef __cplusplus
|
|
33
34
|
extern "C" {
|
|
@@ -39,26 +40,35 @@ extern "C" {
|
|
|
39
40
|
* By default they use 32-bit arithmetic, unless the arguments themselves contain 64-bit floats.
|
|
40
41
|
* For double-precision computation check out the "*_accurate" variants of those "*_serial" functions.
|
|
41
42
|
*/
|
|
43
|
+
SIMSIMD_PUBLIC void simsimd_l2_f64_serial(simsimd_f64_t const* a, simsimd_f64_t const*, simsimd_size_t n, simsimd_distance_t* d);
|
|
42
44
|
SIMSIMD_PUBLIC void simsimd_l2sq_f64_serial(simsimd_f64_t const* a, simsimd_f64_t const*, simsimd_size_t n, simsimd_distance_t* d);
|
|
43
45
|
SIMSIMD_PUBLIC void simsimd_cos_f64_serial(simsimd_f64_t const* a, simsimd_f64_t const*, simsimd_size_t n, simsimd_distance_t* d);
|
|
46
|
+
SIMSIMD_PUBLIC void simsimd_l2_f32_serial(simsimd_f32_t const* a, simsimd_f32_t const*, simsimd_size_t n, simsimd_distance_t* d);
|
|
44
47
|
SIMSIMD_PUBLIC void simsimd_l2sq_f32_serial(simsimd_f32_t const* a, simsimd_f32_t const*, simsimd_size_t n, simsimd_distance_t* d);
|
|
45
48
|
SIMSIMD_PUBLIC void simsimd_cos_f32_serial(simsimd_f32_t const* a, simsimd_f32_t const*, simsimd_size_t n, simsimd_distance_t* d);
|
|
49
|
+
SIMSIMD_PUBLIC void simsimd_l2_f16_serial(simsimd_f16_t const* a, simsimd_f16_t const*, simsimd_size_t n, simsimd_distance_t* d);
|
|
46
50
|
SIMSIMD_PUBLIC void simsimd_l2sq_f16_serial(simsimd_f16_t const* a, simsimd_f16_t const*, simsimd_size_t n, simsimd_distance_t* d);
|
|
47
51
|
SIMSIMD_PUBLIC void simsimd_cos_f16_serial(simsimd_f16_t const* a, simsimd_f16_t const*, simsimd_size_t n, simsimd_distance_t* d);
|
|
52
|
+
SIMSIMD_PUBLIC void simsimd_l2_bf16_serial(simsimd_bf16_t const* a, simsimd_bf16_t const*, simsimd_size_t n, simsimd_distance_t* d);
|
|
48
53
|
SIMSIMD_PUBLIC void simsimd_l2sq_bf16_serial(simsimd_bf16_t const* a, simsimd_bf16_t const*, simsimd_size_t n, simsimd_distance_t* d);
|
|
49
54
|
SIMSIMD_PUBLIC void simsimd_cos_bf16_serial(simsimd_bf16_t const* a, simsimd_bf16_t const*, simsimd_size_t n, simsimd_distance_t* d);
|
|
55
|
+
SIMSIMD_PUBLIC void simsimd_l2_i8_serial(simsimd_i8_t const* a, simsimd_i8_t const*, simsimd_size_t n, simsimd_distance_t* d);
|
|
50
56
|
SIMSIMD_PUBLIC void simsimd_l2sq_i8_serial(simsimd_i8_t const* a, simsimd_i8_t const*, simsimd_size_t n, simsimd_distance_t* d);
|
|
51
57
|
SIMSIMD_PUBLIC void simsimd_cos_i8_serial(simsimd_i8_t const* a, simsimd_i8_t const*, simsimd_size_t n, simsimd_distance_t* d);
|
|
52
58
|
|
|
53
59
|
/* Double-precision serial backends for all numeric types.
|
|
54
60
|
* For single-precision computation check out the "*_serial" counterparts of those "*_accurate" functions.
|
|
55
61
|
*/
|
|
62
|
+
SIMSIMD_PUBLIC void simsimd_l2_f32_accurate(simsimd_f32_t const* a, simsimd_f32_t const*, simsimd_size_t n, simsimd_distance_t* d);
|
|
56
63
|
SIMSIMD_PUBLIC void simsimd_l2sq_f32_accurate(simsimd_f32_t const* a, simsimd_f32_t const*, simsimd_size_t n, simsimd_distance_t* d);
|
|
57
64
|
SIMSIMD_PUBLIC void simsimd_cos_f32_accurate(simsimd_f32_t const* a, simsimd_f32_t const*, simsimd_size_t n, simsimd_distance_t* d);
|
|
65
|
+
SIMSIMD_PUBLIC void simsimd_l2_f16_accurate(simsimd_f16_t const* a, simsimd_f16_t const*, simsimd_size_t n, simsimd_distance_t* d);
|
|
58
66
|
SIMSIMD_PUBLIC void simsimd_l2sq_f16_accurate(simsimd_f16_t const* a, simsimd_f16_t const*, simsimd_size_t n, simsimd_distance_t* d);
|
|
59
67
|
SIMSIMD_PUBLIC void simsimd_cos_f16_accurate(simsimd_f16_t const* a, simsimd_f16_t const*, simsimd_size_t n, simsimd_distance_t* d);
|
|
68
|
+
SIMSIMD_PUBLIC void simsimd_l2_bf16_accurate(simsimd_bf16_t const* a, simsimd_bf16_t const*, simsimd_size_t n, simsimd_distance_t* d);
|
|
60
69
|
SIMSIMD_PUBLIC void simsimd_l2sq_bf16_accurate(simsimd_bf16_t const* a, simsimd_bf16_t const*, simsimd_size_t n, simsimd_distance_t* d);
|
|
61
70
|
SIMSIMD_PUBLIC void simsimd_cos_bf16_accurate(simsimd_bf16_t const* a, simsimd_bf16_t const*, simsimd_size_t n, simsimd_distance_t* d);
|
|
71
|
+
SIMSIMD_PUBLIC void simsimd_l2_i8_accurate(simsimd_i8_t const* a, simsimd_i8_t const*, simsimd_size_t n, simsimd_distance_t* d);
|
|
62
72
|
SIMSIMD_PUBLIC void simsimd_l2sq_i8_accurate(simsimd_i8_t const* a, simsimd_i8_t const*, simsimd_size_t n, simsimd_distance_t* d);
|
|
63
73
|
SIMSIMD_PUBLIC void simsimd_cos_i8_accurate(simsimd_i8_t const* a, simsimd_i8_t const*, simsimd_size_t n, simsimd_distance_t* d);
|
|
64
74
|
|
|
@@ -66,22 +76,37 @@ SIMSIMD_PUBLIC void simsimd_cos_i8_accurate(simsimd_i8_t const* a, simsimd_i8_t
|
|
|
66
76
|
* By far the most portable backend, covering most Arm v8 devices, over a billion phones, and almost all
|
|
67
77
|
* server CPUs produced before 2023.
|
|
68
78
|
*/
|
|
79
|
+
SIMSIMD_PUBLIC void simsimd_l2_f64_neon(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
80
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_f64_neon(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
81
|
+
SIMSIMD_PUBLIC void simsimd_cos_f64_neon(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
82
|
+
SIMSIMD_PUBLIC void simsimd_l2_f32_neon(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
69
83
|
SIMSIMD_PUBLIC void simsimd_l2sq_f32_neon(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
70
84
|
SIMSIMD_PUBLIC void simsimd_cos_f32_neon(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
85
|
+
SIMSIMD_PUBLIC void simsimd_l2_f16_neon(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
71
86
|
SIMSIMD_PUBLIC void simsimd_l2sq_f16_neon(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
72
87
|
SIMSIMD_PUBLIC void simsimd_cos_f16_neon(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
88
|
+
SIMSIMD_PUBLIC void simsimd_l2_bf16_neon(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
89
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_bf16_neon(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
90
|
+
SIMSIMD_PUBLIC void simsimd_cos_bf16_neon(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
91
|
+
SIMSIMD_PUBLIC void simsimd_l2_i8_neon(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
73
92
|
SIMSIMD_PUBLIC void simsimd_l2sq_i8_neon(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
74
93
|
SIMSIMD_PUBLIC void simsimd_cos_i8_neon(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
75
94
|
|
|
76
95
|
/* SIMD-powered backends for Arm SVE, mostly using 32-bit arithmetic over variable-length platform-defined word sizes.
|
|
77
96
|
* Designed for Arm Graviton 3, Microsoft Cobalt, as well as Nvidia Grace and newer Ampere Altra CPUs.
|
|
78
97
|
*/
|
|
79
|
-
SIMSIMD_PUBLIC void
|
|
80
|
-
SIMSIMD_PUBLIC void
|
|
81
|
-
SIMSIMD_PUBLIC void
|
|
82
|
-
SIMSIMD_PUBLIC void
|
|
83
|
-
SIMSIMD_PUBLIC void
|
|
84
|
-
SIMSIMD_PUBLIC void
|
|
98
|
+
SIMSIMD_PUBLIC void simsimd_l2_f32_sve(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
99
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_f32_sve(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
100
|
+
SIMSIMD_PUBLIC void simsimd_cos_f32_sve(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
101
|
+
SIMSIMD_PUBLIC void simsimd_l2_f16_sve(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
102
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_f16_sve(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
103
|
+
SIMSIMD_PUBLIC void simsimd_cos_f16_sve(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
104
|
+
SIMSIMD_PUBLIC void simsimd_l2_bf16_sve(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
105
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_bf16_sve(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
106
|
+
SIMSIMD_PUBLIC void simsimd_cos_bf16_sve(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
107
|
+
SIMSIMD_PUBLIC void simsimd_l2_f64_sve(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
108
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_f64_sve(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
109
|
+
SIMSIMD_PUBLIC void simsimd_cos_f64_sve(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
85
110
|
|
|
86
111
|
/* SIMD-powered backends for AVX2 CPUs of Haswell generation and newer, using 32-bit arithmetic over 256-bit words.
|
|
87
112
|
* First demonstrated in 2011, at least one Haswell-based processor was still being sold in 2022 — the Pentium G3420.
|
|
@@ -89,98 +114,189 @@ SIMSIMD_PUBLIC void simsimd_cos_f64_sve(simsimd_f64_t const* a, simsimd_f64_t co
|
|
|
89
114
|
* On other hand, there is no need to implement AVX2 versions of `f32` and `f64` functions, as those are
|
|
90
115
|
* properly vectorized by recent compilers.
|
|
91
116
|
*/
|
|
92
|
-
SIMSIMD_PUBLIC void
|
|
93
|
-
SIMSIMD_PUBLIC void
|
|
94
|
-
SIMSIMD_PUBLIC void
|
|
95
|
-
SIMSIMD_PUBLIC void
|
|
96
|
-
SIMSIMD_PUBLIC void
|
|
97
|
-
SIMSIMD_PUBLIC void
|
|
98
|
-
SIMSIMD_PUBLIC void
|
|
99
|
-
SIMSIMD_PUBLIC void
|
|
117
|
+
SIMSIMD_PUBLIC void simsimd_l2_i8_haswell(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
118
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_i8_haswell(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
119
|
+
SIMSIMD_PUBLIC void simsimd_cos_i8_haswell(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
120
|
+
SIMSIMD_PUBLIC void simsimd_l2_f16_haswell(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
121
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_f16_haswell(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
122
|
+
SIMSIMD_PUBLIC void simsimd_cos_f16_haswell(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
123
|
+
SIMSIMD_PUBLIC void simsimd_l2_bf16_haswell(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
124
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_bf16_haswell(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
125
|
+
SIMSIMD_PUBLIC void simsimd_cos_bf16_haswell(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
126
|
+
SIMSIMD_PUBLIC void simsimd_l2_f32_haswell(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
127
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_f32_haswell(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
128
|
+
SIMSIMD_PUBLIC void simsimd_cos_f32_haswell(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
100
129
|
|
|
101
130
|
/* SIMD-powered backends for AVX512 CPUs of Skylake generation and newer, using 32-bit arithmetic over 512-bit words.
|
|
102
131
|
* Skylake was launched in 2015, and discontinued in 2019. Skylake had support for F, CD, VL, DQ, and BW extensions,
|
|
103
132
|
* as well as masked operations. This is enough to supersede auto-vectorization on `f32` and `f64` types.
|
|
104
133
|
*/
|
|
105
|
-
SIMSIMD_PUBLIC void
|
|
106
|
-
SIMSIMD_PUBLIC void
|
|
107
|
-
SIMSIMD_PUBLIC void
|
|
108
|
-
SIMSIMD_PUBLIC void
|
|
134
|
+
SIMSIMD_PUBLIC void simsimd_l2_f32_skylake(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
135
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_f32_skylake(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
136
|
+
SIMSIMD_PUBLIC void simsimd_cos_f32_skylake(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
137
|
+
SIMSIMD_PUBLIC void simsimd_l2_f64_skylake(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
138
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_f64_skylake(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
139
|
+
SIMSIMD_PUBLIC void simsimd_cos_f64_skylake(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
109
140
|
|
|
110
141
|
/* SIMD-powered backends for AVX512 CPUs of Ice Lake generation and newer, using mixed arithmetic over 512-bit words.
|
|
111
142
|
* Ice Lake added VNNI, VPOPCNTDQ, IFMA, VBMI, VAES, GFNI, VBMI2, BITALG, VPCLMULQDQ, and other extensions for integral operations.
|
|
112
143
|
* Sapphire Rapids added tiled matrix operations, but we are most interested in the new mixed-precision FMA instructions.
|
|
113
144
|
*/
|
|
114
|
-
SIMSIMD_PUBLIC void
|
|
115
|
-
SIMSIMD_PUBLIC void
|
|
116
|
-
SIMSIMD_PUBLIC void
|
|
117
|
-
SIMSIMD_PUBLIC void
|
|
118
|
-
SIMSIMD_PUBLIC void
|
|
119
|
-
SIMSIMD_PUBLIC void
|
|
145
|
+
SIMSIMD_PUBLIC void simsimd_l2_i8_ice(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
146
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_i8_ice(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
147
|
+
SIMSIMD_PUBLIC void simsimd_cos_i8_ice(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
148
|
+
SIMSIMD_PUBLIC void simsimd_l2_bf16_genoa(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
149
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_bf16_genoa(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
150
|
+
SIMSIMD_PUBLIC void simsimd_cos_bf16_genoa(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
151
|
+
SIMSIMD_PUBLIC void simsimd_l2_f16_sapphire(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
152
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_f16_sapphire(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
153
|
+
SIMSIMD_PUBLIC void simsimd_cos_f16_sapphire(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
120
154
|
|
|
121
155
|
// clang-format on
|
|
122
156
|
|
|
123
|
-
#define SIMSIMD_MAKE_L2SQ(name, input_type, accumulator_type,
|
|
157
|
+
#define SIMSIMD_MAKE_L2SQ(name, input_type, accumulator_type, load_and_convert) \
|
|
124
158
|
SIMSIMD_PUBLIC void simsimd_l2sq_##input_type##_##name(simsimd_##input_type##_t const* a, \
|
|
125
159
|
simsimd_##input_type##_t const* b, simsimd_size_t n, \
|
|
126
160
|
simsimd_distance_t* result) { \
|
|
127
161
|
simsimd_##accumulator_type##_t d2 = 0; \
|
|
128
162
|
for (simsimd_size_t i = 0; i != n; ++i) { \
|
|
129
|
-
simsimd_##accumulator_type##_t ai =
|
|
130
|
-
simsimd_##accumulator_type##_t bi =
|
|
163
|
+
simsimd_##accumulator_type##_t ai = load_and_convert(a + i); \
|
|
164
|
+
simsimd_##accumulator_type##_t bi = load_and_convert(b + i); \
|
|
131
165
|
d2 += (ai - bi) * (ai - bi); \
|
|
132
166
|
} \
|
|
133
167
|
*result = d2; \
|
|
134
168
|
}
|
|
135
169
|
|
|
136
|
-
#define
|
|
170
|
+
#define SIMSIMD_MAKE_L2(name, input_type, accumulator_type, load_and_convert) \
|
|
171
|
+
SIMSIMD_PUBLIC void simsimd_l2_##input_type##_##name(simsimd_##input_type##_t const* a, \
|
|
172
|
+
simsimd_##input_type##_t const* b, simsimd_size_t n, \
|
|
173
|
+
simsimd_distance_t* result) { \
|
|
174
|
+
simsimd_l2sq_##input_type##_##name(a, b, n, result); \
|
|
175
|
+
*result = SIMSIMD_SQRT(*result); \
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
#define SIMSIMD_MAKE_COS(name, input_type, accumulator_type, load_and_convert) \
|
|
137
179
|
SIMSIMD_PUBLIC void simsimd_cos_##input_type##_##name(simsimd_##input_type##_t const* a, \
|
|
138
180
|
simsimd_##input_type##_t const* b, simsimd_size_t n, \
|
|
139
181
|
simsimd_distance_t* result) { \
|
|
140
182
|
simsimd_##accumulator_type##_t ab = 0, a2 = 0, b2 = 0; \
|
|
141
183
|
for (simsimd_size_t i = 0; i != n; ++i) { \
|
|
142
|
-
simsimd_##accumulator_type##_t ai =
|
|
143
|
-
simsimd_##accumulator_type##_t bi =
|
|
184
|
+
simsimd_##accumulator_type##_t ai = load_and_convert(a + i); \
|
|
185
|
+
simsimd_##accumulator_type##_t bi = load_and_convert(b + i); \
|
|
144
186
|
ab += ai * bi; \
|
|
145
187
|
a2 += ai * ai; \
|
|
146
188
|
b2 += bi * bi; \
|
|
147
189
|
} \
|
|
148
|
-
|
|
190
|
+
if (a2 == 0 && b2 == 0) { \
|
|
191
|
+
*result = 0; \
|
|
192
|
+
} else if (ab == 0) { \
|
|
193
|
+
*result = 1; \
|
|
194
|
+
} else { \
|
|
195
|
+
simsimd_distance_t unclipped_result = 1 - ab * SIMSIMD_RSQRT(a2) * SIMSIMD_RSQRT(b2); \
|
|
196
|
+
*result = unclipped_result > 0 ? unclipped_result : 0; \
|
|
197
|
+
} \
|
|
149
198
|
}
|
|
150
199
|
|
|
151
|
-
|
|
152
|
-
|
|
200
|
+
SIMSIMD_MAKE_COS(serial, f64, f64, SIMSIMD_DEREFERENCE) // simsimd_cos_f64_serial
|
|
201
|
+
SIMSIMD_MAKE_L2SQ(serial, f64, f64, SIMSIMD_DEREFERENCE) // simsimd_l2sq_f64_serial
|
|
202
|
+
SIMSIMD_MAKE_L2(serial, f64, f64, SIMSIMD_DEREFERENCE) // simsimd_l2_f64_serial
|
|
153
203
|
|
|
154
|
-
|
|
155
|
-
|
|
204
|
+
SIMSIMD_MAKE_COS(serial, f32, f32, SIMSIMD_DEREFERENCE) // simsimd_cos_f32_serial
|
|
205
|
+
SIMSIMD_MAKE_L2SQ(serial, f32, f32, SIMSIMD_DEREFERENCE) // simsimd_l2sq_f32_serial
|
|
206
|
+
SIMSIMD_MAKE_L2(serial, f32, f32, SIMSIMD_DEREFERENCE) // simsimd_l2_f32_serial
|
|
156
207
|
|
|
157
|
-
|
|
158
|
-
|
|
208
|
+
SIMSIMD_MAKE_COS(serial, f16, f32, SIMSIMD_F16_TO_F32) // simsimd_cos_f16_serial
|
|
209
|
+
SIMSIMD_MAKE_L2SQ(serial, f16, f32, SIMSIMD_F16_TO_F32) // simsimd_l2sq_f16_serial
|
|
210
|
+
SIMSIMD_MAKE_L2(serial, f16, f32, SIMSIMD_F16_TO_F32) // simsimd_l2_f16_serial
|
|
159
211
|
|
|
160
|
-
|
|
161
|
-
|
|
212
|
+
SIMSIMD_MAKE_COS(serial, bf16, f32, SIMSIMD_BF16_TO_F32) // simsimd_cos_bf16_serial
|
|
213
|
+
SIMSIMD_MAKE_L2SQ(serial, bf16, f32, SIMSIMD_BF16_TO_F32) // simsimd_l2sq_bf16_serial
|
|
214
|
+
SIMSIMD_MAKE_L2(serial, bf16, f32, SIMSIMD_BF16_TO_F32) // simsimd_l2_bf16_serial
|
|
162
215
|
|
|
163
|
-
|
|
164
|
-
|
|
216
|
+
SIMSIMD_MAKE_COS(serial, i8, i32, SIMSIMD_DEREFERENCE) // simsimd_cos_i8_serial
|
|
217
|
+
SIMSIMD_MAKE_L2SQ(serial, i8, i32, SIMSIMD_DEREFERENCE) // simsimd_l2sq_i8_serial
|
|
218
|
+
SIMSIMD_MAKE_L2(serial, i8, i32, SIMSIMD_DEREFERENCE) // simsimd_l2_i8_serial
|
|
165
219
|
|
|
166
|
-
|
|
167
|
-
|
|
220
|
+
SIMSIMD_MAKE_COS(accurate, f32, f64, SIMSIMD_DEREFERENCE) // simsimd_cos_f32_accurate
|
|
221
|
+
SIMSIMD_MAKE_L2SQ(accurate, f32, f64, SIMSIMD_DEREFERENCE) // simsimd_l2sq_f32_accurate
|
|
222
|
+
SIMSIMD_MAKE_L2(accurate, f32, f64, SIMSIMD_DEREFERENCE) // simsimd_l2_f32_accurate
|
|
168
223
|
|
|
169
|
-
|
|
170
|
-
|
|
224
|
+
SIMSIMD_MAKE_COS(accurate, f16, f64, SIMSIMD_F16_TO_F32) // simsimd_cos_f16_accurate
|
|
225
|
+
SIMSIMD_MAKE_L2SQ(accurate, f16, f64, SIMSIMD_F16_TO_F32) // simsimd_l2sq_f16_accurate
|
|
226
|
+
SIMSIMD_MAKE_L2(accurate, f16, f64, SIMSIMD_F16_TO_F32) // simsimd_l2_f16_accurate
|
|
171
227
|
|
|
172
|
-
|
|
173
|
-
|
|
228
|
+
SIMSIMD_MAKE_COS(accurate, bf16, f64, SIMSIMD_BF16_TO_F32) // simsimd_cos_bf16_accurate
|
|
229
|
+
SIMSIMD_MAKE_L2SQ(accurate, bf16, f64, SIMSIMD_BF16_TO_F32) // simsimd_l2sq_bf16_accurate
|
|
230
|
+
SIMSIMD_MAKE_L2(accurate, bf16, f64, SIMSIMD_BF16_TO_F32) // simsimd_l2_bf16_accurate
|
|
174
231
|
|
|
175
|
-
|
|
176
|
-
|
|
232
|
+
SIMSIMD_MAKE_COS(accurate, i8, i32, SIMSIMD_DEREFERENCE) // simsimd_cos_i8_accurate
|
|
233
|
+
SIMSIMD_MAKE_L2SQ(accurate, i8, i32, SIMSIMD_DEREFERENCE) // simsimd_l2sq_i8_accurate
|
|
234
|
+
SIMSIMD_MAKE_L2(accurate, i8, i32, SIMSIMD_DEREFERENCE) // simsimd_l2_i8_accurate
|
|
177
235
|
|
|
178
236
|
#if SIMSIMD_TARGET_ARM
|
|
179
237
|
#if SIMSIMD_TARGET_NEON
|
|
180
238
|
#pragma GCC push_options
|
|
181
|
-
#pragma GCC target("+simd")
|
|
182
|
-
#pragma clang attribute push(__attribute__((target("+simd"))), apply_to = function)
|
|
239
|
+
#pragma GCC target("arch=armv8.2-a+simd")
|
|
240
|
+
#pragma clang attribute push(__attribute__((target("arch=armv8.2-a+simd"))), apply_to = function)
|
|
241
|
+
|
|
242
|
+
SIMSIMD_INTERNAL simsimd_f32_t _simsimd_sqrt_f32_neon(simsimd_f32_t x) {
|
|
243
|
+
return vget_lane_f32(vsqrt_f32(vdup_n_f32(x)), 0);
|
|
244
|
+
}
|
|
245
|
+
SIMSIMD_INTERNAL simsimd_f64_t _simsimd_sqrt_f64_neon(simsimd_f64_t x) {
|
|
246
|
+
return vget_lane_f64(vsqrt_f64(vdup_n_f64(x)), 0);
|
|
247
|
+
}
|
|
248
|
+
SIMSIMD_INTERNAL simsimd_distance_t _simsimd_cos_normalize_f32_neon(simsimd_f32_t ab, simsimd_f32_t a2,
|
|
249
|
+
simsimd_f32_t b2) {
|
|
250
|
+
if (a2 == 0 && b2 == 0)
|
|
251
|
+
return 0;
|
|
252
|
+
if (ab == 0)
|
|
253
|
+
return 1;
|
|
254
|
+
simsimd_f32_t squares_arr[2] = {a2, b2};
|
|
255
|
+
float32x2_t squares = vld1_f32(squares_arr);
|
|
256
|
+
// Unlike x86, Arm NEON manuals don't explicitly mention the accuracy of their `rsqrt` approximation.
|
|
257
|
+
// Third party research suggests, that it's less accurate than SSE instructions, having an error of 1.5*2^-12.
|
|
258
|
+
// One or two rounds of Newton-Raphson refinement are recommended to improve the accuracy.
|
|
259
|
+
// https://github.com/lighttransport/embree-aarch64/issues/24
|
|
260
|
+
// https://github.com/lighttransport/embree-aarch64/blob/3f75f8cb4e553d13dced941b5fefd4c826835a6b/common/math/math.h#L137-L145
|
|
261
|
+
float32x2_t rsqrts = vrsqrte_f32(squares);
|
|
262
|
+
// Perform two rounds of Newton-Raphson refinement:
|
|
263
|
+
// https://en.wikipedia.org/wiki/Newton%27s_method
|
|
264
|
+
rsqrts = vmul_f32(rsqrts, vrsqrts_f32(vmul_f32(squares, rsqrts), rsqrts));
|
|
265
|
+
rsqrts = vmul_f32(rsqrts, vrsqrts_f32(vmul_f32(squares, rsqrts), rsqrts));
|
|
266
|
+
vst1_f32(squares_arr, rsqrts);
|
|
267
|
+
simsimd_distance_t result = 1 - ab * squares_arr[0] * squares_arr[1];
|
|
268
|
+
return result > 0 ? result : 0;
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
SIMSIMD_INTERNAL simsimd_distance_t _simsimd_cos_normalize_f64_neon(simsimd_f64_t ab, simsimd_f64_t a2,
|
|
272
|
+
simsimd_f64_t b2) {
|
|
273
|
+
if (a2 == 0 && b2 == 0)
|
|
274
|
+
return 0;
|
|
275
|
+
if (ab == 0)
|
|
276
|
+
return 1;
|
|
277
|
+
simsimd_f64_t squares_arr[2] = {a2, b2};
|
|
278
|
+
float64x2_t squares = vld1q_f64(squares_arr);
|
|
279
|
+
|
|
280
|
+
// Unlike x86, Arm NEON manuals don't explicitly mention the accuracy of their `rsqrt` approximation.
|
|
281
|
+
// Third party research suggests, that it's less accurate than SSE instructions, having an error of 1.5*2^-12.
|
|
282
|
+
// One or two rounds of Newton-Raphson refinement are recommended to improve the accuracy.
|
|
283
|
+
// https://github.com/lighttransport/embree-aarch64/issues/24
|
|
284
|
+
// https://github.com/lighttransport/embree-aarch64/blob/3f75f8cb4e553d13dced941b5fefd4c826835a6b/common/math/math.h#L137-L145
|
|
285
|
+
float64x2_t rsqrts = vrsqrteq_f64(squares);
|
|
286
|
+
// Perform two rounds of Newton-Raphson refinement:
|
|
287
|
+
// https://en.wikipedia.org/wiki/Newton%27s_method
|
|
288
|
+
rsqrts = vmulq_f64(rsqrts, vrsqrtsq_f64(vmulq_f64(squares, rsqrts), rsqrts));
|
|
289
|
+
rsqrts = vmulq_f64(rsqrts, vrsqrtsq_f64(vmulq_f64(squares, rsqrts), rsqrts));
|
|
290
|
+
vst1q_f64(squares_arr, rsqrts);
|
|
291
|
+
simsimd_distance_t result = 1 - ab * squares_arr[0] * squares_arr[1];
|
|
292
|
+
return result > 0 ? result : 0;
|
|
293
|
+
}
|
|
183
294
|
|
|
295
|
+
SIMSIMD_PUBLIC void simsimd_l2_f32_neon(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n,
|
|
296
|
+
simsimd_distance_t* result) {
|
|
297
|
+
simsimd_l2sq_f32_neon(a, b, n, result);
|
|
298
|
+
*result = _simsimd_sqrt_f64_neon(*result);
|
|
299
|
+
}
|
|
184
300
|
SIMSIMD_PUBLIC void simsimd_l2sq_f32_neon(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n,
|
|
185
301
|
simsimd_distance_t* result) {
|
|
186
302
|
float32x4_t sum_vec = vdupq_n_f32(0);
|
|
@@ -216,95 +332,117 @@ SIMSIMD_PUBLIC void simsimd_cos_f32_neon(simsimd_f32_t const* a, simsimd_f32_t c
|
|
|
216
332
|
ab += ai * bi, a2 += ai * ai, b2 += bi * bi;
|
|
217
333
|
}
|
|
218
334
|
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
335
|
+
*result = _simsimd_cos_normalize_f64_neon(ab, a2, b2);
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
SIMSIMD_PUBLIC void simsimd_l2_f64_neon(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n,
|
|
339
|
+
simsimd_distance_t* result) {
|
|
340
|
+
simsimd_l2sq_f64_neon(a, b, n, result);
|
|
341
|
+
*result = _simsimd_sqrt_f64_neon(*result);
|
|
342
|
+
}
|
|
343
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_f64_neon(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n,
|
|
344
|
+
simsimd_distance_t* result) {
|
|
345
|
+
float64x2_t sum_vec = vdupq_n_f64(0);
|
|
346
|
+
simsimd_size_t i = 0;
|
|
347
|
+
for (; i + 2 <= n; i += 2) {
|
|
348
|
+
float64x2_t a_vec = vld1q_f64(a + i);
|
|
349
|
+
float64x2_t b_vec = vld1q_f64(b + i);
|
|
350
|
+
float64x2_t diff_vec = vsubq_f64(a_vec, b_vec);
|
|
351
|
+
sum_vec = vfmaq_f64(sum_vec, diff_vec, diff_vec);
|
|
352
|
+
}
|
|
353
|
+
simsimd_f64_t sum = vaddvq_f64(sum_vec);
|
|
354
|
+
for (; i < n; ++i) {
|
|
355
|
+
simsimd_f64_t diff = a[i] - b[i];
|
|
356
|
+
sum += diff * diff;
|
|
357
|
+
}
|
|
358
|
+
*result = sum;
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
SIMSIMD_PUBLIC void simsimd_cos_f64_neon(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n,
|
|
362
|
+
simsimd_distance_t* result) {
|
|
363
|
+
float64x2_t ab_vec = vdupq_n_f64(0), a2_vec = vdupq_n_f64(0), b2_vec = vdupq_n_f64(0);
|
|
364
|
+
simsimd_size_t i = 0;
|
|
365
|
+
for (; i + 2 <= n; i += 2) {
|
|
366
|
+
float64x2_t a_vec = vld1q_f64(a + i);
|
|
367
|
+
float64x2_t b_vec = vld1q_f64(b + i);
|
|
368
|
+
ab_vec = vfmaq_f64(ab_vec, a_vec, b_vec);
|
|
369
|
+
a2_vec = vfmaq_f64(a2_vec, a_vec, a_vec);
|
|
370
|
+
b2_vec = vfmaq_f64(b2_vec, b_vec, b_vec);
|
|
371
|
+
}
|
|
372
|
+
simsimd_f64_t ab = vaddvq_f64(ab_vec), a2 = vaddvq_f64(a2_vec), b2 = vaddvq_f64(b2_vec);
|
|
373
|
+
for (; i < n; ++i) {
|
|
374
|
+
simsimd_f64_t ai = a[i], bi = b[i];
|
|
375
|
+
ab += ai * bi, a2 += ai * ai, b2 += bi * bi;
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
*result = _simsimd_cos_normalize_f64_neon(ab, a2, b2);
|
|
223
379
|
}
|
|
224
380
|
|
|
225
381
|
#pragma clang attribute pop
|
|
226
382
|
#pragma GCC pop_options
|
|
383
|
+
#endif // SIMSIMD_TARGET_NEON
|
|
227
384
|
|
|
385
|
+
#if SIMSIMD_TARGET_NEON_F16
|
|
228
386
|
#pragma GCC push_options
|
|
229
|
-
#pragma GCC target("+simd+fp16")
|
|
230
|
-
#pragma clang attribute push(__attribute__((target("+simd+fp16"))), apply_to = function)
|
|
387
|
+
#pragma GCC target("arch=armv8.2-a+simd+fp16")
|
|
388
|
+
#pragma clang attribute push(__attribute__((target("arch=armv8.2-a+simd+fp16"))), apply_to = function)
|
|
231
389
|
|
|
390
|
+
SIMSIMD_PUBLIC void simsimd_l2_f16_neon(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n,
|
|
391
|
+
simsimd_distance_t* result) {
|
|
392
|
+
simsimd_l2sq_f16_neon(a, b, n, result);
|
|
393
|
+
*result = _simsimd_sqrt_f32_neon(*result);
|
|
394
|
+
}
|
|
232
395
|
SIMSIMD_PUBLIC void simsimd_l2sq_f16_neon(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n,
|
|
233
396
|
simsimd_distance_t* result) {
|
|
397
|
+
float32x4_t a_vec, b_vec;
|
|
234
398
|
float32x4_t sum_vec = vdupq_n_f32(0);
|
|
235
|
-
simsimd_size_t i = 0;
|
|
236
|
-
for (; i + 4 <= n; i += 4) {
|
|
237
|
-
float32x4_t a_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const*)a + i));
|
|
238
|
-
float32x4_t b_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const*)b + i));
|
|
239
|
-
float32x4_t diff_vec = vsubq_f32(a_vec, b_vec);
|
|
240
|
-
sum_vec = vfmaq_f32(sum_vec, diff_vec, diff_vec);
|
|
241
|
-
}
|
|
242
399
|
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
for (; i < n; ++i, ++j)
|
|
253
|
-
a_padded_tail.f16[j] = a[i], b_padded_tail.f16[j] = b[i];
|
|
254
|
-
for (; j < 4; ++j)
|
|
255
|
-
a_padded_tail.f16[j] = 0, b_padded_tail.f16[j] = 0;
|
|
256
|
-
float32x4_t diff_vec = vsubq_f32(vcvt_f32_f16(a_padded_tail.f16_vec), vcvt_f32_f16(b_padded_tail.f16_vec));
|
|
257
|
-
sum_vec = vfmaq_f32(sum_vec, diff_vec, diff_vec);
|
|
400
|
+
simsimd_l2sq_f16_neon_cycle:
|
|
401
|
+
if (n < 4) {
|
|
402
|
+
a_vec = vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(a, n));
|
|
403
|
+
b_vec = vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(b, n));
|
|
404
|
+
n = 0;
|
|
405
|
+
} else {
|
|
406
|
+
a_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const*)a));
|
|
407
|
+
b_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const*)b));
|
|
408
|
+
n -= 4, a += 4, b += 4;
|
|
258
409
|
}
|
|
410
|
+
float32x4_t diff_vec = vsubq_f32(a_vec, b_vec);
|
|
411
|
+
sum_vec = vfmaq_f32(sum_vec, diff_vec, diff_vec);
|
|
412
|
+
if (n)
|
|
413
|
+
goto simsimd_l2sq_f16_neon_cycle;
|
|
259
414
|
|
|
260
|
-
|
|
261
|
-
*result = sum;
|
|
415
|
+
*result = vaddvq_f32(sum_vec);
|
|
262
416
|
}
|
|
263
417
|
|
|
264
418
|
SIMSIMD_PUBLIC void simsimd_cos_f16_neon(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n,
|
|
265
419
|
simsimd_distance_t* result) {
|
|
266
420
|
float32x4_t ab_vec = vdupq_n_f32(0), a2_vec = vdupq_n_f32(0), b2_vec = vdupq_n_f32(0);
|
|
267
|
-
|
|
268
|
-
for (; i + 4 <= n; i += 4) {
|
|
269
|
-
float32x4_t a_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const*)a + i));
|
|
270
|
-
float32x4_t b_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const*)b + i));
|
|
271
|
-
ab_vec = vfmaq_f32(ab_vec, a_vec, b_vec);
|
|
272
|
-
a2_vec = vfmaq_f32(a2_vec, a_vec, a_vec);
|
|
273
|
-
b2_vec = vfmaq_f32(b2_vec, b_vec, b_vec);
|
|
274
|
-
}
|
|
421
|
+
float32x4_t a_vec, b_vec;
|
|
275
422
|
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
for (; i < n; ++i, ++j)
|
|
286
|
-
a_padded_tail.f16[j] = a[i], b_padded_tail.f16[j] = b[i];
|
|
287
|
-
for (; j < 4; ++j)
|
|
288
|
-
a_padded_tail.f16[j] = 0, b_padded_tail.f16[j] = 0;
|
|
289
|
-
float32x4_t a_vec = vcvt_f32_f16(a_padded_tail.f16_vec);
|
|
290
|
-
float32x4_t b_vec = vcvt_f32_f16(b_padded_tail.f16_vec);
|
|
291
|
-
ab_vec = vfmaq_f32(ab_vec, a_vec, b_vec);
|
|
292
|
-
a2_vec = vfmaq_f32(a2_vec, a_vec, a_vec);
|
|
293
|
-
b2_vec = vfmaq_f32(b2_vec, b_vec, b_vec);
|
|
423
|
+
simsimd_cos_f16_neon_cycle:
|
|
424
|
+
if (n < 4) {
|
|
425
|
+
a_vec = vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(a, n));
|
|
426
|
+
b_vec = vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(b, n));
|
|
427
|
+
n = 0;
|
|
428
|
+
} else {
|
|
429
|
+
a_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const*)a));
|
|
430
|
+
b_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const*)b));
|
|
431
|
+
n -= 4, a += 4, b += 4;
|
|
294
432
|
}
|
|
433
|
+
ab_vec = vfmaq_f32(ab_vec, a_vec, b_vec);
|
|
434
|
+
a2_vec = vfmaq_f32(a2_vec, a_vec, a_vec);
|
|
435
|
+
b2_vec = vfmaq_f32(b2_vec, b_vec, b_vec);
|
|
436
|
+
if (n)
|
|
437
|
+
goto simsimd_cos_f16_neon_cycle;
|
|
295
438
|
|
|
296
|
-
// Avoid `simsimd_approximate_inverse_square_root` on Arm NEON
|
|
297
439
|
simsimd_f32_t ab = vaddvq_f32(ab_vec), a2 = vaddvq_f32(a2_vec), b2 = vaddvq_f32(b2_vec);
|
|
298
|
-
|
|
299
|
-
float32x2_t a2_b2 = vld1_f32(a2_b2_arr);
|
|
300
|
-
a2_b2 = vrsqrte_f32(a2_b2);
|
|
301
|
-
vst1_f32(a2_b2_arr, a2_b2);
|
|
302
|
-
*result = ab != 0 ? 1 - ab * a2_b2_arr[0] * a2_b2_arr[1] : 1;
|
|
440
|
+
*result = _simsimd_cos_normalize_f32_neon(ab, a2, b2);
|
|
303
441
|
}
|
|
304
442
|
|
|
305
443
|
#pragma clang attribute pop
|
|
306
444
|
#pragma GCC pop_options
|
|
307
|
-
#endif //
|
|
445
|
+
#endif // SIMSIMD_TARGET_NEON_F16
|
|
308
446
|
|
|
309
447
|
#if SIMSIMD_TARGET_NEON_BF16
|
|
310
448
|
#pragma GCC push_options
|
|
@@ -341,95 +479,76 @@ SIMSIMD_PUBLIC void simsimd_cos_bf16_neon(simsimd_bf16_t const* a, simsimd_bf16_
|
|
|
341
479
|
// }
|
|
342
480
|
// float32x4_t products_vec = vaddq_f32(products_high_vec, products_low_vec);
|
|
343
481
|
// simsimd_f32_t a2 = products_vec[0], ab = products_vec[1], b2 = products_vec[3];
|
|
482
|
+
//
|
|
483
|
+
// Another way of accomplishing the same thing is to process the odd and even elements separately,
|
|
484
|
+
// using special `vbfmlaltq_f32` and `vbfmlalbq_f32` intrinsics:
|
|
485
|
+
//
|
|
486
|
+
// ab_high_vec = vbfmlaltq_f32(ab_high_vec, a_vec, b_vec);
|
|
487
|
+
// ab_low_vec = vbfmlalbq_f32(ab_low_vec, a_vec, b_vec);
|
|
488
|
+
// a2_high_vec = vbfmlaltq_f32(a2_high_vec, a_vec, a_vec);
|
|
489
|
+
// a2_low_vec = vbfmlalbq_f32(a2_low_vec, a_vec, a_vec);
|
|
490
|
+
// b2_high_vec = vbfmlaltq_f32(b2_high_vec, b_vec, b_vec);
|
|
491
|
+
// b2_low_vec = vbfmlalbq_f32(b2_low_vec, b_vec, b_vec);
|
|
492
|
+
//
|
|
344
493
|
|
|
345
|
-
float32x4_t
|
|
346
|
-
float32x4_t
|
|
347
|
-
float32x4_t
|
|
348
|
-
|
|
349
|
-
for (; i + 8 <= n; i += 8) {
|
|
350
|
-
bfloat16x8_t a_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const*)a + i);
|
|
351
|
-
bfloat16x8_t b_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const*)b + i);
|
|
352
|
-
ab_high_vec = vbfmlaltq_f32(ab_high_vec, a_vec, b_vec);
|
|
353
|
-
ab_low_vec = vbfmlalbq_f32(ab_low_vec, a_vec, b_vec);
|
|
354
|
-
a2_high_vec = vbfmlaltq_f32(a2_high_vec, a_vec, a_vec);
|
|
355
|
-
a2_low_vec = vbfmlalbq_f32(a2_low_vec, a_vec, a_vec);
|
|
356
|
-
b2_high_vec = vbfmlaltq_f32(b2_high_vec, b_vec, b_vec);
|
|
357
|
-
b2_low_vec = vbfmlalbq_f32(b2_low_vec, b_vec, b_vec);
|
|
358
|
-
}
|
|
494
|
+
float32x4_t ab_vec = vdupq_n_f32(0);
|
|
495
|
+
float32x4_t a2_vec = vdupq_n_f32(0);
|
|
496
|
+
float32x4_t b2_vec = vdupq_n_f32(0);
|
|
497
|
+
bfloat16x8_t a_vec, b_vec;
|
|
359
498
|
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
for (; i < n; ++i, ++j)
|
|
370
|
-
a_padded_tail.bf16[j] = a[i], b_padded_tail.bf16[j] = b[i];
|
|
371
|
-
for (; j < 8; ++j)
|
|
372
|
-
a_padded_tail.bf16[j] = 0, b_padded_tail.bf16[j] = 0;
|
|
373
|
-
ab_high_vec = vbfmlaltq_f32(ab_high_vec, a_padded_tail.bf16_vec, b_padded_tail.bf16_vec);
|
|
374
|
-
ab_low_vec = vbfmlalbq_f32(ab_low_vec, a_padded_tail.bf16_vec, b_padded_tail.bf16_vec);
|
|
375
|
-
a2_high_vec = vbfmlaltq_f32(a2_high_vec, a_padded_tail.bf16_vec, a_padded_tail.bf16_vec);
|
|
376
|
-
a2_low_vec = vbfmlalbq_f32(a2_low_vec, a_padded_tail.bf16_vec, a_padded_tail.bf16_vec);
|
|
377
|
-
b2_high_vec = vbfmlaltq_f32(b2_high_vec, b_padded_tail.bf16_vec, b_padded_tail.bf16_vec);
|
|
378
|
-
b2_low_vec = vbfmlalbq_f32(b2_low_vec, b_padded_tail.bf16_vec, b_padded_tail.bf16_vec);
|
|
499
|
+
simsimd_cos_bf16_neon_cycle:
|
|
500
|
+
if (n < 8) {
|
|
501
|
+
a_vec = _simsimd_partial_load_bf16x8_neon(a, n);
|
|
502
|
+
b_vec = _simsimd_partial_load_bf16x8_neon(b, n);
|
|
503
|
+
n = 0;
|
|
504
|
+
} else {
|
|
505
|
+
a_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const*)a);
|
|
506
|
+
b_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const*)b);
|
|
507
|
+
n -= 8, a += 8, b += 8;
|
|
379
508
|
}
|
|
509
|
+
ab_vec = vbfdotq_f32(ab_vec, a_vec, b_vec);
|
|
510
|
+
a2_vec = vbfdotq_f32(a2_vec, a_vec, a_vec);
|
|
511
|
+
b2_vec = vbfdotq_f32(b2_vec, b_vec, b_vec);
|
|
512
|
+
if (n)
|
|
513
|
+
goto simsimd_cos_bf16_neon_cycle;
|
|
380
514
|
|
|
381
515
|
// Avoid `simsimd_approximate_inverse_square_root` on Arm NEON
|
|
382
|
-
simsimd_f32_t ab = vaddvq_f32(
|
|
383
|
-
|
|
384
|
-
b2 = vaddvq_f32(vaddq_f32(b2_high_vec, b2_low_vec));
|
|
385
|
-
simsimd_f32_t a2_b2_arr[2] = {a2, b2};
|
|
386
|
-
float32x2_t a2_b2 = vld1_f32(a2_b2_arr);
|
|
387
|
-
a2_b2 = vrsqrte_f32(a2_b2);
|
|
388
|
-
vst1_f32(a2_b2_arr, a2_b2);
|
|
389
|
-
*result = ab != 0 ? 1 - ab * a2_b2_arr[0] * a2_b2_arr[1] : 1;
|
|
516
|
+
simsimd_f32_t ab = vaddvq_f32(ab_vec), a2 = vaddvq_f32(a2_vec), b2 = vaddvq_f32(b2_vec);
|
|
517
|
+
*result = _simsimd_cos_normalize_f32_neon(ab, a2, b2);
|
|
390
518
|
}
|
|
391
519
|
|
|
520
|
+
SIMSIMD_PUBLIC void simsimd_l2_bf16_neon(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n,
|
|
521
|
+
simsimd_distance_t* result) {
|
|
522
|
+
simsimd_l2sq_bf16_neon(a, b, n, result);
|
|
523
|
+
*result = _simsimd_sqrt_f64_neon(*result);
|
|
524
|
+
}
|
|
392
525
|
SIMSIMD_PUBLIC void simsimd_l2sq_bf16_neon(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n,
|
|
393
526
|
simsimd_distance_t* result) {
|
|
394
|
-
float32x4_t diff_high_vec
|
|
527
|
+
float32x4_t diff_high_vec, diff_low_vec;
|
|
395
528
|
float32x4_t sum_high_vec = vdupq_n_f32(0), sum_low_vec = vdupq_n_f32(0);
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
bfloat16x8_t
|
|
400
|
-
|
|
401
|
-
// and then subtract, converting back to `bf16` for computing the squared difference.
|
|
529
|
+
|
|
530
|
+
simsimd_l2sq_bf16_neon_cycle:
|
|
531
|
+
if (n < 8) {
|
|
532
|
+
bfloat16x8_t a_vec = _simsimd_partial_load_bf16x8_neon(a, n);
|
|
533
|
+
bfloat16x8_t b_vec = _simsimd_partial_load_bf16x8_neon(b, n);
|
|
402
534
|
diff_high_vec = vsubq_f32(vcvt_f32_bf16(vget_high_bf16(a_vec)), vcvt_f32_bf16(vget_high_bf16(b_vec)));
|
|
403
535
|
diff_low_vec = vsubq_f32(vcvt_f32_bf16(vget_low_bf16(a_vec)), vcvt_f32_bf16(vget_low_bf16(b_vec)));
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
if (i < n) {
|
|
412
|
-
union {
|
|
413
|
-
bfloat16x8_t bf16_vec;
|
|
414
|
-
simsimd_bf16_t bf16[8];
|
|
415
|
-
} a_padded_tail, b_padded_tail;
|
|
416
|
-
simsimd_size_t j = 0;
|
|
417
|
-
for (; i < n; ++i, ++j)
|
|
418
|
-
a_padded_tail.bf16[j] = a[i], b_padded_tail.bf16[j] = b[i];
|
|
419
|
-
for (; j < 8; ++j)
|
|
420
|
-
a_padded_tail.bf16[j] = 0, b_padded_tail.bf16[j] = 0;
|
|
421
|
-
diff_high_vec = vsubq_f32(vcvt_f32_bf16(vget_high_bf16(a_padded_tail.bf16_vec)),
|
|
422
|
-
vcvt_f32_bf16(vget_high_bf16(b_padded_tail.bf16_vec)));
|
|
423
|
-
diff_low_vec = vsubq_f32(vcvt_f32_bf16(vget_low_bf16(a_padded_tail.bf16_vec)),
|
|
424
|
-
vcvt_f32_bf16(vget_low_bf16(b_padded_tail.bf16_vec)));
|
|
425
|
-
sum_high_vec = vfmaq_f32(sum_high_vec, diff_high_vec, diff_high_vec);
|
|
426
|
-
sum_low_vec = vfmaq_f32(sum_low_vec, diff_low_vec, diff_low_vec);
|
|
536
|
+
n = 0;
|
|
537
|
+
} else {
|
|
538
|
+
bfloat16x8_t a_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const*)a);
|
|
539
|
+
bfloat16x8_t b_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const*)b);
|
|
540
|
+
diff_high_vec = vsubq_f32(vcvt_f32_bf16(vget_high_bf16(a_vec)), vcvt_f32_bf16(vget_high_bf16(b_vec)));
|
|
541
|
+
diff_low_vec = vsubq_f32(vcvt_f32_bf16(vget_low_bf16(a_vec)), vcvt_f32_bf16(vget_low_bf16(b_vec)));
|
|
542
|
+
n -= 8, a += 8, b += 8;
|
|
427
543
|
}
|
|
544
|
+
sum_high_vec = vfmaq_f32(sum_high_vec, diff_high_vec, diff_high_vec);
|
|
545
|
+
sum_low_vec = vfmaq_f32(sum_low_vec, diff_low_vec, diff_low_vec);
|
|
546
|
+
if (n)
|
|
547
|
+
goto simsimd_l2sq_bf16_neon_cycle;
|
|
428
548
|
|
|
429
|
-
|
|
430
|
-
simsimd_f32_t sum = vaddvq_f32(vaddq_f32(sum_high_vec, sum_low_vec));
|
|
431
|
-
*result = sum;
|
|
549
|
+
*result = vaddvq_f32(vaddq_f32(sum_high_vec, sum_low_vec));
|
|
432
550
|
}
|
|
551
|
+
|
|
433
552
|
#pragma clang attribute pop
|
|
434
553
|
#pragma GCC pop_options
|
|
435
554
|
#endif // SIMSIMD_TARGET_NEON_BF16
|
|
@@ -439,6 +558,11 @@ SIMSIMD_PUBLIC void simsimd_l2sq_bf16_neon(simsimd_bf16_t const* a, simsimd_bf16
|
|
|
439
558
|
#pragma GCC target("arch=armv8.2-a+dotprod+i8mm")
|
|
440
559
|
#pragma clang attribute push(__attribute__((target("arch=armv8.2-a+dotprod+i8mm"))), apply_to = function)
|
|
441
560
|
|
|
561
|
+
SIMSIMD_PUBLIC void simsimd_l2_i8_neon(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n,
|
|
562
|
+
simsimd_distance_t* result) {
|
|
563
|
+
simsimd_l2sq_i8_neon(a, b, n, result);
|
|
564
|
+
*result = _simsimd_sqrt_f32_neon(*result);
|
|
565
|
+
}
|
|
442
566
|
SIMSIMD_PUBLIC void simsimd_l2sq_i8_neon(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n,
|
|
443
567
|
simsimd_distance_t* result) {
|
|
444
568
|
int32x4_t d2_vec = vdupq_n_s32(0);
|
|
@@ -455,7 +579,7 @@ SIMSIMD_PUBLIC void simsimd_l2sq_i8_neon(simsimd_i8_t const* a, simsimd_i8_t con
|
|
|
455
579
|
}
|
|
456
580
|
int32_t d2 = vaddvq_s32(d2_vec);
|
|
457
581
|
for (; i < n; ++i) {
|
|
458
|
-
int32_t n = a[i] - b[i];
|
|
582
|
+
int32_t n = (int32_t)a[i] - b[i];
|
|
459
583
|
d2 += n * n;
|
|
460
584
|
}
|
|
461
585
|
*result = d2;
|
|
@@ -577,12 +701,7 @@ SIMSIMD_PUBLIC void simsimd_cos_i8_neon(simsimd_i8_t const* a, simsimd_i8_t cons
|
|
|
577
701
|
ab += ai * bi, a2 += ai * ai, b2 += bi * bi;
|
|
578
702
|
}
|
|
579
703
|
|
|
580
|
-
|
|
581
|
-
simsimd_f32_t a2_b2_arr[2] = {(simsimd_f32_t)a2, (simsimd_f32_t)b2};
|
|
582
|
-
float32x2_t a2_b2 = vld1_f32(a2_b2_arr);
|
|
583
|
-
a2_b2 = vrsqrte_f32(a2_b2);
|
|
584
|
-
vst1_f32(a2_b2_arr, a2_b2);
|
|
585
|
-
*result = ab != 0 ? 1 - ab * a2_b2_arr[0] * a2_b2_arr[1] : 1;
|
|
704
|
+
*result = _simsimd_cos_normalize_f32_neon(ab, a2, b2);
|
|
586
705
|
}
|
|
587
706
|
|
|
588
707
|
#pragma clang attribute pop
|
|
@@ -591,9 +710,14 @@ SIMSIMD_PUBLIC void simsimd_cos_i8_neon(simsimd_i8_t const* a, simsimd_i8_t cons
|
|
|
591
710
|
|
|
592
711
|
#if SIMSIMD_TARGET_SVE
|
|
593
712
|
#pragma GCC push_options
|
|
594
|
-
#pragma GCC target("+sve")
|
|
595
|
-
#pragma clang attribute push(__attribute__((target("+sve"))), apply_to = function)
|
|
713
|
+
#pragma GCC target("arch=armv8.2-a+sve")
|
|
714
|
+
#pragma clang attribute push(__attribute__((target("arch=armv8.2-a+sve"))), apply_to = function)
|
|
596
715
|
|
|
716
|
+
SIMSIMD_PUBLIC void simsimd_l2_f32_sve(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n,
|
|
717
|
+
simsimd_distance_t* result) {
|
|
718
|
+
simsimd_l2sq_f32_sve(a, b, n, result);
|
|
719
|
+
*result = _simsimd_sqrt_f64_neon(*result);
|
|
720
|
+
}
|
|
597
721
|
SIMSIMD_PUBLIC void simsimd_l2sq_f32_sve(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n,
|
|
598
722
|
simsimd_distance_t* result) {
|
|
599
723
|
simsimd_size_t i = 0;
|
|
@@ -629,15 +753,14 @@ SIMSIMD_PUBLIC void simsimd_cos_f32_sve(simsimd_f32_t const* a, simsimd_f32_t co
|
|
|
629
753
|
simsimd_f32_t ab = svaddv_f32(svptrue_b32(), ab_vec);
|
|
630
754
|
simsimd_f32_t a2 = svaddv_f32(svptrue_b32(), a2_vec);
|
|
631
755
|
simsimd_f32_t b2 = svaddv_f32(svptrue_b32(), b2_vec);
|
|
632
|
-
|
|
633
|
-
// Avoid `simsimd_approximate_inverse_square_root` on Arm NEON
|
|
634
|
-
simsimd_f32_t a2_b2_arr[2] = {a2, b2};
|
|
635
|
-
float32x2_t a2_b2 = vld1_f32(a2_b2_arr);
|
|
636
|
-
a2_b2 = vrsqrte_f32(a2_b2);
|
|
637
|
-
vst1_f32(a2_b2_arr, a2_b2);
|
|
638
|
-
*result = ab != 0 ? 1 - ab * a2_b2_arr[0] * a2_b2_arr[1] : 1;
|
|
756
|
+
*result = _simsimd_cos_normalize_f64_neon(ab, a2, b2);
|
|
639
757
|
}
|
|
640
758
|
|
|
759
|
+
SIMSIMD_PUBLIC void simsimd_l2_f64_sve(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n,
|
|
760
|
+
simsimd_distance_t* result) {
|
|
761
|
+
simsimd_l2sq_f64_sve(a, b, n, result);
|
|
762
|
+
*result = _simsimd_sqrt_f64_neon(*result);
|
|
763
|
+
}
|
|
641
764
|
SIMSIMD_PUBLIC void simsimd_l2sq_f64_sve(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n,
|
|
642
765
|
simsimd_distance_t* result) {
|
|
643
766
|
simsimd_size_t i = 0;
|
|
@@ -673,22 +796,23 @@ SIMSIMD_PUBLIC void simsimd_cos_f64_sve(simsimd_f64_t const* a, simsimd_f64_t co
|
|
|
673
796
|
simsimd_f64_t ab = svaddv_f64(svptrue_b32(), ab_vec);
|
|
674
797
|
simsimd_f64_t a2 = svaddv_f64(svptrue_b32(), a2_vec);
|
|
675
798
|
simsimd_f64_t b2 = svaddv_f64(svptrue_b32(), b2_vec);
|
|
676
|
-
|
|
677
|
-
// Avoid `simsimd_approximate_inverse_square_root` on Arm NEON
|
|
678
|
-
simsimd_f64_t a2_b2_arr[2] = {a2, b2};
|
|
679
|
-
float64x2_t a2_b2 = vld1q_f64(a2_b2_arr);
|
|
680
|
-
a2_b2 = vrsqrteq_f64(a2_b2);
|
|
681
|
-
vst1q_f64(a2_b2_arr, a2_b2);
|
|
682
|
-
*result = ab != 0 ? 1 - ab * a2_b2_arr[0] * a2_b2_arr[1] : 1;
|
|
799
|
+
*result = _simsimd_cos_normalize_f64_neon(ab, a2, b2);
|
|
683
800
|
}
|
|
684
801
|
|
|
685
802
|
#pragma clang attribute pop
|
|
686
803
|
#pragma GCC pop_options
|
|
804
|
+
#endif // SIMSIMD_TARGET_SVE
|
|
687
805
|
|
|
806
|
+
#if SIMSIMD_TARGET_SVE_F16
|
|
688
807
|
#pragma GCC push_options
|
|
689
|
-
#pragma GCC target("+sve+fp16")
|
|
690
|
-
#pragma clang attribute push(__attribute__((target("+sve+fp16"))), apply_to = function)
|
|
808
|
+
#pragma GCC target("arch=armv8.2-a+sve+fp16")
|
|
809
|
+
#pragma clang attribute push(__attribute__((target("arch=armv8.2-a+sve+fp16"))), apply_to = function)
|
|
691
810
|
|
|
811
|
+
SIMSIMD_PUBLIC void simsimd_l2_f16_sve(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n,
|
|
812
|
+
simsimd_distance_t* result) {
|
|
813
|
+
simsimd_l2sq_f16_sve(a, b, n, result);
|
|
814
|
+
*result = _simsimd_sqrt_f32_neon(*result);
|
|
815
|
+
}
|
|
692
816
|
SIMSIMD_PUBLIC void simsimd_l2sq_f16_sve(simsimd_f16_t const* a_enum, simsimd_f16_t const* b_enum, simsimd_size_t n,
|
|
693
817
|
simsimd_distance_t* result) {
|
|
694
818
|
simsimd_size_t i = 0;
|
|
@@ -728,230 +852,283 @@ SIMSIMD_PUBLIC void simsimd_cos_f16_sve(simsimd_f16_t const* a_enum, simsimd_f16
|
|
|
728
852
|
simsimd_f16_for_arm_simd_t ab = svaddv_f16(svptrue_b16(), ab_vec);
|
|
729
853
|
simsimd_f16_for_arm_simd_t a2 = svaddv_f16(svptrue_b16(), a2_vec);
|
|
730
854
|
simsimd_f16_for_arm_simd_t b2 = svaddv_f16(svptrue_b16(), b2_vec);
|
|
855
|
+
*result = _simsimd_cos_normalize_f32_neon(ab, a2, b2);
|
|
856
|
+
}
|
|
731
857
|
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
858
|
+
#pragma clang attribute pop
|
|
859
|
+
#pragma GCC pop_options
|
|
860
|
+
#endif // SIMSIMD_TARGET_SVE_F16
|
|
861
|
+
|
|
862
|
+
#if SIMSIMD_TARGET_SVE_BF16
|
|
863
|
+
#pragma GCC push_options
|
|
864
|
+
#pragma GCC target("arch=armv8.2-a+sve+bf16")
|
|
865
|
+
#pragma clang attribute push(__attribute__((target("arch=armv8.2-a+sve+bf16"))), apply_to = function)
|
|
866
|
+
|
|
867
|
+
SIMSIMD_PUBLIC void simsimd_l2_bf16_sve(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n,
|
|
868
|
+
simsimd_distance_t* result) {
|
|
869
|
+
simsimd_l2sq_bf16_sve(a, b, n, result);
|
|
870
|
+
*result = _simsimd_sqrt_f32_neon(*result);
|
|
871
|
+
}
|
|
872
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_bf16_sve(simsimd_bf16_t const* a_enum, simsimd_bf16_t const* b_enum, simsimd_size_t n,
|
|
873
|
+
simsimd_distance_t* result) {
|
|
874
|
+
simsimd_size_t i = 0;
|
|
875
|
+
svfloat32_t d2_low_vec = svdupq_n_f32(0.f, 0.f, 0.f, 0.f);
|
|
876
|
+
svfloat32_t d2_high_vec = svdupq_n_f32(0.f, 0.f, 0.f, 0.f);
|
|
877
|
+
simsimd_u16_t const* a = (simsimd_u16_t const*)(a_enum);
|
|
878
|
+
simsimd_u16_t const* b = (simsimd_u16_t const*)(b_enum);
|
|
879
|
+
do {
|
|
880
|
+
svbool_t pg_vec = svwhilelt_b16((unsigned int)i, (unsigned int)n);
|
|
881
|
+
svuint16_t a_vec = svld1_u16(pg_vec, a + i);
|
|
882
|
+
svuint16_t b_vec = svld1_u16(pg_vec, b + i);
|
|
883
|
+
|
|
884
|
+
// There is no `bf16` subtraction in SVE, so we need to convert to `u32` and shift.
|
|
885
|
+
svbool_t pg_low_vec = svwhilelt_b32((unsigned int)(i), (unsigned int)n);
|
|
886
|
+
svbool_t pg_high_vec = svwhilelt_b32((unsigned int)(i + svcnth() / 2), (unsigned int)n);
|
|
887
|
+
svfloat32_t a_low_vec = svreinterpret_f32_u32(svlsl_n_u32_x(pg_low_vec, svunpklo_u32(a_vec), 16));
|
|
888
|
+
svfloat32_t a_high_vec = svreinterpret_f32_u32(svlsl_n_u32_x(pg_high_vec, svunpkhi_u32(a_vec), 16));
|
|
889
|
+
svfloat32_t b_low_vec = svreinterpret_f32_u32(svlsl_n_u32_x(pg_low_vec, svunpklo_u32(b_vec), 16));
|
|
890
|
+
svfloat32_t b_high_vec = svreinterpret_f32_u32(svlsl_n_u32_x(pg_high_vec, svunpkhi_u32(b_vec), 16));
|
|
891
|
+
|
|
892
|
+
svfloat32_t a_minus_b_low_vec = svsub_f32_x(pg_low_vec, a_low_vec, b_low_vec);
|
|
893
|
+
svfloat32_t a_minus_b_high_vec = svsub_f32_x(pg_high_vec, a_high_vec, b_high_vec);
|
|
894
|
+
d2_low_vec = svmla_f32_x(pg_vec, d2_low_vec, a_minus_b_low_vec, a_minus_b_low_vec);
|
|
895
|
+
d2_high_vec = svmla_f32_x(pg_vec, d2_high_vec, a_minus_b_low_vec, a_minus_b_low_vec);
|
|
896
|
+
i += svcnth();
|
|
897
|
+
} while (i < n);
|
|
898
|
+
simsimd_f32_t d2 = svaddv_f32(svptrue_b32(), d2_low_vec) + svaddv_f32(svptrue_b32(), d2_high_vec);
|
|
899
|
+
*result = d2;
|
|
900
|
+
}
|
|
901
|
+
|
|
902
|
+
SIMSIMD_PUBLIC void simsimd_cos_bf16_sve(simsimd_bf16_t const* a_enum, simsimd_bf16_t const* b_enum, simsimd_size_t n,
|
|
903
|
+
simsimd_distance_t* result) {
|
|
904
|
+
simsimd_size_t i = 0;
|
|
905
|
+
svfloat32_t ab_vec = svdupq_n_f32(0.f, 0.f, 0.f, 0.f);
|
|
906
|
+
svfloat32_t a2_vec = svdupq_n_f32(0.f, 0.f, 0.f, 0.f);
|
|
907
|
+
svfloat32_t b2_vec = svdupq_n_f32(0.f, 0.f, 0.f, 0.f);
|
|
908
|
+
simsimd_bf16_for_arm_simd_t const* a = (simsimd_bf16_for_arm_simd_t const*)(a_enum);
|
|
909
|
+
simsimd_bf16_for_arm_simd_t const* b = (simsimd_bf16_for_arm_simd_t const*)(b_enum);
|
|
910
|
+
do {
|
|
911
|
+
svbool_t pg_vec = svwhilelt_b16((unsigned int)i, (unsigned int)n);
|
|
912
|
+
svbfloat16_t a_vec = svld1_bf16(pg_vec, a + i);
|
|
913
|
+
svbfloat16_t b_vec = svld1_bf16(pg_vec, b + i);
|
|
914
|
+
ab_vec = svbfdot_f32(ab_vec, a_vec, b_vec);
|
|
915
|
+
a2_vec = svbfdot_f32(a2_vec, a_vec, a_vec);
|
|
916
|
+
b2_vec = svbfdot_f32(b2_vec, b_vec, b_vec);
|
|
917
|
+
i += svcnth();
|
|
918
|
+
} while (i < n);
|
|
919
|
+
|
|
920
|
+
simsimd_f32_t ab = svaddv_f32(svptrue_b32(), ab_vec);
|
|
921
|
+
simsimd_f32_t a2 = svaddv_f32(svptrue_b32(), a2_vec);
|
|
922
|
+
simsimd_f32_t b2 = svaddv_f32(svptrue_b32(), b2_vec);
|
|
923
|
+
*result = _simsimd_cos_normalize_f32_neon(ab, a2, b2);
|
|
738
924
|
}
|
|
739
925
|
|
|
740
926
|
#pragma clang attribute pop
|
|
741
927
|
#pragma GCC pop_options
|
|
742
|
-
#endif //
|
|
928
|
+
#endif // SIMSIMD_TARGET_SVE_BF16
|
|
743
929
|
#endif // SIMSIMD_TARGET_ARM
|
|
744
930
|
|
|
931
|
+
#if SIMSIMD_TARGET_X86
|
|
932
|
+
#if SIMSIMD_TARGET_HASWELL
|
|
933
|
+
#pragma GCC push_options
|
|
934
|
+
#pragma GCC target("avx2")
|
|
935
|
+
#pragma clang attribute push(__attribute__((target("avx2"))), apply_to = function)
|
|
936
|
+
|
|
937
|
+
SIMSIMD_INTERNAL simsimd_f32_t _simsimd_sqrt_f32_haswell(simsimd_f32_t x) {
|
|
938
|
+
return _mm_cvtss_f32(_mm_sqrt_ps(_mm_set_ss(x)));
|
|
939
|
+
}
|
|
940
|
+
SIMSIMD_INTERNAL simsimd_f64_t _simsimd_sqrt_f64_haswell(simsimd_f64_t x) {
|
|
941
|
+
return _mm_cvtsd_f64(_mm_sqrt_pd(_mm_set_sd(x)));
|
|
942
|
+
}
|
|
943
|
+
|
|
944
|
+
SIMSIMD_INTERNAL simsimd_distance_t _simsimd_cos_normalize_f64_haswell(simsimd_f64_t ab, simsimd_f64_t a2,
|
|
945
|
+
simsimd_f64_t b2) {
|
|
946
|
+
|
|
947
|
+
// If both vectors have magnitude 0, the distance is 0.
|
|
948
|
+
if (a2 == 0 && b2 == 0)
|
|
949
|
+
return 0;
|
|
950
|
+
// If any one of the vectors is 0, the square root of the product is 0,
|
|
951
|
+
// the division is illformed, and the result is 1.
|
|
952
|
+
else if (ab == 0)
|
|
953
|
+
return 1;
|
|
954
|
+
// We want to avoid the `simsimd_approximate_inverse_square_root` due to high latency:
|
|
955
|
+
// https://web.archive.org/web/20210208132927/http://assemblyrequired.crashworks.org/timing-square-root/
|
|
956
|
+
// The latency of the native instruction is 4 cycles and it's broadly supported.
|
|
957
|
+
// For single-precision floats it has a maximum relative error of 1.5*2^-12.
|
|
958
|
+
// Higher precision isn't implemented on older CPUs. See `_simsimd_cos_normalize_f64_skylake` for that.
|
|
959
|
+
__m128d squares = _mm_set_pd(a2, b2);
|
|
960
|
+
__m128d rsqrts = _mm_cvtps_pd(_mm_rsqrt_ps(_mm_cvtpd_ps(squares)));
|
|
961
|
+
// Newton-Raphson iteration for reciprocal square root:
|
|
962
|
+
// https://en.wikipedia.org/wiki/Newton%27s_method
|
|
963
|
+
rsqrts = _mm_add_pd( //
|
|
964
|
+
_mm_mul_pd(_mm_set1_pd(1.5), rsqrts),
|
|
965
|
+
_mm_mul_pd(_mm_mul_pd(_mm_mul_pd(squares, _mm_set1_pd(-0.5)), rsqrts), _mm_mul_pd(rsqrts, rsqrts)));
|
|
966
|
+
|
|
967
|
+
simsimd_f64_t a2_reciprocal = _mm_cvtsd_f64(_mm_unpackhi_pd(rsqrts, rsqrts));
|
|
968
|
+
simsimd_f64_t b2_reciprocal = _mm_cvtsd_f64(rsqrts);
|
|
969
|
+
simsimd_distance_t result = 1 - ab * a2_reciprocal * b2_reciprocal;
|
|
970
|
+
return result > 0 ? result : 0;
|
|
971
|
+
}
|
|
972
|
+
|
|
973
|
+
SIMSIMD_INTERNAL simsimd_distance_t _simsimd_cos_normalize_f32_haswell(simsimd_f32_t ab, simsimd_f32_t a2,
|
|
974
|
+
simsimd_f32_t b2) {
|
|
975
|
+
|
|
976
|
+
// If both vectors have magnitude 0, the distance is 0.
|
|
977
|
+
if (a2 == 0.0f && b2 == 0.0f)
|
|
978
|
+
return 0.0f;
|
|
979
|
+
// If any one of the vectors is 0, the square root of the product is 0,
|
|
980
|
+
// the division is illformed, and the result is 1.
|
|
981
|
+
else if (ab == 0.0f)
|
|
982
|
+
return 1.0f;
|
|
983
|
+
|
|
984
|
+
// Load the squares into an __m128 register for single-precision floating-point operations
|
|
985
|
+
__m128 squares = _mm_set_ps(a2, b2, a2, b2); // We replicate to make use of full register
|
|
986
|
+
|
|
987
|
+
// Compute the reciprocal square root of the squares using _mm_rsqrt_ps (single-precision)
|
|
988
|
+
__m128 rsqrts = _mm_rsqrt_ps(squares);
|
|
989
|
+
|
|
990
|
+
// Perform one iteration of Newton-Raphson refinement to improve the precision of rsqrt:
|
|
991
|
+
// Formula: y' = y * (1.5 - 0.5 * x * y * y)
|
|
992
|
+
__m128 half = _mm_set1_ps(0.5f);
|
|
993
|
+
__m128 three_halves = _mm_set1_ps(1.5f);
|
|
994
|
+
rsqrts =
|
|
995
|
+
_mm_mul_ps(rsqrts, _mm_sub_ps(three_halves, _mm_mul_ps(half, _mm_mul_ps(squares, _mm_mul_ps(rsqrts, rsqrts)))));
|
|
996
|
+
|
|
997
|
+
// Extract the reciprocal square roots of a2 and b2 from the __m128 register
|
|
998
|
+
simsimd_f32_t a2_reciprocal = _mm_cvtss_f32(_mm_shuffle_ps(rsqrts, rsqrts, _MM_SHUFFLE(0, 0, 0, 1)));
|
|
999
|
+
simsimd_f32_t b2_reciprocal = _mm_cvtss_f32(rsqrts);
|
|
1000
|
+
|
|
1001
|
+
// Calculate the cosine distance: 1 - ab * a2_reciprocal * b2_reciprocal
|
|
1002
|
+
simsimd_distance_t result = 1.0f - ab * a2_reciprocal * b2_reciprocal;
|
|
1003
|
+
return result > 0 ? result : 0;
|
|
1004
|
+
}
|
|
1005
|
+
|
|
1006
|
+
#pragma clang attribute pop
|
|
1007
|
+
#pragma GCC pop_options
|
|
1008
|
+
#endif // SIMSIMD_TARGET_HASWELL
|
|
1009
|
+
#endif // SIMSIMD_TARGET_X86
|
|
1010
|
+
|
|
745
1011
|
#if SIMSIMD_TARGET_X86
|
|
746
1012
|
#if SIMSIMD_TARGET_HASWELL
|
|
747
1013
|
#pragma GCC push_options
|
|
748
1014
|
#pragma GCC target("avx2", "f16c", "fma")
|
|
749
1015
|
#pragma clang attribute push(__attribute__((target("avx2,f16c,fma"))), apply_to = function)
|
|
750
1016
|
|
|
1017
|
+
SIMSIMD_PUBLIC void simsimd_l2_f16_haswell(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n,
|
|
1018
|
+
simsimd_distance_t* result) {
|
|
1019
|
+
simsimd_l2sq_f16_haswell(a, b, n, result);
|
|
1020
|
+
*result = _simsimd_sqrt_f32_haswell(*result);
|
|
1021
|
+
}
|
|
751
1022
|
SIMSIMD_PUBLIC void simsimd_l2sq_f16_haswell(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n,
|
|
752
1023
|
simsimd_distance_t* result) {
|
|
1024
|
+
__m256 a_vec, b_vec;
|
|
753
1025
|
__m256 d2_vec = _mm256_setzero_ps();
|
|
754
|
-
simsimd_size_t i = 0;
|
|
755
|
-
for (; i + 8 <= n; i += 8) {
|
|
756
|
-
__m256 a_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)(a + i)));
|
|
757
|
-
__m256 b_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)(b + i)));
|
|
758
|
-
__m256 d_vec = _mm256_sub_ps(a_vec, b_vec);
|
|
759
|
-
d2_vec = _mm256_fmadd_ps(d_vec, d_vec, d2_vec);
|
|
760
|
-
}
|
|
761
1026
|
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
for (; i < n; ++i, ++j)
|
|
772
|
-
a_padded_tail.f16[j] = a[i], b_padded_tail.f16[j] = b[i];
|
|
773
|
-
for (; j < 8; ++j)
|
|
774
|
-
a_padded_tail.f16[j] = 0, b_padded_tail.f16[j] = 0;
|
|
775
|
-
__m256 a_vec = _mm256_cvtph_ps(a_padded_tail.f16_vec);
|
|
776
|
-
__m256 b_vec = _mm256_cvtph_ps(b_padded_tail.f16_vec);
|
|
777
|
-
__m256 d_vec = _mm256_sub_ps(a_vec, b_vec);
|
|
778
|
-
d2_vec = _mm256_fmadd_ps(d_vec, d_vec, d2_vec);
|
|
1027
|
+
simsimd_l2sq_f16_haswell_cycle:
|
|
1028
|
+
if (n < 8) {
|
|
1029
|
+
a_vec = _simsimd_partial_load_f16x8_haswell(a, n);
|
|
1030
|
+
b_vec = _simsimd_partial_load_f16x8_haswell(b, n);
|
|
1031
|
+
n = 0;
|
|
1032
|
+
} else {
|
|
1033
|
+
a_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)a));
|
|
1034
|
+
b_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)b));
|
|
1035
|
+
n -= 8, a += 8, b += 8;
|
|
779
1036
|
}
|
|
1037
|
+
__m256 d_vec = _mm256_sub_ps(a_vec, b_vec);
|
|
1038
|
+
d2_vec = _mm256_fmadd_ps(d_vec, d_vec, d2_vec);
|
|
1039
|
+
if (n)
|
|
1040
|
+
goto simsimd_l2sq_f16_haswell_cycle;
|
|
780
1041
|
|
|
781
|
-
|
|
782
|
-
d2_vec = _mm256_hadd_ps(d2_vec, d2_vec);
|
|
783
|
-
d2_vec = _mm256_hadd_ps(d2_vec, d2_vec);
|
|
784
|
-
|
|
785
|
-
simsimd_f32_t f32_result;
|
|
786
|
-
_mm_store_ss(&f32_result, _mm256_castps256_ps128(d2_vec));
|
|
787
|
-
*result = f32_result;
|
|
1042
|
+
*result = _simsimd_reduce_f32x8_haswell(d2_vec);
|
|
788
1043
|
}
|
|
789
1044
|
|
|
790
1045
|
SIMSIMD_PUBLIC void simsimd_cos_f16_haswell(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n,
|
|
791
1046
|
simsimd_distance_t* result) {
|
|
792
|
-
|
|
1047
|
+
__m256 a_vec, b_vec;
|
|
793
1048
|
__m256 ab_vec = _mm256_setzero_ps(), a2_vec = _mm256_setzero_ps(), b2_vec = _mm256_setzero_ps();
|
|
794
|
-
simsimd_size_t i = 0;
|
|
795
|
-
for (; i + 8 <= n; i += 8) {
|
|
796
|
-
__m256 a_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)(a + i)));
|
|
797
|
-
__m256 b_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)(b + i)));
|
|
798
|
-
ab_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_vec);
|
|
799
|
-
a2_vec = _mm256_fmadd_ps(a_vec, a_vec, a2_vec);
|
|
800
|
-
b2_vec = _mm256_fmadd_ps(b_vec, b_vec, b2_vec);
|
|
801
|
-
}
|
|
802
1049
|
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
for (; i < n; ++i, ++j)
|
|
813
|
-
a_padded_tail.f16[j] = a[i], b_padded_tail.f16[j] = b[i];
|
|
814
|
-
for (; j < 8; ++j)
|
|
815
|
-
a_padded_tail.f16[j] = 0, b_padded_tail.f16[j] = 0;
|
|
816
|
-
__m256 a_vec = _mm256_cvtph_ps(a_padded_tail.f16_vec);
|
|
817
|
-
__m256 b_vec = _mm256_cvtph_ps(b_padded_tail.f16_vec);
|
|
818
|
-
ab_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_vec);
|
|
819
|
-
a2_vec = _mm256_fmadd_ps(a_vec, a_vec, a2_vec);
|
|
820
|
-
b2_vec = _mm256_fmadd_ps(b_vec, b_vec, b2_vec);
|
|
1050
|
+
simsimd_cos_f16_haswell_cycle:
|
|
1051
|
+
if (n < 8) {
|
|
1052
|
+
a_vec = _simsimd_partial_load_f16x8_haswell(a, n);
|
|
1053
|
+
b_vec = _simsimd_partial_load_f16x8_haswell(b, n);
|
|
1054
|
+
n = 0;
|
|
1055
|
+
} else {
|
|
1056
|
+
a_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)a));
|
|
1057
|
+
b_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)b));
|
|
1058
|
+
n -= 8, a += 8, b += 8;
|
|
821
1059
|
}
|
|
1060
|
+
ab_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_vec);
|
|
1061
|
+
a2_vec = _mm256_fmadd_ps(a_vec, a_vec, a2_vec);
|
|
1062
|
+
b2_vec = _mm256_fmadd_ps(b_vec, b_vec, b2_vec);
|
|
1063
|
+
if (n)
|
|
1064
|
+
goto simsimd_cos_f16_haswell_cycle;
|
|
822
1065
|
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
a2_vec = _mm256_add_ps(_mm256_permute2f128_ps(a2_vec, a2_vec, 1), a2_vec);
|
|
829
|
-
a2_vec = _mm256_hadd_ps(a2_vec, a2_vec);
|
|
830
|
-
a2_vec = _mm256_hadd_ps(a2_vec, a2_vec);
|
|
831
|
-
|
|
832
|
-
b2_vec = _mm256_add_ps(_mm256_permute2f128_ps(b2_vec, b2_vec, 1), b2_vec);
|
|
833
|
-
b2_vec = _mm256_hadd_ps(b2_vec, b2_vec);
|
|
834
|
-
b2_vec = _mm256_hadd_ps(b2_vec, b2_vec);
|
|
835
|
-
|
|
836
|
-
simsimd_f32_t ab, a2, b2;
|
|
837
|
-
_mm_store_ss(&ab, _mm256_castps256_ps128(ab_vec));
|
|
838
|
-
_mm_store_ss(&a2, _mm256_castps256_ps128(a2_vec));
|
|
839
|
-
_mm_store_ss(&b2, _mm256_castps256_ps128(b2_vec));
|
|
840
|
-
|
|
841
|
-
// Replace simsimd_approximate_inverse_square_root with `rsqrtss`
|
|
842
|
-
__m128 a2_sqrt_recip = _mm_rsqrt_ss(_mm_set_ss((float)a2));
|
|
843
|
-
__m128 b2_sqrt_recip = _mm_rsqrt_ss(_mm_set_ss((float)b2));
|
|
844
|
-
__m128 result_vec = _mm_mul_ss(a2_sqrt_recip, b2_sqrt_recip); // Multiply the reciprocal square roots
|
|
845
|
-
result_vec = _mm_mul_ss(result_vec, _mm_set_ss((float)ab)); // Multiply by ab
|
|
846
|
-
*result = ab != 0 ? 1 - _mm_cvtss_f32(result_vec) : 1; // Extract the final result
|
|
1066
|
+
simsimd_f32_t ab = _simsimd_reduce_f32x8_haswell(ab_vec);
|
|
1067
|
+
simsimd_f32_t a2 = _simsimd_reduce_f32x8_haswell(a2_vec);
|
|
1068
|
+
simsimd_f32_t b2 = _simsimd_reduce_f32x8_haswell(b2_vec);
|
|
1069
|
+
*result = _simsimd_cos_normalize_f32_haswell(ab, a2, b2);
|
|
847
1070
|
}
|
|
848
1071
|
|
|
1072
|
+
SIMSIMD_PUBLIC void simsimd_l2_bf16_haswell(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n,
|
|
1073
|
+
simsimd_distance_t* result) {
|
|
1074
|
+
simsimd_l2sq_bf16_haswell(a, b, n, result);
|
|
1075
|
+
*result = _simsimd_sqrt_f32_haswell(*result);
|
|
1076
|
+
}
|
|
849
1077
|
SIMSIMD_PUBLIC void simsimd_l2sq_bf16_haswell(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n,
|
|
850
1078
|
simsimd_distance_t* result) {
|
|
1079
|
+
__m256 a_vec, b_vec;
|
|
851
1080
|
__m256 d2_vec = _mm256_setzero_ps();
|
|
852
|
-
simsimd_size_t i = 0;
|
|
853
|
-
for (; i + 8 <= n; i += 8) {
|
|
854
|
-
// Upcasting from `bf16` to `f32` is done by shifting the `bf16` values by 16 bits to the left, like:
|
|
855
|
-
// x = _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(x), 16))
|
|
856
|
-
__m256 a_vec =
|
|
857
|
-
_mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(_mm_loadu_si128((__m128i const*)(a + i))), 16));
|
|
858
|
-
__m256 b_vec =
|
|
859
|
-
_mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(_mm_loadu_si128((__m128i const*)(b + i))), 16));
|
|
860
|
-
__m256 d_vec = _mm256_sub_ps(a_vec, b_vec);
|
|
861
|
-
d2_vec = _mm256_fmadd_ps(d_vec, d_vec, d2_vec);
|
|
862
|
-
}
|
|
863
1081
|
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
for (; i < n; ++i, ++j)
|
|
874
|
-
a_padded_tail.bf16[j] = a[i], b_padded_tail.bf16[j] = b[i];
|
|
875
|
-
for (; j < 8; ++j)
|
|
876
|
-
a_padded_tail.bf16[j] = 0, b_padded_tail.bf16[j] = 0;
|
|
877
|
-
__m256 a_vec = _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(a_padded_tail.bf16_vec), 16));
|
|
878
|
-
__m256 b_vec = _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(b_padded_tail.bf16_vec), 16));
|
|
879
|
-
__m256 d_vec = _mm256_sub_ps(a_vec, b_vec);
|
|
880
|
-
d2_vec = _mm256_fmadd_ps(d_vec, d_vec, d2_vec);
|
|
1082
|
+
simsimd_l2sq_bf16_haswell_cycle:
|
|
1083
|
+
if (n < 8) {
|
|
1084
|
+
a_vec = _simsimd_bf16x8_to_f32x8_haswell(_simsimd_partial_load_bf16x8_haswell(a, n));
|
|
1085
|
+
b_vec = _simsimd_bf16x8_to_f32x8_haswell(_simsimd_partial_load_bf16x8_haswell(b, n));
|
|
1086
|
+
n = 0;
|
|
1087
|
+
} else {
|
|
1088
|
+
a_vec = _simsimd_bf16x8_to_f32x8_haswell(_mm_loadu_si128((__m128i const*)a));
|
|
1089
|
+
b_vec = _simsimd_bf16x8_to_f32x8_haswell(_mm_loadu_si128((__m128i const*)b));
|
|
1090
|
+
n -= 8, a += 8, b += 8;
|
|
881
1091
|
}
|
|
1092
|
+
__m256 d_vec = _mm256_sub_ps(a_vec, b_vec);
|
|
1093
|
+
d2_vec = _mm256_fmadd_ps(d_vec, d_vec, d2_vec);
|
|
1094
|
+
if (n)
|
|
1095
|
+
goto simsimd_l2sq_bf16_haswell_cycle;
|
|
882
1096
|
|
|
883
|
-
|
|
884
|
-
d2_vec = _mm256_hadd_ps(d2_vec, d2_vec);
|
|
885
|
-
d2_vec = _mm256_hadd_ps(d2_vec, d2_vec);
|
|
886
|
-
|
|
887
|
-
simsimd_f32_t f32_result;
|
|
888
|
-
_mm_store_ss(&f32_result, _mm256_castps256_ps128(d2_vec));
|
|
889
|
-
*result = f32_result;
|
|
1097
|
+
*result = _simsimd_reduce_f32x8_haswell(d2_vec);
|
|
890
1098
|
}
|
|
891
1099
|
|
|
892
1100
|
SIMSIMD_PUBLIC void simsimd_cos_bf16_haswell(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n,
|
|
893
1101
|
simsimd_distance_t* result) {
|
|
894
|
-
|
|
1102
|
+
__m256 a_vec, b_vec;
|
|
895
1103
|
__m256 ab_vec = _mm256_setzero_ps(), a2_vec = _mm256_setzero_ps(), b2_vec = _mm256_setzero_ps();
|
|
896
|
-
simsimd_size_t i = 0;
|
|
897
|
-
for (; i + 8 <= n; i += 8) {
|
|
898
|
-
// Upcasting from `bf16` to `f32` is done by shifting the `bf16` values by 16 bits to the left, like:
|
|
899
|
-
// x = _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(x), 16))
|
|
900
|
-
__m256 a_vec =
|
|
901
|
-
_mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(_mm_loadu_si128((__m128i const*)(a + i))), 16));
|
|
902
|
-
__m256 b_vec =
|
|
903
|
-
_mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(_mm_loadu_si128((__m128i const*)(b + i))), 16));
|
|
904
|
-
ab_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_vec);
|
|
905
|
-
a2_vec = _mm256_fmadd_ps(a_vec, a_vec, a2_vec);
|
|
906
|
-
b2_vec = _mm256_fmadd_ps(b_vec, b_vec, b2_vec);
|
|
907
|
-
}
|
|
908
1104
|
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
for (; i < n; ++i, ++j)
|
|
919
|
-
a_padded_tail.bf16[j] = a[i], b_padded_tail.bf16[j] = b[i];
|
|
920
|
-
for (; j < 8; ++j)
|
|
921
|
-
a_padded_tail.bf16[j] = 0, b_padded_tail.bf16[j] = 0;
|
|
922
|
-
__m256 a_vec = _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(a_padded_tail.bf16_vec), 16));
|
|
923
|
-
__m256 b_vec = _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(b_padded_tail.bf16_vec), 16));
|
|
924
|
-
ab_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_vec);
|
|
925
|
-
a2_vec = _mm256_fmadd_ps(a_vec, a_vec, a2_vec);
|
|
926
|
-
b2_vec = _mm256_fmadd_ps(b_vec, b_vec, b2_vec);
|
|
1105
|
+
simsimd_cos_bf16_haswell_cycle:
|
|
1106
|
+
if (n < 8) {
|
|
1107
|
+
a_vec = _simsimd_bf16x8_to_f32x8_haswell(_simsimd_partial_load_bf16x8_haswell(a, n));
|
|
1108
|
+
b_vec = _simsimd_bf16x8_to_f32x8_haswell(_simsimd_partial_load_bf16x8_haswell(b, n));
|
|
1109
|
+
n = 0;
|
|
1110
|
+
} else {
|
|
1111
|
+
a_vec = _simsimd_bf16x8_to_f32x8_haswell(_mm_loadu_si128((__m128i const*)a));
|
|
1112
|
+
b_vec = _simsimd_bf16x8_to_f32x8_haswell(_mm_loadu_si128((__m128i const*)b));
|
|
1113
|
+
n -= 8, a += 8, b += 8;
|
|
927
1114
|
}
|
|
1115
|
+
ab_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_vec);
|
|
1116
|
+
a2_vec = _mm256_fmadd_ps(a_vec, a_vec, a2_vec);
|
|
1117
|
+
b2_vec = _mm256_fmadd_ps(b_vec, b_vec, b2_vec);
|
|
1118
|
+
if (n)
|
|
1119
|
+
goto simsimd_cos_bf16_haswell_cycle;
|
|
928
1120
|
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
a2_vec = _mm256_add_ps(_mm256_permute2f128_ps(a2_vec, a2_vec, 1), a2_vec);
|
|
935
|
-
a2_vec = _mm256_hadd_ps(a2_vec, a2_vec);
|
|
936
|
-
a2_vec = _mm256_hadd_ps(a2_vec, a2_vec);
|
|
937
|
-
|
|
938
|
-
b2_vec = _mm256_add_ps(_mm256_permute2f128_ps(b2_vec, b2_vec, 1), b2_vec);
|
|
939
|
-
b2_vec = _mm256_hadd_ps(b2_vec, b2_vec);
|
|
940
|
-
b2_vec = _mm256_hadd_ps(b2_vec, b2_vec);
|
|
941
|
-
|
|
942
|
-
simsimd_f32_t ab, a2, b2;
|
|
943
|
-
_mm_store_ss(&ab, _mm256_castps256_ps128(ab_vec));
|
|
944
|
-
_mm_store_ss(&a2, _mm256_castps256_ps128(a2_vec));
|
|
945
|
-
_mm_store_ss(&b2, _mm256_castps256_ps128(b2_vec));
|
|
946
|
-
|
|
947
|
-
// Replace simsimd_approximate_inverse_square_root with `rsqrtss`
|
|
948
|
-
__m128 a2_sqrt_recip = _mm_rsqrt_ss(_mm_set_ss((float)a2));
|
|
949
|
-
__m128 b2_sqrt_recip = _mm_rsqrt_ss(_mm_set_ss((float)b2));
|
|
950
|
-
__m128 result_vec = _mm_mul_ss(a2_sqrt_recip, b2_sqrt_recip); // Multiply the reciprocal square roots
|
|
951
|
-
result_vec = _mm_mul_ss(result_vec, _mm_set_ss((float)ab)); // Multiply by ab
|
|
952
|
-
*result = ab != 0 ? 1 - _mm_cvtss_f32(result_vec) : 1; // Extract the final result
|
|
1121
|
+
simsimd_f32_t ab = _simsimd_reduce_f32x8_haswell(ab_vec);
|
|
1122
|
+
simsimd_f32_t a2 = _simsimd_reduce_f32x8_haswell(a2_vec);
|
|
1123
|
+
simsimd_f32_t b2 = _simsimd_reduce_f32x8_haswell(b2_vec);
|
|
1124
|
+
*result = _simsimd_cos_normalize_f32_haswell(ab, a2, b2);
|
|
953
1125
|
}
|
|
954
1126
|
|
|
1127
|
+
SIMSIMD_PUBLIC void simsimd_l2_i8_haswell(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n,
|
|
1128
|
+
simsimd_distance_t* result) {
|
|
1129
|
+
simsimd_l2sq_i8_haswell(a, b, n, result);
|
|
1130
|
+
*result = _simsimd_sqrt_f32_haswell(*result);
|
|
1131
|
+
}
|
|
955
1132
|
SIMSIMD_PUBLIC void simsimd_l2sq_i8_haswell(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n,
|
|
956
1133
|
simsimd_distance_t* result) {
|
|
957
1134
|
|
|
@@ -1053,16 +1230,14 @@ SIMSIMD_PUBLIC void simsimd_cos_i8_haswell(simsimd_i8_t const* a, simsimd_i8_t c
|
|
|
1053
1230
|
ab += ai * bi, a2 += ai * ai, b2 += bi * bi;
|
|
1054
1231
|
}
|
|
1055
1232
|
|
|
1056
|
-
|
|
1057
|
-
__m128 a2_sqrt_recip = _mm_rsqrt_ss(_mm_set_ss((float)a2));
|
|
1058
|
-
__m128 b2_sqrt_recip = _mm_rsqrt_ss(_mm_set_ss((float)b2));
|
|
1059
|
-
|
|
1060
|
-
// Compute cosine similarity: ab / sqrt(a2 * b2)
|
|
1061
|
-
__m128 denom = _mm_mul_ss(a2_sqrt_recip, b2_sqrt_recip); // Reciprocal of sqrt(a2 * b2)
|
|
1062
|
-
__m128 result_vec = _mm_mul_ss(_mm_set_ss((float)ab), denom); // ab * reciprocal of sqrt(a2 * b2)
|
|
1063
|
-
*result = ab != 0 ? 1 - _mm_cvtss_f32(result_vec) : 1; // Extract the final result
|
|
1233
|
+
*result = _simsimd_cos_normalize_f32_haswell(ab, a2, b2);
|
|
1064
1234
|
}
|
|
1065
1235
|
|
|
1236
|
+
SIMSIMD_PUBLIC void simsimd_l2_f32_haswell(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n,
|
|
1237
|
+
simsimd_distance_t* result) {
|
|
1238
|
+
simsimd_l2sq_f32_haswell(a, b, n, result);
|
|
1239
|
+
*result = _simsimd_sqrt_f32_haswell(*result);
|
|
1240
|
+
}
|
|
1066
1241
|
SIMSIMD_PUBLIC void simsimd_l2sq_f32_haswell(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n,
|
|
1067
1242
|
simsimd_distance_t* result) {
|
|
1068
1243
|
|
|
@@ -1075,7 +1250,7 @@ SIMSIMD_PUBLIC void simsimd_l2sq_f32_haswell(simsimd_f32_t const* a, simsimd_f32
|
|
|
1075
1250
|
d2_vec = _mm256_fmadd_ps(d_vec, d_vec, d2_vec);
|
|
1076
1251
|
}
|
|
1077
1252
|
|
|
1078
|
-
simsimd_f64_t d2 =
|
|
1253
|
+
simsimd_f64_t d2 = _simsimd_reduce_f32x8_haswell(d2_vec);
|
|
1079
1254
|
for (; i < n; ++i) {
|
|
1080
1255
|
float d = a[i] - b[i];
|
|
1081
1256
|
d2 += d * d;
|
|
@@ -1098,22 +1273,15 @@ SIMSIMD_PUBLIC void simsimd_cos_f32_haswell(simsimd_f32_t const* a, simsimd_f32_
|
|
|
1098
1273
|
a2_vec = _mm256_fmadd_ps(a_vec, a_vec, a2_vec);
|
|
1099
1274
|
b2_vec = _mm256_fmadd_ps(b_vec, b_vec, b2_vec);
|
|
1100
1275
|
}
|
|
1101
|
-
|
|
1102
|
-
simsimd_f64_t
|
|
1103
|
-
simsimd_f64_t
|
|
1276
|
+
|
|
1277
|
+
simsimd_f64_t ab = _simsimd_reduce_f32x8_haswell(ab_vec);
|
|
1278
|
+
simsimd_f64_t a2 = _simsimd_reduce_f32x8_haswell(a2_vec);
|
|
1279
|
+
simsimd_f64_t b2 = _simsimd_reduce_f32x8_haswell(b2_vec);
|
|
1104
1280
|
for (; i < n; ++i) {
|
|
1105
1281
|
float ai = a[i], bi = b[i];
|
|
1106
1282
|
ab += ai * bi, a2 += ai * ai, b2 += bi * bi;
|
|
1107
1283
|
}
|
|
1108
|
-
|
|
1109
|
-
// Compute the reciprocal of the square roots
|
|
1110
|
-
__m128 a2_sqrt_recip = _mm_rsqrt_ss(_mm_set_ss((float)a2));
|
|
1111
|
-
__m128 b2_sqrt_recip = _mm_rsqrt_ss(_mm_set_ss((float)b2));
|
|
1112
|
-
|
|
1113
|
-
// Compute cosine similarity: ab / sqrt(a2 * b2)
|
|
1114
|
-
__m128 denom = _mm_mul_ss(a2_sqrt_recip, b2_sqrt_recip); // Reciprocal of sqrt(a2 * b2)
|
|
1115
|
-
__m128 result_vec = _mm_mul_ss(_mm_set_ss((float)ab), denom); // ab * reciprocal of sqrt(a2 * b2)
|
|
1116
|
-
*result = ab != 0 ? 1 - _mm_cvtss_f32(result_vec) : 1; // Extract the final result
|
|
1284
|
+
*result = _simsimd_cos_normalize_f64_haswell(ab, a2, b2);
|
|
1117
1285
|
}
|
|
1118
1286
|
|
|
1119
1287
|
#pragma clang attribute pop
|
|
@@ -1122,9 +1290,14 @@ SIMSIMD_PUBLIC void simsimd_cos_f32_haswell(simsimd_f32_t const* a, simsimd_f32_
|
|
|
1122
1290
|
|
|
1123
1291
|
#if SIMSIMD_TARGET_SKYLAKE
|
|
1124
1292
|
#pragma GCC push_options
|
|
1125
|
-
#pragma GCC target("avx512f", "avx512vl", "bmi2")
|
|
1126
|
-
#pragma clang attribute push(__attribute__((target("avx512f,avx512vl,bmi2"))), apply_to = function)
|
|
1293
|
+
#pragma GCC target("avx2", "avx512f", "avx512bw", "avx512vl", "bmi2")
|
|
1294
|
+
#pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512bw,avx512vl,bmi2"))), apply_to = function)
|
|
1127
1295
|
|
|
1296
|
+
SIMSIMD_PUBLIC void simsimd_l2_f32_skylake(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n,
|
|
1297
|
+
simsimd_distance_t* result) {
|
|
1298
|
+
simsimd_l2sq_f32_skylake(a, b, n, result);
|
|
1299
|
+
*result = _simsimd_sqrt_f64_haswell(*result);
|
|
1300
|
+
}
|
|
1128
1301
|
SIMSIMD_PUBLIC void simsimd_l2sq_f32_skylake(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n,
|
|
1129
1302
|
simsimd_distance_t* result) {
|
|
1130
1303
|
__m512 d2_vec = _mm512_setzero();
|
|
@@ -1146,7 +1319,51 @@ simsimd_l2sq_f32_skylake_cycle:
|
|
|
1146
1319
|
if (n)
|
|
1147
1320
|
goto simsimd_l2sq_f32_skylake_cycle;
|
|
1148
1321
|
|
|
1149
|
-
*result =
|
|
1322
|
+
*result = _simsimd_reduce_f32x16_skylake(d2_vec);
|
|
1323
|
+
}
|
|
1324
|
+
|
|
1325
|
+
SIMSIMD_INTERNAL simsimd_distance_t _simsimd_cos_normalize_f64_skylake(simsimd_f64_t ab, simsimd_f64_t a2,
|
|
1326
|
+
simsimd_f64_t b2) {
|
|
1327
|
+
|
|
1328
|
+
// If both vectors have magnitude 0, the distance is 0.
|
|
1329
|
+
if (a2 == 0 && b2 == 0)
|
|
1330
|
+
return 0;
|
|
1331
|
+
// If any one of the vectors is 0, the square root of the product is 0,
|
|
1332
|
+
// the division is illformed, and the result is 1.
|
|
1333
|
+
else if (ab == 0)
|
|
1334
|
+
return 1;
|
|
1335
|
+
|
|
1336
|
+
// We want to avoid the `simsimd_approximate_inverse_square_root` due to high latency:
|
|
1337
|
+
// https://web.archive.org/web/20210208132927/http://assemblyrequired.crashworks.org/timing-square-root/
|
|
1338
|
+
// The maximum relative error for this approximation is less than 2^-14, which is 6x lower than
|
|
1339
|
+
// for single-precision floats in the `_simsimd_cos_normalize_f64_haswell` implementation.
|
|
1340
|
+
// Mysteriously, MSVC has no `_mm_rsqrt14_pd` intrinsic, but has it's masked variants,
|
|
1341
|
+
// so let's use `_mm_maskz_rsqrt14_pd(0xFF, ...)` instead.
|
|
1342
|
+
__m128d squares = _mm_set_pd(a2, b2);
|
|
1343
|
+
__m128d rsqrts = _mm_maskz_rsqrt14_pd(0xFF, squares);
|
|
1344
|
+
|
|
1345
|
+
// Let's implement a single Newton-Raphson iteration to refine the result.
|
|
1346
|
+
// This is how it affects downstream applications:
|
|
1347
|
+
//
|
|
1348
|
+
// +--------+------+----------+---------------------+---------------------+---------------------+
|
|
1349
|
+
// | Metric | NDim | DType | Baseline Error | Old SimSIMD Error | New SimSIMD Error |
|
|
1350
|
+
// +--------+------+----------+---------------------+---------------------+---------------------+
|
|
1351
|
+
// | cosine | 1536 | bfloat16 | 1.89e-08 ± 1.59e-08 | 3.07e-07 ± 3.09e-07 | 3.53e-09 ± 2.70e-09 |
|
|
1352
|
+
// | cosine | 1536 | float16 | 1.67e-02 ± 1.44e-02 | 2.68e-05 ± 1.95e-05 | 2.02e-05 ± 1.39e-05 |
|
|
1353
|
+
// | cosine | 1536 | float32 | 2.21e-08 ± 1.65e-08 | 3.47e-07 ± 3.49e-07 | 3.77e-09 ± 2.84e-09 |
|
|
1354
|
+
// | cosine | 1536 | float64 | 0.00e+00 ± 0.00e+00 | 3.80e-07 ± 4.50e-07 | 1.35e-11 ± 1.85e-11 |
|
|
1355
|
+
// | cosine | 1536 | int8 | 0.00e+00 ± 0.00e+00 | 4.60e-04 ± 3.36e-04 | 4.20e-04 ± 4.88e-04 |
|
|
1356
|
+
// +--------+------+----------+---------------------+---------------------+---------------------+
|
|
1357
|
+
//
|
|
1358
|
+
// https://en.wikipedia.org/wiki/Newton%27s_method
|
|
1359
|
+
rsqrts = _mm_add_pd( //
|
|
1360
|
+
_mm_mul_pd(_mm_set1_pd(1.5), rsqrts),
|
|
1361
|
+
_mm_mul_pd(_mm_mul_pd(_mm_mul_pd(squares, _mm_set1_pd(-0.5)), rsqrts), _mm_mul_pd(rsqrts, rsqrts)));
|
|
1362
|
+
|
|
1363
|
+
simsimd_f64_t a2_reciprocal = _mm_cvtsd_f64(_mm_unpackhi_pd(rsqrts, rsqrts));
|
|
1364
|
+
simsimd_f64_t b2_reciprocal = _mm_cvtsd_f64(rsqrts);
|
|
1365
|
+
simsimd_distance_t result = 1 - ab * a2_reciprocal * b2_reciprocal;
|
|
1366
|
+
return result > 0 ? result : 0;
|
|
1150
1367
|
}
|
|
1151
1368
|
|
|
1152
1369
|
SIMSIMD_PUBLIC void simsimd_cos_f32_skylake(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n,
|
|
@@ -1173,19 +1390,17 @@ simsimd_cos_f32_skylake_cycle:
|
|
|
1173
1390
|
if (n)
|
|
1174
1391
|
goto simsimd_cos_f32_skylake_cycle;
|
|
1175
1392
|
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
// Compute the reciprocal square roots of a2 and b2
|
|
1181
|
-
// Mysteriously, MSVC has no `_mm_rsqrt14_ps` intrinsic, but has it's masked variants,
|
|
1182
|
-
// so let's use `_mm_maskz_rsqrt14_ps(0xFF, ...)` instead.
|
|
1183
|
-
__m128 rsqrts = _mm_maskz_rsqrt14_ps(0xFF, _mm_set_ps(0.f, 0.f, a2 + 1.e-9f, b2 + 1.e-9f));
|
|
1184
|
-
simsimd_f32_t rsqrt_a2 = _mm_cvtss_f32(rsqrts);
|
|
1185
|
-
simsimd_f32_t rsqrt_b2 = _mm_cvtss_f32(_mm_shuffle_ps(rsqrts, rsqrts, _MM_SHUFFLE(0, 0, 0, 1)));
|
|
1186
|
-
*result = 1 - ab * rsqrt_a2 * rsqrt_b2;
|
|
1393
|
+
simsimd_f64_t ab = _simsimd_reduce_f32x16_skylake(ab_vec);
|
|
1394
|
+
simsimd_f64_t a2 = _simsimd_reduce_f32x16_skylake(a2_vec);
|
|
1395
|
+
simsimd_f64_t b2 = _simsimd_reduce_f32x16_skylake(b2_vec);
|
|
1396
|
+
*result = _simsimd_cos_normalize_f64_skylake(ab, a2, b2);
|
|
1187
1397
|
}
|
|
1188
1398
|
|
|
1399
|
+
SIMSIMD_PUBLIC void simsimd_l2_f64_skylake(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n,
|
|
1400
|
+
simsimd_distance_t* result) {
|
|
1401
|
+
simsimd_l2sq_f64_skylake(a, b, n, result);
|
|
1402
|
+
*result = _simsimd_sqrt_f64_haswell(*result);
|
|
1403
|
+
}
|
|
1189
1404
|
SIMSIMD_PUBLIC void simsimd_l2sq_f64_skylake(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n,
|
|
1190
1405
|
simsimd_distance_t* result) {
|
|
1191
1406
|
__m512d d2_vec = _mm512_setzero_pd();
|
|
@@ -1234,17 +1449,10 @@ simsimd_cos_f64_skylake_cycle:
|
|
|
1234
1449
|
if (n)
|
|
1235
1450
|
goto simsimd_cos_f64_skylake_cycle;
|
|
1236
1451
|
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
// Compute the reciprocal square roots of a2 and b2
|
|
1242
|
-
// Mysteriously, MSVC has no `_mm_rsqrt14_ps` intrinsic, but has it's masked variants,
|
|
1243
|
-
// so let's use `_mm_maskz_rsqrt14_ps(0xFF, ...)` instead.
|
|
1244
|
-
__m128 rsqrts = _mm_maskz_rsqrt14_ps(0xFF, _mm_set_ps(0.f, 0.f, a2 + 1.e-9f, b2 + 1.e-9f));
|
|
1245
|
-
simsimd_f32_t rsqrt_a2 = _mm_cvtss_f32(rsqrts);
|
|
1246
|
-
simsimd_f32_t rsqrt_b2 = _mm_cvtss_f32(_mm_shuffle_ps(rsqrts, rsqrts, _MM_SHUFFLE(0, 0, 0, 1)));
|
|
1247
|
-
*result = 1 - ab * rsqrt_a2 * rsqrt_b2;
|
|
1452
|
+
simsimd_f64_t ab = _mm512_reduce_add_pd(ab_vec);
|
|
1453
|
+
simsimd_f64_t a2 = _mm512_reduce_add_pd(a2_vec);
|
|
1454
|
+
simsimd_f64_t b2 = _mm512_reduce_add_pd(b2_vec);
|
|
1455
|
+
*result = _simsimd_cos_normalize_f64_skylake(ab, a2, b2);
|
|
1248
1456
|
}
|
|
1249
1457
|
|
|
1250
1458
|
#pragma clang attribute pop
|
|
@@ -1253,15 +1461,71 @@ simsimd_cos_f64_skylake_cycle:
|
|
|
1253
1461
|
|
|
1254
1462
|
#if SIMSIMD_TARGET_GENOA
|
|
1255
1463
|
#pragma GCC push_options
|
|
1256
|
-
#pragma GCC target("avx512f", "avx512vl", "bmi2", "avx512bw", "avx512bf16")
|
|
1257
|
-
#pragma clang attribute push(__attribute__((target("avx512f,avx512vl,bmi2,avx512bw,avx512bf16"))),
|
|
1464
|
+
#pragma GCC target("avx2", "avx512f", "avx512vl", "bmi2", "avx512bw", "avx512bf16")
|
|
1465
|
+
#pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,bmi2,avx512bw,avx512bf16"))), \
|
|
1466
|
+
apply_to = function)
|
|
1467
|
+
|
|
1468
|
+
SIMSIMD_INTERNAL __m512i _simsimd_substract_bf16x32_genoa(__m512i a_i16, __m512i b_i16) {
|
|
1469
|
+
|
|
1470
|
+
union {
|
|
1471
|
+
__m512 fvec;
|
|
1472
|
+
__m512i ivec;
|
|
1473
|
+
simsimd_f32_t f32[16];
|
|
1474
|
+
simsimd_u16_t u16[32];
|
|
1475
|
+
simsimd_bf16_t bf16[32];
|
|
1476
|
+
} d_odd, d_even, d, a_f32_even, b_f32_even, d_f32_even, a_f32_odd, b_f32_odd, d_f32_odd, a, b;
|
|
1477
|
+
a.ivec = a_i16;
|
|
1478
|
+
b.ivec = b_i16;
|
|
1479
|
+
|
|
1480
|
+
// There are several approaches to perform subtraction in `bf16`. The first one is:
|
|
1481
|
+
//
|
|
1482
|
+
// Perform a couple of casts - each is a bitshift. To convert `bf16` to `f32`,
|
|
1483
|
+
// expand it to 32-bit integers, then shift the bits by 16 to the left.
|
|
1484
|
+
// Then subtract as floats, and shift back. During expansion, we will double the space,
|
|
1485
|
+
// and should use separate registers for top and bottom halves.
|
|
1486
|
+
// Some compilers don't have `_mm512_extracti32x8_epi32`, so we use `_mm512_extracti64x4_epi64`:
|
|
1487
|
+
//
|
|
1488
|
+
// a_f32_bot.fvec = _mm512_castsi512_ps(_mm512_slli_epi32(
|
|
1489
|
+
// _mm512_cvtepu16_epi32(_mm512_castsi512_si256(a_i16)), 16));
|
|
1490
|
+
// b_f32_bot.fvec = _mm512_castsi512_ps(_mm512_slli_epi32(
|
|
1491
|
+
// _mm512_cvtepu16_epi32(_mm512_castsi512_si256(b_i16)), 16));
|
|
1492
|
+
// a_f32_top.fvec =_mm512_castsi512_ps(
|
|
1493
|
+
// _mm512_slli_epi32(_mm512_cvtepu16_epi32(_mm512_extracti64x4_epi64(a_i16, 1)), 16));
|
|
1494
|
+
// b_f32_top.fvec =_mm512_castsi512_ps(
|
|
1495
|
+
// _mm512_slli_epi32(_mm512_cvtepu16_epi32(_mm512_extracti64x4_epi64(b_i16, 1)), 16));
|
|
1496
|
+
// d_f32_top.fvec = _mm512_sub_ps(a_f32_top.fvec, b_f32_top.fvec);
|
|
1497
|
+
// d_f32_bot.fvec = _mm512_sub_ps(a_f32_bot.fvec, b_f32_bot.fvec);
|
|
1498
|
+
// d.ivec = _mm512_castsi256_si512(_mm512_cvtepi32_epi16(
|
|
1499
|
+
// _mm512_srli_epi32(_mm512_castps_si512(d_f32_bot.fvec), 16)));
|
|
1500
|
+
// d.ivec = _mm512_inserti64x4(d.ivec, _mm512_cvtepi32_epi16(
|
|
1501
|
+
// _mm512_srli_epi32(_mm512_castps_si512(d_f32_top.fvec), 16)), 1);
|
|
1502
|
+
//
|
|
1503
|
+
// Instead of using multple shifts and an insertion, we can achieve similar result with fewer expensive
|
|
1504
|
+
// calls to `_mm512_permutex2var_epi16`, or a cheap `_mm512_mask_shuffle_epi8` and blend:
|
|
1505
|
+
//
|
|
1506
|
+
a_f32_odd.ivec = _mm512_and_si512(a_i16, _mm512_set1_epi32(0xFFFF0000));
|
|
1507
|
+
a_f32_even.ivec = _mm512_slli_epi32(a_i16, 16);
|
|
1508
|
+
b_f32_odd.ivec = _mm512_and_si512(b_i16, _mm512_set1_epi32(0xFFFF0000));
|
|
1509
|
+
b_f32_even.ivec = _mm512_slli_epi32(b_i16, 16);
|
|
1258
1510
|
|
|
1511
|
+
d_f32_odd.fvec = _mm512_sub_ps(a_f32_odd.fvec, b_f32_odd.fvec);
|
|
1512
|
+
d_f32_even.fvec = _mm512_sub_ps(a_f32_even.fvec, b_f32_even.fvec);
|
|
1513
|
+
|
|
1514
|
+
d_f32_even.ivec = _mm512_srli_epi32(d_f32_even.ivec, 16);
|
|
1515
|
+
d.ivec = _mm512_mask_blend_epi16(0x55555555, d_f32_odd.ivec, d_f32_even.ivec);
|
|
1516
|
+
|
|
1517
|
+
return d.ivec;
|
|
1518
|
+
}
|
|
1519
|
+
|
|
1520
|
+
SIMSIMD_PUBLIC void simsimd_l2_bf16_genoa(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n,
|
|
1521
|
+
simsimd_distance_t* result) {
|
|
1522
|
+
simsimd_l2sq_bf16_genoa(a, b, n, result);
|
|
1523
|
+
*result = _simsimd_sqrt_f32_haswell(*result);
|
|
1524
|
+
}
|
|
1259
1525
|
SIMSIMD_PUBLIC void simsimd_l2sq_bf16_genoa(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n,
|
|
1260
1526
|
simsimd_distance_t* result) {
|
|
1261
|
-
__m512
|
|
1262
|
-
|
|
1263
|
-
__m512 a_f32_top_vec, a_f32_bot_vec, b_f32_top_vec, b_f32_bot_vec;
|
|
1264
|
-
__m512i a_i16_vec, b_i16_vec;
|
|
1527
|
+
__m512 d2_vec = _mm512_setzero_ps();
|
|
1528
|
+
__m512i a_i16_vec, b_i16_vec, d_i16_vec;
|
|
1265
1529
|
|
|
1266
1530
|
simsimd_l2sq_bf16_genoa_cycle:
|
|
1267
1531
|
if (n < 32) {
|
|
@@ -1274,35 +1538,12 @@ simsimd_l2sq_bf16_genoa_cycle:
|
|
|
1274
1538
|
b_i16_vec = _mm512_loadu_epi16(b);
|
|
1275
1539
|
a += 32, b += 32, n -= 32;
|
|
1276
1540
|
}
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
// expand it to 32-bit integers, then shift the bits by 16 to the left. Then subtract as floats,
|
|
1280
|
-
// and shift back. During expansion, we will double the space, and should use separate registers
|
|
1281
|
-
// for top and bottom halves.
|
|
1282
|
-
a_f32_bot_vec =
|
|
1283
|
-
_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(_mm512_castsi512_si256(a_i16_vec)), 16));
|
|
1284
|
-
b_f32_bot_vec =
|
|
1285
|
-
_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(_mm512_castsi512_si256(b_i16_vec)), 16));
|
|
1286
|
-
|
|
1287
|
-
// Some compilers don't have `_mm512_extracti32x8_epi32`, so we need to use `_mm512_extracti64x4_epi64`
|
|
1288
|
-
a_f32_top_vec =
|
|
1289
|
-
_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(_mm512_extracti64x4_epi64(a_i16_vec, 1)), 16));
|
|
1290
|
-
b_f32_top_vec =
|
|
1291
|
-
_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(_mm512_extracti64x4_epi64(b_i16_vec, 1)), 16));
|
|
1292
|
-
|
|
1293
|
-
// Subtract and cast back
|
|
1294
|
-
d_top_vec = _mm512_sub_ps(a_f32_top_vec, b_f32_top_vec);
|
|
1295
|
-
d_bot_vec = _mm512_sub_ps(a_f32_bot_vec, b_f32_bot_vec);
|
|
1296
|
-
d_top_vec = _mm512_castsi512_ps(_mm512_srli_epi32(_mm512_castps_si512(d_top_vec), 16));
|
|
1297
|
-
d_bot_vec = _mm512_castsi512_ps(_mm512_srli_epi32(_mm512_castps_si512(d_bot_vec), 16));
|
|
1298
|
-
|
|
1299
|
-
// Square and accumulate
|
|
1300
|
-
d2_top_vec = _mm512_dpbf16_ps(d2_top_vec, (__m512bh)(d_top_vec), (__m512bh)(d_top_vec));
|
|
1301
|
-
d2_bot_vec = _mm512_dpbf16_ps(d2_bot_vec, (__m512bh)(d_bot_vec), (__m512bh)(d_bot_vec));
|
|
1541
|
+
d_i16_vec = _simsimd_substract_bf16x32_genoa(a_i16_vec, b_i16_vec);
|
|
1542
|
+
d2_vec = _mm512_dpbf16_ps(d2_vec, (__m512bh)(d_i16_vec), (__m512bh)(d_i16_vec));
|
|
1302
1543
|
if (n)
|
|
1303
1544
|
goto simsimd_l2sq_bf16_genoa_cycle;
|
|
1304
1545
|
|
|
1305
|
-
*result =
|
|
1546
|
+
*result = _simsimd_reduce_f32x16_skylake(d2_vec);
|
|
1306
1547
|
}
|
|
1307
1548
|
|
|
1308
1549
|
SIMSIMD_PUBLIC void simsimd_cos_bf16_genoa(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n,
|
|
@@ -1329,15 +1570,10 @@ simsimd_cos_bf16_genoa_cycle:
|
|
|
1329
1570
|
if (n)
|
|
1330
1571
|
goto simsimd_cos_bf16_genoa_cycle;
|
|
1331
1572
|
|
|
1332
|
-
simsimd_f32_t ab =
|
|
1333
|
-
simsimd_f32_t a2 =
|
|
1334
|
-
simsimd_f32_t b2 =
|
|
1335
|
-
|
|
1336
|
-
// Compute the reciprocal square roots of a2 and b2
|
|
1337
|
-
__m128 rsqrts = _mm_rsqrt14_ps(_mm_set_ps(0.f, 0.f, a2 + 1.e-9f, b2 + 1.e-9f));
|
|
1338
|
-
simsimd_f32_t rsqrt_a2 = _mm_cvtss_f32(rsqrts);
|
|
1339
|
-
simsimd_f32_t rsqrt_b2 = _mm_cvtss_f32(_mm_shuffle_ps(rsqrts, rsqrts, _MM_SHUFFLE(0, 0, 0, 1)));
|
|
1340
|
-
*result = ab != 0 ? 1 - ab * rsqrt_a2 * rsqrt_b2 : 1;
|
|
1573
|
+
simsimd_f32_t ab = _simsimd_reduce_f32x16_skylake(ab_vec);
|
|
1574
|
+
simsimd_f32_t a2 = _simsimd_reduce_f32x16_skylake(a2_vec);
|
|
1575
|
+
simsimd_f32_t b2 = _simsimd_reduce_f32x16_skylake(b2_vec);
|
|
1576
|
+
*result = _simsimd_cos_normalize_f32_haswell(ab, a2, b2);
|
|
1341
1577
|
}
|
|
1342
1578
|
|
|
1343
1579
|
#pragma clang attribute pop
|
|
@@ -1346,9 +1582,14 @@ simsimd_cos_bf16_genoa_cycle:
|
|
|
1346
1582
|
|
|
1347
1583
|
#if SIMSIMD_TARGET_SAPPHIRE
|
|
1348
1584
|
#pragma GCC push_options
|
|
1349
|
-
#pragma GCC target("avx512f", "avx512vl", "bmi2", "avx512fp16")
|
|
1350
|
-
#pragma clang attribute push(__attribute__((target("avx512f,avx512vl,bmi2,avx512fp16"))), apply_to = function)
|
|
1585
|
+
#pragma GCC target("avx2", "avx512f", "avx512vl", "bmi2", "avx512fp16")
|
|
1586
|
+
#pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,bmi2,avx512fp16"))), apply_to = function)
|
|
1351
1587
|
|
|
1588
|
+
SIMSIMD_PUBLIC void simsimd_l2_f16_sapphire(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n,
|
|
1589
|
+
simsimd_distance_t* result) {
|
|
1590
|
+
simsimd_l2sq_f16_sapphire(a, b, n, result);
|
|
1591
|
+
*result = _simsimd_sqrt_f32_haswell(*result);
|
|
1592
|
+
}
|
|
1352
1593
|
SIMSIMD_PUBLIC void simsimd_l2sq_f16_sapphire(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n,
|
|
1353
1594
|
simsimd_distance_t* result) {
|
|
1354
1595
|
__m512h d2_vec = _mm512_setzero_ph();
|
|
@@ -1400,12 +1641,7 @@ simsimd_cos_f16_sapphire_cycle:
|
|
|
1400
1641
|
simsimd_f32_t ab = _mm512_reduce_add_ph(ab_vec);
|
|
1401
1642
|
simsimd_f32_t a2 = _mm512_reduce_add_ph(a2_vec);
|
|
1402
1643
|
simsimd_f32_t b2 = _mm512_reduce_add_ph(b2_vec);
|
|
1403
|
-
|
|
1404
|
-
// Compute the reciprocal square roots of a2 and b2
|
|
1405
|
-
__m128 rsqrts = _mm_rsqrt14_ps(_mm_set_ps(0.f, 0.f, a2 + 1.e-9f, b2 + 1.e-9f));
|
|
1406
|
-
simsimd_f32_t rsqrt_a2 = _mm_cvtss_f32(rsqrts);
|
|
1407
|
-
simsimd_f32_t rsqrt_b2 = _mm_cvtss_f32(_mm_shuffle_ps(rsqrts, rsqrts, _MM_SHUFFLE(0, 0, 0, 1)));
|
|
1408
|
-
*result = ab != 0 ? 1 - ab * rsqrt_a2 * rsqrt_b2 : 1;
|
|
1644
|
+
*result = _simsimd_cos_normalize_f32_haswell(ab, a2, b2);
|
|
1409
1645
|
}
|
|
1410
1646
|
|
|
1411
1647
|
#pragma clang attribute pop
|
|
@@ -1414,16 +1650,22 @@ simsimd_cos_f16_sapphire_cycle:
|
|
|
1414
1650
|
|
|
1415
1651
|
#if SIMSIMD_TARGET_ICE
|
|
1416
1652
|
#pragma GCC push_options
|
|
1417
|
-
#pragma GCC target("avx512f", "avx512vl", "bmi2", "avx512bw", "avx512vnni")
|
|
1418
|
-
#pragma clang attribute push(__attribute__((target("avx512f,avx512vl,bmi2,avx512bw,avx512vnni"))),
|
|
1419
|
-
|
|
1653
|
+
#pragma GCC target("avx2", "avx512f", "avx512vl", "bmi2", "avx512bw", "avx512vnni")
|
|
1654
|
+
#pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,bmi2,avx512bw,avx512vnni"))), \
|
|
1655
|
+
apply_to = function)
|
|
1656
|
+
|
|
1657
|
+
SIMSIMD_PUBLIC void simsimd_l2_i8_ice(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n,
|
|
1658
|
+
simsimd_distance_t* result) {
|
|
1659
|
+
simsimd_l2sq_i8_ice(a, b, n, result);
|
|
1660
|
+
*result = _simsimd_sqrt_f32_haswell(*result);
|
|
1661
|
+
}
|
|
1420
1662
|
SIMSIMD_PUBLIC void simsimd_l2sq_i8_ice(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n,
|
|
1421
1663
|
simsimd_distance_t* result) {
|
|
1422
1664
|
__m512i d2_i32s_vec = _mm512_setzero_si512();
|
|
1423
1665
|
__m512i a_vec, b_vec, d_i16s_vec;
|
|
1424
1666
|
|
|
1425
1667
|
simsimd_l2sq_i8_ice_cycle:
|
|
1426
|
-
if (n < 32) {
|
|
1668
|
+
if (n < 32) { // TODO: Avoid easrly i16 upcast to step through 64 values at a time
|
|
1427
1669
|
__mmask32 mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, n);
|
|
1428
1670
|
a_vec = _mm512_cvtepi8_epi16(_mm256_maskz_loadu_epi8(mask, a));
|
|
1429
1671
|
b_vec = _mm512_cvtepi8_epi16(_mm256_maskz_loadu_epi8(mask, b));
|
|
@@ -1488,14 +1730,7 @@ simsimd_cos_i8_ice_cycle:
|
|
|
1488
1730
|
int ab = _mm512_reduce_add_epi32(_mm512_add_epi32(ab_low_i32s_vec, ab_high_i32s_vec));
|
|
1489
1731
|
int a2 = _mm512_reduce_add_epi32(a2_i32s_vec);
|
|
1490
1732
|
int b2 = _mm512_reduce_add_epi32(b2_i32s_vec);
|
|
1491
|
-
|
|
1492
|
-
// Compute the reciprocal square roots of a2 and b2
|
|
1493
|
-
// Mysteriously, MSVC has no `_mm_rsqrt14_ps` intrinsic, but has it's masked variants,
|
|
1494
|
-
// so let's use `_mm_maskz_rsqrt14_ps(0xFF, ...)` instead.
|
|
1495
|
-
__m128 rsqrts = _mm_maskz_rsqrt14_ps(0xFF, _mm_set_ps(0.f, 0.f, a2 + 1.e-9f, b2 + 1.e-9f));
|
|
1496
|
-
simsimd_f32_t rsqrt_a2 = _mm_cvtss_f32(rsqrts);
|
|
1497
|
-
simsimd_f32_t rsqrt_b2 = _mm_cvtss_f32(_mm_shuffle_ps(rsqrts, rsqrts, _MM_SHUFFLE(0, 0, 0, 1)));
|
|
1498
|
-
*result = ab != 0 ? 1 - ab * rsqrt_a2 * rsqrt_b2 : 1;
|
|
1733
|
+
*result = _simsimd_cos_normalize_f32_haswell(ab, a2, b2);
|
|
1499
1734
|
}
|
|
1500
1735
|
|
|
1501
1736
|
#pragma clang attribute pop
|