usearch 2.15.1 → 2.15.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/binding.gyp +1 -1
- package/c/CMakeLists.txt +10 -3
- package/cpp/README.md +1 -1
- package/cpp/bench.cpp +2 -2
- package/include/usearch/index.hpp +27 -13
- package/include/usearch/index_dense.hpp +45 -88
- package/include/usearch/index_plugins.hpp +128 -40
- package/javascript/usearch.test.js +15 -13
- package/package.json +1 -1
- package/prebuilds/darwin-arm64+x64/usearch.node +0 -0
- package/prebuilds/linux-arm64/usearch.node +0 -0
- package/prebuilds/linux-x64/usearch.node +0 -0
- package/simsimd/CODE_OF_CONDUCT.md +128 -0
- package/simsimd/CONTRIBUTING.md +18 -8
- package/simsimd/Cargo.lock +1 -1
- package/simsimd/Cargo.toml +1 -1
- package/simsimd/MANIFEST.in +2 -0
- package/simsimd/README.md +446 -158
- package/simsimd/build.rs +11 -1
- package/simsimd/c/lib.c +172 -40
- package/simsimd/include/simsimd/binary.h +11 -11
- package/simsimd/include/simsimd/curved.h +804 -0
- package/simsimd/include/simsimd/dot.h +333 -334
- package/simsimd/include/simsimd/geospatial.h +9 -3
- package/simsimd/include/simsimd/mesh.h +69 -0
- package/simsimd/include/simsimd/probability.h +178 -150
- package/simsimd/include/simsimd/simsimd.h +581 -107
- package/simsimd/include/simsimd/sparse.h +807 -0
- package/simsimd/include/simsimd/spatial.h +737 -502
- package/simsimd/include/simsimd/types.h +145 -75
- package/simsimd/javascript/simsimd.ts +4 -4
- package/simsimd/package.json +1 -1
- package/simsimd/rust/lib.rs +165 -0
|
@@ -6,15 +6,21 @@
|
|
|
6
6
|
*
|
|
7
7
|
* Contains:
|
|
8
8
|
* - Haversine (Great Circle) distance
|
|
9
|
-
* -
|
|
9
|
+
* - Vincenty's distance function for Oblate Spheroid Geodesics
|
|
10
10
|
*
|
|
11
11
|
* For datatypes:
|
|
12
12
|
* - 32-bit IEEE-754 floating point
|
|
13
13
|
* - 64-bit IEEE-754 floating point
|
|
14
14
|
*
|
|
15
15
|
* For hardware architectures:
|
|
16
|
-
* - Arm (NEON
|
|
17
|
-
* - x86 (
|
|
16
|
+
* - Arm (NEON)
|
|
17
|
+
* - x86 (AVX2)
|
|
18
|
+
*
|
|
19
|
+
* In most cases, for distance computations, we don't need the exact Haversine formula.
|
|
20
|
+
* The very last part of the computation applies `asin(sqrt(x))` non-linear transformation.
|
|
21
|
+
* Both `asin` and `sqrt` are monotonically increasing functions, so their product is also
|
|
22
|
+
* monotonically increasing. This means, for relative similarity/closeness computation we
|
|
23
|
+
* can avoid that expensive last step.
|
|
18
24
|
*
|
|
19
25
|
* x86 intrinsics: https://www.intel.com/content/www/us/en/docs/intrinsics-guide/
|
|
20
26
|
* Arm intrinsics: https://developer.arm.com/architectures/instruction-sets/intrinsics/
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file mesh.h
|
|
3
|
+
* @brief SIMD-accelerated similarity measures for meshes and rigid 3D bodies.
|
|
4
|
+
* @author Ash Vardanian
|
|
5
|
+
* @date June 19, 2024
|
|
6
|
+
*
|
|
7
|
+
* Contains:
|
|
8
|
+
* - Root Mean Square Deviation (RMSD) for rigid body superposition
|
|
9
|
+
* - Kabsch algorithm for optimal rigid body superposition
|
|
10
|
+
*
|
|
11
|
+
* For datatypes:
|
|
12
|
+
* - 64-bit IEEE-754 floating point
|
|
13
|
+
* - 32-bit IEEE-754 floating point
|
|
14
|
+
* - 16-bit IEEE-754 floating point
|
|
15
|
+
* - 16-bit brain-floating point
|
|
16
|
+
*
|
|
17
|
+
* For hardware architectures:
|
|
18
|
+
* - Arm (NEON, SVE)
|
|
19
|
+
* - x86 (AVX512)
|
|
20
|
+
*
|
|
21
|
+
* x86 intrinsics: https://www.intel.com/content/www/us/en/docs/intrinsics-guide/
|
|
22
|
+
* Arm intrinsics: https://developer.arm.com/architectures/instruction-sets/intrinsics/
|
|
23
|
+
*/
|
|
24
|
+
#ifndef SIMSIMD_MESH_H
|
|
25
|
+
#define SIMSIMD_MESH_H
|
|
26
|
+
|
|
27
|
+
#include "types.h"
|
|
28
|
+
|
|
29
|
+
#ifdef __cplusplus
|
|
30
|
+
extern "C" {
|
|
31
|
+
#endif
|
|
32
|
+
|
|
33
|
+
// clang-format off
|
|
34
|
+
|
|
35
|
+
/* Serial backends for all numeric types.
|
|
36
|
+
* By default they use 32-bit arithmetic, unless the arguments themselves contain 64-bit floats.
|
|
37
|
+
* For double-precision computation check out the "*_accurate" variants of those "*_serial" functions.
|
|
38
|
+
*/
|
|
39
|
+
SIMSIMD_PUBLIC void simsimd_rmsd_f64_serial(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_f64_t* a_centroid, simsimd_f64_t* b_centroid, simsimd_distance_t* result);
|
|
40
|
+
SIMSIMD_PUBLIC void simsimd_kabsch_f64_serial(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_f64_t* a_centroid, simsimd_f64_t* b_centroid, simsimd_distance_t* result);
|
|
41
|
+
|
|
42
|
+
SIMSIMD_PUBLIC void simsimd_rmsd_f32_serial(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_f32_t* a_centroid, simsimd_f32_t* b_centroid, simsimd_distance_t* result);
|
|
43
|
+
SIMSIMD_PUBLIC void simsimd_kabsch_f32_serial(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_f32_t* a_centroid, simsimd_f32_t* b_centroid, simsimd_distance_t* result);
|
|
44
|
+
|
|
45
|
+
SIMSIMD_PUBLIC void simsimd_rmsd_f16_serial(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_f16_t* a_centroid, simsimd_f16_t* b_centroid, simsimd_distance_t* result);
|
|
46
|
+
SIMSIMD_PUBLIC void simsimd_kabsch_f16_serial(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_f16_t* a_centroid, simsimd_f16_t* b_centroid, simsimd_distance_t* result);
|
|
47
|
+
|
|
48
|
+
SIMSIMD_PUBLIC void simsimd_rmsd_bf16_serial(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_bf16_t* a_centroid, simsimd_bf16_t* b_centroid, simsimd_distance_t* result);
|
|
49
|
+
SIMSIMD_PUBLIC void simsimd_kabsch_bf16_serial(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_bf16_t* a_centroid, simsimd_bf16_t* b_centroid, simsimd_distance_t* result);
|
|
50
|
+
|
|
51
|
+
/* Double-precision serial backends for all numeric types.
|
|
52
|
+
* For single-precision computation check out the "*_serial" counterparts of those "*_accurate" functions.
|
|
53
|
+
*/
|
|
54
|
+
SIMSIMD_PUBLIC void simsimd_rmsd_f32_accurate(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_f32_t* a_centroid, simsimd_f32_t* b_centroid, simsimd_distance_t* result);
|
|
55
|
+
SIMSIMD_PUBLIC void simsimd_kabsch_f32_accurate(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_f32_t* a_centroid, simsimd_f32_t* b_centroid, simsimd_distance_t* result);
|
|
56
|
+
|
|
57
|
+
SIMSIMD_PUBLIC void simsimd_rmsd_f16_accurate(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_f16_t* a_centroid, simsimd_f16_t* b_centroid, simsimd_distance_t* result);
|
|
58
|
+
SIMSIMD_PUBLIC void simsimd_kabsch_f16_accurate(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_f16_t* a_centroid, simsimd_f16_t* b_centroid, simsimd_distance_t* result);
|
|
59
|
+
|
|
60
|
+
SIMSIMD_PUBLIC void simsimd_rmsd_bf16_accurate(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_bf16_t* a_centroid, simsimd_bf16_t* b_centroid, simsimd_distance_t* result);
|
|
61
|
+
SIMSIMD_PUBLIC void simsimd_kabsch_bf16_accurate(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_bf16_t* a_centroid, simsimd_bf16_t* b_centroid, simsimd_distance_t* result);
|
|
62
|
+
|
|
63
|
+
// clang-format on
|
|
64
|
+
|
|
65
|
+
#ifdef __cplusplus
|
|
66
|
+
}
|
|
67
|
+
#endif
|
|
68
|
+
|
|
69
|
+
#endif
|
|
@@ -11,6 +11,7 @@
|
|
|
11
11
|
* For datatypes:
|
|
12
12
|
* - 32-bit floating point numbers
|
|
13
13
|
* - 16-bit floating point numbers
|
|
14
|
+
* - 16-bit brain-floating point numbers
|
|
14
15
|
*
|
|
15
16
|
* For hardware architectures:
|
|
16
17
|
* - Arm (NEON, SVE)
|
|
@@ -82,27 +83,27 @@ SIMSIMD_PUBLIC void simsimd_kl_f16_sapphire(simsimd_f16_t const* a, simsimd_f16_
|
|
|
82
83
|
SIMSIMD_PUBLIC void simsimd_js_f16_sapphire(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* divergence);
|
|
83
84
|
// clang-format on
|
|
84
85
|
|
|
85
|
-
#define SIMSIMD_MAKE_KL(name, input_type, accumulator_type,
|
|
86
|
+
#define SIMSIMD_MAKE_KL(name, input_type, accumulator_type, load_and_convert, epsilon) \
|
|
86
87
|
SIMSIMD_PUBLIC void simsimd_kl_##input_type##_##name(simsimd_##input_type##_t const* a, \
|
|
87
88
|
simsimd_##input_type##_t const* b, simsimd_size_t n, \
|
|
88
89
|
simsimd_distance_t* result) { \
|
|
89
90
|
simsimd_##accumulator_type##_t d = 0; \
|
|
90
91
|
for (simsimd_size_t i = 0; i != n; ++i) { \
|
|
91
|
-
simsimd_##accumulator_type##_t ai =
|
|
92
|
-
simsimd_##accumulator_type##_t bi =
|
|
92
|
+
simsimd_##accumulator_type##_t ai = load_and_convert(a + i); \
|
|
93
|
+
simsimd_##accumulator_type##_t bi = load_and_convert(b + i); \
|
|
93
94
|
d += ai * SIMSIMD_LOG((ai + epsilon) / (bi + epsilon)); \
|
|
94
95
|
} \
|
|
95
96
|
*result = (simsimd_distance_t)d; \
|
|
96
97
|
}
|
|
97
98
|
|
|
98
|
-
#define SIMSIMD_MAKE_JS(name, input_type, accumulator_type,
|
|
99
|
+
#define SIMSIMD_MAKE_JS(name, input_type, accumulator_type, load_and_convert, epsilon) \
|
|
99
100
|
SIMSIMD_PUBLIC void simsimd_js_##input_type##_##name(simsimd_##input_type##_t const* a, \
|
|
100
101
|
simsimd_##input_type##_t const* b, simsimd_size_t n, \
|
|
101
102
|
simsimd_distance_t* result) { \
|
|
102
103
|
simsimd_##accumulator_type##_t d = 0; \
|
|
103
104
|
for (simsimd_size_t i = 0; i != n; ++i) { \
|
|
104
|
-
simsimd_##accumulator_type##_t ai =
|
|
105
|
-
simsimd_##accumulator_type##_t bi =
|
|
105
|
+
simsimd_##accumulator_type##_t ai = load_and_convert(a + i); \
|
|
106
|
+
simsimd_##accumulator_type##_t bi = load_and_convert(b + i); \
|
|
106
107
|
simsimd_##accumulator_type##_t mi = (ai + bi) / 2; \
|
|
107
108
|
d += ai * SIMSIMD_LOG((ai + epsilon) / (mi + epsilon)); \
|
|
108
109
|
d += bi * SIMSIMD_LOG((bi + epsilon) / (mi + epsilon)); \
|
|
@@ -110,34 +111,34 @@ SIMSIMD_PUBLIC void simsimd_js_f16_sapphire(simsimd_f16_t const* a, simsimd_f16_
|
|
|
110
111
|
*result = (simsimd_distance_t)d / 2; \
|
|
111
112
|
}
|
|
112
113
|
|
|
113
|
-
SIMSIMD_MAKE_KL(serial, f64, f64,
|
|
114
|
-
SIMSIMD_MAKE_JS(serial, f64, f64,
|
|
114
|
+
SIMSIMD_MAKE_KL(serial, f64, f64, SIMSIMD_DEREFERENCE, SIMSIMD_F32_DIVISION_EPSILON) // simsimd_kl_f64_serial
|
|
115
|
+
SIMSIMD_MAKE_JS(serial, f64, f64, SIMSIMD_DEREFERENCE, SIMSIMD_F32_DIVISION_EPSILON) // simsimd_js_f64_serial
|
|
115
116
|
|
|
116
|
-
SIMSIMD_MAKE_KL(serial, f32, f32,
|
|
117
|
-
SIMSIMD_MAKE_JS(serial, f32, f32,
|
|
117
|
+
SIMSIMD_MAKE_KL(serial, f32, f32, SIMSIMD_DEREFERENCE, SIMSIMD_F32_DIVISION_EPSILON) // simsimd_kl_f32_serial
|
|
118
|
+
SIMSIMD_MAKE_JS(serial, f32, f32, SIMSIMD_DEREFERENCE, SIMSIMD_F32_DIVISION_EPSILON) // simsimd_js_f32_serial
|
|
118
119
|
|
|
119
|
-
SIMSIMD_MAKE_KL(serial, f16, f32,
|
|
120
|
-
SIMSIMD_MAKE_JS(serial, f16, f32,
|
|
120
|
+
SIMSIMD_MAKE_KL(serial, f16, f32, SIMSIMD_F16_TO_F32, SIMSIMD_F32_DIVISION_EPSILON) // simsimd_kl_f16_serial
|
|
121
|
+
SIMSIMD_MAKE_JS(serial, f16, f32, SIMSIMD_F16_TO_F32, SIMSIMD_F32_DIVISION_EPSILON) // simsimd_js_f16_serial
|
|
121
122
|
|
|
122
|
-
SIMSIMD_MAKE_KL(serial, bf16, f32,
|
|
123
|
-
SIMSIMD_MAKE_JS(serial, bf16, f32,
|
|
123
|
+
SIMSIMD_MAKE_KL(serial, bf16, f32, SIMSIMD_BF16_TO_F32, SIMSIMD_F32_DIVISION_EPSILON) // simsimd_kl_bf16_serial
|
|
124
|
+
SIMSIMD_MAKE_JS(serial, bf16, f32, SIMSIMD_BF16_TO_F32, SIMSIMD_F32_DIVISION_EPSILON) // simsimd_js_bf16_serial
|
|
124
125
|
|
|
125
|
-
SIMSIMD_MAKE_KL(accurate, f32, f64,
|
|
126
|
-
SIMSIMD_MAKE_JS(accurate, f32, f64,
|
|
126
|
+
SIMSIMD_MAKE_KL(accurate, f32, f64, SIMSIMD_DEREFERENCE, SIMSIMD_F32_DIVISION_EPSILON) // simsimd_kl_f32_accurate
|
|
127
|
+
SIMSIMD_MAKE_JS(accurate, f32, f64, SIMSIMD_DEREFERENCE, SIMSIMD_F32_DIVISION_EPSILON) // simsimd_js_f32_accurate
|
|
127
128
|
|
|
128
|
-
SIMSIMD_MAKE_KL(accurate, f16, f64,
|
|
129
|
-
SIMSIMD_MAKE_JS(accurate, f16, f64,
|
|
129
|
+
SIMSIMD_MAKE_KL(accurate, f16, f64, SIMSIMD_F16_TO_F32, SIMSIMD_F32_DIVISION_EPSILON) // simsimd_kl_f16_accurate
|
|
130
|
+
SIMSIMD_MAKE_JS(accurate, f16, f64, SIMSIMD_F16_TO_F32, SIMSIMD_F32_DIVISION_EPSILON) // simsimd_js_f16_accurate
|
|
130
131
|
|
|
131
|
-
SIMSIMD_MAKE_KL(accurate, bf16, f64,
|
|
132
|
-
SIMSIMD_MAKE_JS(accurate, bf16, f64,
|
|
132
|
+
SIMSIMD_MAKE_KL(accurate, bf16, f64, SIMSIMD_BF16_TO_F32, SIMSIMD_F32_DIVISION_EPSILON) // simsimd_kl_bf16_accurate
|
|
133
|
+
SIMSIMD_MAKE_JS(accurate, bf16, f64, SIMSIMD_BF16_TO_F32, SIMSIMD_F32_DIVISION_EPSILON) // simsimd_js_bf16_accurate
|
|
133
134
|
|
|
134
135
|
#if SIMSIMD_TARGET_ARM
|
|
135
136
|
#if SIMSIMD_TARGET_NEON
|
|
136
137
|
#pragma GCC push_options
|
|
137
|
-
#pragma GCC target("+simd")
|
|
138
|
-
#pragma clang attribute push(__attribute__((target("+simd"))), apply_to = function)
|
|
138
|
+
#pragma GCC target("arch=armv8.2-a+simd")
|
|
139
|
+
#pragma clang attribute push(__attribute__((target("arch=armv8.2-a+simd"))), apply_to = function)
|
|
139
140
|
|
|
140
|
-
SIMSIMD_PUBLIC float32x4_t
|
|
141
|
+
SIMSIMD_PUBLIC float32x4_t _simsimd_log2_f32_neon(float32x4_t x) {
|
|
141
142
|
// Extracting the exponent
|
|
142
143
|
int32x4_t i = vreinterpretq_s32_f32(x);
|
|
143
144
|
int32x4_t e = vsubq_s32(vshrq_n_s32(vandq_s32(i, vdupq_n_s32(0x7F800000)), 23), vdupq_n_s32(127));
|
|
@@ -164,79 +165,104 @@ SIMSIMD_PUBLIC float32x4_t simsimd_log2_f32_neon(float32x4_t x) {
|
|
|
164
165
|
|
|
165
166
|
SIMSIMD_PUBLIC void simsimd_kl_f32_neon(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n,
|
|
166
167
|
simsimd_distance_t* result) {
|
|
167
|
-
float32x4_t sum_vec = vdupq_n_f32(0);
|
|
168
168
|
simsimd_f32_t epsilon = SIMSIMD_F32_DIVISION_EPSILON;
|
|
169
169
|
float32x4_t epsilon_vec = vdupq_n_f32(epsilon);
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
170
|
+
float32x4_t sum_vec = vdupq_n_f32(0);
|
|
171
|
+
float32x4_t a_vec, b_vec;
|
|
172
|
+
|
|
173
|
+
simsimd_kl_f32_neon_cycle:
|
|
174
|
+
if (n < 4) {
|
|
175
|
+
a_vec = _simsimd_partial_load_f32x4_neon(a, n);
|
|
176
|
+
b_vec = _simsimd_partial_load_f32x4_neon(b, n);
|
|
177
|
+
n = 0;
|
|
178
|
+
} else {
|
|
179
|
+
a_vec = vld1q_f32(a);
|
|
180
|
+
b_vec = vld1q_f32(b);
|
|
181
|
+
n -= 4, a += 4, b += 4;
|
|
178
182
|
}
|
|
183
|
+
|
|
184
|
+
float32x4_t ratio_vec = vdivq_f32(vaddq_f32(a_vec, epsilon_vec), vaddq_f32(b_vec, epsilon_vec));
|
|
185
|
+
float32x4_t log_ratio_vec = _simsimd_log2_f32_neon(ratio_vec);
|
|
186
|
+
float32x4_t prod_vec = vmulq_f32(a_vec, log_ratio_vec);
|
|
187
|
+
sum_vec = vaddq_f32(sum_vec, prod_vec);
|
|
188
|
+
if (n != 0)
|
|
189
|
+
goto simsimd_kl_f32_neon_cycle;
|
|
190
|
+
|
|
179
191
|
simsimd_f32_t log2_normalizer = 0.693147181f;
|
|
180
192
|
simsimd_f32_t sum = vaddvq_f32(sum_vec) * log2_normalizer;
|
|
181
|
-
for (; i < n; ++i)
|
|
182
|
-
sum += a[i] * SIMSIMD_LOG((a[i] + epsilon) / (b[i] + epsilon));
|
|
183
193
|
*result = sum;
|
|
184
194
|
}
|
|
185
195
|
|
|
186
196
|
SIMSIMD_PUBLIC void simsimd_js_f32_neon(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n,
|
|
187
197
|
simsimd_distance_t* result) {
|
|
188
|
-
float32x4_t sum_vec = vdupq_n_f32(0);
|
|
189
198
|
simsimd_f32_t epsilon = SIMSIMD_F32_DIVISION_EPSILON;
|
|
190
199
|
float32x4_t epsilon_vec = vdupq_n_f32(epsilon);
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
200
|
+
float32x4_t sum_vec = vdupq_n_f32(0);
|
|
201
|
+
float32x4_t a_vec, b_vec;
|
|
202
|
+
|
|
203
|
+
simsimd_js_f32_neon_cycle:
|
|
204
|
+
if (n < 4) {
|
|
205
|
+
a_vec = _simsimd_partial_load_f32x4_neon(a, n);
|
|
206
|
+
b_vec = _simsimd_partial_load_f32x4_neon(b, n);
|
|
207
|
+
n = 0;
|
|
208
|
+
} else {
|
|
209
|
+
a_vec = vld1q_f32(a);
|
|
210
|
+
b_vec = vld1q_f32(b);
|
|
211
|
+
n -= 4, a += 4, b += 4;
|
|
203
212
|
}
|
|
213
|
+
|
|
214
|
+
float32x4_t m_vec = vmulq_f32(vaddq_f32(a_vec, b_vec), vdupq_n_f32(0.5));
|
|
215
|
+
float32x4_t ratio_a_vec = vdivq_f32(vaddq_f32(a_vec, epsilon_vec), vaddq_f32(m_vec, epsilon_vec));
|
|
216
|
+
float32x4_t ratio_b_vec = vdivq_f32(vaddq_f32(b_vec, epsilon_vec), vaddq_f32(m_vec, epsilon_vec));
|
|
217
|
+
float32x4_t log_ratio_a_vec = _simsimd_log2_f32_neon(ratio_a_vec);
|
|
218
|
+
float32x4_t log_ratio_b_vec = _simsimd_log2_f32_neon(ratio_b_vec);
|
|
219
|
+
float32x4_t prod_a_vec = vmulq_f32(a_vec, log_ratio_a_vec);
|
|
220
|
+
float32x4_t prod_b_vec = vmulq_f32(b_vec, log_ratio_b_vec);
|
|
221
|
+
sum_vec = vaddq_f32(sum_vec, vaddq_f32(prod_a_vec, prod_b_vec));
|
|
222
|
+
if (n != 0)
|
|
223
|
+
goto simsimd_js_f32_neon_cycle;
|
|
224
|
+
|
|
204
225
|
simsimd_f32_t log2_normalizer = 0.693147181f;
|
|
205
226
|
simsimd_f32_t sum = vaddvq_f32(sum_vec) * log2_normalizer;
|
|
206
|
-
|
|
207
|
-
simsimd_f32_t mi = 0.5f * (a[i] + b[i]);
|
|
208
|
-
sum += a[i] * SIMSIMD_LOG((a[i] + epsilon) / (mi + epsilon));
|
|
209
|
-
sum += b[i] * SIMSIMD_LOG((b[i] + epsilon) / (mi + epsilon));
|
|
210
|
-
}
|
|
211
|
-
*result = sum;
|
|
227
|
+
*result = sum / 2;
|
|
212
228
|
}
|
|
213
229
|
|
|
214
230
|
#pragma clang attribute pop
|
|
215
231
|
#pragma GCC pop_options
|
|
232
|
+
#endif // SIMSIMD_TARGET_NEON
|
|
216
233
|
|
|
234
|
+
#if SIMSIMD_TARGET_NEON_F16
|
|
217
235
|
#pragma GCC push_options
|
|
218
|
-
#pragma GCC target("+simd+fp16")
|
|
219
|
-
#pragma clang attribute push(__attribute__((target("+simd+fp16"))), apply_to = function)
|
|
236
|
+
#pragma GCC target("arch=armv8.2-a+simd+fp16")
|
|
237
|
+
#pragma clang attribute push(__attribute__((target("arch=armv8.2-a+simd+fp16"))), apply_to = function)
|
|
220
238
|
|
|
221
239
|
SIMSIMD_PUBLIC void simsimd_kl_f16_neon(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n,
|
|
222
240
|
simsimd_distance_t* result) {
|
|
223
241
|
float32x4_t sum_vec = vdupq_n_f32(0);
|
|
224
242
|
simsimd_f32_t epsilon = SIMSIMD_F32_DIVISION_EPSILON;
|
|
225
243
|
float32x4_t epsilon_vec = vdupq_n_f32(epsilon);
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
244
|
+
float32x4_t a_vec, b_vec;
|
|
245
|
+
|
|
246
|
+
simsimd_kl_f16_neon_cycle:
|
|
247
|
+
if (n < 4) {
|
|
248
|
+
a_vec = vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(a, n));
|
|
249
|
+
b_vec = vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(b, n));
|
|
250
|
+
n = 0;
|
|
251
|
+
} else {
|
|
252
|
+
a_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const*)a));
|
|
253
|
+
b_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const*)b));
|
|
254
|
+
n -= 4, a += 4, b += 4;
|
|
234
255
|
}
|
|
256
|
+
|
|
257
|
+
float32x4_t ratio_vec = vdivq_f32(vaddq_f32(a_vec, epsilon_vec), vaddq_f32(b_vec, epsilon_vec));
|
|
258
|
+
float32x4_t log_ratio_vec = _simsimd_log2_f32_neon(ratio_vec);
|
|
259
|
+
float32x4_t prod_vec = vmulq_f32(a_vec, log_ratio_vec);
|
|
260
|
+
sum_vec = vaddq_f32(sum_vec, prod_vec);
|
|
261
|
+
if (n)
|
|
262
|
+
goto simsimd_kl_f16_neon_cycle;
|
|
263
|
+
|
|
235
264
|
simsimd_f32_t log2_normalizer = 0.693147181f;
|
|
236
265
|
simsimd_f32_t sum = vaddvq_f32(sum_vec) * log2_normalizer;
|
|
237
|
-
for (; i < n; ++i)
|
|
238
|
-
sum += SIMSIMD_UNCOMPRESS_F16(a[i]) *
|
|
239
|
-
SIMSIMD_LOG((SIMSIMD_UNCOMPRESS_F16(a[i]) + epsilon) / (SIMSIMD_UNCOMPRESS_F16(b[i]) + epsilon));
|
|
240
266
|
*result = sum;
|
|
241
267
|
}
|
|
242
268
|
|
|
@@ -245,34 +271,38 @@ SIMSIMD_PUBLIC void simsimd_js_f16_neon(simsimd_f16_t const* a, simsimd_f16_t co
|
|
|
245
271
|
float32x4_t sum_vec = vdupq_n_f32(0);
|
|
246
272
|
simsimd_f32_t epsilon = SIMSIMD_F32_DIVISION_EPSILON;
|
|
247
273
|
float32x4_t epsilon_vec = vdupq_n_f32(epsilon);
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
sum_vec = vaddq_f32(sum_vec, vaddq_f32(prod_a_vec, prod_b_vec));
|
|
274
|
+
float32x4_t a_vec, b_vec;
|
|
275
|
+
|
|
276
|
+
simsimd_js_f16_neon_cycle:
|
|
277
|
+
if (n < 4) {
|
|
278
|
+
a_vec = vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(a, n));
|
|
279
|
+
b_vec = vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(b, n));
|
|
280
|
+
n = 0;
|
|
281
|
+
} else {
|
|
282
|
+
a_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const*)a));
|
|
283
|
+
b_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const*)b));
|
|
284
|
+
n -= 4, a += 4, b += 4;
|
|
260
285
|
}
|
|
286
|
+
|
|
287
|
+
float32x4_t m_vec = vmulq_f32(vaddq_f32(a_vec, b_vec), vdupq_n_f32(0.5));
|
|
288
|
+
float32x4_t ratio_a_vec = vdivq_f32(vaddq_f32(a_vec, epsilon_vec), vaddq_f32(m_vec, epsilon_vec));
|
|
289
|
+
float32x4_t ratio_b_vec = vdivq_f32(vaddq_f32(b_vec, epsilon_vec), vaddq_f32(m_vec, epsilon_vec));
|
|
290
|
+
float32x4_t log_ratio_a_vec = _simsimd_log2_f32_neon(ratio_a_vec);
|
|
291
|
+
float32x4_t log_ratio_b_vec = _simsimd_log2_f32_neon(ratio_b_vec);
|
|
292
|
+
float32x4_t prod_a_vec = vmulq_f32(a_vec, log_ratio_a_vec);
|
|
293
|
+
float32x4_t prod_b_vec = vmulq_f32(b_vec, log_ratio_b_vec);
|
|
294
|
+
sum_vec = vaddq_f32(sum_vec, vaddq_f32(prod_a_vec, prod_b_vec));
|
|
295
|
+
if (n)
|
|
296
|
+
goto simsimd_js_f16_neon_cycle;
|
|
297
|
+
|
|
261
298
|
simsimd_f32_t log2_normalizer = 0.693147181f;
|
|
262
299
|
simsimd_f32_t sum = vaddvq_f32(sum_vec) * log2_normalizer;
|
|
263
|
-
|
|
264
|
-
simsimd_f32_t ai = SIMSIMD_UNCOMPRESS_F16(a[i]);
|
|
265
|
-
simsimd_f32_t bi = SIMSIMD_UNCOMPRESS_F16(b[i]);
|
|
266
|
-
simsimd_f32_t mi = 0.5f * (ai + bi);
|
|
267
|
-
sum += ai * SIMSIMD_LOG((ai + epsilon) / (mi + epsilon));
|
|
268
|
-
sum += bi * SIMSIMD_LOG((bi + epsilon) / (mi + epsilon));
|
|
269
|
-
}
|
|
270
|
-
*result = sum;
|
|
300
|
+
*result = sum / 2;
|
|
271
301
|
}
|
|
272
302
|
|
|
273
303
|
#pragma clang attribute pop
|
|
274
304
|
#pragma GCC pop_options
|
|
275
|
-
#endif //
|
|
305
|
+
#endif // SIMSIMD_TARGET_NEON_F16
|
|
276
306
|
#endif // SIMSIMD_TARGET_ARM
|
|
277
307
|
|
|
278
308
|
#if SIMSIMD_TARGET_X86
|
|
@@ -281,7 +311,7 @@ SIMSIMD_PUBLIC void simsimd_js_f16_neon(simsimd_f16_t const* a, simsimd_f16_t co
|
|
|
281
311
|
#pragma GCC target("avx2", "f16c", "fma")
|
|
282
312
|
#pragma clang attribute push(__attribute__((target("avx2,f16c,fma"))), apply_to = function)
|
|
283
313
|
|
|
284
|
-
|
|
314
|
+
SIMSIMD_INTERNAL __m256 _simsimd_log2_f32_haswell(__m256 x) {
|
|
285
315
|
// Extracting the exponent
|
|
286
316
|
__m256i i = _mm256_castps_si256(x);
|
|
287
317
|
__m256i e = _mm256_srli_epi32(_mm256_and_si256(i, _mm256_set1_epi32(0x7F800000)), 23);
|
|
@@ -313,69 +343,67 @@ SIMSIMD_PUBLIC void simsimd_kl_f16_haswell(simsimd_f16_t const* a, simsimd_f16_t
|
|
|
313
343
|
__m256 sum_vec = _mm256_setzero_ps();
|
|
314
344
|
simsimd_f32_t epsilon = SIMSIMD_F32_DIVISION_EPSILON;
|
|
315
345
|
__m256 epsilon_vec = _mm256_set1_ps(epsilon);
|
|
316
|
-
|
|
317
|
-
for (; i + 8 <= n; i += 8) {
|
|
318
|
-
__m256 a_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)(a + i)));
|
|
319
|
-
__m256 b_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)(b + i)));
|
|
320
|
-
__m256 ratio_vec = _mm256_div_ps(_mm256_add_ps(a_vec, epsilon_vec), _mm256_add_ps(b_vec, epsilon_vec));
|
|
321
|
-
__m256 log_ratio_vec = simsimd_log2_f32_haswell(ratio_vec);
|
|
322
|
-
__m256 prod_vec = _mm256_mul_ps(a_vec, log_ratio_vec);
|
|
323
|
-
sum_vec = _mm256_add_ps(sum_vec, prod_vec);
|
|
324
|
-
}
|
|
346
|
+
__m256 a_vec, b_vec;
|
|
325
347
|
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
348
|
+
simsimd_kl_f16_haswell_cycle:
|
|
349
|
+
if (n < 8) {
|
|
350
|
+
a_vec = _simsimd_partial_load_f16x8_haswell(a, n);
|
|
351
|
+
b_vec = _simsimd_partial_load_f16x8_haswell(b, n);
|
|
352
|
+
n = 0;
|
|
353
|
+
} else {
|
|
354
|
+
a_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)a));
|
|
355
|
+
b_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)b));
|
|
356
|
+
n -= 8, a += 8, b += 8;
|
|
357
|
+
}
|
|
358
|
+
a_vec = _mm256_add_ps(a_vec, epsilon_vec);
|
|
359
|
+
b_vec = _mm256_add_ps(b_vec, epsilon_vec);
|
|
360
|
+
__m256 ratio_vec = _mm256_div_ps(a_vec, b_vec);
|
|
361
|
+
__m256 log_ratio_vec = _simsimd_log2_f32_haswell(ratio_vec);
|
|
362
|
+
__m256 prod_vec = _mm256_mul_ps(a_vec, log_ratio_vec);
|
|
363
|
+
sum_vec = _mm256_add_ps(sum_vec, prod_vec);
|
|
364
|
+
if (n)
|
|
365
|
+
goto simsimd_kl_f16_haswell_cycle;
|
|
329
366
|
|
|
330
367
|
simsimd_f32_t log2_normalizer = 0.693147181f;
|
|
331
|
-
simsimd_f32_t sum;
|
|
332
|
-
_mm_store_ss(&sum, _mm256_castps256_ps128(sum_vec));
|
|
368
|
+
simsimd_f32_t sum = _simsimd_reduce_f32x8_haswell(sum_vec);
|
|
333
369
|
sum *= log2_normalizer;
|
|
334
|
-
|
|
335
|
-
// Accumulate the tail:
|
|
336
|
-
for (; i < n; ++i)
|
|
337
|
-
sum += SIMSIMD_UNCOMPRESS_F16(a[i]) *
|
|
338
|
-
SIMSIMD_LOG((SIMSIMD_UNCOMPRESS_F16(a[i]) + epsilon) / (SIMSIMD_UNCOMPRESS_F16(b[i]) + epsilon));
|
|
339
370
|
*result = sum;
|
|
340
371
|
}
|
|
341
372
|
|
|
342
373
|
SIMSIMD_PUBLIC void simsimd_js_f16_haswell(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n,
|
|
343
374
|
simsimd_distance_t* result) {
|
|
344
|
-
__m256 sum_vec = _mm256_setzero_ps();
|
|
345
375
|
simsimd_f32_t epsilon = SIMSIMD_F32_DIVISION_EPSILON;
|
|
346
376
|
__m256 epsilon_vec = _mm256_set1_ps(epsilon);
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
__m256 a_vec = _mm256_add_ps(_mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)(a + i))), epsilon_vec);
|
|
350
|
-
__m256 b_vec = _mm256_add_ps(_mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)(b + i))), epsilon_vec);
|
|
351
|
-
__m256 m_vec = _mm256_mul_ps(_mm256_add_ps(a_vec, b_vec), _mm256_set1_ps(0.5f)); // M = (P + Q) / 2
|
|
352
|
-
__m256 ratio_a_vec = _mm256_div_ps(a_vec, m_vec);
|
|
353
|
-
__m256 ratio_b_vec = _mm256_div_ps(b_vec, m_vec);
|
|
354
|
-
__m256 log_ratio_a_vec = simsimd_log2_f32_haswell(ratio_a_vec);
|
|
355
|
-
__m256 log_ratio_b_vec = simsimd_log2_f32_haswell(ratio_b_vec);
|
|
356
|
-
__m256 prod_a_vec = _mm256_mul_ps(a_vec, log_ratio_a_vec);
|
|
357
|
-
__m256 prod_b_vec = _mm256_mul_ps(b_vec, log_ratio_b_vec);
|
|
358
|
-
sum_vec = _mm256_add_ps(sum_vec, prod_a_vec);
|
|
359
|
-
sum_vec = _mm256_add_ps(sum_vec, prod_b_vec);
|
|
360
|
-
}
|
|
377
|
+
__m256 sum_vec = _mm256_setzero_ps();
|
|
378
|
+
__m256 a_vec, b_vec;
|
|
361
379
|
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
380
|
+
simsimd_js_f16_haswell_cycle:
|
|
381
|
+
if (n < 8) {
|
|
382
|
+
a_vec = _simsimd_partial_load_f16x8_haswell(a, n);
|
|
383
|
+
b_vec = _simsimd_partial_load_f16x8_haswell(b, n);
|
|
384
|
+
n = 0;
|
|
385
|
+
} else {
|
|
386
|
+
a_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)a));
|
|
387
|
+
b_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)b));
|
|
388
|
+
n -= 8, a += 8, b += 8;
|
|
389
|
+
}
|
|
390
|
+
a_vec = _mm256_add_ps(a_vec, epsilon_vec);
|
|
391
|
+
b_vec = _mm256_add_ps(b_vec, epsilon_vec);
|
|
392
|
+
__m256 m_vec = _mm256_mul_ps(_mm256_add_ps(a_vec, b_vec), _mm256_set1_ps(0.5f)); // M = (P + Q) / 2
|
|
393
|
+
__m256 ratio_a_vec = _mm256_div_ps(a_vec, m_vec);
|
|
394
|
+
__m256 ratio_b_vec = _mm256_div_ps(b_vec, m_vec);
|
|
395
|
+
__m256 log_ratio_a_vec = _simsimd_log2_f32_haswell(ratio_a_vec);
|
|
396
|
+
__m256 log_ratio_b_vec = _simsimd_log2_f32_haswell(ratio_b_vec);
|
|
397
|
+
__m256 prod_a_vec = _mm256_mul_ps(a_vec, log_ratio_a_vec);
|
|
398
|
+
__m256 prod_b_vec = _mm256_mul_ps(b_vec, log_ratio_b_vec);
|
|
399
|
+
sum_vec = _mm256_add_ps(sum_vec, prod_a_vec);
|
|
400
|
+
sum_vec = _mm256_add_ps(sum_vec, prod_b_vec);
|
|
401
|
+
if (n)
|
|
402
|
+
goto simsimd_js_f16_haswell_cycle;
|
|
365
403
|
|
|
366
404
|
simsimd_f32_t log2_normalizer = 0.693147181f;
|
|
367
|
-
simsimd_f32_t sum;
|
|
368
|
-
_mm_store_ss(&sum, _mm256_castps256_ps128(sum_vec));
|
|
405
|
+
simsimd_f32_t sum = _simsimd_reduce_f32x8_haswell(sum_vec);
|
|
369
406
|
sum *= log2_normalizer;
|
|
370
|
-
|
|
371
|
-
// Accumulate the tail:
|
|
372
|
-
for (; i < n; ++i) {
|
|
373
|
-
simsimd_f32_t ai = SIMSIMD_UNCOMPRESS_F16(a[i]);
|
|
374
|
-
simsimd_f32_t bi = SIMSIMD_UNCOMPRESS_F16(b[i]);
|
|
375
|
-
simsimd_f32_t mi = ai + bi;
|
|
376
|
-
sum += ai * SIMSIMD_LOG((ai + epsilon) / (mi + epsilon));
|
|
377
|
-
sum += bi * SIMSIMD_LOG((bi + epsilon) / (mi + epsilon));
|
|
378
|
-
}
|
|
379
407
|
*result = sum / 2;
|
|
380
408
|
}
|
|
381
409
|
|
|
@@ -385,10 +413,10 @@ SIMSIMD_PUBLIC void simsimd_js_f16_haswell(simsimd_f16_t const* a, simsimd_f16_t
|
|
|
385
413
|
|
|
386
414
|
#if SIMSIMD_TARGET_SKYLAKE
|
|
387
415
|
#pragma GCC push_options
|
|
388
|
-
#pragma GCC target("avx512f", "avx512vl", "bmi2")
|
|
389
|
-
#pragma clang attribute push(__attribute__((target("avx512f,avx512vl,bmi2"))), apply_to = function)
|
|
416
|
+
#pragma GCC target("avx2", "avx512f", "avx512vl", "bmi2")
|
|
417
|
+
#pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,bmi2"))), apply_to = function)
|
|
390
418
|
|
|
391
|
-
|
|
419
|
+
SIMSIMD_INTERNAL __m512 _simsimd_log2_f32_skylake(__m512 x) {
|
|
392
420
|
// Extract the exponent and mantissa
|
|
393
421
|
__m512 one = _mm512_set1_ps(1.0f);
|
|
394
422
|
__m512 e = _mm512_getexp_ps(x);
|
|
@@ -424,7 +452,7 @@ simsimd_kl_f32_skylake_cycle:
|
|
|
424
452
|
a += 16, b += 16, n -= 16;
|
|
425
453
|
}
|
|
426
454
|
__m512 ratio_vec = _mm512_div_ps(a_vec, b_vec);
|
|
427
|
-
__m512 log_ratio_vec =
|
|
455
|
+
__m512 log_ratio_vec = _simsimd_log2_f32_skylake(ratio_vec);
|
|
428
456
|
__m512 prod_vec = _mm512_mul_ps(a_vec, log_ratio_vec);
|
|
429
457
|
sum_vec = _mm512_add_ps(sum_vec, prod_vec);
|
|
430
458
|
if (n)
|
|
@@ -460,15 +488,15 @@ simsimd_js_f32_skylake_cycle:
|
|
|
460
488
|
__m512 m_recip_approx = _mm512_rcp14_ps(m_vec);
|
|
461
489
|
__m512 ratio_a_vec = _mm512_mul_ps(a_vec, m_recip_approx);
|
|
462
490
|
__m512 ratio_b_vec = _mm512_mul_ps(b_vec, m_recip_approx);
|
|
463
|
-
__m512 log_ratio_a_vec =
|
|
464
|
-
__m512 log_ratio_b_vec =
|
|
491
|
+
__m512 log_ratio_a_vec = _simsimd_log2_f32_skylake(ratio_a_vec);
|
|
492
|
+
__m512 log_ratio_b_vec = _simsimd_log2_f32_skylake(ratio_b_vec);
|
|
465
493
|
sum_a_vec = _mm512_maskz_fmadd_ps(nonzero_mask, a_vec, log_ratio_a_vec, sum_a_vec);
|
|
466
494
|
sum_b_vec = _mm512_maskz_fmadd_ps(nonzero_mask, b_vec, log_ratio_b_vec, sum_b_vec);
|
|
467
495
|
if (n)
|
|
468
496
|
goto simsimd_js_f32_skylake_cycle;
|
|
469
497
|
|
|
470
498
|
simsimd_f32_t log2_normalizer = 0.693147181f;
|
|
471
|
-
*result = _mm512_reduce_add_ps(_mm512_add_ps(sum_a_vec, sum_b_vec)) *
|
|
499
|
+
*result = _mm512_reduce_add_ps(_mm512_add_ps(sum_a_vec, sum_b_vec)) * log2_normalizer / 2;
|
|
472
500
|
}
|
|
473
501
|
|
|
474
502
|
#pragma clang attribute pop
|
|
@@ -477,10 +505,10 @@ simsimd_js_f32_skylake_cycle:
|
|
|
477
505
|
|
|
478
506
|
#if SIMSIMD_TARGET_SAPPHIRE
|
|
479
507
|
#pragma GCC push_options
|
|
480
|
-
#pragma GCC target("avx512f", "avx512vl", "bmi2", "avx512fp16")
|
|
481
|
-
#pragma clang attribute push(__attribute__((target("avx512f,avx512vl,bmi2,avx512fp16"))), apply_to = function)
|
|
508
|
+
#pragma GCC target("avx2", "avx512f", "avx512vl", "bmi2", "avx512fp16")
|
|
509
|
+
#pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,bmi2,avx512fp16"))), apply_to = function)
|
|
482
510
|
|
|
483
|
-
|
|
511
|
+
SIMSIMD_INTERNAL __m512h _simsimd_log2_f16_sapphire(__m512h x) {
|
|
484
512
|
// Extract the exponent and mantissa
|
|
485
513
|
__m512h one = _mm512_set1_ph((simsimd_f16_t)1);
|
|
486
514
|
__m512h e = _mm512_getexp_ph(x);
|
|
@@ -515,7 +543,7 @@ simsimd_kl_f16_sapphire_cycle:
|
|
|
515
543
|
a += 32, b += 32, n -= 32;
|
|
516
544
|
}
|
|
517
545
|
__m512h ratio_vec = _mm512_div_ph(a_vec, b_vec);
|
|
518
|
-
__m512h log_ratio_vec =
|
|
546
|
+
__m512h log_ratio_vec = _simsimd_log2_f16_sapphire(ratio_vec);
|
|
519
547
|
__m512h prod_vec = _mm512_mul_ph(a_vec, log_ratio_vec);
|
|
520
548
|
sum_vec = _mm512_add_ph(sum_vec, prod_vec);
|
|
521
549
|
if (n)
|
|
@@ -550,15 +578,15 @@ simsimd_js_f16_sapphire_cycle:
|
|
|
550
578
|
__m512h m_recip_approx = _mm512_rcp_ph(m_vec);
|
|
551
579
|
__m512h ratio_a_vec = _mm512_mul_ph(a_vec, m_recip_approx);
|
|
552
580
|
__m512h ratio_b_vec = _mm512_mul_ph(b_vec, m_recip_approx);
|
|
553
|
-
__m512h log_ratio_a_vec =
|
|
554
|
-
__m512h log_ratio_b_vec =
|
|
581
|
+
__m512h log_ratio_a_vec = _simsimd_log2_f16_sapphire(ratio_a_vec);
|
|
582
|
+
__m512h log_ratio_b_vec = _simsimd_log2_f16_sapphire(ratio_b_vec);
|
|
555
583
|
sum_a_vec = _mm512_maskz_fmadd_ph(nonzero_mask, a_vec, log_ratio_a_vec, sum_a_vec);
|
|
556
584
|
sum_b_vec = _mm512_maskz_fmadd_ph(nonzero_mask, b_vec, log_ratio_b_vec, sum_b_vec);
|
|
557
585
|
if (n)
|
|
558
586
|
goto simsimd_js_f16_sapphire_cycle;
|
|
559
587
|
|
|
560
588
|
simsimd_f32_t log2_normalizer = 0.693147181f;
|
|
561
|
-
*result = _mm512_reduce_add_ph(_mm512_add_ph(sum_a_vec, sum_b_vec)) *
|
|
589
|
+
*result = _mm512_reduce_add_ph(_mm512_add_ph(sum_a_vec, sum_b_vec)) * log2_normalizer / 2;
|
|
562
590
|
}
|
|
563
591
|
|
|
564
592
|
#pragma clang attribute pop
|