usearch 2.9.1 → 2.9.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CITATION.cff +1 -1
- package/CONTRIBUTING.md +10 -2
- package/Cargo.toml +1 -1
- package/README.md +12 -15
- package/VERSION +1 -1
- package/c/test.c +58 -50
- package/conanfile.py +1 -1
- package/include/usearch/index.hpp +1 -1
- package/include/usearch/index_plugins.hpp +20 -10
- package/javascript/types/usearch.d.ts.map +1 -1
- package/javascript/usearch.js +4 -1
- package/package.json +1 -1
- package/simsimd/CONTRIBUTING.md +88 -0
- package/simsimd/Cargo.lock +4 -3
- package/simsimd/Cargo.toml +10 -2
- package/simsimd/README.md +102 -60
- package/simsimd/include/simsimd/binary.h +44 -0
- package/simsimd/include/simsimd/complex.h +174 -0
- package/simsimd/include/simsimd/simsimd.h +18 -9
- package/simsimd/include/simsimd/spatial.h +128 -36
- package/simsimd/include/simsimd/types.h +3 -2
- package/simsimd/package-lock.json +5 -5
- package/simsimd/package.json +1 -1
- package/simsimd/rust/benches/native.rs +12 -9
- package/simsimd/rust/lib.c +32 -55
- package/simsimd/rust/lib.rs +374 -56
- package/stringzilla/CONTRIBUTING.md +17 -0
- package/stringzilla/Cargo.lock +1 -1
- package/stringzilla/Cargo.toml +1 -1
- package/stringzilla/README.md +175 -34
- package/stringzilla/VERSION +1 -1
- package/stringzilla/build.rs +77 -4
- package/stringzilla/c/lib.c +33 -0
- package/stringzilla/cli/README.md +30 -13
- package/stringzilla/cli/__init__.py +0 -0
- package/stringzilla/cli/split.py +66 -33
- package/stringzilla/cli/wc.py +101 -21
- package/stringzilla/include/stringzilla/experimental.h +122 -1
- package/stringzilla/include/stringzilla/stringzilla.h +262 -132
- package/stringzilla/include/stringzilla/stringzilla.hpp +13 -5
- package/stringzilla/package.json +1 -1
- package/stringzilla/rust/lib.rs +673 -162
- package/wasmer.toml +1 -1
package/simsimd/rust/lib.rs
CHANGED
|
@@ -1,27 +1,19 @@
|
|
|
1
1
|
//! # SimSIMD - Hardware-Accelerated Similarity Metrics and Distance Functions
|
|
2
2
|
//!
|
|
3
|
-
//! * Zero-dependency header-only C 99 library with bindings for Python, JavaScript and Rust.
|
|
4
3
|
//! * Targets ARM NEON, SVE, x86 AVX2, AVX-512 (VNNI, FP16) hardware backends.
|
|
5
|
-
//! *
|
|
6
|
-
//! *
|
|
7
|
-
//! * Up to 200x faster than scipy.spatial.distance and numpy.inner.
|
|
8
|
-
//! * Used in USearch and several DBMS products.
|
|
4
|
+
//! * Handles `f64` double-, `f32` single-, and `f16` half-precision, `i8` integral, and binary vectors.
|
|
5
|
+
//! * Zero-dependency header-only C 99 library with bindings for Rust and other langauges.
|
|
9
6
|
//!
|
|
10
7
|
//! ## Implemented distance functions include:
|
|
11
8
|
//!
|
|
12
9
|
//! * Euclidean (L2), Inner Distance, and Cosine (Angular) spatial distances.
|
|
13
10
|
//! * Hamming (~ Manhattan) and Jaccard (~ Tanimoto) binary distances.
|
|
14
|
-
//! * Kullback-Leibler and Jensen
|
|
11
|
+
//! * Kullback-Leibler and Jensen-Shannon divergences for probability distributions.
|
|
15
12
|
//!
|
|
16
|
-
//!
|
|
17
|
-
//! implemented for types with SIMD support (e.g., i8 and f32). The trait provides
|
|
18
|
-
//! methods for computing cosine similarity, inner product, and squared Euclidean
|
|
19
|
-
//! distance between slices of these types.
|
|
20
|
-
//!
|
|
21
|
-
//! # Example
|
|
13
|
+
//! ## Example
|
|
22
14
|
//!
|
|
23
15
|
//! ```rust
|
|
24
|
-
//! use simsimd::
|
|
16
|
+
//! use simsimd::SpatialSimilarity;
|
|
25
17
|
//!
|
|
26
18
|
//! let a = &[1, 2, 3];
|
|
27
19
|
//! let b = &[4, 5, 6];
|
|
@@ -29,74 +21,165 @@
|
|
|
29
21
|
//! // Compute cosine similarity
|
|
30
22
|
//! let cosine_sim = i8::cosine(a, b);
|
|
31
23
|
//!
|
|
32
|
-
//! // Compute inner product
|
|
24
|
+
//! // Compute inner product distance
|
|
33
25
|
//! let inner_product = i8::inner(a, b);
|
|
34
26
|
//!
|
|
35
27
|
//! // Compute squared Euclidean distance
|
|
36
28
|
//! let sqeuclidean_dist = i8::sqeuclidean(a, b);
|
|
37
29
|
//! ```
|
|
38
30
|
//!
|
|
39
|
-
//!
|
|
31
|
+
//! ## Traits
|
|
32
|
+
//!
|
|
33
|
+
//! The `SpatialSimilarity` trait covers following methods:
|
|
40
34
|
//!
|
|
41
|
-
//!
|
|
42
|
-
//!
|
|
43
|
-
//!
|
|
35
|
+
//! - `cosine(a: &[Self], b: &[Self]) -> Option<f32>`: Computes cosine similarity between two slices.
|
|
36
|
+
//! - `inner(a: &[Self], b: &[Self]) -> Option<f32>`: Computes inner product distance between two slices.
|
|
37
|
+
//! - `sqeuclidean(a: &[Self], b: &[Self]) -> Option<f32>`: Computes squared Euclidean distance between two slices.
|
|
44
38
|
//!
|
|
39
|
+
//! The `BinarySimilarity` trait covers following methods:
|
|
45
40
|
//!
|
|
46
|
-
//!
|
|
41
|
+
//! - `hamming(a: &[Self], b: &[Self]) -> Option<f32>`: Computes Hamming distance between two slices.
|
|
42
|
+
//! - `jaccard(a: &[Self], b: &[Self]) -> Option<f32>`: Computes Jaccard index between two slices.
|
|
47
43
|
//!
|
|
48
|
-
//! The `
|
|
49
|
-
//! three associated methods:
|
|
44
|
+
//! The `ProbabilitySimilarity` trait covers following methods:
|
|
50
45
|
//!
|
|
51
|
-
//! - `
|
|
52
|
-
//! - `
|
|
53
|
-
//! - `sqeuclidean(a: &[Self], b: &[Self]) -> Option<f32>`: Computes squared Euclidean distance between two slices.
|
|
46
|
+
//! - `jensenshannon(a: &[Self], b: &[Self]) -> Option<f32>`: Computes Jensen-Shannon divergence between two slices.
|
|
47
|
+
//! - `kullbackleibler(a: &[Self], b: &[Self]) -> Option<f32>`: Computes Kullback-Leibler divergence between two slices.
|
|
54
48
|
//!
|
|
55
49
|
#![allow(non_camel_case_types)]
|
|
56
50
|
|
|
57
51
|
extern "C" {
|
|
58
|
-
/// Computes cosine similarity for i8 types.
|
|
59
52
|
fn cosine_i8(a: *const i8, b: *const i8, c: usize) -> f32;
|
|
60
|
-
|
|
61
|
-
/// Computes cosine similarity for f32 types.
|
|
53
|
+
fn cosine_f16(a: *const u16, b: *const u16, c: usize) -> f32;
|
|
62
54
|
fn cosine_f32(a: *const f32, b: *const f32, c: usize) -> f32;
|
|
55
|
+
fn cosine_f64(a: *const f64, b: *const f64, c: usize) -> f32;
|
|
63
56
|
|
|
64
|
-
/// Computes inner product for i8 types.
|
|
65
57
|
fn inner_i8(a: *const i8, b: *const i8, c: usize) -> f32;
|
|
66
|
-
|
|
67
|
-
/// Computes inner product for f32 types.
|
|
58
|
+
fn inner_f16(a: *const u16, b: *const u16, c: usize) -> f32;
|
|
68
59
|
fn inner_f32(a: *const f32, b: *const f32, c: usize) -> f32;
|
|
60
|
+
fn inner_f64(a: *const f64, b: *const f64, c: usize) -> f32;
|
|
69
61
|
|
|
70
|
-
/// Computes squared Euclidean distance for i8 types.
|
|
71
62
|
fn sqeuclidean_i8(a: *const i8, b: *const i8, c: usize) -> f32;
|
|
72
|
-
|
|
73
|
-
/// Computes squared Euclidean distance for f32 types.
|
|
63
|
+
fn sqeuclidean_f16(a: *const u16, b: *const u16, c: usize) -> f32;
|
|
74
64
|
fn sqeuclidean_f32(a: *const f32, b: *const f32, c: usize) -> f32;
|
|
65
|
+
fn sqeuclidean_f64(a: *const f64, b: *const f64, c: usize) -> f32;
|
|
66
|
+
|
|
67
|
+
fn hamming_b8(a: *const u8, b: *const u8, c: usize) -> f32;
|
|
68
|
+
fn jaccard_b8(a: *const u8, b: *const u8, c: usize) -> f32;
|
|
69
|
+
|
|
70
|
+
fn js_f16(a: *const u16, b: *const u16, c: usize) -> f32;
|
|
71
|
+
fn js_f32(a: *const f32, b: *const f32, c: usize) -> f32;
|
|
72
|
+
fn js_f64(a: *const f64, b: *const f64, c: usize) -> f32;
|
|
73
|
+
|
|
74
|
+
fn kl_f16(a: *const u16, b: *const u16, c: usize) -> f32;
|
|
75
|
+
fn kl_f32(a: *const f32, b: *const f32, c: usize) -> f32;
|
|
76
|
+
fn kl_f64(a: *const f64, b: *const f64, c: usize) -> f32;
|
|
75
77
|
}
|
|
76
78
|
|
|
77
|
-
/// A
|
|
78
|
-
|
|
79
|
+
/// A half-precision floating point number.
|
|
80
|
+
#[repr(transparent)]
|
|
81
|
+
pub struct f16(u16);
|
|
82
|
+
|
|
83
|
+
impl f16 {}
|
|
84
|
+
|
|
85
|
+
/// `SpatialSimilarity` provides a set of trait methods for computing similarity
|
|
86
|
+
/// or distance between spatial data vectors in SIMD (Single Instruction, Multiple Data) context.
|
|
87
|
+
/// These methods can be used to calculate metrics like cosine similarity, inner product,
|
|
88
|
+
/// and squared Euclidean distance between two slices of data.
|
|
89
|
+
///
|
|
90
|
+
/// Each method takes two slices of data (a and b) and returns an Option<f32>.
|
|
91
|
+
/// The result is `None` if the slices are not of the same length, as these operations
|
|
92
|
+
/// require one-to-one correspondence between the elements of the slices.
|
|
93
|
+
/// Otherwise, it returns the computed similarity or distance as `Some(f32)`.
|
|
94
|
+
pub trait SpatialSimilarity
|
|
79
95
|
where
|
|
80
96
|
Self: Sized,
|
|
81
97
|
{
|
|
82
|
-
/// Computes cosine similarity between two slices.
|
|
98
|
+
/// Computes the cosine similarity between two slices.
|
|
99
|
+
/// The cosine similarity is a measure of similarity between two non-zero vectors
|
|
100
|
+
/// of an inner product space that measures the cosine of the angle between them.
|
|
83
101
|
fn cosine(a: &[Self], b: &[Self]) -> Option<f32>;
|
|
84
102
|
|
|
85
|
-
/// Computes inner product between two slices.
|
|
103
|
+
/// Computes the inner product (also known as dot product) between two slices.
|
|
104
|
+
/// The inner product is the sum of the products of the corresponding entries
|
|
105
|
+
/// of the two sequences of numbers.
|
|
86
106
|
fn inner(a: &[Self], b: &[Self]) -> Option<f32>;
|
|
87
107
|
|
|
88
|
-
/// Computes squared Euclidean distance between two slices.
|
|
108
|
+
/// Computes the squared Euclidean distance between two slices.
|
|
109
|
+
/// The squared Euclidean distance is the sum of the squared differences
|
|
110
|
+
/// between corresponding elements of the two slices.
|
|
89
111
|
fn sqeuclidean(a: &[Self], b: &[Self]) -> Option<f32>;
|
|
90
112
|
}
|
|
91
113
|
|
|
92
|
-
|
|
93
|
-
|
|
114
|
+
/// `BinarySimilarity` provides trait methods for computing similarity metrics
|
|
115
|
+
/// that are commonly used with binary data vectors, such as Hamming distance
|
|
116
|
+
/// and Jaccard index.
|
|
117
|
+
///
|
|
118
|
+
/// The methods accept two slices of binary data and return an Option<f32>
|
|
119
|
+
/// indicating the computed similarity or distance, with `None` returned if the
|
|
120
|
+
/// slices differ in length.
|
|
121
|
+
pub trait BinarySimilarity
|
|
122
|
+
where
|
|
123
|
+
Self: Sized,
|
|
124
|
+
{
|
|
125
|
+
/// Computes the Hamming distance between two binary data slices.
|
|
126
|
+
/// The Hamming distance between two strings of equal length is the number of
|
|
127
|
+
/// bits at which the corresponding values are different.
|
|
128
|
+
fn hamming(a: &[Self], b: &[Self]) -> Option<f32>;
|
|
129
|
+
|
|
130
|
+
/// Computes the Jaccard index between two bitsets represented by binary data slices.
|
|
131
|
+
/// The Jaccard index, also known as the Jaccard similarity coefficient, is a statistic
|
|
132
|
+
/// used for gauging the similarity and diversity of sample sets.
|
|
133
|
+
fn jaccard(a: &[Self], b: &[Self]) -> Option<f32>;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/// `ProbabilitySimilarity` provides trait methods for computing similarity or divergence
|
|
137
|
+
/// measures between probability distributions, such as the Jensen-Shannon divergence
|
|
138
|
+
/// and the Kullback-Leibler divergence.
|
|
139
|
+
///
|
|
140
|
+
/// These methods are particularly useful in contexts such as information theory and
|
|
141
|
+
/// machine learning, where one often needs to measure how one probability distribution
|
|
142
|
+
/// differs from a second, reference probability distribution.
|
|
143
|
+
pub trait ProbabilitySimilarity
|
|
144
|
+
where
|
|
145
|
+
Self: Sized,
|
|
146
|
+
{
|
|
147
|
+
/// Computes the Jensen-Shannon divergence between two probability distributions.
|
|
148
|
+
/// The Jensen-Shannon divergence is a method of measuring the similarity between
|
|
149
|
+
/// two probability distributions. It is based on the Kullback-Leibler divergence,
|
|
150
|
+
/// but is symmetric and always has a finite value.
|
|
151
|
+
fn jensenshannon(a: &[Self], b: &[Self]) -> Option<f32>;
|
|
152
|
+
|
|
153
|
+
/// Computes the Kullback-Leibler divergence between two probability distributions.
|
|
154
|
+
/// The Kullback-Leibler divergence is a measure of how one probability distribution
|
|
155
|
+
/// diverges from a second, expected probability distribution.
|
|
156
|
+
fn kullbackleibler(a: &[Self], b: &[Self]) -> Option<f32>;
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
impl BinarySimilarity for u8 {
|
|
160
|
+
fn hamming(a: &[Self], b: &[Self]) -> Option<f32> {
|
|
94
161
|
if a.len() != b.len() {
|
|
95
162
|
return None;
|
|
96
163
|
}
|
|
164
|
+
let operation = unsafe { hamming_b8(a.as_ptr(), b.as_ptr(), a.len()) };
|
|
165
|
+
Some(operation)
|
|
166
|
+
}
|
|
97
167
|
|
|
98
|
-
|
|
168
|
+
fn jaccard(a: &[Self], b: &[Self]) -> Option<f32> {
|
|
169
|
+
if a.len() != b.len() {
|
|
170
|
+
return None;
|
|
171
|
+
}
|
|
172
|
+
let operation = unsafe { jaccard_b8(a.as_ptr(), b.as_ptr(), a.len()) };
|
|
173
|
+
Some(operation)
|
|
174
|
+
}
|
|
175
|
+
}
|
|
99
176
|
|
|
177
|
+
impl SpatialSimilarity for i8 {
|
|
178
|
+
fn cosine(a: &[Self], b: &[Self]) -> Option<f32> {
|
|
179
|
+
if a.len() != b.len() {
|
|
180
|
+
return None;
|
|
181
|
+
}
|
|
182
|
+
let operation = unsafe { cosine_i8(a.as_ptr(), b.as_ptr(), a.len()) };
|
|
100
183
|
Some(operation)
|
|
101
184
|
}
|
|
102
185
|
|
|
@@ -104,9 +187,7 @@ impl SimSIMD for i8 {
|
|
|
104
187
|
if a.len() != b.len() {
|
|
105
188
|
return None;
|
|
106
189
|
}
|
|
107
|
-
|
|
108
190
|
let operation = unsafe { inner_i8(a.as_ptr(), b.as_ptr(), a.len()) };
|
|
109
|
-
|
|
110
191
|
Some(operation)
|
|
111
192
|
}
|
|
112
193
|
|
|
@@ -114,21 +195,21 @@ impl SimSIMD for i8 {
|
|
|
114
195
|
if a.len() != b.len() {
|
|
115
196
|
return None;
|
|
116
197
|
}
|
|
117
|
-
|
|
118
198
|
let operation = unsafe { sqeuclidean_i8(a.as_ptr(), b.as_ptr(), a.len()) };
|
|
119
|
-
|
|
120
199
|
Some(operation)
|
|
121
200
|
}
|
|
122
201
|
}
|
|
123
202
|
|
|
124
|
-
impl
|
|
203
|
+
impl SpatialSimilarity for f16 {
|
|
125
204
|
fn cosine(a: &[Self], b: &[Self]) -> Option<f32> {
|
|
126
205
|
if a.len() != b.len() {
|
|
127
206
|
return None;
|
|
128
207
|
}
|
|
129
208
|
|
|
130
|
-
|
|
131
|
-
|
|
209
|
+
// Explicitly cast `*const f16` to `*const u16`
|
|
210
|
+
let a_ptr = a.as_ptr() as *const u16;
|
|
211
|
+
let b_ptr = b.as_ptr() as *const u16;
|
|
212
|
+
let operation = unsafe { cosine_f16(a_ptr, b_ptr, a.len()) };
|
|
132
213
|
Some(operation)
|
|
133
214
|
}
|
|
134
215
|
|
|
@@ -137,8 +218,66 @@ impl SimSIMD for f32 {
|
|
|
137
218
|
return None;
|
|
138
219
|
}
|
|
139
220
|
|
|
221
|
+
// Explicitly cast `*const f16` to `*const u16`
|
|
222
|
+
let a_ptr = a.as_ptr() as *const u16;
|
|
223
|
+
let b_ptr = b.as_ptr() as *const u16;
|
|
224
|
+
let operation = unsafe { inner_f16(a_ptr, b_ptr, a.len()) };
|
|
225
|
+
Some(operation)
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
fn sqeuclidean(a: &[Self], b: &[Self]) -> Option<f32> {
|
|
229
|
+
if a.len() != b.len() {
|
|
230
|
+
return None;
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
// Explicitly cast `*const f16` to `*const u16`
|
|
234
|
+
let a_ptr = a.as_ptr() as *const u16;
|
|
235
|
+
let b_ptr = b.as_ptr() as *const u16;
|
|
236
|
+
let operation = unsafe { sqeuclidean_f16(a_ptr, b_ptr, a.len()) };
|
|
237
|
+
Some(operation)
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
impl SpatialSimilarity for f32 {
|
|
242
|
+
fn cosine(a: &[Self], b: &[Self]) -> Option<f32> {
|
|
243
|
+
if a.len() != b.len() {
|
|
244
|
+
return None;
|
|
245
|
+
}
|
|
246
|
+
let operation = unsafe { cosine_f32(a.as_ptr(), b.as_ptr(), a.len()) };
|
|
247
|
+
Some(operation)
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
fn inner(a: &[Self], b: &[Self]) -> Option<f32> {
|
|
251
|
+
if a.len() != b.len() {
|
|
252
|
+
return None;
|
|
253
|
+
}
|
|
140
254
|
let operation = unsafe { inner_f32(a.as_ptr(), b.as_ptr(), a.len()) };
|
|
255
|
+
Some(operation)
|
|
256
|
+
}
|
|
141
257
|
|
|
258
|
+
fn sqeuclidean(a: &[Self], b: &[Self]) -> Option<f32> {
|
|
259
|
+
if a.len() != b.len() {
|
|
260
|
+
return None;
|
|
261
|
+
}
|
|
262
|
+
let operation = unsafe { sqeuclidean_f32(a.as_ptr(), b.as_ptr(), a.len()) };
|
|
263
|
+
Some(operation)
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
impl SpatialSimilarity for f64 {
|
|
268
|
+
fn cosine(a: &[Self], b: &[Self]) -> Option<f32> {
|
|
269
|
+
if a.len() != b.len() {
|
|
270
|
+
return None;
|
|
271
|
+
}
|
|
272
|
+
let operation = unsafe { cosine_f64(a.as_ptr(), b.as_ptr(), a.len()) };
|
|
273
|
+
Some(operation)
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
fn inner(a: &[Self], b: &[Self]) -> Option<f32> {
|
|
277
|
+
if a.len() != b.len() {
|
|
278
|
+
return None;
|
|
279
|
+
}
|
|
280
|
+
let operation = unsafe { inner_f64(a.as_ptr(), b.as_ptr(), a.len()) };
|
|
142
281
|
Some(operation)
|
|
143
282
|
}
|
|
144
283
|
|
|
@@ -146,16 +285,101 @@ impl SimSIMD for f32 {
|
|
|
146
285
|
if a.len() != b.len() {
|
|
147
286
|
return None;
|
|
148
287
|
}
|
|
288
|
+
let operation = unsafe { sqeuclidean_f64(a.as_ptr(), b.as_ptr(), a.len()) };
|
|
289
|
+
Some(operation)
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
impl ProbabilitySimilarity for f16 {
|
|
294
|
+
fn jensenshannon(a: &[Self], b: &[Self]) -> Option<f32> {
|
|
295
|
+
if a.len() != b.len() {
|
|
296
|
+
return None;
|
|
297
|
+
}
|
|
149
298
|
|
|
150
|
-
|
|
299
|
+
// Explicitly cast `*const f16` to `*const u16`
|
|
300
|
+
let a_ptr = a.as_ptr() as *const u16;
|
|
301
|
+
let b_ptr = b.as_ptr() as *const u16;
|
|
302
|
+
let operation = unsafe { js_f16(a_ptr, b_ptr, a.len()) };
|
|
303
|
+
Some(operation)
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
fn kullbackleibler(a: &[Self], b: &[Self]) -> Option<f32> {
|
|
307
|
+
if a.len() != b.len() {
|
|
308
|
+
return None;
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
// Explicitly cast `*const f16` to `*const u16`
|
|
312
|
+
let a_ptr = a.as_ptr() as *const u16;
|
|
313
|
+
let b_ptr = b.as_ptr() as *const u16;
|
|
314
|
+
let operation = unsafe { kl_f16(a_ptr, b_ptr, a.len()) };
|
|
315
|
+
Some(operation)
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
impl ProbabilitySimilarity for f32 {
|
|
320
|
+
fn jensenshannon(a: &[Self], b: &[Self]) -> Option<f32> {
|
|
321
|
+
if a.len() != b.len() {
|
|
322
|
+
return None;
|
|
323
|
+
}
|
|
324
|
+
let operation = unsafe { js_f32(a.as_ptr(), b.as_ptr(), a.len()) };
|
|
325
|
+
Some(operation)
|
|
326
|
+
}
|
|
151
327
|
|
|
328
|
+
fn kullbackleibler(a: &[Self], b: &[Self]) -> Option<f32> {
|
|
329
|
+
if a.len() != b.len() {
|
|
330
|
+
return None;
|
|
331
|
+
}
|
|
332
|
+
let operation = unsafe { kl_f32(a.as_ptr(), b.as_ptr(), a.len()) };
|
|
152
333
|
Some(operation)
|
|
153
334
|
}
|
|
154
335
|
}
|
|
155
336
|
|
|
337
|
+
impl ProbabilitySimilarity for f64 {
|
|
338
|
+
fn jensenshannon(a: &[Self], b: &[Self]) -> Option<f32> {
|
|
339
|
+
if a.len() != b.len() {
|
|
340
|
+
return None;
|
|
341
|
+
}
|
|
342
|
+
let operation = unsafe { js_f64(a.as_ptr(), b.as_ptr(), a.len()) };
|
|
343
|
+
Some(operation)
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
fn kullbackleibler(a: &[Self], b: &[Self]) -> Option<f32> {
|
|
347
|
+
if a.len() != b.len() {
|
|
348
|
+
return None;
|
|
349
|
+
}
|
|
350
|
+
let operation = unsafe { kl_f64(a.as_ptr(), b.as_ptr(), a.len()) };
|
|
351
|
+
Some(operation)
|
|
352
|
+
}
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
// In the older revisions of the library, the `SpatialSimilarity`
|
|
356
|
+
// trait was called `SimSIMD`. The following trait provides a compatibility layer.
|
|
357
|
+
pub trait SimSIMD
|
|
358
|
+
where
|
|
359
|
+
Self: Sized,
|
|
360
|
+
{
|
|
361
|
+
fn cosine(a: &[Self], b: &[Self]) -> Option<f32>;
|
|
362
|
+
fn inner(a: &[Self], b: &[Self]) -> Option<f32>;
|
|
363
|
+
fn sqeuclidean(a: &[Self], b: &[Self]) -> Option<f32>;
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
impl<T: SpatialSimilarity> SimSIMD for T {
|
|
367
|
+
fn cosine(a: &[Self], b: &[Self]) -> Option<f32> {
|
|
368
|
+
SpatialSimilarity::cosine(a, b)
|
|
369
|
+
}
|
|
370
|
+
fn inner(a: &[Self], b: &[Self]) -> Option<f32> {
|
|
371
|
+
SpatialSimilarity::inner(a, b)
|
|
372
|
+
}
|
|
373
|
+
fn sqeuclidean(a: &[Self], b: &[Self]) -> Option<f32> {
|
|
374
|
+
SpatialSimilarity::sqeuclidean(a, b)
|
|
375
|
+
}
|
|
376
|
+
}
|
|
377
|
+
|
|
156
378
|
#[cfg(test)]
|
|
157
379
|
mod tests {
|
|
158
380
|
use super::*;
|
|
381
|
+
use half::f16 as HalfF16;
|
|
382
|
+
|
|
159
383
|
//
|
|
160
384
|
fn assert_almost_equal(left: f32, right: f32, tolerance: f32) {
|
|
161
385
|
let lower = right - tolerance;
|
|
@@ -170,19 +394,19 @@ mod tests {
|
|
|
170
394
|
let b = &[3, 97, 127];
|
|
171
395
|
|
|
172
396
|
if let Some(result) = SimSIMD::cosine(a, b) {
|
|
173
|
-
assert_almost_equal(0.00012027938, result, 0.01);
|
|
174
397
|
println!("The result of cosine_i8 is {:.8}", result);
|
|
398
|
+
assert_almost_equal(0.00012027938, result, 0.01);
|
|
175
399
|
}
|
|
176
400
|
}
|
|
177
401
|
|
|
178
402
|
#[test]
|
|
179
403
|
fn test_cosine_f32() {
|
|
180
404
|
let a = &[1.0, 2.0, 3.0];
|
|
181
|
-
let b = &[
|
|
405
|
+
let b = &[4.0, 5.0, 6.0];
|
|
182
406
|
|
|
183
407
|
if let Some(result) = SimSIMD::cosine(a, b) {
|
|
184
|
-
assert_almost_equal(0.004930496, result, 0.01);
|
|
185
408
|
println!("The result of cosine_f32 is {:.8}", result);
|
|
409
|
+
assert_almost_equal(0.025, result, 0.01);
|
|
186
410
|
}
|
|
187
411
|
}
|
|
188
412
|
|
|
@@ -192,8 +416,8 @@ mod tests {
|
|
|
192
416
|
let b = &[4, 5, 6];
|
|
193
417
|
|
|
194
418
|
if let Some(result) = SimSIMD::inner(a, b) {
|
|
195
|
-
assert_almost_equal(0.029403687, result, 0.01);
|
|
196
419
|
println!("The result of inner_i8 is {:.8}", result);
|
|
420
|
+
assert_almost_equal(0.029403687, result, 0.01);
|
|
197
421
|
}
|
|
198
422
|
}
|
|
199
423
|
|
|
@@ -203,8 +427,8 @@ mod tests {
|
|
|
203
427
|
let b = &[4.0, 5.0, 6.0];
|
|
204
428
|
|
|
205
429
|
if let Some(result) = SimSIMD::inner(a, b) {
|
|
206
|
-
assert_almost_equal(-31.0, result, 0.01);
|
|
207
430
|
println!("The result of inner_f32 is {:.8}", result);
|
|
431
|
+
assert_almost_equal(-31.0, result, 0.01);
|
|
208
432
|
}
|
|
209
433
|
}
|
|
210
434
|
|
|
@@ -214,8 +438,8 @@ mod tests {
|
|
|
214
438
|
let b = &[4, 5, 6];
|
|
215
439
|
|
|
216
440
|
if let Some(result) = SimSIMD::sqeuclidean(a, b) {
|
|
217
|
-
assert_almost_equal(27.0, result, 0.01);
|
|
218
441
|
println!("The result of sqeuclidean_i8 is {:.8}", result);
|
|
442
|
+
assert_almost_equal(27.0, result, 0.01);
|
|
219
443
|
}
|
|
220
444
|
}
|
|
221
445
|
|
|
@@ -225,8 +449,102 @@ mod tests {
|
|
|
225
449
|
let b = &[4.0, 5.0, 6.0];
|
|
226
450
|
|
|
227
451
|
if let Some(result) = SimSIMD::sqeuclidean(a, b) {
|
|
228
|
-
assert_almost_equal(27.0, result, 0.01);
|
|
229
452
|
println!("The result of sqeuclidean_f32 is {:.8}", result);
|
|
453
|
+
assert_almost_equal(27.0, result, 0.01);
|
|
454
|
+
}
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
// Adding new tests for bit-level distances
|
|
458
|
+
#[test]
|
|
459
|
+
fn test_hamming_u8() {
|
|
460
|
+
let a = &[0b01010101, 0b11110000, 0b10101010]; // Binary representations for clarity
|
|
461
|
+
let b = &[0b01010101, 0b11110000, 0b10101010];
|
|
462
|
+
|
|
463
|
+
if let Some(result) = BinarySimilarity::hamming(a, b) {
|
|
464
|
+
println!("The result of hamming_u8 is {:.8}", result);
|
|
465
|
+
assert_almost_equal(0.0, result, 0.01); // Perfect match
|
|
466
|
+
}
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
#[test]
|
|
470
|
+
fn test_jaccard_u8() {
|
|
471
|
+
// For binary data, treat each byte as a set of bits
|
|
472
|
+
let a = &[0b11110000, 0b00001111, 0b10101010];
|
|
473
|
+
let b = &[0b11110000, 0b00001111, 0b01010101];
|
|
474
|
+
|
|
475
|
+
if let Some(result) = BinarySimilarity::jaccard(a, b) {
|
|
476
|
+
println!("The result of jaccard_u8 is {:.8}", result);
|
|
477
|
+
assert_almost_equal(0.5, result, 0.01); // Example value
|
|
478
|
+
}
|
|
479
|
+
}
|
|
480
|
+
|
|
481
|
+
// Adding new tests for probability similarities
|
|
482
|
+
#[test]
|
|
483
|
+
fn test_js_f32() {
|
|
484
|
+
let a = &[0.1, 0.9, 0.0];
|
|
485
|
+
let b = &[0.2, 0.8, 0.0];
|
|
486
|
+
|
|
487
|
+
if let Some(result) = ProbabilitySimilarity::jensenshannon(a, b) {
|
|
488
|
+
println!("The result of js_f32 is {:.8}", result);
|
|
489
|
+
assert_almost_equal(0.01, result, 0.01); // Example value
|
|
490
|
+
}
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
#[test]
|
|
494
|
+
fn test_kl_f32() {
|
|
495
|
+
let a = &[0.1, 0.9, 0.0];
|
|
496
|
+
let b = &[0.2, 0.8, 0.0];
|
|
497
|
+
|
|
498
|
+
if let Some(result) = ProbabilitySimilarity::kullbackleibler(a, b) {
|
|
499
|
+
println!("The result of kl_f32 is {:.8}", result);
|
|
500
|
+
assert_almost_equal(0.036, result, 0.01); // Example value
|
|
501
|
+
}
|
|
502
|
+
}
|
|
503
|
+
|
|
504
|
+
#[test]
|
|
505
|
+
fn test_cosine_f16_same() {
|
|
506
|
+
// Assuming these u16 values represent f16 bit patterns, and they are identical
|
|
507
|
+
let a_u16: &[u16] = &[15360, 16384, 17408]; // Corresponding to some f16 values
|
|
508
|
+
let b_u16: &[u16] = &[15360, 16384, 17408]; // Same as above for simplicity
|
|
509
|
+
|
|
510
|
+
// Reinterpret cast from &[u16] to &[f16]
|
|
511
|
+
// SAFETY: This is safe as long as the representations are guaranteed to be identical,
|
|
512
|
+
// which they are for transparent structs wrapping the same type.
|
|
513
|
+
let a_f16: &[f16] =
|
|
514
|
+
unsafe { std::slice::from_raw_parts(a_u16.as_ptr() as *const f16, a_u16.len()) };
|
|
515
|
+
let b_f16: &[f16] =
|
|
516
|
+
unsafe { std::slice::from_raw_parts(b_u16.as_ptr() as *const f16, b_u16.len()) };
|
|
517
|
+
|
|
518
|
+
if let Some(result) = SimSIMD::cosine(a_f16, b_f16) {
|
|
519
|
+
println!("The result of cosine_f16 is {:.8}", result);
|
|
520
|
+
assert_almost_equal(0.0, result, 0.01); // Example value, adjust according to actual expected value
|
|
521
|
+
}
|
|
522
|
+
}
|
|
523
|
+
|
|
524
|
+
#[test]
|
|
525
|
+
fn test_cosine_f16_interop() {
|
|
526
|
+
let a_half: Vec<HalfF16> = vec![1.0, 2.0, 3.0]
|
|
527
|
+
.iter()
|
|
528
|
+
.map(|&x| HalfF16::from_f32(x))
|
|
529
|
+
.collect();
|
|
530
|
+
let b_half: Vec<HalfF16> = vec![4.0, 5.0, 6.0]
|
|
531
|
+
.iter()
|
|
532
|
+
.map(|&x| HalfF16::from_f32(x))
|
|
533
|
+
.collect();
|
|
534
|
+
|
|
535
|
+
// SAFETY: This is safe as long as the memory representations are guaranteed to be identical,
|
|
536
|
+
// which they are due to both being #[repr(transparent)] wrappers around u16.
|
|
537
|
+
let a_simsimd: &[f16] =
|
|
538
|
+
unsafe { std::slice::from_raw_parts(a_half.as_ptr() as *const f16, a_half.len()) };
|
|
539
|
+
let b_simsimd: &[f16] =
|
|
540
|
+
unsafe { std::slice::from_raw_parts(b_half.as_ptr() as *const f16, b_half.len()) };
|
|
541
|
+
|
|
542
|
+
// Use the reinterpret-casted slices with your SpatialSimilarity implementation
|
|
543
|
+
if let Some(result) = SpatialSimilarity::cosine(a_simsimd, b_simsimd) {
|
|
544
|
+
// Expected value might need adjustment depending on actual cosine functionality
|
|
545
|
+
// Assuming identical vectors yield cosine similarity of 1.0
|
|
546
|
+
println!("The result of cosine_f16 (interop) is {:.8}", result);
|
|
547
|
+
assert_almost_equal(0.025, result, 0.01);
|
|
230
548
|
}
|
|
231
549
|
}
|
|
232
550
|
}
|
|
@@ -136,6 +136,12 @@ cppcheck --project=build_artifacts/compile_commands.json --enable=all
|
|
|
136
136
|
clang-tidy-11 -p build_artifacts
|
|
137
137
|
```
|
|
138
138
|
|
|
139
|
+
I'd recommend putting the following breakpoints:
|
|
140
|
+
|
|
141
|
+
- `__asan::ReportGenericError` - to detect illegal memory accesses.
|
|
142
|
+
- `__GI_exit` - to stop at exit points - the end of running any executable.
|
|
143
|
+
- `__builtin_unreachable` - to catch all the places where the code is expected to be unreachable.
|
|
144
|
+
|
|
139
145
|
### Benchmarking
|
|
140
146
|
|
|
141
147
|
For benchmarks, you can use the following commands:
|
|
@@ -374,6 +380,17 @@ echo "export PATH=/usr/share/swift/usr/bin:$PATH" >> ~/.bashrc
|
|
|
374
380
|
source ~/.bashrc
|
|
375
381
|
```
|
|
376
382
|
|
|
383
|
+
You can check the available images on [`swift.org/download` page](https://www.swift.org/download/#releases).
|
|
384
|
+
For x86 CPUs, the following commands would work:
|
|
385
|
+
|
|
386
|
+
```bash
|
|
387
|
+
wget https://download.swift.org/swift-5.9.2-release/ubuntu2204/swift-5.9.2-RELEASE/swift-5.9.2-RELEASE-ubuntu22.04.tar.gz
|
|
388
|
+
tar xzf swift-5.9.2-RELEASE-ubuntu22.04.tar.gz
|
|
389
|
+
sudo mv swift-5.9.2-RELEASE-ubuntu22.04 /usr/share/swift
|
|
390
|
+
echo "export PATH=/usr/share/swift/usr/bin:$PATH" >> ~/.bashrc
|
|
391
|
+
source ~/.bashrc
|
|
392
|
+
```
|
|
393
|
+
|
|
377
394
|
Alternatively, on Linux, the official Swift Docker image can be used for builds and tests:
|
|
378
395
|
|
|
379
396
|
```bash
|
package/stringzilla/Cargo.lock
CHANGED
package/stringzilla/Cargo.toml
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[package]
|
|
2
2
|
name = "stringzilla"
|
|
3
|
-
version = "3.
|
|
3
|
+
version = "3.5.0"
|
|
4
4
|
authors = ["Ash Vardanian <1983160+ashvardanian@users.noreply.github.com>"]
|
|
5
5
|
description = "Faster SIMD-accelerated string search, sorting, fingerprints, and edit distances"
|
|
6
6
|
edition = "2021"
|