usearch 2.9.2 → 2.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +39 -38
- package/javascript/{types → dist/cjs}/usearch.d.ts +90 -115
- package/javascript/dist/cjs/usearch.js +531 -0
- package/javascript/dist/esm/usearch.d.ts +212 -0
- package/javascript/{usearch.js → dist/esm/usearch.js} +199 -153
- package/package.json +25 -23
- package/prebuilds/darwin-arm64+x64/usearch.node +0 -0
- package/prebuilds/linux-arm64/usearch.node +0 -0
- package/prebuilds/linux-x64/usearch.node +0 -0
- package/prebuilds/win32-ia32/usearch.node +0 -0
- package/prebuilds/win32-x64/usearch.node +0 -0
- package/.clang-format +0 -10
- package/.clang-tidy +0 -60
- package/.cmake-format.py +0 -19
- package/.dockerignore +0 -93
- package/.gitattributes +0 -9
- package/.github/ISSUE_TEMPLATE/bug_report.yml +0 -90
- package/.github/ISSUE_TEMPLATE/config.yml +0 -5
- package/.github/ISSUE_TEMPLATE/feature_request.yml +0 -64
- package/.github/workflows/Dockerfile.libs +0 -36
- package/.github/workflows/prerelease.yml +0 -331
- package/.github/workflows/release.yml +0 -722
- package/.github/workflows/update_version.sh +0 -15
- package/.gitmodules +0 -9
- package/.releaserc +0 -90
- package/CITATION.cff +0 -11
- package/CMakeLists.txt +0 -324
- package/CODE_OF_CONDUCT.md +0 -128
- package/CONTRIBUTING.md +0 -361
- package/Cargo.toml +0 -19
- package/Dockerfile +0 -16
- package/VERSION +0 -1
- package/assets/unum.png +0 -0
- package/assets/usearch-approaches-transparent.png +0 -0
- package/assets/usearch-approaches-white.png +0 -0
- package/assets/usearch-neighbor-types.png +0 -0
- package/binding.gyp +0 -87
- package/build.gradle +0 -204
- package/build.rs +0 -18
- package/c/CMakeLists.txt +0 -25
- package/c/README.md +0 -34
- package/c/lib.cpp +0 -358
- package/c/test.c +0 -382
- package/c/usearch.h +0 -311
- package/cmake/clipp.cmake +0 -7
- package/cmake/config.cmake.in +0 -15
- package/cmake/emscripten.cmake +0 -8
- package/cmake/jemalloc.cmake +0 -62
- package/cmake/pkg-config.pc.in +0 -4
- package/cmake/usearchConfigVersion.cmake.in +0 -20
- package/conanfile.py +0 -26
- package/cpp/CMakeLists.txt +0 -14
- package/cpp/README.md +0 -161
- package/cpp/bench.cpp +0 -618
- package/cpp/test.cpp +0 -323
- package/fp16/CMakeLists.txt +0 -176
- package/fp16/LICENSE +0 -11
- package/fp16/README.md +0 -20
- package/fp16/bench/alt-element.cc +0 -89
- package/fp16/bench/from-alt-array.cc +0 -175
- package/fp16/bench/from-ieee-array.cc +0 -399
- package/fp16/bench/ieee-element.cc +0 -272
- package/fp16/bench/to-alt-array.cc +0 -91
- package/fp16/bench/to-ieee-array.cc +0 -303
- package/fp16/cmake/DownloadGoogleBenchmark.cmake +0 -15
- package/fp16/cmake/DownloadGoogleTest.cmake +0 -15
- package/fp16/cmake/DownloadPSimd.cmake +0 -15
- package/fp16/configure.py +0 -65
- package/fp16/confu.yaml +0 -8
- package/fp16/include/fp16/__init__.py +0 -0
- package/fp16/include/fp16/avx.py +0 -50
- package/fp16/include/fp16/avx2.py +0 -53
- package/fp16/include/fp16/bitcasts.h +0 -92
- package/fp16/include/fp16/fp16.h +0 -451
- package/fp16/include/fp16/psimd.h +0 -131
- package/fp16/include/fp16.h +0 -11
- package/fp16/jni/Android.mk +0 -6
- package/fp16/jni/Application.mk +0 -4
- package/fp16/test/alt-from-fp32-value.cc +0 -498
- package/fp16/test/alt-to-fp32-bits.cc +0 -263
- package/fp16/test/alt-to-fp32-psimd.cc +0 -145
- package/fp16/test/alt-to-fp32-value.cc +0 -348
- package/fp16/test/alt-to-fp32x2-psimd.cc +0 -245
- package/fp16/test/bitcasts.cc +0 -56
- package/fp16/test/ieee-from-fp32-value.cc +0 -523
- package/fp16/test/ieee-to-fp32-bits.cc +0 -333
- package/fp16/test/ieee-to-fp32-psimd.cc +0 -312
- package/fp16/test/ieee-to-fp32-value.cc +0 -420
- package/fp16/test/ieee-to-fp32x2-psimd.cc +0 -541
- package/fp16/test/peachpy/alt-xmm-to-fp32-xmm-avx.cc +0 -96
- package/fp16/test/peachpy/alt-xmm-to-fp32-ymm-avx2.cc +0 -96
- package/fp16/test/peachpy/stubs.py +0 -38
- package/fp16/test/tables.cc +0 -530
- package/fp16/test/tables.h +0 -13
- package/fp16/third-party/THHalf.h +0 -92
- package/fp16/third-party/eigen-half.h +0 -170
- package/fp16/third-party/float16-compressor.h +0 -80
- package/fp16/third-party/half.hpp +0 -2916
- package/fp16/third-party/npy-halffloat.h +0 -328
- package/include/usearch/index.hpp +0 -3836
- package/include/usearch/index_dense.hpp +0 -2004
- package/include/usearch/index_plugins.hpp +0 -2213
- package/javascript/lib.cpp +0 -341
- package/javascript/types/usearch.d.ts.map +0 -1
- package/javascript/usearch.test.js +0 -74
- package/package-ci.json +0 -10
- package/pyproject.toml +0 -51
- package/setup.py +0 -201
- package/simsimd/.releaserc +0 -86
- package/simsimd/CONTRIBUTING.md +0 -88
- package/simsimd/Cargo.lock +0 -709
- package/simsimd/Cargo.toml +0 -44
- package/simsimd/LICENSE +0 -201
- package/simsimd/README.md +0 -317
- package/simsimd/binding.gyp +0 -36
- package/simsimd/build.rs +0 -20
- package/simsimd/include/simsimd/binary.h +0 -245
- package/simsimd/include/simsimd/complex.h +0 -174
- package/simsimd/include/simsimd/probability.h +0 -548
- package/simsimd/include/simsimd/simsimd.h +0 -465
- package/simsimd/include/simsimd/spatial.h +0 -1267
- package/simsimd/include/simsimd/types.h +0 -208
- package/simsimd/javascript/bench.js +0 -127
- package/simsimd/javascript/fallback.ts +0 -193
- package/simsimd/javascript/lib.c +0 -102
- package/simsimd/javascript/node-gyp-build.d.ts +0 -1
- package/simsimd/javascript/simsimd.ts +0 -124
- package/simsimd/javascript/test.mjs +0 -153
- package/simsimd/javascript/tsconfig-base.json +0 -25
- package/simsimd/javascript/tsconfig-cjs.json +0 -8
- package/simsimd/javascript/tsconfig-esm.json +0 -8
- package/simsimd/package-lock.json +0 -1688
- package/simsimd/package.json +0 -61
- package/simsimd/rust/benches/cosine.rs +0 -27
- package/simsimd/rust/benches/native.rs +0 -30
- package/simsimd/rust/benches/sqeuclidean.rs +0 -27
- package/simsimd/rust/lib.c +0 -45
- package/simsimd/rust/lib.rs +0 -550
- package/sqlite/README.md +0 -178
- package/sqlite/lib.cpp +0 -325
- package/stringzilla/CONTRIBUTING.md +0 -446
- package/stringzilla/Cargo.lock +0 -25
- package/stringzilla/Cargo.toml +0 -26
- package/stringzilla/LICENSE +0 -201
- package/stringzilla/README.md +0 -1422
- package/stringzilla/VERSION +0 -1
- package/stringzilla/assets/cover-strinzilla.jpeg +0 -0
- package/stringzilla/assets/meme-stringzilla-v2.jpeg +0 -0
- package/stringzilla/assets/meme-stringzilla-v3.jpeg +0 -0
- package/stringzilla/binding.gyp +0 -16
- package/stringzilla/build.rs +0 -87
- package/stringzilla/c/lib.c +0 -337
- package/stringzilla/cli/README.md +0 -65
- package/stringzilla/cli/__init__.py +0 -0
- package/stringzilla/cli/split.py +0 -105
- package/stringzilla/cli/wc.py +0 -117
- package/stringzilla/include/stringzilla/experimental.h +0 -845
- package/stringzilla/include/stringzilla/module.modulemap +0 -9
- package/stringzilla/include/stringzilla/stringzilla.h +0 -5370
- package/stringzilla/include/stringzilla/stringzilla.hpp +0 -3838
- package/stringzilla/javascript/lib.c +0 -125
- package/stringzilla/javascript/stringzilla.js +0 -22
- package/stringzilla/package-ci.json +0 -10
- package/stringzilla/package.json +0 -32
- package/stringzilla/rust/lib.rs +0 -873
- package/stringzilla/swift/StringProtocol+StringZilla.swift +0 -280
- package/stringzilla/swift/Test.swift +0 -60
- package/tsconfig.json +0 -21
- package/wasmer.toml +0 -12
- package/winlibinstaller.bat +0 -42
- /package/{simsimd/javascript/dist-package-cjs.json → javascript/dist/cjs/package.json} +0 -0
- /package/{simsimd/javascript/dist-package-esm.json → javascript/dist/esm/package.json} +0 -0
package/sqlite/README.md
DELETED
|
@@ -1,178 +0,0 @@
|
|
|
1
|
-
# USearch Extensions for SQLite
|
|
2
|
-
|
|
3
|
-
USearch exposes the SIMD-accelerated distance functions for SQLite databases aimed to accelerate vector search and fuzzy string matching operations.
|
|
4
|
-
This includes:
|
|
5
|
-
|
|
6
|
-
- [x] Cosine and Euclidean distances for float vectors,
|
|
7
|
-
- [x] Jaccard and Hamming distances for binary vectors,
|
|
8
|
-
- [x] Levenshtein and Hamming distances for strings,
|
|
9
|
-
- [x] Haversine distance for geographical coordinates.
|
|
10
|
-
|
|
11
|
-
The SIMD-acceleration covers AVX2, most subsets of AVX512, ARM NEON, and Arm SVE instruction sets, more than most BLAS libraries.
|
|
12
|
-
The implementations are coming from [SimSIMD](https://github.com/ashvardanian/simsimd) and [StringZilla](https://github.com/ashvardanian/stringzilla).
|
|
13
|
-
They are most efficient when vectors are stored as BLOBs, but for broader compatibility can also handle JSONs, and even separate columns containing vector elements.
|
|
14
|
-
|
|
15
|
-
## Installation
|
|
16
|
-
|
|
17
|
-
USearch currently ships the SQLite extensions as part of the Python wheels, so they can be obtained from PyPi.
|
|
18
|
-
|
|
19
|
-
```sh
|
|
20
|
-
pip install usearch # brings sqlite extensions
|
|
21
|
-
```
|
|
22
|
-
|
|
23
|
-
## Quickstart
|
|
24
|
-
|
|
25
|
-
To USearch extensions to your SQLite database, you can use the following commands:
|
|
26
|
-
|
|
27
|
-
```py
|
|
28
|
-
import sqlite3
|
|
29
|
-
import usearch
|
|
30
|
-
|
|
31
|
-
conn = sqlite3.connect(":memory:")
|
|
32
|
-
conn.enable_load_extension(True)
|
|
33
|
-
conn.load_extension(usearch.sqlite)
|
|
34
|
-
```
|
|
35
|
-
|
|
36
|
-
Afterwards, the following script should work fine.
|
|
37
|
-
|
|
38
|
-
```sql
|
|
39
|
-
-- Create a table with a JSON column for vectors
|
|
40
|
-
CREATE TABLE vectors_table (
|
|
41
|
-
id SERIAL PRIMARY KEY,
|
|
42
|
-
vector JSON NOT NULL
|
|
43
|
-
);
|
|
44
|
-
|
|
45
|
-
-- Insert two 3D vectors
|
|
46
|
-
INSERT INTO vectors_table (id, vector)
|
|
47
|
-
VALUES
|
|
48
|
-
(42, '[1.0, 2.0, 3.0]'),
|
|
49
|
-
(43, '[4.0, 5.0, 6.0]');
|
|
50
|
-
|
|
51
|
-
-- Compute the distances to [7.0, 8.0, 9.0] using
|
|
52
|
-
-- the `distance_cosine_f32` extension function
|
|
53
|
-
SELECT
|
|
54
|
-
id,
|
|
55
|
-
distance_cosine_f32(vt.vector, '[7.0, 8.0, 9.0]') AS distance
|
|
56
|
-
FROM vectors_table AS vt;
|
|
57
|
-
```
|
|
58
|
-
|
|
59
|
-
### Functionality
|
|
60
|
-
|
|
61
|
-
#### Strings
|
|
62
|
-
|
|
63
|
-
String functions accept BLOBs and TEXTs, and return the distance as an integer.
|
|
64
|
-
This includes:
|
|
65
|
-
|
|
66
|
-
- `distance_levenshtein_bytes`,
|
|
67
|
-
- `distance_levenshtein_unicode`,
|
|
68
|
-
- `distance_hamming_bytes`,
|
|
69
|
-
- `distance_hamming_unicode`.
|
|
70
|
-
|
|
71
|
-
The Levenshtein distance would report the number of insertions, deletions, and substitutions required to transform one string into another, while the Hamming distance would only report the substitutons.
|
|
72
|
-
When applied to strings of different length, the Hamming distance would evalute the prefix of the longer string and will add the length difference to the result.
|
|
73
|
-
The `_bytes` variants compute the distance in the number of bytes, while the `_unicode` variants compute the distance in the number of Unicode code points, assuming the inputs are UTF8 encoded.
|
|
74
|
-
Take a look at the following three similar looking, but distinct strings:
|
|
75
|
-
|
|
76
|
-
- `école` - 6 codepoints (runes), 7 bytes.
|
|
77
|
-
- `école` - 5 codepoints (runes), 6 bytes.
|
|
78
|
-
- `écolé` - 5 codepoints (runes), 7 bytes.
|
|
79
|
-
|
|
80
|
-
Those have three different letter "e" variants, including etter "é" as a single character, etter as "e" and an accent "´", as well as the plain letter "e".
|
|
81
|
-
This would result in different distances between those strings, depending on the chosen metric.
|
|
82
|
-
|
|
83
|
-
```sql
|
|
84
|
-
CREATE TABLE strings_table (
|
|
85
|
-
id SERIAL PRIMARY KEY,
|
|
86
|
-
word TEXT NOT NULL
|
|
87
|
-
);
|
|
88
|
-
|
|
89
|
-
INSERT INTO strings_table (id, word)
|
|
90
|
-
VALUES
|
|
91
|
-
(42, 'école'),
|
|
92
|
-
(43, 'école');
|
|
93
|
-
|
|
94
|
-
SELECT
|
|
95
|
-
st.id,
|
|
96
|
-
|
|
97
|
-
distance_levenshtein_bytes(st.word, 'écolé') AS lb,
|
|
98
|
-
distance_levenshtein_unicode(st.word, 'écolé') AS lu,
|
|
99
|
-
distance_hamming_bytes(st.word, 'écolé') AS hb,
|
|
100
|
-
distance_hamming_unicode(st.word, 'écolé') AS hu,
|
|
101
|
-
|
|
102
|
-
distance_levenshtein_bytes(st.word, 'écolé', 2) AS lbb,
|
|
103
|
-
distance_levenshtein_unicode(st.word, 'écolé', 2) AS lub,
|
|
104
|
-
distance_hamming_bytes(st.word, 'écolé', 2) AS hbb,
|
|
105
|
-
distance_hamming_unicode(st.word, 'écolé', 2) AS hub
|
|
106
|
-
|
|
107
|
-
FROM strings_table AS st;
|
|
108
|
-
```
|
|
109
|
-
|
|
110
|
-
The last 4 columns will contain the bounded versions of the distance, which are faster to compute, thanks to the early stopping condition.
|
|
111
|
-
They might be handy, if you want to accelerate search for something like an "autocomplete" feature in a search bar.
|
|
112
|
-
The output will look like this:
|
|
113
|
-
|
|
114
|
-
| id | lb | lu | hb | hu | lbb | lub | hbb | hub |
|
|
115
|
-
| --- | --- | --- | --- | --- | --- | --- | --- | --- |
|
|
116
|
-
| 42 | 5 | 3 | 7 | 6 | 2 | 2 | 2 | 2 |
|
|
117
|
-
| 43 | 2 | 1 | 2 | 1 | 2 | 1 | 2 | 1 |
|
|
118
|
-
|
|
119
|
-
#### Bit Vectors
|
|
120
|
-
|
|
121
|
-
If using SQLite as a feature store, you might be dealing with single-bit vectors.
|
|
122
|
-
|
|
123
|
-
- `distance_hamming_binary` - the Hamming distance, meaning number of bits that differ between two vectors,
|
|
124
|
-
- `distance_jaccard_binary` - the Jaccard distance, meaning the number of bits that differ between two vectors divided by the number of bits that are set in at least one of the vectors.
|
|
125
|
-
|
|
126
|
-
Here is an example:
|
|
127
|
-
|
|
128
|
-
```sql
|
|
129
|
-
CREATE TABLE binary_vectors (
|
|
130
|
-
id SERIAL PRIMARY KEY,
|
|
131
|
-
vector BLOB NOT NULL
|
|
132
|
-
);
|
|
133
|
-
INSERT INTO binary_vectors (id, vector)
|
|
134
|
-
VALUES
|
|
135
|
-
(42, X'FFFFFF'), -- 111111111111111111111111 in binary
|
|
136
|
-
(43, X'000000'); -- 000000000000000000000000 in binary
|
|
137
|
-
SELECT bv.id,
|
|
138
|
-
distance_hamming_binary(bv.vector, X'FFFF00') AS hamming_distance,
|
|
139
|
-
distance_jaccard_binary(bv.vector, X'FFFF00') AS jaccard_distance
|
|
140
|
-
FROM binary_vectors AS bv;
|
|
141
|
-
```
|
|
142
|
-
|
|
143
|
-
#### Dense Vectors
|
|
144
|
-
|
|
145
|
-
Distance functions for desnse vectors can be used on both BLOBs and JSONs.
|
|
146
|
-
Every name is structured as `distance_<metric>_<type>`, where
|
|
147
|
-
|
|
148
|
-
- `<metric>` is the name of the metric, like `cosine`, `inner`, `sqeuclidean`, and `divergence`,
|
|
149
|
-
- `<type>` is the type of the vector elements, like `f64`, `f32`, `f16`, and `i8`.
|
|
150
|
-
|
|
151
|
-
The `cosine` metric is the cosine similarity, the `inner` metric is the inner (dot) product, the `sqeuclidean` metric is the squared Euclidean distance, and the `divergence` metric is the Jensen-Shannon divergence - the symmetric variant of the Kullback-Leibler divergence.
|
|
152
|
-
|
|
153
|
-
- `distance_sqeuclidean_f64`
|
|
154
|
-
- `distance_cosine_f64`
|
|
155
|
-
- `distance_inner_f64`
|
|
156
|
-
- `distance_divergence_f64`
|
|
157
|
-
- `distance_sqeuclidean_f32`
|
|
158
|
-
- `distance_cosine_f32`
|
|
159
|
-
- `distance_inner_f32`
|
|
160
|
-
- `distance_divergence_f32`
|
|
161
|
-
- `distance_sqeuclidean_f16`
|
|
162
|
-
- `distance_cosine_f16`
|
|
163
|
-
- `distance_inner_f16`
|
|
164
|
-
- `distance_divergence_f16`
|
|
165
|
-
- `distance_sqeuclidean_i8`
|
|
166
|
-
- `distance_cosine_i8`
|
|
167
|
-
- `distance_inner_i8`
|
|
168
|
-
- `distance_divergence_i8`
|
|
169
|
-
|
|
170
|
-
#### Geographical Coordinates
|
|
171
|
-
|
|
172
|
-
When dealing with geographical or other spherical coordinates, you can use:
|
|
173
|
-
|
|
174
|
-
- `distance_haversine_meters` - the Haversine distance on the sphere multiplied by the Earth's radius in meters.
|
|
175
|
-
|
|
176
|
-
---
|
|
177
|
-
|
|
178
|
-
Feel free to contribute more examples to this file 🤗
|
package/sqlite/lib.cpp
DELETED
|
@@ -1,325 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @brief SQLite3 bindings for USearch.
|
|
3
|
-
* @file lib.cpp
|
|
4
|
-
* @author Ash Vardanian
|
|
5
|
-
* @date November 28, 2023
|
|
6
|
-
* @copyright Copyright (c) 2023
|
|
7
|
-
*/
|
|
8
|
-
|
|
9
|
-
#include <charconv> // `std::from_chars`
|
|
10
|
-
#include <cstdlib> // `std::strtod`
|
|
11
|
-
|
|
12
|
-
#include <sqlite3ext.h>
|
|
13
|
-
SQLITE_EXTENSION_INIT1
|
|
14
|
-
|
|
15
|
-
#define SZ_DYNAMIC_DISPATCH 1
|
|
16
|
-
#include <stringzilla/stringzilla.hpp>
|
|
17
|
-
|
|
18
|
-
#include <usearch/index_dense.hpp>
|
|
19
|
-
#include <usearch/index_plugins.hpp>
|
|
20
|
-
|
|
21
|
-
using namespace unum::usearch;
|
|
22
|
-
using namespace unum;
|
|
23
|
-
|
|
24
|
-
namespace sz = ashvardanian::stringzilla;
|
|
25
|
-
|
|
26
|
-
using metric_t = metric_punned_t;
|
|
27
|
-
using distance_t = distance_punned_t;
|
|
28
|
-
|
|
29
|
-
template <scalar_kind_t scalar_kind_ak> struct parsed_scalar_kind_gt {
|
|
30
|
-
using type = f32_t;
|
|
31
|
-
static constexpr scalar_kind_t kind = scalar_kind_t::f32_k;
|
|
32
|
-
};
|
|
33
|
-
|
|
34
|
-
template <> struct parsed_scalar_kind_gt<scalar_kind_t::f64_k> {
|
|
35
|
-
using type = f64_t;
|
|
36
|
-
static constexpr scalar_kind_t kind = scalar_kind_t::f64_k;
|
|
37
|
-
};
|
|
38
|
-
|
|
39
|
-
template <scalar_kind_t scalar_kind_ak, metric_kind_t metric_kind_ak>
|
|
40
|
-
static void sqlite_dense(sqlite3_context* context, int argc, sqlite3_value** argv) {
|
|
41
|
-
|
|
42
|
-
if (argc < 2) {
|
|
43
|
-
sqlite3_result_error(context, "Distance function expects at least two arguments", -1);
|
|
44
|
-
return;
|
|
45
|
-
}
|
|
46
|
-
|
|
47
|
-
int type1 = sqlite3_value_type(argv[0]);
|
|
48
|
-
int type2 = sqlite3_value_type(argv[1]);
|
|
49
|
-
|
|
50
|
-
// Our primary case is having two BLOBs containing dense vector representations.
|
|
51
|
-
if (argc == 2 && type1 == SQLITE_BLOB && type2 == SQLITE_BLOB) {
|
|
52
|
-
// The pointer obtained from `sqlite3_value_text` or `sqlite3_value_blob` can
|
|
53
|
-
// be invalidated by the next call to `sqlite3_value_bytes` according to docs, so read it first.
|
|
54
|
-
int bytes1 = sqlite3_value_bytes(argv[0]);
|
|
55
|
-
int bytes2 = sqlite3_value_bytes(argv[1]);
|
|
56
|
-
if (bytes1 != bytes2) {
|
|
57
|
-
sqlite3_result_error(context, "Vectors have different number of dimensions", -1);
|
|
58
|
-
return;
|
|
59
|
-
}
|
|
60
|
-
void const* vec1 = sqlite3_value_blob(argv[0]);
|
|
61
|
-
void const* vec2 = sqlite3_value_blob(argv[1]);
|
|
62
|
-
|
|
63
|
-
std::size_t dimensions = (size_t)(bytes1)*CHAR_BIT / bits_per_scalar(scalar_kind_ak);
|
|
64
|
-
metric_t metric = metric_t(dimensions, metric_kind_ak, scalar_kind_ak);
|
|
65
|
-
distance_punned_t distance =
|
|
66
|
-
metric(reinterpret_cast<byte_t const*>(vec1), reinterpret_cast<byte_t const*>(vec2));
|
|
67
|
-
sqlite3_result_double(context, distance);
|
|
68
|
-
}
|
|
69
|
-
|
|
70
|
-
// Worst case is to have JSON arrays or comma-separated values
|
|
71
|
-
else if (argc == 2 && type1 == SQLITE_TEXT && type2 == SQLITE_TEXT) {
|
|
72
|
-
// The pointer obtained from `sqlite3_value_text` or `sqlite3_value_blob` can
|
|
73
|
-
// be invalidated by the next call to `sqlite3_value_bytes` according to docs, so read it first.
|
|
74
|
-
size_t bytes1 = (size_t)sqlite3_value_bytes(argv[0]);
|
|
75
|
-
size_t bytes2 = (size_t)sqlite3_value_bytes(argv[1]);
|
|
76
|
-
char* vec1 = (char*)sqlite3_value_text(argv[0]);
|
|
77
|
-
char* vec2 = (char*)sqlite3_value_text(argv[1]);
|
|
78
|
-
size_t commas1 = sz::string_view(vec1, bytes1).find_all(",").size();
|
|
79
|
-
size_t commas2 = sz::string_view(vec2, bytes2).find_all(",").size();
|
|
80
|
-
if (commas1 != commas2) {
|
|
81
|
-
sqlite3_result_error(context, "Vectors have different number of dimensions", -1);
|
|
82
|
-
return;
|
|
83
|
-
}
|
|
84
|
-
|
|
85
|
-
// Valid JSON array of numbers would be packed into [] square brackets
|
|
86
|
-
if (bytes1 && vec1[0] == '[')
|
|
87
|
-
++vec1, --bytes1;
|
|
88
|
-
if (bytes2 && vec2[0] == '[')
|
|
89
|
-
++vec2, --bytes2;
|
|
90
|
-
// We don't have to trim the end
|
|
91
|
-
// if (bytes1 && vec1[bytes1 - 1] == ']')
|
|
92
|
-
// --bytes1;
|
|
93
|
-
// if (bytes2 && vec2[bytes2 - 1] == ']')
|
|
94
|
-
// --bytes2;
|
|
95
|
-
|
|
96
|
-
// Allocate vectors on stack and parse strings into them
|
|
97
|
-
using scalar_t = typename parsed_scalar_kind_gt<scalar_kind_ak>::type;
|
|
98
|
-
size_t dimensions = commas1 + 1;
|
|
99
|
-
scalar_t parsed1[dimensions], parsed2[dimensions];
|
|
100
|
-
for (size_t i = 0; i != dimensions; ++i) {
|
|
101
|
-
// Skip whitespace
|
|
102
|
-
while (bytes1 && vec1[0] == ' ')
|
|
103
|
-
++vec1, --bytes1;
|
|
104
|
-
while (bytes2 && vec2[0] == ' ')
|
|
105
|
-
++vec2, --bytes2;
|
|
106
|
-
|
|
107
|
-
// Parse the floating-point numbers
|
|
108
|
-
// Sadly, most modern compilers don't support the `std::from_chars` yet
|
|
109
|
-
#if __cpp_lib_to_chars
|
|
110
|
-
std::from_chars_result result1 = std::from_chars(vec1, vec1 + bytes1, parsed1[i]);
|
|
111
|
-
std::from_chars_result result2 = std::from_chars(vec2, vec2 + bytes2, parsed2[i]);
|
|
112
|
-
if (result1.ec != std::errc() || result2.ec != std::errc()) {
|
|
113
|
-
sqlite3_result_error(context, "Number can't be parsed", -1);
|
|
114
|
-
return;
|
|
115
|
-
}
|
|
116
|
-
bytes1 -= result1.ptr - vec1;
|
|
117
|
-
bytes2 -= result2.ptr - vec2;
|
|
118
|
-
vec1 = (char*)result1.ptr;
|
|
119
|
-
vec2 = (char*)result2.ptr;
|
|
120
|
-
#else
|
|
121
|
-
char* parsed1_end = vec1 + bytes1;
|
|
122
|
-
parsed1[i] = std::strtod(vec1, &parsed1_end);
|
|
123
|
-
char* parsed2_end = vec2 + bytes2;
|
|
124
|
-
parsed2[i] = std::strtod(vec2, &parsed2_end);
|
|
125
|
-
if (vec1 == parsed1_end || vec2 == parsed2_end) {
|
|
126
|
-
sqlite3_result_error(context, "Number can't be parsed", -1);
|
|
127
|
-
return;
|
|
128
|
-
}
|
|
129
|
-
bytes1 -= parsed1_end - vec1;
|
|
130
|
-
bytes2 -= parsed2_end - vec2;
|
|
131
|
-
vec1 = parsed1_end;
|
|
132
|
-
vec2 = parsed2_end;
|
|
133
|
-
#endif
|
|
134
|
-
|
|
135
|
-
// Skip the whitespaces and commas
|
|
136
|
-
while (bytes1 && (vec1[0] == ' ' || vec1[0] == ','))
|
|
137
|
-
++vec1, --bytes1;
|
|
138
|
-
while (bytes2 && (vec2[0] == ' ' || vec2[0] == ','))
|
|
139
|
-
++vec2, --bytes2;
|
|
140
|
-
}
|
|
141
|
-
|
|
142
|
-
// Compute the distance itself
|
|
143
|
-
metric_t metric = metric_t(dimensions, metric_kind_ak, parsed_scalar_kind_gt<scalar_kind_ak>::kind);
|
|
144
|
-
distance_punned_t distance =
|
|
145
|
-
metric(reinterpret_cast<byte_t const*>(parsed1), reinterpret_cast<byte_t const*>(parsed2));
|
|
146
|
-
sqlite3_result_double(context, distance);
|
|
147
|
-
}
|
|
148
|
-
|
|
149
|
-
// Less efficient, yet still common case is to have many scalar columns
|
|
150
|
-
else if (argc % 2 == 0) {
|
|
151
|
-
|
|
152
|
-
// Allocate vectors on stack and parse floating-point values into them
|
|
153
|
-
using scalar_t = typename parsed_scalar_kind_gt<scalar_kind_ak>::type;
|
|
154
|
-
size_t dimensions = argc / 2;
|
|
155
|
-
scalar_t parsed1[dimensions], parsed2[dimensions];
|
|
156
|
-
for (size_t i = 0; i != dimensions; ++i) {
|
|
157
|
-
switch (sqlite3_value_type(argv[i])) {
|
|
158
|
-
case SQLITE_FLOAT: parsed1[i] = sqlite3_value_double(argv[i]); break;
|
|
159
|
-
case SQLITE_INTEGER: parsed1[i] = sqlite3_value_int(argv[i]); break;
|
|
160
|
-
case SQLITE_NULL: parsed1[i] = 0; break;
|
|
161
|
-
default:
|
|
162
|
-
sqlite3_result_error(context, "Scalar columns may only contain 32-bit integers, floats, or NULLs.", -1);
|
|
163
|
-
return;
|
|
164
|
-
}
|
|
165
|
-
switch (sqlite3_value_type(argv[dimensions + i])) {
|
|
166
|
-
case SQLITE_FLOAT: parsed2[i] = sqlite3_value_double(argv[dimensions + i]); break;
|
|
167
|
-
case SQLITE_INTEGER: parsed2[i] = sqlite3_value_int(argv[dimensions + i]); break;
|
|
168
|
-
case SQLITE_NULL: parsed2[i] = 0; break;
|
|
169
|
-
default:
|
|
170
|
-
sqlite3_result_error(context, "Scalar columns may only contain 32-bit integers, floats, or NULLs.", -1);
|
|
171
|
-
return;
|
|
172
|
-
}
|
|
173
|
-
}
|
|
174
|
-
|
|
175
|
-
// Compute the distance itself
|
|
176
|
-
metric_t metric = metric_t(dimensions, metric_kind_ak, parsed_scalar_kind_gt<scalar_kind_ak>::kind);
|
|
177
|
-
distance_punned_t distance =
|
|
178
|
-
metric(reinterpret_cast<byte_t const*>(parsed1), reinterpret_cast<byte_t const*>(parsed2));
|
|
179
|
-
sqlite3_result_double(context, distance);
|
|
180
|
-
}
|
|
181
|
-
// Unsupported arguments combination
|
|
182
|
-
else {
|
|
183
|
-
sqlite3_result_error(context, "Number of columns in two vectors must be divisible by two", -1);
|
|
184
|
-
}
|
|
185
|
-
}
|
|
186
|
-
|
|
187
|
-
enum class string_metric_kind_t : std::uint8_t {
|
|
188
|
-
levenshtein_bytes_k,
|
|
189
|
-
levenshtein_unicode_k,
|
|
190
|
-
hamming_bytes_k,
|
|
191
|
-
hamming_unicode_k,
|
|
192
|
-
};
|
|
193
|
-
|
|
194
|
-
template <string_metric_kind_t string_metric_kind_ak>
|
|
195
|
-
static void sqlite_strings(sqlite3_context* context, int argc, sqlite3_value** argv) {
|
|
196
|
-
|
|
197
|
-
if (argc != 2 && argc != 3) {
|
|
198
|
-
sqlite3_result_error(context, "Distance function expects 2 or 3 arguments", -1);
|
|
199
|
-
return;
|
|
200
|
-
}
|
|
201
|
-
|
|
202
|
-
int type1 = sqlite3_value_type(argv[0]);
|
|
203
|
-
int type2 = sqlite3_value_type(argv[1]);
|
|
204
|
-
auto is_text_or_blob = [](int type) { return type == SQLITE_TEXT || type == SQLITE_BLOB; };
|
|
205
|
-
if (!is_text_or_blob(type1) || !is_text_or_blob(type2)) {
|
|
206
|
-
sqlite3_result_error(context, "Distance function expects text or blob arguments", -1);
|
|
207
|
-
return;
|
|
208
|
-
}
|
|
209
|
-
|
|
210
|
-
sz_size_t bound = 0;
|
|
211
|
-
if (argc == 3) {
|
|
212
|
-
int type3 = sqlite3_value_type(argv[2]);
|
|
213
|
-
if (type3 != SQLITE_INTEGER) {
|
|
214
|
-
sqlite3_result_error(context, "Distance function expects integer as the third argument", -1);
|
|
215
|
-
return;
|
|
216
|
-
}
|
|
217
|
-
std::int64_t signed_bound = sqlite3_value_int64(argv[2]);
|
|
218
|
-
if (signed_bound < 0) {
|
|
219
|
-
sqlite3_result_error(context, "Distance function expects non-negative integer as the third argument", -1);
|
|
220
|
-
return;
|
|
221
|
-
}
|
|
222
|
-
bound = (sz_size_t)signed_bound;
|
|
223
|
-
}
|
|
224
|
-
|
|
225
|
-
// The pointer obtained from `sqlite3_value_text` or `sqlite3_value_blob` can
|
|
226
|
-
// be invalidated by the next call to `sqlite3_value_bytes` according to docs, so read it first.
|
|
227
|
-
sz_size_t bytes1 = (sz_size_t)sqlite3_value_bytes(argv[0]);
|
|
228
|
-
sz_size_t bytes2 = (sz_size_t)sqlite3_value_bytes(argv[1]);
|
|
229
|
-
auto extract_data_pointer = [](int type, sqlite3_value* value) -> sz_cptr_t {
|
|
230
|
-
return type == SQLITE_BLOB ? (sz_cptr_t)sqlite3_value_blob(value) : (sz_cptr_t)sqlite3_value_text(value);
|
|
231
|
-
};
|
|
232
|
-
sz_cptr_t vec1 = extract_data_pointer(type1, argv[0]);
|
|
233
|
-
sz_cptr_t vec2 = extract_data_pointer(type2, argv[1]);
|
|
234
|
-
sz::string_view str1(vec1, bytes1);
|
|
235
|
-
sz::string_view str2(vec2, bytes2);
|
|
236
|
-
|
|
237
|
-
// Dispatch the right backend
|
|
238
|
-
sz_size_t result = SZ_SIZE_MAX;
|
|
239
|
-
switch (string_metric_kind_ak) {
|
|
240
|
-
case string_metric_kind_t::levenshtein_bytes_k: result = sz::edit_distance(str1, str2, bound); break;
|
|
241
|
-
case string_metric_kind_t::levenshtein_unicode_k: result = sz::edit_distance_utf8(str1, str2, bound); break;
|
|
242
|
-
case string_metric_kind_t::hamming_bytes_k: result = sz::hamming_distance(str1, str2, bound); break;
|
|
243
|
-
case string_metric_kind_t::hamming_unicode_k: result = sz::hamming_distance_utf8(str1, str2, bound); break;
|
|
244
|
-
}
|
|
245
|
-
|
|
246
|
-
// Check for errors
|
|
247
|
-
if (result == SZ_SIZE_MAX) {
|
|
248
|
-
sqlite3_result_error(context, "Distance function failed to compute the result", -1);
|
|
249
|
-
return;
|
|
250
|
-
}
|
|
251
|
-
|
|
252
|
-
sqlite3_result_int64(context, (std::int64_t)result);
|
|
253
|
-
}
|
|
254
|
-
|
|
255
|
-
extern "C" SZ_DYNAMIC int sqlite3_compiled_init( //
|
|
256
|
-
sqlite3* db, //
|
|
257
|
-
char** error_message, //
|
|
258
|
-
sqlite3_api_routines const* api) {
|
|
259
|
-
SQLITE_EXTENSION_INIT2(api)
|
|
260
|
-
|
|
261
|
-
int flags = SQLITE_UTF8 | SQLITE_DETERMINISTIC | SQLITE_INNOCUOUS;
|
|
262
|
-
int num_params = -1; // Any number will be accepted
|
|
263
|
-
|
|
264
|
-
// String similarity metrics
|
|
265
|
-
sqlite3_create_function(db, "distance_levenshtein_bytes", num_params, flags, NULL,
|
|
266
|
-
sqlite_strings<string_metric_kind_t::levenshtein_bytes_k>, NULL, NULL);
|
|
267
|
-
sqlite3_create_function(db, "distance_levenshtein_unicode", num_params, flags, NULL,
|
|
268
|
-
sqlite_strings<string_metric_kind_t::levenshtein_unicode_k>, NULL, NULL);
|
|
269
|
-
sqlite3_create_function(db, "distance_hamming_bytes", num_params, flags, NULL,
|
|
270
|
-
sqlite_strings<string_metric_kind_t::hamming_bytes_k>, NULL, NULL);
|
|
271
|
-
sqlite3_create_function(db, "distance_hamming_unicode", num_params, flags, NULL,
|
|
272
|
-
sqlite_strings<string_metric_kind_t::hamming_unicode_k>, NULL, NULL);
|
|
273
|
-
|
|
274
|
-
// Bit-wise metrics
|
|
275
|
-
sqlite3_create_function(db, "distance_hamming_binary", num_params, flags, NULL,
|
|
276
|
-
sqlite_dense<scalar_kind_t::b1x8_k, metric_kind_t::hamming_k>, NULL, NULL);
|
|
277
|
-
sqlite3_create_function(db, "distance_jaccard_binary", num_params, flags, NULL,
|
|
278
|
-
sqlite_dense<scalar_kind_t::b1x8_k, metric_kind_t::jaccard_k>, NULL, NULL);
|
|
279
|
-
sqlite3_create_function(db, "distance_haversine_meters", num_params, flags, NULL,
|
|
280
|
-
sqlite_dense<scalar_kind_t::f64_k, metric_kind_t::haversine_k>, NULL, NULL);
|
|
281
|
-
|
|
282
|
-
// Double-precision metrics
|
|
283
|
-
sqlite3_create_function(db, "distance_sqeuclidean_f64", num_params, flags, NULL,
|
|
284
|
-
sqlite_dense<scalar_kind_t::f64_k, metric_kind_t::l2sq_k>, NULL, NULL);
|
|
285
|
-
sqlite3_create_function(db, "distance_cosine_f64", num_params, flags, NULL,
|
|
286
|
-
sqlite_dense<scalar_kind_t::f64_k, metric_kind_t::cos_k>, NULL, NULL);
|
|
287
|
-
sqlite3_create_function(db, "distance_inner_f64", num_params, flags, NULL,
|
|
288
|
-
sqlite_dense<scalar_kind_t::f64_k, metric_kind_t::ip_k>, NULL, NULL);
|
|
289
|
-
sqlite3_create_function(db, "distance_divergence_f64", num_params, flags, NULL,
|
|
290
|
-
sqlite_dense<scalar_kind_t::f64_k, metric_kind_t::divergence_k>, NULL, NULL);
|
|
291
|
-
|
|
292
|
-
// Single-precision metrics
|
|
293
|
-
sqlite3_create_function(db, "distance_sqeuclidean_f32", num_params, flags, NULL,
|
|
294
|
-
sqlite_dense<scalar_kind_t::f32_k, metric_kind_t::l2sq_k>, NULL, NULL);
|
|
295
|
-
sqlite3_create_function(db, "distance_cosine_f32", num_params, flags, NULL,
|
|
296
|
-
sqlite_dense<scalar_kind_t::f32_k, metric_kind_t::cos_k>, NULL, NULL);
|
|
297
|
-
sqlite3_create_function(db, "distance_inner_f32", num_params, flags, NULL,
|
|
298
|
-
sqlite_dense<scalar_kind_t::f32_k, metric_kind_t::ip_k>, NULL, NULL);
|
|
299
|
-
sqlite3_create_function(db, "distance_divergence_f32", num_params, flags, NULL,
|
|
300
|
-
sqlite_dense<scalar_kind_t::f32_k, metric_kind_t::divergence_k>, NULL, NULL);
|
|
301
|
-
|
|
302
|
-
// Half-precision metrics
|
|
303
|
-
sqlite3_create_function(db, "distance_sqeuclidean_f16", num_params, flags, NULL,
|
|
304
|
-
sqlite_dense<scalar_kind_t::f16_k, metric_kind_t::l2sq_k>, NULL, NULL);
|
|
305
|
-
sqlite3_create_function(db, "distance_cosine_f16", num_params, flags, NULL,
|
|
306
|
-
sqlite_dense<scalar_kind_t::f16_k, metric_kind_t::cos_k>, NULL, NULL);
|
|
307
|
-
sqlite3_create_function(db, "distance_inner_f16", num_params, flags, NULL,
|
|
308
|
-
sqlite_dense<scalar_kind_t::f16_k, metric_kind_t::ip_k>, NULL, NULL);
|
|
309
|
-
sqlite3_create_function(db, "distance_divergence_f16", num_params, flags, NULL,
|
|
310
|
-
sqlite_dense<scalar_kind_t::f16_k, metric_kind_t::divergence_k>, NULL, NULL);
|
|
311
|
-
|
|
312
|
-
// Integer metrics
|
|
313
|
-
sqlite3_create_function(db, "distance_sqeuclidean_i8", num_params, flags, NULL,
|
|
314
|
-
sqlite_dense<scalar_kind_t::i8_k, metric_kind_t::l2sq_k>, NULL, NULL);
|
|
315
|
-
sqlite3_create_function(db, "distance_cosine_i8", num_params, flags, NULL,
|
|
316
|
-
sqlite_dense<scalar_kind_t::i8_k, metric_kind_t::cos_k>, NULL, NULL);
|
|
317
|
-
sqlite3_create_function(db, "distance_inner_i8", num_params, flags, NULL,
|
|
318
|
-
sqlite_dense<scalar_kind_t::i8_k, metric_kind_t::ip_k>, NULL, NULL);
|
|
319
|
-
sqlite3_create_function(db, "distance_divergence_i8", num_params, flags, NULL,
|
|
320
|
-
sqlite_dense<scalar_kind_t::i8_k, metric_kind_t::divergence_k>, NULL, NULL);
|
|
321
|
-
|
|
322
|
-
return SQLITE_OK;
|
|
323
|
-
}
|
|
324
|
-
|
|
325
|
-
#include "../stringzilla/c/lib.c"
|