usearch 2.9.2 → 2.10.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (172) hide show
  1. package/README.md +39 -38
  2. package/javascript/{types → dist/cjs}/usearch.d.ts +90 -115
  3. package/javascript/dist/cjs/usearch.js +531 -0
  4. package/javascript/dist/esm/usearch.d.ts +212 -0
  5. package/javascript/{usearch.js → dist/esm/usearch.js} +199 -153
  6. package/package.json +25 -23
  7. package/prebuilds/darwin-arm64+x64/usearch.node +0 -0
  8. package/prebuilds/linux-arm64/usearch.node +0 -0
  9. package/prebuilds/linux-x64/usearch.node +0 -0
  10. package/prebuilds/win32-ia32/usearch.node +0 -0
  11. package/prebuilds/win32-x64/usearch.node +0 -0
  12. package/.clang-format +0 -10
  13. package/.clang-tidy +0 -60
  14. package/.cmake-format.py +0 -19
  15. package/.dockerignore +0 -93
  16. package/.gitattributes +0 -9
  17. package/.github/ISSUE_TEMPLATE/bug_report.yml +0 -90
  18. package/.github/ISSUE_TEMPLATE/config.yml +0 -5
  19. package/.github/ISSUE_TEMPLATE/feature_request.yml +0 -64
  20. package/.github/workflows/Dockerfile.libs +0 -36
  21. package/.github/workflows/prerelease.yml +0 -331
  22. package/.github/workflows/release.yml +0 -722
  23. package/.github/workflows/update_version.sh +0 -15
  24. package/.gitmodules +0 -9
  25. package/.releaserc +0 -90
  26. package/CITATION.cff +0 -11
  27. package/CMakeLists.txt +0 -324
  28. package/CODE_OF_CONDUCT.md +0 -128
  29. package/CONTRIBUTING.md +0 -361
  30. package/Cargo.toml +0 -19
  31. package/Dockerfile +0 -16
  32. package/VERSION +0 -1
  33. package/assets/unum.png +0 -0
  34. package/assets/usearch-approaches-transparent.png +0 -0
  35. package/assets/usearch-approaches-white.png +0 -0
  36. package/assets/usearch-neighbor-types.png +0 -0
  37. package/binding.gyp +0 -87
  38. package/build.gradle +0 -204
  39. package/build.rs +0 -18
  40. package/c/CMakeLists.txt +0 -25
  41. package/c/README.md +0 -34
  42. package/c/lib.cpp +0 -358
  43. package/c/test.c +0 -382
  44. package/c/usearch.h +0 -311
  45. package/cmake/clipp.cmake +0 -7
  46. package/cmake/config.cmake.in +0 -15
  47. package/cmake/emscripten.cmake +0 -8
  48. package/cmake/jemalloc.cmake +0 -62
  49. package/cmake/pkg-config.pc.in +0 -4
  50. package/cmake/usearchConfigVersion.cmake.in +0 -20
  51. package/conanfile.py +0 -26
  52. package/cpp/CMakeLists.txt +0 -14
  53. package/cpp/README.md +0 -161
  54. package/cpp/bench.cpp +0 -618
  55. package/cpp/test.cpp +0 -323
  56. package/fp16/CMakeLists.txt +0 -176
  57. package/fp16/LICENSE +0 -11
  58. package/fp16/README.md +0 -20
  59. package/fp16/bench/alt-element.cc +0 -89
  60. package/fp16/bench/from-alt-array.cc +0 -175
  61. package/fp16/bench/from-ieee-array.cc +0 -399
  62. package/fp16/bench/ieee-element.cc +0 -272
  63. package/fp16/bench/to-alt-array.cc +0 -91
  64. package/fp16/bench/to-ieee-array.cc +0 -303
  65. package/fp16/cmake/DownloadGoogleBenchmark.cmake +0 -15
  66. package/fp16/cmake/DownloadGoogleTest.cmake +0 -15
  67. package/fp16/cmake/DownloadPSimd.cmake +0 -15
  68. package/fp16/configure.py +0 -65
  69. package/fp16/confu.yaml +0 -8
  70. package/fp16/include/fp16/__init__.py +0 -0
  71. package/fp16/include/fp16/avx.py +0 -50
  72. package/fp16/include/fp16/avx2.py +0 -53
  73. package/fp16/include/fp16/bitcasts.h +0 -92
  74. package/fp16/include/fp16/fp16.h +0 -451
  75. package/fp16/include/fp16/psimd.h +0 -131
  76. package/fp16/include/fp16.h +0 -11
  77. package/fp16/jni/Android.mk +0 -6
  78. package/fp16/jni/Application.mk +0 -4
  79. package/fp16/test/alt-from-fp32-value.cc +0 -498
  80. package/fp16/test/alt-to-fp32-bits.cc +0 -263
  81. package/fp16/test/alt-to-fp32-psimd.cc +0 -145
  82. package/fp16/test/alt-to-fp32-value.cc +0 -348
  83. package/fp16/test/alt-to-fp32x2-psimd.cc +0 -245
  84. package/fp16/test/bitcasts.cc +0 -56
  85. package/fp16/test/ieee-from-fp32-value.cc +0 -523
  86. package/fp16/test/ieee-to-fp32-bits.cc +0 -333
  87. package/fp16/test/ieee-to-fp32-psimd.cc +0 -312
  88. package/fp16/test/ieee-to-fp32-value.cc +0 -420
  89. package/fp16/test/ieee-to-fp32x2-psimd.cc +0 -541
  90. package/fp16/test/peachpy/alt-xmm-to-fp32-xmm-avx.cc +0 -96
  91. package/fp16/test/peachpy/alt-xmm-to-fp32-ymm-avx2.cc +0 -96
  92. package/fp16/test/peachpy/stubs.py +0 -38
  93. package/fp16/test/tables.cc +0 -530
  94. package/fp16/test/tables.h +0 -13
  95. package/fp16/third-party/THHalf.h +0 -92
  96. package/fp16/third-party/eigen-half.h +0 -170
  97. package/fp16/third-party/float16-compressor.h +0 -80
  98. package/fp16/third-party/half.hpp +0 -2916
  99. package/fp16/third-party/npy-halffloat.h +0 -328
  100. package/include/usearch/index.hpp +0 -3836
  101. package/include/usearch/index_dense.hpp +0 -2004
  102. package/include/usearch/index_plugins.hpp +0 -2213
  103. package/javascript/lib.cpp +0 -341
  104. package/javascript/types/usearch.d.ts.map +0 -1
  105. package/javascript/usearch.test.js +0 -74
  106. package/package-ci.json +0 -10
  107. package/pyproject.toml +0 -51
  108. package/setup.py +0 -201
  109. package/simsimd/.releaserc +0 -86
  110. package/simsimd/CONTRIBUTING.md +0 -88
  111. package/simsimd/Cargo.lock +0 -709
  112. package/simsimd/Cargo.toml +0 -44
  113. package/simsimd/LICENSE +0 -201
  114. package/simsimd/README.md +0 -317
  115. package/simsimd/binding.gyp +0 -36
  116. package/simsimd/build.rs +0 -20
  117. package/simsimd/include/simsimd/binary.h +0 -245
  118. package/simsimd/include/simsimd/complex.h +0 -174
  119. package/simsimd/include/simsimd/probability.h +0 -548
  120. package/simsimd/include/simsimd/simsimd.h +0 -465
  121. package/simsimd/include/simsimd/spatial.h +0 -1267
  122. package/simsimd/include/simsimd/types.h +0 -208
  123. package/simsimd/javascript/bench.js +0 -127
  124. package/simsimd/javascript/fallback.ts +0 -193
  125. package/simsimd/javascript/lib.c +0 -102
  126. package/simsimd/javascript/node-gyp-build.d.ts +0 -1
  127. package/simsimd/javascript/simsimd.ts +0 -124
  128. package/simsimd/javascript/test.mjs +0 -153
  129. package/simsimd/javascript/tsconfig-base.json +0 -25
  130. package/simsimd/javascript/tsconfig-cjs.json +0 -8
  131. package/simsimd/javascript/tsconfig-esm.json +0 -8
  132. package/simsimd/package-lock.json +0 -1688
  133. package/simsimd/package.json +0 -61
  134. package/simsimd/rust/benches/cosine.rs +0 -27
  135. package/simsimd/rust/benches/native.rs +0 -30
  136. package/simsimd/rust/benches/sqeuclidean.rs +0 -27
  137. package/simsimd/rust/lib.c +0 -45
  138. package/simsimd/rust/lib.rs +0 -550
  139. package/sqlite/README.md +0 -178
  140. package/sqlite/lib.cpp +0 -325
  141. package/stringzilla/CONTRIBUTING.md +0 -446
  142. package/stringzilla/Cargo.lock +0 -25
  143. package/stringzilla/Cargo.toml +0 -26
  144. package/stringzilla/LICENSE +0 -201
  145. package/stringzilla/README.md +0 -1422
  146. package/stringzilla/VERSION +0 -1
  147. package/stringzilla/assets/cover-strinzilla.jpeg +0 -0
  148. package/stringzilla/assets/meme-stringzilla-v2.jpeg +0 -0
  149. package/stringzilla/assets/meme-stringzilla-v3.jpeg +0 -0
  150. package/stringzilla/binding.gyp +0 -16
  151. package/stringzilla/build.rs +0 -87
  152. package/stringzilla/c/lib.c +0 -337
  153. package/stringzilla/cli/README.md +0 -65
  154. package/stringzilla/cli/__init__.py +0 -0
  155. package/stringzilla/cli/split.py +0 -105
  156. package/stringzilla/cli/wc.py +0 -117
  157. package/stringzilla/include/stringzilla/experimental.h +0 -845
  158. package/stringzilla/include/stringzilla/module.modulemap +0 -9
  159. package/stringzilla/include/stringzilla/stringzilla.h +0 -5370
  160. package/stringzilla/include/stringzilla/stringzilla.hpp +0 -3838
  161. package/stringzilla/javascript/lib.c +0 -125
  162. package/stringzilla/javascript/stringzilla.js +0 -22
  163. package/stringzilla/package-ci.json +0 -10
  164. package/stringzilla/package.json +0 -32
  165. package/stringzilla/rust/lib.rs +0 -873
  166. package/stringzilla/swift/StringProtocol+StringZilla.swift +0 -280
  167. package/stringzilla/swift/Test.swift +0 -60
  168. package/tsconfig.json +0 -21
  169. package/wasmer.toml +0 -12
  170. package/winlibinstaller.bat +0 -42
  171. /package/{simsimd/javascript/dist-package-cjs.json → javascript/dist/cjs/package.json} +0 -0
  172. /package/{simsimd/javascript/dist-package-esm.json → javascript/dist/esm/package.json} +0 -0
package/sqlite/README.md DELETED
@@ -1,178 +0,0 @@
1
- # USearch Extensions for SQLite
2
-
3
- USearch exposes the SIMD-accelerated distance functions for SQLite databases aimed to accelerate vector search and fuzzy string matching operations.
4
- This includes:
5
-
6
- - [x] Cosine and Euclidean distances for float vectors,
7
- - [x] Jaccard and Hamming distances for binary vectors,
8
- - [x] Levenshtein and Hamming distances for strings,
9
- - [x] Haversine distance for geographical coordinates.
10
-
11
- The SIMD-acceleration covers AVX2, most subsets of AVX512, ARM NEON, and Arm SVE instruction sets, more than most BLAS libraries.
12
- The implementations are coming from [SimSIMD](https://github.com/ashvardanian/simsimd) and [StringZilla](https://github.com/ashvardanian/stringzilla).
13
- They are most efficient when vectors are stored as BLOBs, but for broader compatibility can also handle JSONs, and even separate columns containing vector elements.
14
-
15
- ## Installation
16
-
17
- USearch currently ships the SQLite extensions as part of the Python wheels, so they can be obtained from PyPi.
18
-
19
- ```sh
20
- pip install usearch # brings sqlite extensions
21
- ```
22
-
23
- ## Quickstart
24
-
25
- To USearch extensions to your SQLite database, you can use the following commands:
26
-
27
- ```py
28
- import sqlite3
29
- import usearch
30
-
31
- conn = sqlite3.connect(":memory:")
32
- conn.enable_load_extension(True)
33
- conn.load_extension(usearch.sqlite)
34
- ```
35
-
36
- Afterwards, the following script should work fine.
37
-
38
- ```sql
39
- -- Create a table with a JSON column for vectors
40
- CREATE TABLE vectors_table (
41
- id SERIAL PRIMARY KEY,
42
- vector JSON NOT NULL
43
- );
44
-
45
- -- Insert two 3D vectors
46
- INSERT INTO vectors_table (id, vector)
47
- VALUES
48
- (42, '[1.0, 2.0, 3.0]'),
49
- (43, '[4.0, 5.0, 6.0]');
50
-
51
- -- Compute the distances to [7.0, 8.0, 9.0] using
52
- -- the `distance_cosine_f32` extension function
53
- SELECT
54
- id,
55
- distance_cosine_f32(vt.vector, '[7.0, 8.0, 9.0]') AS distance
56
- FROM vectors_table AS vt;
57
- ```
58
-
59
- ### Functionality
60
-
61
- #### Strings
62
-
63
- String functions accept BLOBs and TEXTs, and return the distance as an integer.
64
- This includes:
65
-
66
- - `distance_levenshtein_bytes`,
67
- - `distance_levenshtein_unicode`,
68
- - `distance_hamming_bytes`,
69
- - `distance_hamming_unicode`.
70
-
71
- The Levenshtein distance would report the number of insertions, deletions, and substitutions required to transform one string into another, while the Hamming distance would only report the substitutons.
72
- When applied to strings of different length, the Hamming distance would evalute the prefix of the longer string and will add the length difference to the result.
73
- The `_bytes` variants compute the distance in the number of bytes, while the `_unicode` variants compute the distance in the number of Unicode code points, assuming the inputs are UTF8 encoded.
74
- Take a look at the following three similar looking, but distinct strings:
75
-
76
- - `école` - 6 codepoints (runes), 7 bytes.
77
- - `école` - 5 codepoints (runes), 6 bytes.
78
- - `écolé` - 5 codepoints (runes), 7 bytes.
79
-
80
- Those have three different letter "e" variants, including etter "é" as a single character, etter as "e" and an accent "´", as well as the plain letter "e".
81
- This would result in different distances between those strings, depending on the chosen metric.
82
-
83
- ```sql
84
- CREATE TABLE strings_table (
85
- id SERIAL PRIMARY KEY,
86
- word TEXT NOT NULL
87
- );
88
-
89
- INSERT INTO strings_table (id, word)
90
- VALUES
91
- (42, 'école'),
92
- (43, 'école');
93
-
94
- SELECT
95
- st.id,
96
-
97
- distance_levenshtein_bytes(st.word, 'écolé') AS lb,
98
- distance_levenshtein_unicode(st.word, 'écolé') AS lu,
99
- distance_hamming_bytes(st.word, 'écolé') AS hb,
100
- distance_hamming_unicode(st.word, 'écolé') AS hu,
101
-
102
- distance_levenshtein_bytes(st.word, 'écolé', 2) AS lbb,
103
- distance_levenshtein_unicode(st.word, 'écolé', 2) AS lub,
104
- distance_hamming_bytes(st.word, 'écolé', 2) AS hbb,
105
- distance_hamming_unicode(st.word, 'écolé', 2) AS hub
106
-
107
- FROM strings_table AS st;
108
- ```
109
-
110
- The last 4 columns will contain the bounded versions of the distance, which are faster to compute, thanks to the early stopping condition.
111
- They might be handy, if you want to accelerate search for something like an "autocomplete" feature in a search bar.
112
- The output will look like this:
113
-
114
- | id | lb | lu | hb | hu | lbb | lub | hbb | hub |
115
- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
116
- | 42 | 5 | 3 | 7 | 6 | 2 | 2 | 2 | 2 |
117
- | 43 | 2 | 1 | 2 | 1 | 2 | 1 | 2 | 1 |
118
-
119
- #### Bit Vectors
120
-
121
- If using SQLite as a feature store, you might be dealing with single-bit vectors.
122
-
123
- - `distance_hamming_binary` - the Hamming distance, meaning number of bits that differ between two vectors,
124
- - `distance_jaccard_binary` - the Jaccard distance, meaning the number of bits that differ between two vectors divided by the number of bits that are set in at least one of the vectors.
125
-
126
- Here is an example:
127
-
128
- ```sql
129
- CREATE TABLE binary_vectors (
130
- id SERIAL PRIMARY KEY,
131
- vector BLOB NOT NULL
132
- );
133
- INSERT INTO binary_vectors (id, vector)
134
- VALUES
135
- (42, X'FFFFFF'), -- 111111111111111111111111 in binary
136
- (43, X'000000'); -- 000000000000000000000000 in binary
137
- SELECT bv.id,
138
- distance_hamming_binary(bv.vector, X'FFFF00') AS hamming_distance,
139
- distance_jaccard_binary(bv.vector, X'FFFF00') AS jaccard_distance
140
- FROM binary_vectors AS bv;
141
- ```
142
-
143
- #### Dense Vectors
144
-
145
- Distance functions for desnse vectors can be used on both BLOBs and JSONs.
146
- Every name is structured as `distance_<metric>_<type>`, where
147
-
148
- - `<metric>` is the name of the metric, like `cosine`, `inner`, `sqeuclidean`, and `divergence`,
149
- - `<type>` is the type of the vector elements, like `f64`, `f32`, `f16`, and `i8`.
150
-
151
- The `cosine` metric is the cosine similarity, the `inner` metric is the inner (dot) product, the `sqeuclidean` metric is the squared Euclidean distance, and the `divergence` metric is the Jensen-Shannon divergence - the symmetric variant of the Kullback-Leibler divergence.
152
-
153
- - `distance_sqeuclidean_f64`
154
- - `distance_cosine_f64`
155
- - `distance_inner_f64`
156
- - `distance_divergence_f64`
157
- - `distance_sqeuclidean_f32`
158
- - `distance_cosine_f32`
159
- - `distance_inner_f32`
160
- - `distance_divergence_f32`
161
- - `distance_sqeuclidean_f16`
162
- - `distance_cosine_f16`
163
- - `distance_inner_f16`
164
- - `distance_divergence_f16`
165
- - `distance_sqeuclidean_i8`
166
- - `distance_cosine_i8`
167
- - `distance_inner_i8`
168
- - `distance_divergence_i8`
169
-
170
- #### Geographical Coordinates
171
-
172
- When dealing with geographical or other spherical coordinates, you can use:
173
-
174
- - `distance_haversine_meters` - the Haversine distance on the sphere multiplied by the Earth's radius in meters.
175
-
176
- ---
177
-
178
- Feel free to contribute more examples to this file 🤗
package/sqlite/lib.cpp DELETED
@@ -1,325 +0,0 @@
1
- /**
2
- * @brief SQLite3 bindings for USearch.
3
- * @file lib.cpp
4
- * @author Ash Vardanian
5
- * @date November 28, 2023
6
- * @copyright Copyright (c) 2023
7
- */
8
-
9
- #include <charconv> // `std::from_chars`
10
- #include <cstdlib> // `std::strtod`
11
-
12
- #include <sqlite3ext.h>
13
- SQLITE_EXTENSION_INIT1
14
-
15
- #define SZ_DYNAMIC_DISPATCH 1
16
- #include <stringzilla/stringzilla.hpp>
17
-
18
- #include <usearch/index_dense.hpp>
19
- #include <usearch/index_plugins.hpp>
20
-
21
- using namespace unum::usearch;
22
- using namespace unum;
23
-
24
- namespace sz = ashvardanian::stringzilla;
25
-
26
- using metric_t = metric_punned_t;
27
- using distance_t = distance_punned_t;
28
-
29
- template <scalar_kind_t scalar_kind_ak> struct parsed_scalar_kind_gt {
30
- using type = f32_t;
31
- static constexpr scalar_kind_t kind = scalar_kind_t::f32_k;
32
- };
33
-
34
- template <> struct parsed_scalar_kind_gt<scalar_kind_t::f64_k> {
35
- using type = f64_t;
36
- static constexpr scalar_kind_t kind = scalar_kind_t::f64_k;
37
- };
38
-
39
- template <scalar_kind_t scalar_kind_ak, metric_kind_t metric_kind_ak>
40
- static void sqlite_dense(sqlite3_context* context, int argc, sqlite3_value** argv) {
41
-
42
- if (argc < 2) {
43
- sqlite3_result_error(context, "Distance function expects at least two arguments", -1);
44
- return;
45
- }
46
-
47
- int type1 = sqlite3_value_type(argv[0]);
48
- int type2 = sqlite3_value_type(argv[1]);
49
-
50
- // Our primary case is having two BLOBs containing dense vector representations.
51
- if (argc == 2 && type1 == SQLITE_BLOB && type2 == SQLITE_BLOB) {
52
- // The pointer obtained from `sqlite3_value_text` or `sqlite3_value_blob` can
53
- // be invalidated by the next call to `sqlite3_value_bytes` according to docs, so read it first.
54
- int bytes1 = sqlite3_value_bytes(argv[0]);
55
- int bytes2 = sqlite3_value_bytes(argv[1]);
56
- if (bytes1 != bytes2) {
57
- sqlite3_result_error(context, "Vectors have different number of dimensions", -1);
58
- return;
59
- }
60
- void const* vec1 = sqlite3_value_blob(argv[0]);
61
- void const* vec2 = sqlite3_value_blob(argv[1]);
62
-
63
- std::size_t dimensions = (size_t)(bytes1)*CHAR_BIT / bits_per_scalar(scalar_kind_ak);
64
- metric_t metric = metric_t(dimensions, metric_kind_ak, scalar_kind_ak);
65
- distance_punned_t distance =
66
- metric(reinterpret_cast<byte_t const*>(vec1), reinterpret_cast<byte_t const*>(vec2));
67
- sqlite3_result_double(context, distance);
68
- }
69
-
70
- // Worst case is to have JSON arrays or comma-separated values
71
- else if (argc == 2 && type1 == SQLITE_TEXT && type2 == SQLITE_TEXT) {
72
- // The pointer obtained from `sqlite3_value_text` or `sqlite3_value_blob` can
73
- // be invalidated by the next call to `sqlite3_value_bytes` according to docs, so read it first.
74
- size_t bytes1 = (size_t)sqlite3_value_bytes(argv[0]);
75
- size_t bytes2 = (size_t)sqlite3_value_bytes(argv[1]);
76
- char* vec1 = (char*)sqlite3_value_text(argv[0]);
77
- char* vec2 = (char*)sqlite3_value_text(argv[1]);
78
- size_t commas1 = sz::string_view(vec1, bytes1).find_all(",").size();
79
- size_t commas2 = sz::string_view(vec2, bytes2).find_all(",").size();
80
- if (commas1 != commas2) {
81
- sqlite3_result_error(context, "Vectors have different number of dimensions", -1);
82
- return;
83
- }
84
-
85
- // Valid JSON array of numbers would be packed into [] square brackets
86
- if (bytes1 && vec1[0] == '[')
87
- ++vec1, --bytes1;
88
- if (bytes2 && vec2[0] == '[')
89
- ++vec2, --bytes2;
90
- // We don't have to trim the end
91
- // if (bytes1 && vec1[bytes1 - 1] == ']')
92
- // --bytes1;
93
- // if (bytes2 && vec2[bytes2 - 1] == ']')
94
- // --bytes2;
95
-
96
- // Allocate vectors on stack and parse strings into them
97
- using scalar_t = typename parsed_scalar_kind_gt<scalar_kind_ak>::type;
98
- size_t dimensions = commas1 + 1;
99
- scalar_t parsed1[dimensions], parsed2[dimensions];
100
- for (size_t i = 0; i != dimensions; ++i) {
101
- // Skip whitespace
102
- while (bytes1 && vec1[0] == ' ')
103
- ++vec1, --bytes1;
104
- while (bytes2 && vec2[0] == ' ')
105
- ++vec2, --bytes2;
106
-
107
- // Parse the floating-point numbers
108
- // Sadly, most modern compilers don't support the `std::from_chars` yet
109
- #if __cpp_lib_to_chars
110
- std::from_chars_result result1 = std::from_chars(vec1, vec1 + bytes1, parsed1[i]);
111
- std::from_chars_result result2 = std::from_chars(vec2, vec2 + bytes2, parsed2[i]);
112
- if (result1.ec != std::errc() || result2.ec != std::errc()) {
113
- sqlite3_result_error(context, "Number can't be parsed", -1);
114
- return;
115
- }
116
- bytes1 -= result1.ptr - vec1;
117
- bytes2 -= result2.ptr - vec2;
118
- vec1 = (char*)result1.ptr;
119
- vec2 = (char*)result2.ptr;
120
- #else
121
- char* parsed1_end = vec1 + bytes1;
122
- parsed1[i] = std::strtod(vec1, &parsed1_end);
123
- char* parsed2_end = vec2 + bytes2;
124
- parsed2[i] = std::strtod(vec2, &parsed2_end);
125
- if (vec1 == parsed1_end || vec2 == parsed2_end) {
126
- sqlite3_result_error(context, "Number can't be parsed", -1);
127
- return;
128
- }
129
- bytes1 -= parsed1_end - vec1;
130
- bytes2 -= parsed2_end - vec2;
131
- vec1 = parsed1_end;
132
- vec2 = parsed2_end;
133
- #endif
134
-
135
- // Skip the whitespaces and commas
136
- while (bytes1 && (vec1[0] == ' ' || vec1[0] == ','))
137
- ++vec1, --bytes1;
138
- while (bytes2 && (vec2[0] == ' ' || vec2[0] == ','))
139
- ++vec2, --bytes2;
140
- }
141
-
142
- // Compute the distance itself
143
- metric_t metric = metric_t(dimensions, metric_kind_ak, parsed_scalar_kind_gt<scalar_kind_ak>::kind);
144
- distance_punned_t distance =
145
- metric(reinterpret_cast<byte_t const*>(parsed1), reinterpret_cast<byte_t const*>(parsed2));
146
- sqlite3_result_double(context, distance);
147
- }
148
-
149
- // Less efficient, yet still common case is to have many scalar columns
150
- else if (argc % 2 == 0) {
151
-
152
- // Allocate vectors on stack and parse floating-point values into them
153
- using scalar_t = typename parsed_scalar_kind_gt<scalar_kind_ak>::type;
154
- size_t dimensions = argc / 2;
155
- scalar_t parsed1[dimensions], parsed2[dimensions];
156
- for (size_t i = 0; i != dimensions; ++i) {
157
- switch (sqlite3_value_type(argv[i])) {
158
- case SQLITE_FLOAT: parsed1[i] = sqlite3_value_double(argv[i]); break;
159
- case SQLITE_INTEGER: parsed1[i] = sqlite3_value_int(argv[i]); break;
160
- case SQLITE_NULL: parsed1[i] = 0; break;
161
- default:
162
- sqlite3_result_error(context, "Scalar columns may only contain 32-bit integers, floats, or NULLs.", -1);
163
- return;
164
- }
165
- switch (sqlite3_value_type(argv[dimensions + i])) {
166
- case SQLITE_FLOAT: parsed2[i] = sqlite3_value_double(argv[dimensions + i]); break;
167
- case SQLITE_INTEGER: parsed2[i] = sqlite3_value_int(argv[dimensions + i]); break;
168
- case SQLITE_NULL: parsed2[i] = 0; break;
169
- default:
170
- sqlite3_result_error(context, "Scalar columns may only contain 32-bit integers, floats, or NULLs.", -1);
171
- return;
172
- }
173
- }
174
-
175
- // Compute the distance itself
176
- metric_t metric = metric_t(dimensions, metric_kind_ak, parsed_scalar_kind_gt<scalar_kind_ak>::kind);
177
- distance_punned_t distance =
178
- metric(reinterpret_cast<byte_t const*>(parsed1), reinterpret_cast<byte_t const*>(parsed2));
179
- sqlite3_result_double(context, distance);
180
- }
181
- // Unsupported arguments combination
182
- else {
183
- sqlite3_result_error(context, "Number of columns in two vectors must be divisible by two", -1);
184
- }
185
- }
186
-
187
- enum class string_metric_kind_t : std::uint8_t {
188
- levenshtein_bytes_k,
189
- levenshtein_unicode_k,
190
- hamming_bytes_k,
191
- hamming_unicode_k,
192
- };
193
-
194
- template <string_metric_kind_t string_metric_kind_ak>
195
- static void sqlite_strings(sqlite3_context* context, int argc, sqlite3_value** argv) {
196
-
197
- if (argc != 2 && argc != 3) {
198
- sqlite3_result_error(context, "Distance function expects 2 or 3 arguments", -1);
199
- return;
200
- }
201
-
202
- int type1 = sqlite3_value_type(argv[0]);
203
- int type2 = sqlite3_value_type(argv[1]);
204
- auto is_text_or_blob = [](int type) { return type == SQLITE_TEXT || type == SQLITE_BLOB; };
205
- if (!is_text_or_blob(type1) || !is_text_or_blob(type2)) {
206
- sqlite3_result_error(context, "Distance function expects text or blob arguments", -1);
207
- return;
208
- }
209
-
210
- sz_size_t bound = 0;
211
- if (argc == 3) {
212
- int type3 = sqlite3_value_type(argv[2]);
213
- if (type3 != SQLITE_INTEGER) {
214
- sqlite3_result_error(context, "Distance function expects integer as the third argument", -1);
215
- return;
216
- }
217
- std::int64_t signed_bound = sqlite3_value_int64(argv[2]);
218
- if (signed_bound < 0) {
219
- sqlite3_result_error(context, "Distance function expects non-negative integer as the third argument", -1);
220
- return;
221
- }
222
- bound = (sz_size_t)signed_bound;
223
- }
224
-
225
- // The pointer obtained from `sqlite3_value_text` or `sqlite3_value_blob` can
226
- // be invalidated by the next call to `sqlite3_value_bytes` according to docs, so read it first.
227
- sz_size_t bytes1 = (sz_size_t)sqlite3_value_bytes(argv[0]);
228
- sz_size_t bytes2 = (sz_size_t)sqlite3_value_bytes(argv[1]);
229
- auto extract_data_pointer = [](int type, sqlite3_value* value) -> sz_cptr_t {
230
- return type == SQLITE_BLOB ? (sz_cptr_t)sqlite3_value_blob(value) : (sz_cptr_t)sqlite3_value_text(value);
231
- };
232
- sz_cptr_t vec1 = extract_data_pointer(type1, argv[0]);
233
- sz_cptr_t vec2 = extract_data_pointer(type2, argv[1]);
234
- sz::string_view str1(vec1, bytes1);
235
- sz::string_view str2(vec2, bytes2);
236
-
237
- // Dispatch the right backend
238
- sz_size_t result = SZ_SIZE_MAX;
239
- switch (string_metric_kind_ak) {
240
- case string_metric_kind_t::levenshtein_bytes_k: result = sz::edit_distance(str1, str2, bound); break;
241
- case string_metric_kind_t::levenshtein_unicode_k: result = sz::edit_distance_utf8(str1, str2, bound); break;
242
- case string_metric_kind_t::hamming_bytes_k: result = sz::hamming_distance(str1, str2, bound); break;
243
- case string_metric_kind_t::hamming_unicode_k: result = sz::hamming_distance_utf8(str1, str2, bound); break;
244
- }
245
-
246
- // Check for errors
247
- if (result == SZ_SIZE_MAX) {
248
- sqlite3_result_error(context, "Distance function failed to compute the result", -1);
249
- return;
250
- }
251
-
252
- sqlite3_result_int64(context, (std::int64_t)result);
253
- }
254
-
255
- extern "C" SZ_DYNAMIC int sqlite3_compiled_init( //
256
- sqlite3* db, //
257
- char** error_message, //
258
- sqlite3_api_routines const* api) {
259
- SQLITE_EXTENSION_INIT2(api)
260
-
261
- int flags = SQLITE_UTF8 | SQLITE_DETERMINISTIC | SQLITE_INNOCUOUS;
262
- int num_params = -1; // Any number will be accepted
263
-
264
- // String similarity metrics
265
- sqlite3_create_function(db, "distance_levenshtein_bytes", num_params, flags, NULL,
266
- sqlite_strings<string_metric_kind_t::levenshtein_bytes_k>, NULL, NULL);
267
- sqlite3_create_function(db, "distance_levenshtein_unicode", num_params, flags, NULL,
268
- sqlite_strings<string_metric_kind_t::levenshtein_unicode_k>, NULL, NULL);
269
- sqlite3_create_function(db, "distance_hamming_bytes", num_params, flags, NULL,
270
- sqlite_strings<string_metric_kind_t::hamming_bytes_k>, NULL, NULL);
271
- sqlite3_create_function(db, "distance_hamming_unicode", num_params, flags, NULL,
272
- sqlite_strings<string_metric_kind_t::hamming_unicode_k>, NULL, NULL);
273
-
274
- // Bit-wise metrics
275
- sqlite3_create_function(db, "distance_hamming_binary", num_params, flags, NULL,
276
- sqlite_dense<scalar_kind_t::b1x8_k, metric_kind_t::hamming_k>, NULL, NULL);
277
- sqlite3_create_function(db, "distance_jaccard_binary", num_params, flags, NULL,
278
- sqlite_dense<scalar_kind_t::b1x8_k, metric_kind_t::jaccard_k>, NULL, NULL);
279
- sqlite3_create_function(db, "distance_haversine_meters", num_params, flags, NULL,
280
- sqlite_dense<scalar_kind_t::f64_k, metric_kind_t::haversine_k>, NULL, NULL);
281
-
282
- // Double-precision metrics
283
- sqlite3_create_function(db, "distance_sqeuclidean_f64", num_params, flags, NULL,
284
- sqlite_dense<scalar_kind_t::f64_k, metric_kind_t::l2sq_k>, NULL, NULL);
285
- sqlite3_create_function(db, "distance_cosine_f64", num_params, flags, NULL,
286
- sqlite_dense<scalar_kind_t::f64_k, metric_kind_t::cos_k>, NULL, NULL);
287
- sqlite3_create_function(db, "distance_inner_f64", num_params, flags, NULL,
288
- sqlite_dense<scalar_kind_t::f64_k, metric_kind_t::ip_k>, NULL, NULL);
289
- sqlite3_create_function(db, "distance_divergence_f64", num_params, flags, NULL,
290
- sqlite_dense<scalar_kind_t::f64_k, metric_kind_t::divergence_k>, NULL, NULL);
291
-
292
- // Single-precision metrics
293
- sqlite3_create_function(db, "distance_sqeuclidean_f32", num_params, flags, NULL,
294
- sqlite_dense<scalar_kind_t::f32_k, metric_kind_t::l2sq_k>, NULL, NULL);
295
- sqlite3_create_function(db, "distance_cosine_f32", num_params, flags, NULL,
296
- sqlite_dense<scalar_kind_t::f32_k, metric_kind_t::cos_k>, NULL, NULL);
297
- sqlite3_create_function(db, "distance_inner_f32", num_params, flags, NULL,
298
- sqlite_dense<scalar_kind_t::f32_k, metric_kind_t::ip_k>, NULL, NULL);
299
- sqlite3_create_function(db, "distance_divergence_f32", num_params, flags, NULL,
300
- sqlite_dense<scalar_kind_t::f32_k, metric_kind_t::divergence_k>, NULL, NULL);
301
-
302
- // Half-precision metrics
303
- sqlite3_create_function(db, "distance_sqeuclidean_f16", num_params, flags, NULL,
304
- sqlite_dense<scalar_kind_t::f16_k, metric_kind_t::l2sq_k>, NULL, NULL);
305
- sqlite3_create_function(db, "distance_cosine_f16", num_params, flags, NULL,
306
- sqlite_dense<scalar_kind_t::f16_k, metric_kind_t::cos_k>, NULL, NULL);
307
- sqlite3_create_function(db, "distance_inner_f16", num_params, flags, NULL,
308
- sqlite_dense<scalar_kind_t::f16_k, metric_kind_t::ip_k>, NULL, NULL);
309
- sqlite3_create_function(db, "distance_divergence_f16", num_params, flags, NULL,
310
- sqlite_dense<scalar_kind_t::f16_k, metric_kind_t::divergence_k>, NULL, NULL);
311
-
312
- // Integer metrics
313
- sqlite3_create_function(db, "distance_sqeuclidean_i8", num_params, flags, NULL,
314
- sqlite_dense<scalar_kind_t::i8_k, metric_kind_t::l2sq_k>, NULL, NULL);
315
- sqlite3_create_function(db, "distance_cosine_i8", num_params, flags, NULL,
316
- sqlite_dense<scalar_kind_t::i8_k, metric_kind_t::cos_k>, NULL, NULL);
317
- sqlite3_create_function(db, "distance_inner_i8", num_params, flags, NULL,
318
- sqlite_dense<scalar_kind_t::i8_k, metric_kind_t::ip_k>, NULL, NULL);
319
- sqlite3_create_function(db, "distance_divergence_i8", num_params, flags, NULL,
320
- sqlite_dense<scalar_kind_t::i8_k, metric_kind_t::divergence_k>, NULL, NULL);
321
-
322
- return SQLITE_OK;
323
- }
324
-
325
- #include "../stringzilla/c/lib.c"