usearch 1.1.1 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/prerelease.yml +3 -3
- package/.github/workflows/release.yml +3 -3
- package/.github/workflows/update_version.sh +1 -1
- package/.gitmodules +1 -4
- package/Cargo.toml +1 -1
- package/Package.swift +2 -1
- package/README.md +86 -42
- package/binding.gyp +16 -12
- package/build.gradle +1 -1
- package/build.rs +0 -1
- package/c/CMakeLists.txt +11 -7
- package/c/lib.cpp +36 -24
- package/c/usearch.h +42 -12
- package/cluster.ipynb +167 -0
- package/conanfile.py +1 -1
- package/cpp/CMakeLists.txt +8 -3
- package/cpp/bench.cpp +2 -2
- package/cpp/test.cpp +42 -25
- package/docs/benchmarks.md +8 -0
- package/docs/compilation.md +7 -7
- package/docs/conf.py +27 -24
- package/docs/javascript/reference.rst +3 -0
- package/golang/README.md +31 -7
- package/golang/lib.go +1 -1
- package/include/usearch/index.hpp +348 -69
- package/include/usearch/index_dense.hpp +587 -179
- package/include/usearch/index_plugins.hpp +194 -16
- package/java/cloud/unum/usearch/Index.java +13 -0
- package/java/cloud/unum/usearch/cloud_unum_usearch_Index.cpp +40 -9
- package/java/cloud/unum/usearch/cloud_unum_usearch_Index.h +16 -0
- package/javascript/docs.js +111 -0
- package/javascript/lib.cpp +29 -9
- package/javascript/usearch.d.ts +9 -1
- package/objc/USearchObjective.mm +11 -11
- package/package.json +2 -2
- package/pyproject.toml +1 -1
- package/python/lib.cpp +488 -158
- package/python/scripts/index_faiss.py +3 -3
- package/python/scripts/join.py +6 -6
- package/python/scripts/test_index.py +219 -0
- package/python/scripts/test_sparse.py +21 -21
- package/python/scripts/test_tooling.py +111 -0
- package/python/usearch/eval.py +9 -5
- package/python/usearch/index.py +491 -216
- package/rust/lib.cpp +6 -24
- package/rust/lib.hpp +1 -3
- package/rust/lib.rs +89 -0
- package/setup.py +5 -6
- package/wasm/CMakeLists.txt +0 -1
- package/wasmer.toml +1 -1
- package/python/scripts/test.py +0 -364
- package/robin-map/.codecov.yml +0 -5
- package/robin-map/.github/workflows/ci.yml +0 -136
- package/robin-map/CMakeLists.txt +0 -86
- package/robin-map/LICENSE +0 -21
- package/robin-map/README.md +0 -483
- package/robin-map/cmake/tsl-robin-mapConfig.cmake.in +0 -9
- package/robin-map/doxygen.conf +0 -2483
- package/robin-map/include/tsl/robin_growth_policy.h +0 -406
- package/robin-map/include/tsl/robin_hash.h +0 -1639
- package/robin-map/include/tsl/robin_map.h +0 -807
- package/robin-map/include/tsl/robin_set.h +0 -660
- package/robin-map/tests/CMakeLists.txt +0 -26
- package/robin-map/tests/custom_allocator_tests.cpp +0 -138
- package/robin-map/tests/main.cpp +0 -26
- package/robin-map/tests/policy_tests.cpp +0 -97
- package/robin-map/tests/robin_map_tests.cpp +0 -1451
- package/robin-map/tests/robin_set_tests.cpp +0 -164
- package/robin-map/tests/utils.h +0 -443
- package/robin-map/tsl-robin-map.natvis +0 -78
|
@@ -66,7 +66,7 @@ jobs:
|
|
|
66
66
|
- name: Build locally
|
|
67
67
|
run: python -m pip install .
|
|
68
68
|
- name: Test with PyTest
|
|
69
|
-
run: pytest python/scripts/
|
|
69
|
+
run: pytest python/scripts/
|
|
70
70
|
|
|
71
71
|
|
|
72
72
|
test_python_37:
|
|
@@ -95,7 +95,7 @@ jobs:
|
|
|
95
95
|
run: python -m pip install .
|
|
96
96
|
|
|
97
97
|
- name: Test with PyTest
|
|
98
|
-
run: pytest python/scripts/
|
|
98
|
+
run: pytest python/scripts/
|
|
99
99
|
|
|
100
100
|
|
|
101
101
|
test_javascript:
|
|
@@ -160,7 +160,7 @@ jobs:
|
|
|
160
160
|
sudo apt install -y nodejs
|
|
161
161
|
git clone https://github.com/emscripten-core/emsdk.git
|
|
162
162
|
./emsdk/emsdk install latest
|
|
163
|
-
- name: Build USearch
|
|
163
|
+
- name: Build USearch using Emscripten
|
|
164
164
|
run: |
|
|
165
165
|
./emsdk/emsdk activate latest && source ./emsdk/emsdk_env.sh
|
|
166
166
|
emcmake cmake -DUSEARCH_BUILD_BENCHMARK=0 -DUSEARCH_BUILD_WASM=1 -B ./build -DCMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS} -s TOTAL_MEMORY=64MB" && emmake make -C ./build
|
|
@@ -111,13 +111,13 @@ jobs:
|
|
|
111
111
|
- uses: actions/setup-python@v3
|
|
112
112
|
|
|
113
113
|
- name: Setup Docker
|
|
114
|
-
if: matrix.os
|
|
114
|
+
if: matrix.os == 'ubuntu-20.04'
|
|
115
115
|
uses: crazy-max/ghaction-setup-docker@v1.0.0
|
|
116
116
|
with:
|
|
117
117
|
version: 23.0.1
|
|
118
118
|
|
|
119
119
|
- name: Setup QEMU
|
|
120
|
-
if: matrix.os
|
|
120
|
+
if: matrix.os == 'ubuntu-20.04'
|
|
121
121
|
uses: docker/setup-qemu-action@v2.1.0
|
|
122
122
|
|
|
123
123
|
- name: Install CIBuildWheel
|
|
@@ -319,7 +319,7 @@ jobs:
|
|
|
319
319
|
run: |
|
|
320
320
|
sudo apt update &&
|
|
321
321
|
sudo apt install -y doxygen graphviz dia git &&
|
|
322
|
-
pip install sphinx sphinx-js breathe furo m2r2 sphinxcontrib-googleanalytics==0.2.dev20220708 sphinxcontrib-jquery &&
|
|
322
|
+
pip install sphinx==7.1.2 sphinx-js breathe furo m2r2 sphinxcontrib-googleanalytics==0.2.dev20220708 sphinxcontrib-jquery &&
|
|
323
323
|
npm install -g jsdoc
|
|
324
324
|
- name: Install USearch from PyPi
|
|
325
325
|
run: pip install usearch
|
|
@@ -9,5 +9,5 @@ echo $1 > VERSION &&
|
|
|
9
9
|
sed -i "s/^\(#define USEARCH_VERSION_MINOR \).*/\1$(echo "$1" | cut -d. -f2)/" ./include/usearch/index.hpp &&
|
|
10
10
|
sed -i "s/^\(#define USEARCH_VERSION_PATCH \).*/\1$(echo "$1" | cut -d. -f3)/" ./include/usearch/index.hpp &&
|
|
11
11
|
sed -i "s/<version>[0-9]\+\.[0-9]\+\.[0-9]\+/<version>$1/" README.md &&
|
|
12
|
-
sed -i "s/version = {0
|
|
12
|
+
sed -i "s/version = {[0-9]\+\.[0-9]\+\.[0-9]\+}/version = {$1}/" README.md &&
|
|
13
13
|
sed -i "s/version=\".*\"/version=\"$1\"/" wasmer.toml
|
package/.gitmodules
CHANGED
package/Cargo.toml
CHANGED
package/Package.swift
CHANGED
|
@@ -19,7 +19,6 @@ let package = Package(
|
|
|
19
19
|
cxxSettings: [
|
|
20
20
|
.headerSearchPath("../include/"),
|
|
21
21
|
.headerSearchPath("../fp16/include/"),
|
|
22
|
-
.headerSearchPath("../robin-map/include/"),
|
|
23
22
|
.headerSearchPath("../simismd/include/")
|
|
24
23
|
]
|
|
25
24
|
),
|
|
@@ -27,12 +26,14 @@ let package = Package(
|
|
|
27
26
|
name: "USearch",
|
|
28
27
|
dependencies: ["USearchObjective"],
|
|
29
28
|
path: "swift",
|
|
29
|
+
exclude: ["README.md", "Test.swift"],
|
|
30
30
|
sources: ["USearch.swift", "Index+Sugar.swift"]
|
|
31
31
|
),
|
|
32
32
|
.testTarget(
|
|
33
33
|
name: "USearchTests",
|
|
34
34
|
dependencies: ["USearch"],
|
|
35
35
|
path: "swift",
|
|
36
|
+
exclude: ["USearch.swift", "Index+Sugar.swift", "README.md"],
|
|
36
37
|
sources: ["Test.swift"]
|
|
37
38
|
)
|
|
38
39
|
],
|
package/README.md
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
<h1 align="center">USearch</h1>
|
|
2
2
|
<h3 align="center">
|
|
3
|
-
|
|
4
|
-
|
|
3
|
+
Faster & Smaller Single-File<br/>
|
|
4
|
+
Search Engine for Vectors & Texts<br/>
|
|
5
5
|
</h3>
|
|
6
6
|
<br/>
|
|
7
7
|
|
|
@@ -18,7 +18,7 @@ Vector Search Engine<br/>
|
|
|
18
18
|
</p>
|
|
19
19
|
|
|
20
20
|
<p align="center">
|
|
21
|
-
Euclidean • Angular •
|
|
21
|
+
Euclidean • Angular • Bitwise • Haversine • User-Defined Metrics
|
|
22
22
|
<br/>
|
|
23
23
|
<a href="https://unum-cloud.github.io/usearch/cpp">C++ 11</a> •
|
|
24
24
|
<a href="https://unum-cloud.github.io/usearch/python">Python 3</a> •
|
|
@@ -31,9 +31,16 @@ Euclidean • Angular • Jaccard • Hamming • Haversine • User-Defined Met
|
|
|
31
31
|
<a href="https://unum-cloud.github.io/usearch/golang">GoLang</a> •
|
|
32
32
|
<a href="https://unum-cloud.github.io/usearch/wolfram">Wolfram</a>
|
|
33
33
|
<br/>
|
|
34
|
-
Linux • MacOS • Windows • Docker • WebAssembly
|
|
34
|
+
Linux • MacOS • Windows • iOS • Docker • WebAssembly
|
|
35
35
|
</p>
|
|
36
36
|
|
|
37
|
+
<div align="center">
|
|
38
|
+
<a href="https://pypi.org/project/usearch/"> <img alt="PyPI - Downloads" src="https://img.shields.io/pypi/dm/usearch?label=pypi%20downloads"> </a>
|
|
39
|
+
<a href="https://www.npmjs.com/package/usearch"> <img alt="npm" src="https://img.shields.io/npm/dy/usearch?label=npm%20dowloads"> </a>
|
|
40
|
+
<a href="https://crates.io/crates/usearch"> <img alt="Crates.io" src="https://img.shields.io/crates/d/usearch?label=crate%20downloads"> </a>
|
|
41
|
+
<img alt="GitHub code size in bytes" src="https://img.shields.io/github/languages/code-size/unum-cloud/usearch">
|
|
42
|
+
</div>
|
|
43
|
+
|
|
37
44
|
---
|
|
38
45
|
|
|
39
46
|
- ✅ Benchmark-topping performance.
|
|
@@ -42,10 +49,11 @@ Linux • MacOS • Windows • Docker • WebAssembly
|
|
|
42
49
|
- ✅ Variable dimensionality vectors for unique applications, including search over compressed data.
|
|
43
50
|
- ✅ Bitwise Tanimoto and Sorensen coefficients for [Genomics and Chemistry applications](#usearch--rdkit--molecular-search).
|
|
44
51
|
- ✅ Hardware-agnostic `f16` & `i8` - [half-precision & quarter-precision support](#memory-efficiency-downcasting-and-quantization).
|
|
45
|
-
- ✅ [View large indexes from disk](#
|
|
52
|
+
- ✅ [View large indexes from disk](#serving-index-from-disk) without loading into RAM.
|
|
46
53
|
- ✅ Space-efficient point-clouds with `uint40_t`, accommodating 4B+ size.
|
|
47
54
|
- ✅ Compatible with OpenMP and custom "executors", for fine-grained control over CPU utilization.
|
|
48
55
|
- ✅ Heterogeneous lookups, renaming/relabeling, and on-the-fly deletions.
|
|
56
|
+
- ✅ Near-real-time [clustering and sub-clusterings](#clustering) for Tens or Millions of clusters.
|
|
49
57
|
- ✅ [Semantic Search](#usearch--ai--multi-modal-semantic-search) and [Joins](#joins).
|
|
50
58
|
|
|
51
59
|
[usearch-header]: https://github.com/unum-cloud/usearch/blob/main/include/usearch/index.hpp
|
|
@@ -57,14 +65,15 @@ FAISS is a widely recognized standard for high-performance vector search engines
|
|
|
57
65
|
USearch and FAISS both employ the same HNSW algorithm, but they differ significantly in their design principles.
|
|
58
66
|
USearch is compact and broadly compatible without sacrificing performance, with a primary focus on user-defined metrics and fewer dependencies.
|
|
59
67
|
|
|
60
|
-
|
|
|
61
|
-
|
|
|
62
|
-
| Implementation
|
|
63
|
-
| Supported metrics
|
|
64
|
-
| Supported
|
|
65
|
-
|
|
|
66
|
-
|
|
|
67
|
-
|
|
|
68
|
+
| | FAISS | USearch |
|
|
69
|
+
| :------------------ | :---------------------------- | :--------------------------------- |
|
|
70
|
+
| Implementation | 84 K [SLOC][sloc] in `faiss/` | 3 K [SLOC][sloc] in `usearch/` |
|
|
71
|
+
| Supported metrics | 9 fixed metrics | Any User-Defined metrics |
|
|
72
|
+
| Supported languages | C++, Python | 10 languages |
|
|
73
|
+
| Supported ID types | `uint32_t`, `uint64_t` | `uint32_t`, `uint40_t`, `uint64_t` |
|
|
74
|
+
| Dependencies | BLAS, OpenMP | None |
|
|
75
|
+
| Bindings | SWIG | Native |
|
|
76
|
+
| Acceleration | Learned Quantization | Downcasting |
|
|
68
77
|
|
|
69
78
|
[sloc]: https://en.wikipedia.org/wiki/Source_lines_of_code
|
|
70
79
|
|
|
@@ -96,6 +105,25 @@ assert matches[0].distance <= 0.001
|
|
|
96
105
|
assert np.allclose(index[42], vector)
|
|
97
106
|
```
|
|
98
107
|
|
|
108
|
+
Comparing the performance of FAISS against USearch on 1 Million 96-dimensional vectors from the famous Deep1B dataset, once can expect the following numbers on modern AWS `c7g.metal` instances.
|
|
109
|
+
|
|
110
|
+
| | FAISS, `f32` | USearch, `f32` | USearch, `f16` | USearch, `i8` |
|
|
111
|
+
| :----------- | -----------: | -------------: | -------------: | ----------------: |
|
|
112
|
+
| Batch Insert | 16 K/s | 73 K/s | 100 K/s | 104 K/s **+550%** |
|
|
113
|
+
| Batch Search | 82 K/s | 103 K/s | 113 K/s | 134 K/s **+63%** |
|
|
114
|
+
| Bulk Insert | 76 K/s | 105 K/s | 115 K/s | 202 K/s **+165%** |
|
|
115
|
+
| Bulk Search | 118 K/s | 174 K/s | 173 K/s | 304 K/s **+157%** |
|
|
116
|
+
| Recall @ 10 | 99% | 99.2% | 99.1% | 99.2% |
|
|
117
|
+
|
|
118
|
+
> HNSW was configured with identical hyper-parameters:
|
|
119
|
+
> connectivity `M=16`,
|
|
120
|
+
> expansion @ construction `efConstruction=128`,
|
|
121
|
+
> and expansion @ search `ef=64`.
|
|
122
|
+
> Batch size is 256.
|
|
123
|
+
> Jump to the [Performance Tuning][benchmarking] section to read about the effects of those hyper-parameters.
|
|
124
|
+
|
|
125
|
+
[benchmarking]: https://github.com/unum-cloud/usearch/blob/main/docs/benchmarks.md
|
|
126
|
+
|
|
99
127
|
## User-Defined Functions
|
|
100
128
|
|
|
101
129
|
While most vector search packages concentrate on just a couple of metrics - "Inner Product distance" and "Euclidean distance," USearch extends this list to include any user-defined metrics.
|
|
@@ -124,27 +152,7 @@ Instead, we have focused on high-precision arithmetic over low-precision downcas
|
|
|
124
152
|
The same index, and `add` and `search` operations will automatically down-cast or up-cast between `f32_t`, `f16_t`, `f64_t`, and `i8_t` representations, even if the hardware doesn't natively support it.
|
|
125
153
|
Continuing the topic of memory efficiency, we provide a `uint40_t` to allow collection with over 4B+ vectors without allocating 8 bytes for every neighbor reference in the proximity graph.
|
|
126
154
|
|
|
127
|
-
|
|
128
|
-
| :----------- | -----------: | -------------: | -------------: | ----------------: |
|
|
129
|
-
| Batch Insert | 16 K/s | 73 K/s | 100 K/s | 104 K/s **+550%** |
|
|
130
|
-
| Batch Search | 82 K/s | 103 K/s | 113 K/s | 134 K/s **+63%** |
|
|
131
|
-
| Bulk Insert | 76 K/s | 105 K/s | 115 K/s | 202 K/s **+165%** |
|
|
132
|
-
| Bulk Search | 118 K/s | 174 K/s | 173 K/s | 304 K/s **+157%** |
|
|
133
|
-
| Recall @ 10 | 99% | 99.2% | 99.1% | 99.2% |
|
|
134
|
-
|
|
135
|
-
> Dataset: 1M vectors sample of the Deep1B dataset.
|
|
136
|
-
> Hardware: `c7g.metal` AWS instance with 64 cores and DDR5 memory.
|
|
137
|
-
> HNSW was configured with identical hyper-parameters:
|
|
138
|
-
> connectivity `M=16`,
|
|
139
|
-
> expansion @ construction `efConstruction=128`,
|
|
140
|
-
> and expansion @ search `ef=64`.
|
|
141
|
-
> Batch size is 256.
|
|
142
|
-
> Both libraries were compiled for the target architecture.
|
|
143
|
-
> Jump to the [Performance Tuning][benchmarking] section to read about the effects of those hyper-parameters.
|
|
144
|
-
|
|
145
|
-
[benchmarking]: https://github.com/unum-cloud/usearch/blob/main/docs/benchmarks.md
|
|
146
|
-
|
|
147
|
-
## Disk-based Indexes
|
|
155
|
+
## Serving `Index` from Disk
|
|
148
156
|
|
|
149
157
|
With USearch, you can serve indexes from external memory, enabling you to optimize your server choices for indexing speed and serving costs.
|
|
150
158
|
This can result in **20x cost reduction** on AWS and other public clouds.
|
|
@@ -159,7 +167,7 @@ other_view = Index(ndim=..., metric=CompiledMetric(...))
|
|
|
159
167
|
other_view.view("index.usearch")
|
|
160
168
|
```
|
|
161
169
|
|
|
162
|
-
## Exact
|
|
170
|
+
## Exact vs. Approximate Search
|
|
163
171
|
|
|
164
172
|
Approximate search methods, such as HNSW, are predominantly used when an exact brute-force search becomes too resource-intensive.
|
|
165
173
|
This typically occurs when you have millions of entries in a collection.
|
|
@@ -183,6 +191,8 @@ When compared to FAISS's `IndexFlatL2` in Google Colab, **[USearch may offer up
|
|
|
183
191
|
- `faiss.IndexFlatL2`: **55.3 ms**.
|
|
184
192
|
- `usearch.index.search`: **2.54 ms**.
|
|
185
193
|
|
|
194
|
+
## `Indexes` for Multi-Index Lookups
|
|
195
|
+
|
|
186
196
|
For larger workloads targeting billions or even trillions of vectors, parallel multi-index lookups become invaluable.
|
|
187
197
|
These lookups prevent the need to construct a single, massive index, allowing users to query multiple smaller ones instead.
|
|
188
198
|
|
|
@@ -198,6 +208,36 @@ multi_index = Indexes(
|
|
|
198
208
|
multi_index.search(...)
|
|
199
209
|
```
|
|
200
210
|
|
|
211
|
+
## Clustering
|
|
212
|
+
|
|
213
|
+
Once the index is constructed, it can be used to cluster entries much faster.
|
|
214
|
+
In essense, the `Index` itself can be seen as a clustering, and it allows iterative deepening.
|
|
215
|
+
|
|
216
|
+
```py
|
|
217
|
+
clustering = index.cluster(
|
|
218
|
+
min_count=10, # Optional
|
|
219
|
+
max_count=15, # Optional
|
|
220
|
+
threads=..., # Optional
|
|
221
|
+
)
|
|
222
|
+
|
|
223
|
+
# Get the clusters and their sizes
|
|
224
|
+
centroid_keys, sizes = clustering.centroids_popularity
|
|
225
|
+
|
|
226
|
+
# Use Matplotlib draw a histogram
|
|
227
|
+
clustering.plot_centroids_popularity()
|
|
228
|
+
|
|
229
|
+
# Export a NetworkX graph of the clusters
|
|
230
|
+
g = clustering.network
|
|
231
|
+
|
|
232
|
+
# Get members of a specific cluster
|
|
233
|
+
first_members = clustering.members_of(centroid_keys[0])
|
|
234
|
+
|
|
235
|
+
# Deepen into that cluster spliting it into more parts, all same arguments supported
|
|
236
|
+
sub_clustering = clustering.subcluster(min_count=..., max_count=...)
|
|
237
|
+
```
|
|
238
|
+
|
|
239
|
+
Using Scikit-Learn, on a 1 Million point dataset, one may expect queries to take anywhere from minutes to hours, depending on the number of clusters you want to highlight. For 50'000 clusters the performance difference between USearch and conventional clustering methods may easily reach 100x.
|
|
240
|
+
|
|
201
241
|
## Joins, One-to-One, One-to-Many, and Many-to-Many Mappings
|
|
202
242
|
|
|
203
243
|
One of the big questions these days is how will AI change the world of databases and data management.
|
|
@@ -229,7 +269,7 @@ Broader functionality is ported per request.
|
|
|
229
269
|
| Add, search | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
|
|
230
270
|
| Save, load, view | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
|
|
231
271
|
| User-defined metrics | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ |
|
|
232
|
-
| Joins
|
|
272
|
+
| Joins | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
|
|
233
273
|
| Variable-length vectors | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
|
|
234
274
|
| 4B+ capacities | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
|
|
235
275
|
|
|
@@ -316,13 +356,17 @@ matches = index.search(fingerprints, 10)
|
|
|
316
356
|
[smiles]: https://en.wikipedia.org/wiki/Simplified_molecular-input_line-entry_system
|
|
317
357
|
[rdkit-fingerprints]: https://www.rdkit.org/docs/RDKit_Book.html#additional-information-about-the-fingerprints
|
|
318
358
|
|
|
359
|
+
### USearch + POI Coordinates = GIS Applications... on iOS?
|
|
360
|
+
|
|
361
|
+
With Objective-C and iOS bindings, USearch can be easily used in mobile applications
|
|
362
|
+
|
|
319
363
|
|
|
320
364
|
## Integrations
|
|
321
365
|
|
|
322
|
-
- [x]
|
|
323
|
-
- [x] LangChain.
|
|
324
|
-
- [
|
|
325
|
-
- [ ] Microsoft Semantic Kernel.
|
|
366
|
+
- [x] GPTCache: [Python](https://github.com/zilliztech/GPTCache/releases/tag/0.1.29).
|
|
367
|
+
- [x] LangChain: [Python](https://github.com/langchain-ai/langchain/releases/tag/v0.0.257) and [JavaScipt](https://github.com/hwchase17/langchainjs/releases/tag/0.0.125).
|
|
368
|
+
- [x] ClickHouse: [C++](https://github.com/ClickHouse/ClickHouse/pull/53447).
|
|
369
|
+
- [ ] Microsoft Semantic Kernel: [Python](https://github.com/microsoft/semantic-kernel/pull/2358) and C#.
|
|
326
370
|
|
|
327
371
|
## Citations
|
|
328
372
|
|
|
@@ -332,8 +376,8 @@ doi = {10.5281/zenodo.7949416},
|
|
|
332
376
|
author = {Vardanian, Ash},
|
|
333
377
|
title = {{USearch by Unum Cloud}},
|
|
334
378
|
url = {https://github.com/unum-cloud/usearch},
|
|
335
|
-
version = {0.
|
|
336
|
-
year = {2022}
|
|
379
|
+
version = {1.0.0},
|
|
380
|
+
year = {2022},
|
|
337
381
|
month = jun,
|
|
338
382
|
}
|
|
339
383
|
```
|
package/binding.gyp
CHANGED
|
@@ -2,32 +2,36 @@
|
|
|
2
2
|
"targets": [
|
|
3
3
|
{
|
|
4
4
|
"target_name": "usearch",
|
|
5
|
-
"sources": [
|
|
6
|
-
"javascript/lib.cpp"
|
|
7
|
-
],
|
|
5
|
+
"sources": ["javascript/lib.cpp"],
|
|
8
6
|
"include_dirs": [
|
|
9
7
|
"<!@(node -p \"require('node-addon-api').include\")",
|
|
10
8
|
"include",
|
|
11
9
|
"fp16/include",
|
|
12
|
-
"
|
|
13
|
-
|
|
10
|
+
"simsimd/include",
|
|
11
|
+
],
|
|
12
|
+
"dependencies": ["<!(node -p \"require('node-addon-api').gyp\")"],
|
|
13
|
+
"cflags": [
|
|
14
|
+
"-fexceptions",
|
|
15
|
+
"-Wno-unknown-pragmas",
|
|
16
|
+
"-Wno-maybe-uninitialized",
|
|
14
17
|
],
|
|
15
|
-
"
|
|
16
|
-
"
|
|
18
|
+
"cflags_cc": [
|
|
19
|
+
"-fexceptions",
|
|
20
|
+
"-Wno-unknown-pragmas",
|
|
21
|
+
"-Wno-maybe-uninitialized",
|
|
22
|
+
"-std=c++11",
|
|
17
23
|
],
|
|
18
|
-
"cflags": ["-fexceptions", "-Wno-unknown-pragmas", "-Wno-maybe-uninitialized"],
|
|
19
|
-
"cflags_cc": ["-fexceptions", "-Wno-unknown-pragmas", "-Wno-maybe-uninitialized", "-std=c++11"],
|
|
20
24
|
"xcode_settings": {
|
|
21
25
|
"GCC_ENABLE_CPP_EXCEPTIONS": "YES",
|
|
22
26
|
"CLANG_CXX_LIBRARY": "libc++",
|
|
23
|
-
"MACOSX_DEPLOYMENT_TARGET": "10.15"
|
|
27
|
+
"MACOSX_DEPLOYMENT_TARGET": "10.15",
|
|
24
28
|
},
|
|
25
29
|
"msvs_settings": {
|
|
26
30
|
"VCCLCompilerTool": {
|
|
27
31
|
"ExceptionHandling": 1,
|
|
28
|
-
"AdditionalOptions": ["-std:c++11"]
|
|
32
|
+
"AdditionalOptions": ["-std:c++11"],
|
|
29
33
|
}
|
|
30
|
-
}
|
|
34
|
+
},
|
|
31
35
|
}
|
|
32
36
|
]
|
|
33
37
|
}
|
package/build.gradle
CHANGED
|
@@ -45,7 +45,7 @@ model {
|
|
|
45
45
|
include "**/*.cpp"
|
|
46
46
|
}
|
|
47
47
|
exportedHeaders {
|
|
48
|
-
srcDirs "include", "fp16/include", "
|
|
48
|
+
srcDirs "include", "fp16/include", "simsimd/include", "${Jvm.current().javaHome}/include"
|
|
49
49
|
}
|
|
50
50
|
}
|
|
51
51
|
}
|
package/build.rs
CHANGED
package/c/CMakeLists.txt
CHANGED
|
@@ -3,7 +3,6 @@
|
|
|
3
3
|
set(USEARCH_PUNNED_INCLUDE_DIRS
|
|
4
4
|
"${CMAKE_CURRENT_SOURCE_DIR}/../include"
|
|
5
5
|
"${CMAKE_CURRENT_SOURCE_DIR}/../fp16/include"
|
|
6
|
-
"${CMAKE_CURRENT_SOURCE_DIR}/../robin-map/include"
|
|
7
6
|
"${CMAKE_CURRENT_SOURCE_DIR}/../simsimd/include"
|
|
8
7
|
"${CMAKE_CURRENT_SOURCE_DIR}/"
|
|
9
8
|
)
|
|
@@ -11,14 +10,20 @@ set(USEARCH_PUNNED_INCLUDE_DIRS
|
|
|
11
10
|
# This article discusses a better way to allow building either static or shared libraries
|
|
12
11
|
# https://alexreinking.com/blog/building-a-dual-shared-and-static-library-with-cmake.html
|
|
13
12
|
if (${USEARCH_BUILD_STATIC})
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
13
|
+
if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
|
|
14
|
+
set(CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS} -static-libstdc++")
|
|
15
|
+
elseif(CMAKE_CXX_COMPILER_ID MATCHES "Clang")
|
|
16
|
+
set(CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS} -static")
|
|
17
|
+
endif()
|
|
18
|
+
|
|
19
|
+
add_library(usearch_c STATIC "${CMAKE_CURRENT_SOURCE_DIR}/lib.cpp")
|
|
20
|
+
else()
|
|
21
|
+
add_library(usearch_c SHARED "${CMAKE_CURRENT_SOURCE_DIR}/lib.cpp")
|
|
18
22
|
endif()
|
|
19
23
|
|
|
20
24
|
set_target_properties(usearch_c PROPERTIES POSITION_INDEPENDENT_CODE ON)
|
|
21
25
|
set_target_properties(usearch_c PROPERTIES CXX_STANDARD 11)
|
|
26
|
+
set_target_properties(usearch_c PROPERTIES C_STANDARD 99)
|
|
22
27
|
|
|
23
28
|
target_include_directories(usearch_c PRIVATE ${USEARCH_PUNNED_INCLUDE_DIRS})
|
|
24
29
|
set_target_properties(usearch_c PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
|
|
@@ -41,8 +46,7 @@ if (${USEARCH_LOOKUP_LABEL})
|
|
|
41
46
|
endif()
|
|
42
47
|
|
|
43
48
|
if(${USEARCH_BUILD_TEST})
|
|
44
|
-
add_executable(test_c test.c
|
|
49
|
+
add_executable(test_c "${CMAKE_CURRENT_SOURCE_DIR}/test.c")
|
|
45
50
|
target_link_libraries(test_c usearch_c)
|
|
46
51
|
set_target_properties(test_c PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
|
|
47
|
-
|
|
48
52
|
endif()
|
package/c/lib.cpp
CHANGED
|
@@ -2,10 +2,6 @@
|
|
|
2
2
|
|
|
3
3
|
#include <usearch/index_dense.hpp>
|
|
4
4
|
|
|
5
|
-
#ifndef USEARCH_EXPORT
|
|
6
|
-
#define USEARCH_EXPORT
|
|
7
|
-
#endif
|
|
8
|
-
|
|
9
5
|
extern "C" {
|
|
10
6
|
#include "usearch.h"
|
|
11
7
|
}
|
|
@@ -58,13 +54,13 @@ add_result_t add_(index_dense_t* index, usearch_key_t key, void const* vector, s
|
|
|
58
54
|
}
|
|
59
55
|
}
|
|
60
56
|
|
|
61
|
-
bool get_(index_dense_t* index, usearch_key_t key, void* vector, scalar_kind_t kind) {
|
|
57
|
+
bool get_(index_dense_t* index, usearch_key_t key, size_t count, void* vector, scalar_kind_t kind) {
|
|
62
58
|
switch (kind) {
|
|
63
|
-
case scalar_kind_t::f32_k: return index->get(key, (f32_t*)vector);
|
|
64
|
-
case scalar_kind_t::f64_k: return index->get(key, (f64_t*)vector);
|
|
65
|
-
case scalar_kind_t::f16_k: return index->get(key, (f16_t*)vector);
|
|
66
|
-
case scalar_kind_t::i8_k: return index->get(key, (i8_bits_t*)vector);
|
|
67
|
-
case scalar_kind_t::b1x8_k: return index->get(key, (b1x8_t*)vector);
|
|
59
|
+
case scalar_kind_t::f32_k: return index->get(key, (f32_t*)vector, count);
|
|
60
|
+
case scalar_kind_t::f64_k: return index->get(key, (f64_t*)vector, count);
|
|
61
|
+
case scalar_kind_t::f16_k: return index->get(key, (f16_t*)vector, count);
|
|
62
|
+
case scalar_kind_t::i8_k: return index->get(key, (i8_bits_t*)vector, count);
|
|
63
|
+
case scalar_kind_t::b1x8_k: return index->get(key, (b1x8_t*)vector, count);
|
|
68
64
|
default: return search_result_t().failed("Unknown scalar kind!");
|
|
69
65
|
}
|
|
70
66
|
}
|
|
@@ -87,6 +83,7 @@ USEARCH_EXPORT usearch_index_t usearch_init(usearch_init_options_t* options, use
|
|
|
87
83
|
assert(options && error);
|
|
88
84
|
|
|
89
85
|
index_dense_config_t config(options->connectivity, options->expansion_add, options->expansion_search);
|
|
86
|
+
config.multi = options->multi;
|
|
90
87
|
metric_kind_t metric_kind = to_native_metric(options->metric_kind);
|
|
91
88
|
scalar_kind_t scalar_kind = to_native_scalar(options->quantization);
|
|
92
89
|
|
|
@@ -111,7 +108,7 @@ USEARCH_EXPORT void usearch_save(usearch_index_t index, char const* path, usearc
|
|
|
111
108
|
assert(index && path && error);
|
|
112
109
|
serialization_result_t result = reinterpret_cast<index_dense_t*>(index)->save(path);
|
|
113
110
|
if (!result)
|
|
114
|
-
*error = result.error.
|
|
111
|
+
*error = result.error.release();
|
|
115
112
|
}
|
|
116
113
|
|
|
117
114
|
USEARCH_EXPORT void usearch_load(usearch_index_t index, char const* path, usearch_error_t* error) {
|
|
@@ -119,7 +116,7 @@ USEARCH_EXPORT void usearch_load(usearch_index_t index, char const* path, usearc
|
|
|
119
116
|
assert(index && path && error);
|
|
120
117
|
serialization_result_t result = reinterpret_cast<index_dense_t*>(index)->load(path);
|
|
121
118
|
if (!result)
|
|
122
|
-
*error = result.error.
|
|
119
|
+
*error = result.error.release();
|
|
123
120
|
}
|
|
124
121
|
|
|
125
122
|
USEARCH_EXPORT void usearch_view(usearch_index_t index, char const* path, usearch_error_t* error) {
|
|
@@ -127,7 +124,7 @@ USEARCH_EXPORT void usearch_view(usearch_index_t index, char const* path, usearc
|
|
|
127
124
|
assert(index && path && error);
|
|
128
125
|
serialization_result_t result = reinterpret_cast<index_dense_t*>(index)->view(path);
|
|
129
126
|
if (!result)
|
|
130
|
-
*error = result.error.
|
|
127
|
+
*error = result.error.release();
|
|
131
128
|
}
|
|
132
129
|
|
|
133
130
|
USEARCH_EXPORT size_t usearch_size(usearch_index_t index, usearch_error_t*) { //
|
|
@@ -159,7 +156,7 @@ USEARCH_EXPORT void usearch_add(
|
|
|
159
156
|
assert(index && vector && error);
|
|
160
157
|
add_result_t result = add_(reinterpret_cast<index_dense_t*>(index), key, vector, to_native_scalar(kind));
|
|
161
158
|
if (!result)
|
|
162
|
-
*error = result.error.
|
|
159
|
+
*error = result.error.release();
|
|
163
160
|
}
|
|
164
161
|
|
|
165
162
|
USEARCH_EXPORT bool usearch_contains(usearch_index_t index, usearch_key_t key, usearch_error_t*) {
|
|
@@ -167,35 +164,50 @@ USEARCH_EXPORT bool usearch_contains(usearch_index_t index, usearch_key_t key, u
|
|
|
167
164
|
return reinterpret_cast<index_dense_t*>(index)->contains(key);
|
|
168
165
|
}
|
|
169
166
|
|
|
167
|
+
USEARCH_EXPORT size_t usearch_count(usearch_index_t index, usearch_key_t key, usearch_error_t*) {
|
|
168
|
+
assert(index);
|
|
169
|
+
return reinterpret_cast<index_dense_t*>(index)->count(key);
|
|
170
|
+
}
|
|
171
|
+
|
|
170
172
|
USEARCH_EXPORT size_t usearch_search( //
|
|
171
173
|
usearch_index_t index, void const* vector, usearch_scalar_kind_t kind, size_t results_limit, //
|
|
172
|
-
usearch_key_t*
|
|
174
|
+
usearch_key_t* found_keys, usearch_distance_t* found_distances, usearch_error_t* error) {
|
|
173
175
|
|
|
174
176
|
assert(index && vector && error);
|
|
175
177
|
search_result_t result =
|
|
176
178
|
search_(reinterpret_cast<index_dense_t*>(index), vector, to_native_scalar(kind), results_limit);
|
|
177
179
|
if (!result) {
|
|
178
|
-
*error = result.error.
|
|
180
|
+
*error = result.error.release();
|
|
179
181
|
return 0;
|
|
180
182
|
}
|
|
181
183
|
|
|
182
|
-
return result.dump_to(
|
|
184
|
+
return result.dump_to(found_keys, found_distances);
|
|
183
185
|
}
|
|
184
186
|
|
|
185
|
-
USEARCH_EXPORT
|
|
186
|
-
usearch_index_t index, usearch_key_t key, //
|
|
187
|
-
void*
|
|
187
|
+
USEARCH_EXPORT size_t usearch_get( //
|
|
188
|
+
usearch_index_t index, usearch_key_t key, size_t count, //
|
|
189
|
+
void* vectors, usearch_scalar_kind_t kind, usearch_error_t*) {
|
|
188
190
|
|
|
189
|
-
assert(index &&
|
|
190
|
-
return get_(reinterpret_cast<index_dense_t*>(index), key,
|
|
191
|
+
assert(index && vectors);
|
|
192
|
+
return get_(reinterpret_cast<index_dense_t*>(index), key, count, vectors, to_native_scalar(kind));
|
|
191
193
|
}
|
|
192
194
|
|
|
193
|
-
USEARCH_EXPORT
|
|
195
|
+
USEARCH_EXPORT size_t usearch_remove(usearch_index_t index, usearch_key_t key, usearch_error_t* error) {
|
|
194
196
|
|
|
195
197
|
assert(index && error);
|
|
196
198
|
labeling_result_t result = reinterpret_cast<index_dense_t*>(index)->remove(key);
|
|
197
199
|
if (!result)
|
|
198
|
-
*error = result.error.
|
|
200
|
+
*error = result.error.release();
|
|
201
|
+
return result.completed;
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
USEARCH_EXPORT size_t usearch_rename(usearch_index_t index, usearch_key_t from, usearch_key_t to,
|
|
205
|
+
usearch_error_t* error) {
|
|
206
|
+
|
|
207
|
+
assert(index && error);
|
|
208
|
+
labeling_result_t result = reinterpret_cast<index_dense_t*>(index)->rename(from, to);
|
|
209
|
+
if (!result)
|
|
210
|
+
*error = result.error.release();
|
|
199
211
|
return result.completed;
|
|
200
212
|
}
|
|
201
213
|
}
|