usearch 2.1.0 → 2.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/Dockerfile.libs +10 -10
- package/.github/workflows/prerelease.yml +27 -0
- package/.github/workflows/release.yml +185 -63
- package/Cargo.toml +1 -1
- package/README.md +1 -1
- package/conanfile.py +1 -1
- package/csharp/nuget/nuget-package.props +2 -2
- package/docs/format.md +63 -0
- package/docs/format.rst +5 -0
- package/docs/index.rst +1 -0
- package/golang/README.md +23 -1
- package/include/usearch/index.hpp +17 -15
- package/package.json +1 -1
- package/python/usearch/index.py +9 -6
- package/setup.py +1 -0
- package/wasmer.toml +1 -1
- package/winlibinstaller.bat +42 -0
- package/csharp/nuget/NUGET.md +0 -11
|
@@ -7,7 +7,7 @@ ARG docker_ip
|
|
|
7
7
|
ARG user_pass
|
|
8
8
|
|
|
9
9
|
RUN apt-get update && \
|
|
10
|
-
apt-get install -y --no-install-recommends
|
|
10
|
+
apt-get install -y --no-install-recommends cmake g++-12 build-essential libjemalloc-dev golang openssh-client sshpass && \
|
|
11
11
|
apt-get clean && \
|
|
12
12
|
rm -rf /var/lib/apt/lists/*
|
|
13
13
|
|
|
@@ -23,12 +23,12 @@ RUN if [ "$TARGETPLATFORM" = "linux/amd64" ]; then \
|
|
|
23
23
|
export arch="arm"; \
|
|
24
24
|
fi && \
|
|
25
25
|
version=$(cat VERSION) && \
|
|
26
|
-
mkdir -p
|
|
27
|
-
touch
|
|
28
|
-
mkdir -p
|
|
29
|
-
mkdir
|
|
30
|
-
cp c/usearch.h
|
|
31
|
-
cp golang/libusearch.a
|
|
32
|
-
echo "Package: usearch\nVersion: $version\nMaintainer: Ashot Vardanian\nArchitecture: "$arch"\nDescription: Faster & Smaller Single-File Search Engine for Vectors & Texts" >
|
|
33
|
-
dpkg-deb --build
|
|
34
|
-
sshpass -p "$user_pass" scp -o StrictHostKeyChecking=no
|
|
26
|
+
mkdir -p usearch_linux_"$arch"_"$version"/DEBIAN && \
|
|
27
|
+
touch usearch_linux_"$arch"_"$version"/DEBIAN/control && \
|
|
28
|
+
mkdir -p usearch_linux_"$arch"_"$version"/usr/local/lib && \
|
|
29
|
+
mkdir usearch_linux_"$arch"_"$version"/usr/local/include && \
|
|
30
|
+
cp c/usearch.h usearch_linux_"$arch"_"$version"/usr/local/include/ && \
|
|
31
|
+
cp golang/libusearch.a usearch_linux_"$arch"_"$version"/usr/local/lib/ && \
|
|
32
|
+
echo "Package: usearch\nVersion: $version\nMaintainer: Ashot Vardanian\nArchitecture: "$arch"\nDescription: Faster & Smaller Single-File Search Engine for Vectors & Texts" > usearch_linux_"$arch"_"$version"/DEBIAN/control && \
|
|
33
|
+
dpkg-deb --build usearch_linux_"$arch"_"$version" && \
|
|
34
|
+
sshpass -p "$user_pass" scp -o StrictHostKeyChecking=no usearch_linux_"$arch"_"$version".deb "$user_name"@"$docker_ip":/home/$user_name/work/usearch/usearch/
|
|
@@ -219,6 +219,33 @@ jobs:
|
|
|
219
219
|
shell: bash
|
|
220
220
|
working-directory: ${{ github.workspace }}/csharp/
|
|
221
221
|
|
|
222
|
+
test_golang:
|
|
223
|
+
name: Test Golang
|
|
224
|
+
runs-on: ubuntu-latest
|
|
225
|
+
steps:
|
|
226
|
+
- uses: actions/checkout@v3
|
|
227
|
+
with:
|
|
228
|
+
persist-credentials: false
|
|
229
|
+
ref: 'main-dev'
|
|
230
|
+
- run: git submodule update --init --recursive
|
|
231
|
+
|
|
232
|
+
- name: Prepare environment
|
|
233
|
+
run: |
|
|
234
|
+
sudo apt update &&
|
|
235
|
+
sudo apt install -y cmake g++-12 build-essential libjemalloc-dev
|
|
236
|
+
|
|
237
|
+
- name: Build library
|
|
238
|
+
run: |
|
|
239
|
+
cd golang && make -C ../c libusearch_c.so
|
|
240
|
+
sudo mv ../c/libusearch_c.so /usr/local/lib/libusearch.a
|
|
241
|
+
sudo mv ../c/usearch.h /usr/local/include/usearch.h
|
|
242
|
+
|
|
243
|
+
- name: Run test
|
|
244
|
+
run: |
|
|
245
|
+
cd golang
|
|
246
|
+
go test -v
|
|
247
|
+
|
|
248
|
+
|
|
222
249
|
build_docker:
|
|
223
250
|
name: Test Docker Image
|
|
224
251
|
runs-on: ubuntu-latest
|
|
@@ -14,6 +14,7 @@ permissions:
|
|
|
14
14
|
contents: write
|
|
15
15
|
pages: write
|
|
16
16
|
id-token: write
|
|
17
|
+
packages: write
|
|
17
18
|
|
|
18
19
|
jobs:
|
|
19
20
|
|
|
@@ -29,7 +30,7 @@ jobs:
|
|
|
29
30
|
|
|
30
31
|
|
|
31
32
|
rebase:
|
|
32
|
-
name: Rebase
|
|
33
|
+
name: Rebase Dev. Branch
|
|
33
34
|
needs: versioning
|
|
34
35
|
runs-on: ubuntu-latest
|
|
35
36
|
steps:
|
|
@@ -40,66 +41,23 @@ jobs:
|
|
|
40
41
|
|
|
41
42
|
- name: Perform rebase
|
|
42
43
|
run: |
|
|
44
|
+
git fetch origin main
|
|
43
45
|
git checkout main-dev
|
|
44
|
-
git rebase main
|
|
46
|
+
git rebase origin/main
|
|
45
47
|
|
|
46
48
|
- name: Push changes
|
|
47
49
|
uses: CasperWA/push-protected@v2
|
|
48
50
|
with:
|
|
49
|
-
token: ${{ secrets.
|
|
51
|
+
token: ${{ secrets.SEMANTIC_RELEASE_TOKEN }}
|
|
50
52
|
branch: main-dev
|
|
51
53
|
unprotect_reviews: True
|
|
52
54
|
force: True
|
|
53
55
|
|
|
54
56
|
|
|
55
|
-
|
|
56
|
-
name:
|
|
57
|
+
create_linux_deb_package:
|
|
58
|
+
name: Create Debian Package for ${{ matrix.arch }}
|
|
57
59
|
runs-on: ubuntu-latest
|
|
58
60
|
needs: versioning
|
|
59
|
-
steps:
|
|
60
|
-
- name: Get last release ID
|
|
61
|
-
id: last_release
|
|
62
|
-
run: |
|
|
63
|
-
response=$(curl -s -H "Authorization: Bearer ${{ secrets.SEMANTIC_RELEASE_TOKEN }}" \
|
|
64
|
-
"https://api.github.com/repos/${{ github.repository }}/releases/latest")
|
|
65
|
-
echo "last_release_id=$(echo "$response" | jq -r '.id')" >> $GITHUB_OUTPUT
|
|
66
|
-
|
|
67
|
-
- uses: robinraju/release-downloader@v1.8
|
|
68
|
-
name: Download release assets
|
|
69
|
-
with:
|
|
70
|
-
latest: true
|
|
71
|
-
fileName: "*"
|
|
72
|
-
out-file-path: "downloads"
|
|
73
|
-
tarBall: true
|
|
74
|
-
zipBall: true
|
|
75
|
-
|
|
76
|
-
- name: Calculate source hashes
|
|
77
|
-
id: hashes
|
|
78
|
-
run: |
|
|
79
|
-
source_zip_path=$(find $GITHUB_WORKSPACE/downloads -name "*.zip")
|
|
80
|
-
zip_hash=$(sha256sum "$source_zip_path" | awk '{print $1}')
|
|
81
|
-
source_tar_path=$(find $GITHUB_WORKSPACE/downloads -name "*.tar.gz" ! -name "docs.tar.gz")
|
|
82
|
-
tar_hash=$(sha256sum "$source_tar_path" | awk '{print $1}')
|
|
83
|
-
echo "zip_hash=$zip_hash" >> $GITHUB_OUTPUT
|
|
84
|
-
echo "tar_hash=$tar_hash" >> $GITHUB_OUTPUT
|
|
85
|
-
|
|
86
|
-
- name: Upload hashes to release
|
|
87
|
-
uses: irongut/EditRelease@v1.2.0
|
|
88
|
-
with:
|
|
89
|
-
token: ${{ secrets.SEMANTIC_RELEASE_TOKEN }}
|
|
90
|
-
id: ${{ steps.last_release.outputs.last_release_id }}
|
|
91
|
-
prerelease: false
|
|
92
|
-
replacebody: false
|
|
93
|
-
body: |
|
|
94
|
-
## Hashes
|
|
95
|
-
* Source code (zip) : `${{ steps.hashes.outputs.zip_hash }}`
|
|
96
|
-
* Source code (tar.gz) : `${{ steps.hashes.outputs.tar_hash }}`
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
create_deb_packages_linux:
|
|
100
|
-
name: Create deb package for Linux on ${{ matrix.arch }}
|
|
101
|
-
needs: versioning
|
|
102
|
-
runs-on: ubuntu-latest
|
|
103
61
|
strategy:
|
|
104
62
|
matrix:
|
|
105
63
|
arch: ["linux/amd64", "linux/arm64"]
|
|
@@ -110,26 +68,26 @@ jobs:
|
|
|
110
68
|
ref: 'main'
|
|
111
69
|
- run: git submodule update --init --recursive
|
|
112
70
|
|
|
113
|
-
- name:
|
|
71
|
+
- name: Setup QEMU
|
|
114
72
|
uses: docker/setup-qemu-action@v2
|
|
115
73
|
|
|
116
|
-
- name:
|
|
74
|
+
- name: Setup Docker
|
|
117
75
|
uses: docker/setup-buildx-action@v2
|
|
118
76
|
|
|
119
|
-
- name: Get
|
|
77
|
+
- name: Get Docker daemon IP
|
|
120
78
|
id: dockerip
|
|
121
79
|
run: |
|
|
122
80
|
docker_ip=$(ip addr show docker0 | grep -Po 'inet \K[\d.]+') && \
|
|
123
81
|
echo "docker_ip=$docker_ip" >> "$GITHUB_OUTPUT"
|
|
124
|
-
|
|
125
|
-
- name: Set
|
|
82
|
+
|
|
83
|
+
- name: Set runner user password
|
|
126
84
|
id: userpass
|
|
127
85
|
run: |
|
|
128
86
|
user_pass="1122"
|
|
129
87
|
echo "runner:$user_pass" | sudo chpasswd
|
|
130
88
|
echo "user_pass=$user_pass" >> "$GITHUB_OUTPUT"
|
|
131
89
|
|
|
132
|
-
- name: Build
|
|
90
|
+
- name: Build library
|
|
133
91
|
run: |
|
|
134
92
|
docker buildx create --use
|
|
135
93
|
docker buildx build \
|
|
@@ -139,11 +97,10 @@ jobs:
|
|
|
139
97
|
--file .github/workflows/Dockerfile.libs \
|
|
140
98
|
--tag unum/ustore-deps:latest \
|
|
141
99
|
--load .
|
|
142
|
-
|
|
143
100
|
- name: Clear Docker buildx cache
|
|
144
101
|
run: docker buildx prune --all --force
|
|
145
102
|
|
|
146
|
-
- name: Upload archive
|
|
103
|
+
- name: Upload archive
|
|
147
104
|
uses: xresloader/upload-to-github-release@v1
|
|
148
105
|
env:
|
|
149
106
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
@@ -152,6 +109,165 @@ jobs:
|
|
|
152
109
|
update_latest_release: true
|
|
153
110
|
|
|
154
111
|
|
|
112
|
+
create_windows_dll_library:
|
|
113
|
+
name: Create Dll Library for Windows ${{ matrix.arch }}
|
|
114
|
+
runs-on: windows-latest
|
|
115
|
+
needs: versioning
|
|
116
|
+
strategy:
|
|
117
|
+
matrix:
|
|
118
|
+
arch: [x64, x86]
|
|
119
|
+
steps:
|
|
120
|
+
- uses: actions/checkout@v3
|
|
121
|
+
with:
|
|
122
|
+
persist-credentials: false
|
|
123
|
+
ref: 'main'
|
|
124
|
+
- run: git submodule update --init --recursive
|
|
125
|
+
|
|
126
|
+
- name: Setup Cmake
|
|
127
|
+
uses: jwlawson/actions-setup-cmake@v1.13
|
|
128
|
+
|
|
129
|
+
- uses: actions/setup-go@v4
|
|
130
|
+
with:
|
|
131
|
+
go-version: '^1.19.0'
|
|
132
|
+
|
|
133
|
+
- name: Setup Mingw
|
|
134
|
+
uses: egor-tensin/setup-mingw@v2
|
|
135
|
+
with:
|
|
136
|
+
platform: ${{ matrix.arch }}
|
|
137
|
+
|
|
138
|
+
- name: Get version
|
|
139
|
+
id: set_version
|
|
140
|
+
shell: bash
|
|
141
|
+
run: |
|
|
142
|
+
version=$(cat VERSION)
|
|
143
|
+
echo "version=$version" >> $GITHUB_OUTPUT
|
|
144
|
+
|
|
145
|
+
- name: Build library
|
|
146
|
+
run: |
|
|
147
|
+
cd c/ && g++ -std=c++11 -o libusearch.dll -O3 lib.cpp -I. -I ../include/ -I ../fp16/include/ -shared && cd ..
|
|
148
|
+
cp c/libusearch.dll golang/libusearch.dll && cp c/usearch.h golang/usearch.h
|
|
149
|
+
|
|
150
|
+
- name: Golang test
|
|
151
|
+
run: |
|
|
152
|
+
cd golang/ && go test -v
|
|
153
|
+
if: ${{ matrix.arch == 'x64' }}
|
|
154
|
+
|
|
155
|
+
- name: Compress library
|
|
156
|
+
run: |
|
|
157
|
+
cd golang/
|
|
158
|
+
tar -cvf usearch_windows_${{ matrix.arch }}_${{ steps.set_version.outputs.version }}.tar .\libusearch.dll .\usearch.h
|
|
159
|
+
|
|
160
|
+
- name: Upload archive
|
|
161
|
+
uses: xresloader/upload-to-github-release@v1
|
|
162
|
+
env:
|
|
163
|
+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
164
|
+
with:
|
|
165
|
+
file: "golang/usearch_windows_${{ matrix.arch }}_${{ steps.set_version.outputs.version }}.tar"
|
|
166
|
+
update_latest_release: true
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
create_macos_library:
|
|
170
|
+
name: Create Library for MacOS ${{ matrix.arch }}
|
|
171
|
+
runs-on: macOS-latest
|
|
172
|
+
needs: versioning
|
|
173
|
+
strategy:
|
|
174
|
+
matrix:
|
|
175
|
+
arch: [arm64, x86_64]
|
|
176
|
+
steps:
|
|
177
|
+
- uses: actions/checkout@v3
|
|
178
|
+
with:
|
|
179
|
+
persist-credentials: false
|
|
180
|
+
ref: 'main'
|
|
181
|
+
- run: git submodule update --init --recursive
|
|
182
|
+
|
|
183
|
+
- uses: actions/setup-go@v4
|
|
184
|
+
with:
|
|
185
|
+
go-version: '^1.21.0'
|
|
186
|
+
|
|
187
|
+
- name: Get version
|
|
188
|
+
id: version
|
|
189
|
+
run: |
|
|
190
|
+
version=$(cat VERSION)
|
|
191
|
+
echo "version=$version" >> $GITHUB_OUTPUT
|
|
192
|
+
|
|
193
|
+
- name: Build library
|
|
194
|
+
run: |
|
|
195
|
+
cd c/ && clang++ -std=c++11 -o libusearch.so -O3 lib.cpp -I. -I ../include/ -I ../fp16/include/ -shared -target ${{ matrix.arch }}-apple-macos11 && cd ..
|
|
196
|
+
cp c/usearch.h golang/
|
|
197
|
+
cp c/libusearch.so golang/libusearch.so
|
|
198
|
+
|
|
199
|
+
- name: Golang test
|
|
200
|
+
run: |
|
|
201
|
+
cd golang/
|
|
202
|
+
go test -v
|
|
203
|
+
if: ${{ matrix.arch == 'x86_64' }}
|
|
204
|
+
|
|
205
|
+
- name: Compress library
|
|
206
|
+
run: |
|
|
207
|
+
cd golang/
|
|
208
|
+
zip -r usearch_macOS_${{ matrix.arch }}_${{ steps.version.outputs.version }}.zip libusearch.so usearch.h
|
|
209
|
+
|
|
210
|
+
- name: Upload archive
|
|
211
|
+
uses: xresloader/upload-to-github-release@v1
|
|
212
|
+
env:
|
|
213
|
+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
214
|
+
with:
|
|
215
|
+
file: "golang/*.zip"
|
|
216
|
+
update_latest_release: true
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
publish_hashes:
|
|
220
|
+
name: Add hashes to release notes
|
|
221
|
+
runs-on: ubuntu-latest
|
|
222
|
+
needs: [versioning, create_linux_deb_package, create_windows_dll_library, create_macos_library]
|
|
223
|
+
steps:
|
|
224
|
+
- name: Get last release ID
|
|
225
|
+
id: last_release
|
|
226
|
+
run: |
|
|
227
|
+
response=$(curl -s -H "Authorization: Bearer ${{ secrets.SEMANTIC_RELEASE_TOKEN }}" \
|
|
228
|
+
"https://api.github.com/repos/${{ github.repository }}/releases/latest")
|
|
229
|
+
echo "last_release_id=$(echo "$response" | jq -r '.id')" >> $GITHUB_OUTPUT
|
|
230
|
+
|
|
231
|
+
- uses: robinraju/release-downloader@v1.8
|
|
232
|
+
name: Download release assets
|
|
233
|
+
with:
|
|
234
|
+
latest: true
|
|
235
|
+
fileName: "*"
|
|
236
|
+
out-file-path: "downloads"
|
|
237
|
+
tarBall: true
|
|
238
|
+
zipBall: true
|
|
239
|
+
|
|
240
|
+
- name: Calculate source hashes
|
|
241
|
+
id: hashes
|
|
242
|
+
run: |
|
|
243
|
+
i=1
|
|
244
|
+
cd $GITHUB_WORKSPACE/downloads
|
|
245
|
+
for asset_name in $(ls); do
|
|
246
|
+
asset_hash=$(sha256sum $asset_name | awk '{print $1}')
|
|
247
|
+
echo "asset_name_$i=$asset_name" >> $GITHUB_OUTPUT
|
|
248
|
+
echo "asset_hash_$i=$asset_hash" >> $GITHUB_OUTPUT
|
|
249
|
+
i=$((i+1))
|
|
250
|
+
done
|
|
251
|
+
|
|
252
|
+
- name: Upload hashes to release
|
|
253
|
+
uses: irongut/EditRelease@v1.2.0
|
|
254
|
+
with:
|
|
255
|
+
token: ${{ secrets.SEMANTIC_RELEASE_TOKEN }}
|
|
256
|
+
id: ${{ steps.last_release.outputs.last_release_id }}
|
|
257
|
+
prerelease: false
|
|
258
|
+
replacebody: false
|
|
259
|
+
body: |
|
|
260
|
+
## Hashes
|
|
261
|
+
* ${{ steps.hashes.outputs.asset_name_1 }} : `${{ steps.hashes.outputs.asset_hash_1 }}`
|
|
262
|
+
* ${{ steps.hashes.outputs.asset_name_2 }} : `${{ steps.hashes.outputs.asset_hash_2 }}`
|
|
263
|
+
* ${{ steps.hashes.outputs.asset_name_3 }} : `${{ steps.hashes.outputs.asset_hash_3 }}`
|
|
264
|
+
* ${{ steps.hashes.outputs.asset_name_4 }} : `${{ steps.hashes.outputs.asset_hash_4 }}`
|
|
265
|
+
* ${{ steps.hashes.outputs.asset_name_5 }} : `${{ steps.hashes.outputs.asset_hash_5 }}`
|
|
266
|
+
* ${{ steps.hashes.outputs.asset_name_6 }} : `${{ steps.hashes.outputs.asset_hash_6 }}`
|
|
267
|
+
* ${{ steps.hashes.outputs.asset_name_7 }} : `${{ steps.hashes.outputs.asset_hash_7 }}`
|
|
268
|
+
* ${{ steps.hashes.outputs.asset_name_8 }} : `${{ steps.hashes.outputs.asset_hash_8 }}`
|
|
269
|
+
|
|
270
|
+
|
|
155
271
|
build_wheels:
|
|
156
272
|
name: Build Wheels for ${{ matrix.os }}
|
|
157
273
|
needs: versioning
|
|
@@ -375,11 +491,12 @@ jobs:
|
|
|
375
491
|
runs-on: ubuntu-latest
|
|
376
492
|
needs: [build_csharp]
|
|
377
493
|
env:
|
|
494
|
+
SOLUTION: ${{ github.workspace }}/csharp
|
|
378
495
|
USEARCH_LIBS: ${{ github.workspace }}/csharp/lib
|
|
379
|
-
|
|
496
|
+
NUGET_PACKAGES: ${{ github.workspace }}/csharp/packages
|
|
380
497
|
# NUGET_SOURCE: https://apiint.nugettest.org/v3/index.json
|
|
381
498
|
NUGET_SOURCE: https://api.nuget.org/v3/index.json
|
|
382
|
-
|
|
499
|
+
GITHUB_PACKAGES_SOURCE: https://nuget.pkg.github.com/${{ github.repository_owner }}/index.json
|
|
383
500
|
steps:
|
|
384
501
|
- uses: actions/checkout@v3
|
|
385
502
|
- run: git submodule update --init --recursive
|
|
@@ -398,12 +515,17 @@ jobs:
|
|
|
398
515
|
- name: Pack project
|
|
399
516
|
run: |
|
|
400
517
|
VERSION_CONTENT=$(< VERSION)
|
|
401
|
-
dotnet pack "${{ env.
|
|
518
|
+
dotnet pack "${{ env.SOLUTION }}" -c Release -p:Version="$VERSION_CONTENT" --output ${{ env.NUGET_PACKAGES }}
|
|
402
519
|
working-directory: ${{ github.workspace }}
|
|
403
520
|
|
|
404
|
-
- name: Publish NuGet
|
|
405
|
-
run:
|
|
406
|
-
|
|
521
|
+
- name: Publish to NuGet
|
|
522
|
+
run: dotnet nuget push "${{ env.NUGET_PACKAGES }}/*.nupkg" --api-key "${{ secrets.NUGET_API }}" --source "${{ env.NUGET_SOURCE }}" --skip-duplicate
|
|
523
|
+
|
|
524
|
+
- name: Prepare publish to GitHub Registry
|
|
525
|
+
run: dotnet nuget add source --username ${{ github.repository_owner }} --password ${{ secrets.GITHUB_TOKEN }} --store-password-in-clear-text --name github "${{ env.GITHUB_PACKAGES_SOURCE }}"
|
|
526
|
+
|
|
527
|
+
- name: Publish to GitHub Registry
|
|
528
|
+
run: dotnet nuget push "${{ env.NUGET_PACKAGES }}/*.nupkg" --api-key "${{ secrets.GITHUB_TOKEN }}" --source "github" --skip-duplicate
|
|
407
529
|
|
|
408
530
|
build_docs:
|
|
409
531
|
name: Build Docs
|
package/Cargo.toml
CHANGED
package/README.md
CHANGED
|
@@ -83,7 +83,7 @@ USearch is compact and broadly compatible without sacrificing performance, with
|
|
|
83
83
|
Base functionality is identical to FAISS, and the interface must be familiar if you have ever investigated Approximate Nearest Neighbors search:
|
|
84
84
|
|
|
85
85
|
```py
|
|
86
|
-
$ pip install usearch
|
|
86
|
+
$ pip install usearch
|
|
87
87
|
|
|
88
88
|
import numpy as np
|
|
89
89
|
from usearch.index import Index
|
package/conanfile.py
CHANGED
|
@@ -4,7 +4,7 @@ from conans import ConanFile
|
|
|
4
4
|
class USearchConan(ConanFile):
|
|
5
5
|
|
|
6
6
|
name = 'USearch'
|
|
7
|
-
version = '2.1.
|
|
7
|
+
version = '2.1.2'
|
|
8
8
|
license = 'Apache 2.0'
|
|
9
9
|
url = 'https://github.com/unum-cloud/usearch'
|
|
10
10
|
description = 'Smaller & Faster Single-File Vector Search Engine from Unum'
|
|
@@ -15,14 +15,14 @@
|
|
|
15
15
|
|
|
16
16
|
<PackageIcon>unum.png</PackageIcon>
|
|
17
17
|
<PackageIconUrl>unum.png</PackageIconUrl>
|
|
18
|
-
<PackageReadmeFile>
|
|
18
|
+
<PackageReadmeFile>README.md</PackageReadmeFile>
|
|
19
19
|
</PropertyGroup>
|
|
20
20
|
|
|
21
21
|
<ItemGroup>
|
|
22
22
|
<None Include="$(NativeLibDir)runtimes\**\*.*" Pack="true" PackagePath="runtimes" />
|
|
23
23
|
<None Include="$(RepoRoot)LICENSE" Pack="true" PackagePath=""/>
|
|
24
24
|
<None Include="$(RepoRoot)assets\unum.png" Pack="true" PackagePath=""/>
|
|
25
|
-
<None Include="$(
|
|
25
|
+
<None Include="$(RepoRoot)README.md" Pack="true" PackagePath=""/>
|
|
26
26
|
</ItemGroup>
|
|
27
27
|
|
|
28
28
|
<!-- Package validation https://learn.microsoft.com/en-us/dotnet/fundamentals/package-validation/overview?WT.mc_id=DT-MVP-5003978 -->
|
package/docs/format.md
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
# USearch Format Specification
|
|
2
|
+
|
|
3
|
+
> ⚠️ **Note**: The format is expected to change in the 3.0 release to support sparse indexes and offer even faster startup times when `view`-ing large indexes from disk.
|
|
4
|
+
|
|
5
|
+
## Table of Contents
|
|
6
|
+
|
|
7
|
+
- [Dense Indexes](#dense-indexes)
|
|
8
|
+
- [Core](#core)
|
|
9
|
+
- [Fragmented Storage](#fragmented-storage)
|
|
10
|
+
|
|
11
|
+
## Dense Indexes
|
|
12
|
+
|
|
13
|
+
Dense index files consist of two main parts: [a binary matrix of all vectors](#matrix-blob) and the [index](#index-blob) itself.
|
|
14
|
+
|
|
15
|
+
### Matrix BLOB
|
|
16
|
+
|
|
17
|
+
The matrix BLOB is optionally prepended to the index file. It starts with the number of `rows` and `columns`, which can be either 32-bit or 64-bit unsigned integers.
|
|
18
|
+
|
|
19
|
+
- The number of `rows` corresponds to the number of vectors in the index.
|
|
20
|
+
- The number of `columns` corresponds to the number of bytes in a single vector.
|
|
21
|
+
|
|
22
|
+
### Index BLOB
|
|
23
|
+
|
|
24
|
+
The index BLOB is composed of three main sections:
|
|
25
|
+
|
|
26
|
+
1. **Metadata**: Occupies 64 bytes and contains the following fields:
|
|
27
|
+
1. 7-byte magic string `usearch` to infer the MIME type of the file.
|
|
28
|
+
2. 3-byte version number, containing the `major`, `minor`, and `patch` release numbers.
|
|
29
|
+
3. 1-byte enums for `metric`, `scalar`, `key`, and `compressed_slot` types.
|
|
30
|
+
4. 8-byte integers for the number of present vectors, number of deleted vectors, and number of dimensions.
|
|
31
|
+
5. 1-byte flags for multi-vector support.
|
|
32
|
+
|
|
33
|
+
2. **Levels**: Contains 1-byte integers representing the level each node occupies. This information is used to estimate memory requirements and to enable memory-mapping of the entire file.
|
|
34
|
+
|
|
35
|
+
3. **Core**: Detailed in the [Core section](#core). This format is consistent across all types of indexes.
|
|
36
|
+
|
|
37
|
+
## Fragmented Storage
|
|
38
|
+
|
|
39
|
+
The matrix BLOB can be stored separately and is not required to successfully open the file.
|
|
40
|
+
|
|
41
|
+
## Core
|
|
42
|
+
|
|
43
|
+
The core section of the index is serialized as a sequence of nodes, each representing a point in the index. This section immediately follows the `levels` block in the serialized format.
|
|
44
|
+
|
|
45
|
+
### Header
|
|
46
|
+
|
|
47
|
+
The core section begins with a header containing the following fields:
|
|
48
|
+
|
|
49
|
+
- `uint64_t size`: Number of nodes in the index.
|
|
50
|
+
- `uint64_t connectivity`: Maximum number of edges for a node.
|
|
51
|
+
- `uint64_t connectivity_base`: Base connectivity value.
|
|
52
|
+
- `uint64_t max_level`: Maximum level in the multi-level graph.
|
|
53
|
+
- `uint64_t entry_slot`: Entry slot for the graph.
|
|
54
|
+
|
|
55
|
+
These fields are serialized in a contiguous block of bytes, in the order listed.
|
|
56
|
+
|
|
57
|
+
### Levels
|
|
58
|
+
|
|
59
|
+
Following the header, the `levels` block contains a sequence of 1-byte integers. Each integer represents the level of a node in the index. The number of levels is equal to the `size` field in the header.
|
|
60
|
+
|
|
61
|
+
### Nodes
|
|
62
|
+
|
|
63
|
+
After the `levels` block, the nodes are serialized. Each node is represented as a contiguous block of bytes. The size and format of these blocks are implementation-specific. Nodes are serialized in the same order as their corresponding levels in the `levels` block.
|
package/docs/format.rst
ADDED
package/docs/index.rst
CHANGED
package/golang/README.md
CHANGED
|
@@ -2,14 +2,36 @@
|
|
|
2
2
|
|
|
3
3
|
## Installation
|
|
4
4
|
|
|
5
|
+
### Linux
|
|
6
|
+
|
|
5
7
|
Download and install from the Debian package from the latest release.
|
|
6
8
|
Substitute `<release_tag>`, `<arch>`, and `<usearch_version>` with your settings.
|
|
7
9
|
|
|
8
10
|
```
|
|
9
|
-
wget https://github.com/unum-cloud/usearch/releases/download/<release_tag>/
|
|
11
|
+
wget https://github.com/unum-cloud/usearch/releases/download/<release_tag>/usearch_linux_<arch>_<usearch_version>.deb
|
|
10
12
|
dpkg -i usearch_<arch>_<usearch_version>.deb
|
|
11
13
|
```
|
|
12
14
|
|
|
15
|
+
### Windows
|
|
16
|
+
|
|
17
|
+
Run a `winlibinstaller.bat` script from the main repository in the folder where you will run `go run`.
|
|
18
|
+
That will install the USearch library and include in the same folder` where the script was run.
|
|
19
|
+
|
|
20
|
+
```
|
|
21
|
+
.\usearch\winlibinstaller.bat
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
### MacOS
|
|
25
|
+
|
|
26
|
+
Download and unpack a zip archive from the latest release.
|
|
27
|
+
Move the USearch library and the include file to their respective folders.
|
|
28
|
+
```
|
|
29
|
+
wget https://github.com/unum-cloud/usearch/releases/download/<release_tag>/usearch_macOS_<arch>_<usearch_version>.zip
|
|
30
|
+
unzip usearch_macOS_<arch>_<usearch_version>.zip
|
|
31
|
+
sudo mv libusearch.so /usr/local/lib && sudo mv usearch.h /usr/local/include
|
|
32
|
+
|
|
33
|
+
```
|
|
34
|
+
|
|
13
35
|
## Quickstart
|
|
14
36
|
|
|
15
37
|
1. Create a `go.mod` file:
|
|
@@ -1272,7 +1272,7 @@ struct dummy_executor_t {
|
|
|
1272
1272
|
* @brief An example of what a USearch-compatible key-to-key mapping should look like.
|
|
1273
1273
|
*
|
|
1274
1274
|
* This is particularly helpful for "Semantic Joins", where we map entries of one collection
|
|
1275
|
-
* to entries of another. In
|
|
1275
|
+
* to entries of another. In asymmetric setups, where A -> B is needed, but B -> A is not,
|
|
1276
1276
|
* this can be passed to minimize memory usage.
|
|
1277
1277
|
*/
|
|
1278
1278
|
struct dummy_key_to_key_mapping_t {
|
|
@@ -1284,16 +1284,16 @@ struct dummy_key_to_key_mapping_t {
|
|
|
1284
1284
|
|
|
1285
1285
|
/**
|
|
1286
1286
|
* @brief Checks if the provided object has a dummy type, emulating an interface,
|
|
1287
|
-
* but
|
|
1287
|
+
* but performing no real computation.
|
|
1288
1288
|
*/
|
|
1289
1289
|
template <typename object_at> static constexpr bool is_dummy() {
|
|
1290
1290
|
using object_t = typename std::remove_all_extents<object_at>::type;
|
|
1291
|
-
return std::is_same<object_t, dummy_predicate_t>::value || //
|
|
1292
|
-
std::is_same<object_t, dummy_callback_t>::value || //
|
|
1293
|
-
std::is_same<object_t, dummy_progress_t>::value || //
|
|
1294
|
-
std::is_same<object_t, dummy_prefetch_t>::value || //
|
|
1295
|
-
std::is_same<object_t, dummy_executor_t>::value || //
|
|
1296
|
-
std::is_same<object_t, dummy_key_to_key_mapping_t>::value;
|
|
1291
|
+
return std::is_same<typename std::decay<object_t>::type, dummy_predicate_t>::value || //
|
|
1292
|
+
std::is_same<typename std::decay<object_t>::type, dummy_callback_t>::value || //
|
|
1293
|
+
std::is_same<typename std::decay<object_t>::type, dummy_progress_t>::value || //
|
|
1294
|
+
std::is_same<typename std::decay<object_t>::type, dummy_prefetch_t>::value || //
|
|
1295
|
+
std::is_same<typename std::decay<object_t>::type, dummy_executor_t>::value || //
|
|
1296
|
+
std::is_same<typename std::decay<object_t>::type, dummy_key_to_key_mapping_t>::value;
|
|
1297
1297
|
}
|
|
1298
1298
|
|
|
1299
1299
|
template <typename, typename at> struct has_reset_gt {
|
|
@@ -1513,6 +1513,8 @@ class memory_mapped_file_t {
|
|
|
1513
1513
|
#else
|
|
1514
1514
|
int descriptor = open(path_, O_RDONLY);
|
|
1515
1515
|
#endif
|
|
1516
|
+
if (descriptor < 0)
|
|
1517
|
+
return result.failed(std::strerror(errno));
|
|
1516
1518
|
|
|
1517
1519
|
// Estimate the file size
|
|
1518
1520
|
struct stat file_stat;
|
|
@@ -2502,7 +2504,7 @@ class index_gt {
|
|
|
2502
2504
|
}
|
|
2503
2505
|
|
|
2504
2506
|
/**
|
|
2505
|
-
* @brief Identifies the closest cluster to the
|
|
2507
|
+
* @brief Identifies the closest cluster to the given ::query. Thread-safe.
|
|
2506
2508
|
*
|
|
2507
2509
|
* @param[in] query Content that will be compared against other entries in the index.
|
|
2508
2510
|
* @param[in] level The index level to target. Higher means lower resolution.
|
|
@@ -3325,7 +3327,7 @@ class index_gt {
|
|
|
3325
3327
|
visits.clear();
|
|
3326
3328
|
|
|
3327
3329
|
// Optional prefetching
|
|
3328
|
-
if (!std::is_same<prefetch_at, dummy_prefetch_t>::value)
|
|
3330
|
+
if (!std::is_same<typename std::decay<prefetch_at>::type, dummy_prefetch_t>::value)
|
|
3329
3331
|
prefetch(citerator_at(closest_slot), citerator_at(closest_slot + 1));
|
|
3330
3332
|
|
|
3331
3333
|
distance_t closest_dist = context.measure(query, citerator_at(closest_slot), metric);
|
|
@@ -3337,7 +3339,7 @@ class index_gt {
|
|
|
3337
3339
|
neighbors_ref_t closest_neighbors = neighbors_non_base_(node_at_(closest_slot), level);
|
|
3338
3340
|
|
|
3339
3341
|
// Optional prefetching
|
|
3340
|
-
if (!std::is_same<prefetch_at, dummy_prefetch_t>::value) {
|
|
3342
|
+
if (!std::is_same<typename std::decay<prefetch_at>::type, dummy_prefetch_t>::value) {
|
|
3341
3343
|
candidates_range_t missing_candidates{*this, closest_neighbors, visits};
|
|
3342
3344
|
prefetch(missing_candidates.begin(), missing_candidates.end());
|
|
3343
3345
|
}
|
|
@@ -3379,7 +3381,7 @@ class index_gt {
|
|
|
3379
3381
|
return false;
|
|
3380
3382
|
|
|
3381
3383
|
// Optional prefetching
|
|
3382
|
-
if (!std::is_same<prefetch_at, dummy_prefetch_t>::value)
|
|
3384
|
+
if (!std::is_same<typename std::decay<prefetch_at>::type, dummy_prefetch_t>::value)
|
|
3383
3385
|
prefetch(citerator_at(start_slot), citerator_at(start_slot + 1));
|
|
3384
3386
|
|
|
3385
3387
|
distance_t radius = context.measure(query, citerator_at(start_slot), metric);
|
|
@@ -3404,7 +3406,7 @@ class index_gt {
|
|
|
3404
3406
|
neighbors_ref_t candidate_neighbors = neighbors_(candidate_ref, level);
|
|
3405
3407
|
|
|
3406
3408
|
// Optional prefetching
|
|
3407
|
-
if (!std::is_same<prefetch_at, dummy_prefetch_t>::value) {
|
|
3409
|
+
if (!std::is_same<typename std::decay<prefetch_at>::type, dummy_prefetch_t>::value) {
|
|
3408
3410
|
candidates_range_t missing_candidates{*this, candidate_neighbors, visits};
|
|
3409
3411
|
prefetch(missing_candidates.begin(), missing_candidates.end());
|
|
3410
3412
|
}
|
|
@@ -3453,7 +3455,7 @@ class index_gt {
|
|
|
3453
3455
|
return false;
|
|
3454
3456
|
|
|
3455
3457
|
// Optional prefetching
|
|
3456
|
-
if (!std::is_same<prefetch_at, dummy_prefetch_t>::value)
|
|
3458
|
+
if (!std::is_same<typename std::decay<prefetch_at>::type, dummy_prefetch_t>::value)
|
|
3457
3459
|
prefetch(citerator_at(start_slot), citerator_at(start_slot + 1));
|
|
3458
3460
|
|
|
3459
3461
|
distance_t radius = context.measure(query, citerator_at(start_slot), metric);
|
|
@@ -3473,7 +3475,7 @@ class index_gt {
|
|
|
3473
3475
|
neighbors_ref_t candidate_neighbors = neighbors_base_(node_at_(candidate.slot));
|
|
3474
3476
|
|
|
3475
3477
|
// Optional prefetching
|
|
3476
|
-
if (!std::is_same<prefetch_at, dummy_prefetch_t>::value) {
|
|
3478
|
+
if (!std::is_same<typename std::decay<prefetch_at>::type, dummy_prefetch_t>::value) {
|
|
3477
3479
|
candidates_range_t missing_candidates{*this, candidate_neighbors, visits};
|
|
3478
3480
|
prefetch(missing_candidates.begin(), missing_candidates.end());
|
|
3479
3481
|
}
|
package/package.json
CHANGED
package/python/usearch/index.py
CHANGED
|
@@ -562,12 +562,11 @@ class Index:
|
|
|
562
562
|
)
|
|
563
563
|
|
|
564
564
|
self.path = path
|
|
565
|
-
if path
|
|
566
|
-
path = os.fspath(path)
|
|
565
|
+
if path is not None:
|
|
567
566
|
if view:
|
|
568
|
-
self.
|
|
567
|
+
self.view(path)
|
|
569
568
|
else:
|
|
570
|
-
self.
|
|
569
|
+
self.load(path)
|
|
571
570
|
|
|
572
571
|
@staticmethod
|
|
573
572
|
def metadata(path_or_buffer: PathOrBuffer) -> Optional[dict]:
|
|
@@ -919,7 +918,9 @@ class Index:
|
|
|
919
918
|
def expansion_search(self, v: int):
|
|
920
919
|
self._compiled.expansion_search = v
|
|
921
920
|
|
|
922
|
-
def save(
|
|
921
|
+
def save(
|
|
922
|
+
self, path_or_buffer: Union[str, os.PathLike, NoneType] = None
|
|
923
|
+
) -> Optional[bytes]:
|
|
923
924
|
path_or_buffer = path_or_buffer if path_or_buffer else self.path
|
|
924
925
|
if path_or_buffer is None:
|
|
925
926
|
return self._compiled.save_index_to_buffer()
|
|
@@ -937,7 +938,9 @@ class Index:
|
|
|
937
938
|
else:
|
|
938
939
|
self._compiled.load_index_from_path(os.fspath(path_or_buffer))
|
|
939
940
|
|
|
940
|
-
def view(
|
|
941
|
+
def view(
|
|
942
|
+
self, path_or_buffer: Union[str, os.PathLike, bytes, bytearray, NoneType] = None
|
|
943
|
+
):
|
|
941
944
|
path_or_buffer = path_or_buffer if path_or_buffer else self.path
|
|
942
945
|
if path_or_buffer is None:
|
|
943
946
|
raise Exception("Define the source")
|
package/setup.py
CHANGED
|
@@ -66,6 +66,7 @@ setup(
|
|
|
66
66
|
package_dir={"usearch": "python/usearch"},
|
|
67
67
|
description="Smaller & Faster Single-File Vector Search Engine from Unum",
|
|
68
68
|
author="Ash Vardanian",
|
|
69
|
+
author_email="info@unum.cloud",
|
|
69
70
|
long_description=long_description,
|
|
70
71
|
long_description_content_type="text/markdown",
|
|
71
72
|
license="Apache-2.0",
|
package/wasmer.toml
CHANGED
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
@echo off
|
|
2
|
+
|
|
3
|
+
rem Get latest release tag
|
|
4
|
+
set "repo_owner=unum-cloud"
|
|
5
|
+
set "repo_name=usearch"
|
|
6
|
+
|
|
7
|
+
rem Get the latest tag without quotes
|
|
8
|
+
for /f "delims=" %%i in ('curl -s https://api.github.com/repos/%repo_owner%/%repo_name%/releases/latest ^| findstr "tag_name"') do (
|
|
9
|
+
for /f "tokens=2 delims=:, " %%a in ("%%i") do (
|
|
10
|
+
set "latest_tag=%%~a"
|
|
11
|
+
)
|
|
12
|
+
)
|
|
13
|
+
|
|
14
|
+
rem Get Windows architecture
|
|
15
|
+
for /f "tokens=2 delims= " %%a in ('wmic OS get OSArchitecture ^| find "64"') do set "arch=x64"
|
|
16
|
+
if "%arch%"=="" set "arch=x86"
|
|
17
|
+
|
|
18
|
+
echo Split library version from release version!
|
|
19
|
+
timeout /t 1
|
|
20
|
+
rem Split the first 'v' from the version number.
|
|
21
|
+
for /f "tokens=1 delims=v" %%a in ('echo %latest_tag%') do (
|
|
22
|
+
set lib_version=%%a
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
echo Download usearch library archive!
|
|
26
|
+
rem Download the file.
|
|
27
|
+
timeout /t 2
|
|
28
|
+
curl -LO https://github.com/%repo_owner%/%repo_name%/releases/download/%latest_tag%/usearch_windows_%arch%_%lib_version%.tar
|
|
29
|
+
echo Download is complete!
|
|
30
|
+
|
|
31
|
+
echo Extract archive!
|
|
32
|
+
rem Extract archive.
|
|
33
|
+
timeout /t 1
|
|
34
|
+
tar -xf usearch_windows_%arch%_%lib_version%.tar
|
|
35
|
+
echo Extract is complete!
|
|
36
|
+
|
|
37
|
+
rem Remove the archive.
|
|
38
|
+
del usearch_windows_%arch%_%lib_version%.tar
|
|
39
|
+
|
|
40
|
+
echo Extraction complete and downloaded .tar file removed.
|
|
41
|
+
timeout /t 1
|
|
42
|
+
exit /b
|
package/csharp/nuget/NUGET.md
DELETED
|
@@ -1,11 +0,0 @@
|
|
|
1
|
-
**Cloud.Unum.USearch** is a wrapper for the USearch vector search engine, offering fast searches, multiple metrics, and features like SIMD-optimized metrics.
|
|
2
|
-
|
|
3
|
-
**Supported metrics**: Euclidean, Angular, Jaccard, Hamming, Haversine, User-Defined Metrics
|
|
4
|
-
|
|
5
|
-
**Platforms**: Linux, MacOS, Windows
|
|
6
|
-
|
|
7
|
-
# Getting Started ⚡
|
|
8
|
-
|
|
9
|
-
- [Quickstart](https://github.com/unum-cloud/usearch/blob/main/csharp/README.md)
|
|
10
|
-
- Join the [Discord community](https://discord.gg/A6wxt6dS9j)
|
|
11
|
-
- Check out the [GitHub repository](https://github.com/unum-cloud/usearch)
|