usearch 0.2.3 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/ISSUE_TEMPLATE/bug_report.yml +90 -0
- package/.github/ISSUE_TEMPLATE/config.yml +5 -0
- package/.github/ISSUE_TEMPLATE/feature_request.yml +64 -0
- package/.github/workflows/release.yml +45 -20
- package/.github/workflows/update_version.sh +6 -0
- package/.releaserc +77 -0
- package/.vscode/settings.json +14 -2
- package/CODE_OF_CONDUCT.md +128 -0
- package/CONTRIBUTING.md +12 -0
- package/Cargo.toml +1 -1
- package/README.md +246 -220
- package/VERSION +1 -1
- package/binding.gyp +13 -9
- package/build.rs +1 -0
- package/conanfile.py +1 -1
- package/docs/Compilation.md +112 -0
- package/docs/benchmarks.md +182 -0
- package/docs/benchmarks.rst +5 -0
- package/docs/compilation.rst +5 -0
- package/docs/conf.dox +2668 -0
- package/docs/index.rst +7 -0
- package/include/usearch/usearch.hpp +454 -238
- package/javascript/lib.cpp +82 -8
- package/objc/USearchObjective.mm +2 -2
- package/objc/include/USearchObjective.h +10 -9
- package/package.json +7 -1
- package/python/bench.py +37 -7
- package/python/lib.cpp +70 -43
- package/python/test.py +78 -36
- package/rust/lib.cpp +6 -2
- package/simsimd/include/simsimd/simsimd.h +31 -41
- package/src/advanced.hpp +68 -40
- package/src/bench.cpp +24 -23
- package/swift/Index+Sugar.swift +1 -0
- package/swift/Test.swift +1 -0
- package/.github/workflows/package.json +0 -82
- package/CMakeListsInstructions.md +0 -185
|
@@ -1,82 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"devDependencies": {
|
|
3
|
-
"@semantic-release/exec": "github:semantic-release/exec",
|
|
4
|
-
"@semantic-release/git": "^10.0.1",
|
|
5
|
-
"conventional-changelog-eslint": "^3.0.9",
|
|
6
|
-
"semantic-release": "^20.1.3"
|
|
7
|
-
},
|
|
8
|
-
"release": {
|
|
9
|
-
"branches": [
|
|
10
|
-
"main"
|
|
11
|
-
],
|
|
12
|
-
"debug": true,
|
|
13
|
-
"ci": true,
|
|
14
|
-
"dryRun": false,
|
|
15
|
-
"plugins": [
|
|
16
|
-
[
|
|
17
|
-
"@semantic-release/commit-analyzer",
|
|
18
|
-
{
|
|
19
|
-
"preset": "eslint",
|
|
20
|
-
"releaseRules": [
|
|
21
|
-
{
|
|
22
|
-
"tag": "Add",
|
|
23
|
-
"release": "minor"
|
|
24
|
-
},
|
|
25
|
-
{
|
|
26
|
-
"tag": "Improve",
|
|
27
|
-
"release": "patch"
|
|
28
|
-
},
|
|
29
|
-
{
|
|
30
|
-
"tag": "Make",
|
|
31
|
-
"release": "patch"
|
|
32
|
-
},
|
|
33
|
-
{
|
|
34
|
-
"tag": "Refactor",
|
|
35
|
-
"release": false
|
|
36
|
-
}
|
|
37
|
-
]
|
|
38
|
-
}
|
|
39
|
-
],
|
|
40
|
-
[
|
|
41
|
-
"@semantic-release/release-notes-generator",
|
|
42
|
-
{
|
|
43
|
-
"preset": "eslint",
|
|
44
|
-
"releaseRules": [
|
|
45
|
-
{
|
|
46
|
-
"tag": "Add",
|
|
47
|
-
"release": "minor"
|
|
48
|
-
},
|
|
49
|
-
{
|
|
50
|
-
"tag": "Improve",
|
|
51
|
-
"release": "patch"
|
|
52
|
-
},
|
|
53
|
-
{
|
|
54
|
-
"tag": "Make",
|
|
55
|
-
"release": "patch"
|
|
56
|
-
},
|
|
57
|
-
{
|
|
58
|
-
"tag": "Refactor",
|
|
59
|
-
"release": false
|
|
60
|
-
}
|
|
61
|
-
]
|
|
62
|
-
}
|
|
63
|
-
],
|
|
64
|
-
"@semantic-release/github",
|
|
65
|
-
[
|
|
66
|
-
"@semantic-release/exec",
|
|
67
|
-
{
|
|
68
|
-
"prepareCmd": "echo '${nextRelease.version}' > VERSION"
|
|
69
|
-
}
|
|
70
|
-
],
|
|
71
|
-
[
|
|
72
|
-
"@semantic-release/git",
|
|
73
|
-
{
|
|
74
|
-
"assets": [
|
|
75
|
-
"VERSION"
|
|
76
|
-
],
|
|
77
|
-
"message": "Build: Released ${nextRelease.version} [skip ci]\n\n${nextRelease.notes}"
|
|
78
|
-
}
|
|
79
|
-
]
|
|
80
|
-
]
|
|
81
|
-
}
|
|
82
|
-
}
|
|
@@ -1,185 +0,0 @@
|
|
|
1
|
-
# Compiling USearch
|
|
2
|
-
|
|
3
|
-
## Build
|
|
4
|
-
|
|
5
|
-
Before building the first time, please pull submodules.
|
|
6
|
-
|
|
7
|
-
```sh
|
|
8
|
-
git submodule update --init --recursive
|
|
9
|
-
```
|
|
10
|
-
|
|
11
|
-
### Linux
|
|
12
|
-
|
|
13
|
-
```sh
|
|
14
|
-
cmake -B ./build_release \
|
|
15
|
-
-DCMAKE_CXX_COMPILER="g++-12" \
|
|
16
|
-
-DCMAKE_BUILD_TYPE=Release \
|
|
17
|
-
-DUSEARCH_USE_SIMD=1 \
|
|
18
|
-
-DUSEARCH_USE_OPENMP=1 \
|
|
19
|
-
-DUSEARCH_USE_JEMALLOC=1 && \
|
|
20
|
-
make -C ./build_release -j
|
|
21
|
-
|
|
22
|
-
# To benchmark in C++:
|
|
23
|
-
./build_release/bench \
|
|
24
|
-
--vectors datasets/wiki_1M/base.1M.fbin \
|
|
25
|
-
--queries datasets/wiki_1M/query.public.100K.fbin \
|
|
26
|
-
--neighbors datasets/wiki_1M/groundtruth.public.100K.ibin
|
|
27
|
-
|
|
28
|
-
# Large benchmark on a 1TB dataset:
|
|
29
|
-
./build_release/bench \
|
|
30
|
-
--vectors datasets/t2i_1B/base.1B.fbin \
|
|
31
|
-
--queries datasets/t2i_1B/query.public.100K.fbin \
|
|
32
|
-
--neighbors datasets/t2i_1B/groundtruth.public.100K.ibin \
|
|
33
|
-
--output datasets/t2i_1B/index.usearch \
|
|
34
|
-
--cos
|
|
35
|
-
|
|
36
|
-
# To benchmark in Python:
|
|
37
|
-
python python/bench.py \
|
|
38
|
-
--vectors datasets/wiki_1M/base.1M.fbin \
|
|
39
|
-
--queries datasets/wiki_1M/query.public.100K.fbin \
|
|
40
|
-
--neighbors datasets/wiki_1M/groundtruth.public.100K.ibin
|
|
41
|
-
```
|
|
42
|
-
|
|
43
|
-
### MacOS
|
|
44
|
-
|
|
45
|
-
```sh
|
|
46
|
-
brew install libomp llvm
|
|
47
|
-
cmake \
|
|
48
|
-
-DCMAKE_C_COMPILER="/opt/homebrew/opt/llvm/bin/clang" \
|
|
49
|
-
-DCMAKE_CXX_COMPILER="/opt/homebrew/opt/llvm/bin/clang++" \
|
|
50
|
-
-DUSEARCH_USE_SIMD=1 \
|
|
51
|
-
-DUSEARCH_USE_OPENMP=1 \
|
|
52
|
-
-B ./build_release && \
|
|
53
|
-
make -C ./build_release -j
|
|
54
|
-
```
|
|
55
|
-
|
|
56
|
-
### Python
|
|
57
|
-
|
|
58
|
-
```sh
|
|
59
|
-
pip install -e .
|
|
60
|
-
python python/test.py
|
|
61
|
-
```
|
|
62
|
-
|
|
63
|
-
### JavaScript
|
|
64
|
-
|
|
65
|
-
```sh
|
|
66
|
-
npm install
|
|
67
|
-
node javascript/test.js
|
|
68
|
-
npm publish
|
|
69
|
-
```
|
|
70
|
-
|
|
71
|
-
### Rust
|
|
72
|
-
|
|
73
|
-
```sh
|
|
74
|
-
cargo test -p usearch
|
|
75
|
-
cargo publish
|
|
76
|
-
```
|
|
77
|
-
|
|
78
|
-
### Java
|
|
79
|
-
|
|
80
|
-
```sh
|
|
81
|
-
gradle clean build
|
|
82
|
-
java -cp . -Djava.library.path=/Users/av/github/usearch/build/libs/usearch/shared java/cloud/unum/usearch/Index.java
|
|
83
|
-
```
|
|
84
|
-
|
|
85
|
-
Or step by-step:
|
|
86
|
-
|
|
87
|
-
```sh
|
|
88
|
-
cs java/unum/cloud/usearch
|
|
89
|
-
javac -h . Index.java
|
|
90
|
-
|
|
91
|
-
# Ubuntu:
|
|
92
|
-
g++ -c -fPIC -I${JAVA_HOME}/include -I${JAVA_HOME}/include/linux cloud_unum_usearch_Index.cpp -o cloud_unum_usearch_Index.o
|
|
93
|
-
g++ -shared -fPIC -o libusearch.so cloud_unum_usearch_Index.o -lc
|
|
94
|
-
|
|
95
|
-
# Windows
|
|
96
|
-
g++ -c -I%JAVA_HOME%\include -I%JAVA_HOME%\include\win32 cloud_unum_usearch_Index.cpp -o cloud_unum_usearch_Index.o
|
|
97
|
-
g++ -shared -o USearchJNI.dll cloud_unum_usearch_Index.o -Wl,--add-stdcall-alias
|
|
98
|
-
|
|
99
|
-
# MacOS
|
|
100
|
-
g++ -std=c++11 -c -fPIC \
|
|
101
|
-
-I../../../../include -I../../../../src -I../../../../fp16/include -I../../../../simsimd/include \
|
|
102
|
-
-I${JAVA_HOME}/include -I${JAVA_HOME}/include/darwin cloud_unum_usearch_Index.cpp -o cloud_unum_usearch_Index.o
|
|
103
|
-
g++ -dynamiclib -o libusearch.dylib cloud_unum_usearch_Index.o -lc
|
|
104
|
-
# Run linking to that directory
|
|
105
|
-
java -cp . -Djava.library.path=/Users/av/github/usearch/java/cloud/unum/usearch/ Index.java
|
|
106
|
-
java -cp . -Djava.library.path=/Users/av/github/usearch/java cloud.unum.usearch.Index
|
|
107
|
-
```
|
|
108
|
-
|
|
109
|
-
### Wolfram
|
|
110
|
-
|
|
111
|
-
```sh
|
|
112
|
-
brew install --cask wolfram-engine
|
|
113
|
-
```
|
|
114
|
-
|
|
115
|
-
## Benchmarks
|
|
116
|
-
|
|
117
|
-
### Datasets
|
|
118
|
-
|
|
119
|
-
BigANN benchmark is a good starting point, if you are searching for large collections of high-dimensional vectors.
|
|
120
|
-
Those often come with precomputed ground-truth neighbors, which is handy for recall evaluation.
|
|
121
|
-
|
|
122
|
-
| Dataset | Scalar Type | Dimensions | Metric | Size |
|
|
123
|
-
| :-------------------------------------- | :---------: | :--------: | :----: | :-------: |
|
|
124
|
-
| [Unum UForm Wiki][unum-wiki] | float32 | 256 | IP | 1 GB |
|
|
125
|
-
| [Yandex Text-to-Image Sample][unum-t2i] | float32 | 200 | Cos | 1 GB |
|
|
126
|
-
| | | | | |
|
|
127
|
-
| [Microsoft SPACEV][spacev] | int8 | 100 | L2 | 93 GB |
|
|
128
|
-
| [Microsoft Turing-ANNS][turing] | float32 | 100 | L2 | 373 GB |
|
|
129
|
-
| [Yandex Deep1B][deep] | float32 | 96 | L2 | 358 GB |
|
|
130
|
-
| [Yandex Text-to-Image][t2i] | float32 | 200 | Cos | 750 GB |
|
|
131
|
-
| | | | | |
|
|
132
|
-
| [ViT-L/12 LAION][laion] | float32 | 2048 | Cos | 2 - 10 TB |
|
|
133
|
-
|
|
134
|
-
Luckily, smaller samples of those datasets are available.
|
|
135
|
-
|
|
136
|
-
[unum-wiki]: https://huggingface.co/datasets/unum-cloud/ann-wiki-1m
|
|
137
|
-
[unum-t2i]: https://huggingface.co/datasets/unum-cloud/ann-t2i-1m
|
|
138
|
-
[spacev]: https://github.com/microsoft/SPTAG/tree/main/datasets/SPACEV1B
|
|
139
|
-
[turing]: https://learning2hash.github.io/publications/microsoftturinganns1B/
|
|
140
|
-
[t2i]: https://research.yandex.com/blog/benchmarks-for-billion-scale-similarity-search
|
|
141
|
-
[deep]: https://research.yandex.com/blog/benchmarks-for-billion-scale-similarity-search
|
|
142
|
-
[laion]: https://laion.ai/blog/laion-5b/#download-the-data
|
|
143
|
-
|
|
144
|
-
### Unum UForm Wiki
|
|
145
|
-
|
|
146
|
-
```sh
|
|
147
|
-
mkdir -p datasets/wiki_1M/ && \
|
|
148
|
-
wget -nc https://huggingface.co/datasets/unum-cloud/ann-wiki-1m/resolve/main/base.1M.fbin -P datasets/wiki_1M/ &&
|
|
149
|
-
wget -nc https://huggingface.co/datasets/unum-cloud/ann-wiki-1m/resolve/main/query.public.100K.fbin -P datasets/wiki_1M/ &&
|
|
150
|
-
wget -nc https://huggingface.co/datasets/unum-cloud/ann-wiki-1m/resolve/main/groundtruth.public.100K.ibin -P datasets/wiki_1M/
|
|
151
|
-
```
|
|
152
|
-
|
|
153
|
-
### Yandex Text-to-Image
|
|
154
|
-
|
|
155
|
-
```sh
|
|
156
|
-
mkdir -p datasets/t2i_1B/ && \
|
|
157
|
-
wget -nc https://storage.yandexcloud.net/yandex-research/ann-datasets/T2I/base.1B.fbin -P datasets/t2i_1B/ &&
|
|
158
|
-
wget -nc https://storage.yandexcloud.net/yandex-research/ann-datasets/T2I/base.1M.fbin -P datasets/t2i_1B/ &&
|
|
159
|
-
wget -nc https://storage.yandexcloud.net/yandex-research/ann-datasets/T2I/query.public.100K.fbin -P datasets/t2i_1B/ &&
|
|
160
|
-
wget -nc https://storage.yandexcloud.net/yandex-research/ann-datasets/T2I/groundtruth.public.100K.ibin -P datasets/t2i_1B/
|
|
161
|
-
```
|
|
162
|
-
|
|
163
|
-
### Yandex Deep1B
|
|
164
|
-
|
|
165
|
-
```sh
|
|
166
|
-
mkdir -p datasets/deep_1B/ && \
|
|
167
|
-
wget -nc https://storage.yandexcloud.net/yandex-research/ann-datasets/DEEP/base.1B.fbin -P datasets/deep_1B/ &&
|
|
168
|
-
wget -nc https://storage.yandexcloud.net/yandex-research/ann-datasets/DEEP/base.10M.fbin -P datasets/deep_1B/ &&
|
|
169
|
-
wget -nc https://storage.yandexcloud.net/yandex-research/ann-datasets/DEEP/query.public.10K.fbin -P datasets/deep_1B/ &&
|
|
170
|
-
wget -nc https://storage.yandexcloud.net/yandex-research/ann-datasets/DEEP/groundtruth.public.10K.ibin -P datasets/deep_1B/
|
|
171
|
-
```
|
|
172
|
-
|
|
173
|
-
### Profiling
|
|
174
|
-
|
|
175
|
-
With `perf`:
|
|
176
|
-
|
|
177
|
-
```sh
|
|
178
|
-
# Pass environment variables with `-E`, and `-d` for details
|
|
179
|
-
sudo -E perf stat -d ./build_release/bench ...
|
|
180
|
-
sudo -E perf mem -d ./build_release/bench ...
|
|
181
|
-
# Sample on-CPU functions for the specified command, at 1 Kilo Hertz:
|
|
182
|
-
sudo -E perf record -F 1000 ./build_release/bench ...
|
|
183
|
-
perf record -d -e arm_spe// -- ./build_release/bench ..
|
|
184
|
-
```
|
|
185
|
-
|