usearch 0.22.2 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/prerelease.yml +3 -3
- package/.releaserc +9 -0
- package/.vscode/launch.json +15 -11
- package/.vscode/settings.json +12 -2
- package/.vscode/tasks.json +4 -4
- package/CMakeLists.txt +6 -1
- package/Cargo.toml +2 -2
- package/README.md +39 -46
- package/build.rs +2 -2
- package/c/CMakeLists.txt +48 -0
- package/c/README.md +34 -0
- package/c/lib.cpp +83 -72
- package/c/test.c +20 -20
- package/c/usearch.h +132 -32
- package/conanfile.py +1 -1
- package/cpp/CMakeLists.txt +3 -1
- package/cpp/README.md +16 -16
- package/cpp/bench.cpp +50 -126
- package/cpp/test.cpp +164 -77
- package/docs/benchmarks.md +26 -5
- package/docs/c/reference.rst +3 -3
- package/docs/compilation.md +49 -45
- package/docs/cpp/index.rst +2 -2
- package/docs/cpp/reference.rst +13 -3
- package/docs/index.rst +8 -8
- package/docs/java/reference.rst +1 -1
- package/golang/lib.go +275 -141
- package/golang/lib_test.go +105 -0
- package/include/usearch/index.hpp +1733 -1659
- package/include/usearch/index_dense.hpp +1400 -0
- package/include/usearch/index_plugins.hpp +1263 -0
- package/java/README.md +1 -1
- package/java/cloud/unum/usearch/Index.java +10 -10
- package/java/cloud/unum/usearch/cloud_unum_usearch_Index.cpp +34 -37
- package/java/test/IndexTest.java +1 -1
- package/javascript/README.md +7 -7
- package/javascript/lib.cpp +94 -55
- package/javascript/test.js +25 -11
- package/javascript/usearch.d.ts +87 -0
- package/javascript/usearch.js +2 -118
- package/objc/USearchObjective.mm +57 -31
- package/objc/include/USearchObjective.h +32 -22
- package/package.json +5 -3
- package/pyproject.toml +3 -3
- package/python/README.md +23 -14
- package/python/lib.cpp +472 -444
- package/python/scripts/bench.ipynb +3 -2
- package/python/scripts/bench.py +4 -4
- package/python/scripts/index_faiss.py +5 -5
- package/python/scripts/join.py +4 -4
- package/python/scripts/test.py +152 -61
- package/python/scripts/test_jit.py +10 -12
- package/python/scripts/test_sparse.py +32 -0
- package/python/usearch/client.py +26 -28
- package/python/usearch/eval.py +73 -23
- package/python/usearch/index.py +520 -228
- package/python/usearch/io.py +57 -25
- package/python/usearch/numba.py +4 -4
- package/python/usearch/server.py +10 -10
- package/rust/lib.cpp +22 -27
- package/rust/lib.hpp +11 -12
- package/rust/lib.rs +3 -3
- package/setup.py +5 -2
- package/simsimd/.github/workflows/prerelease.yml +73 -0
- package/simsimd/.github/workflows/release.yml +110 -0
- package/simsimd/.github/workflows/update_version.sh +3 -0
- package/simsimd/.releaserc +77 -0
- package/simsimd/.vscode/launch.json +27 -1
- package/simsimd/.vscode/settings.json +22 -1
- package/simsimd/.vscode/tasks.json +6 -0
- package/simsimd/CMakeLists.txt +3 -2
- package/simsimd/README.md +82 -46
- package/simsimd/bench.cxx +63 -24
- package/simsimd/include/simsimd/simsimd.h +88 -66
- package/simsimd/include/simsimd/simsimd_chem.h +126 -0
- package/simsimd/package.json +8 -0
- package/simsimd/pyproject.toml +42 -0
- package/simsimd/python/python.c +81 -0
- package/simsimd/python/run.sh +1 -0
- package/simsimd/python/test.c +109 -0
- package/simsimd/python/test.py +23 -0
- package/simsimd/setup.py +44 -0
- package/swift/Index+Sugar.swift +75 -23
- package/swift/README.md +26 -2
- package/swift/Test.swift +19 -7
- package/swift/USearch.swift +2 -2
- package/tsconfig.json +8 -0
- package/wasm/CMakeLists.txt +1 -0
- package/wasmer.toml +5 -1
- package/wolfram/CMakeLists.txt +1 -0
- package/wolfram/lib.cpp +11 -13
- package/golang/usearch.h +0 -9
- package/golang/usearch_par_test.go +0 -40
- package/golang/usearch_test.go +0 -82
- package/include/usearch/index_punned_dense.hpp +0 -871
- package/include/usearch/index_punned_helpers.hpp +0 -566
|
@@ -17,8 +17,8 @@ permissions:
|
|
|
17
17
|
|
|
18
18
|
jobs:
|
|
19
19
|
|
|
20
|
-
|
|
21
|
-
name: Test
|
|
20
|
+
test_cpp:
|
|
21
|
+
name: Test C++
|
|
22
22
|
runs-on: ubuntu-latest
|
|
23
23
|
steps:
|
|
24
24
|
- uses: actions/checkout@v3
|
|
@@ -163,7 +163,7 @@ jobs:
|
|
|
163
163
|
- name: Build USearch by Emscripten
|
|
164
164
|
run: |
|
|
165
165
|
./emsdk/emsdk activate latest && source ./emsdk/emsdk_env.sh
|
|
166
|
-
emcmake cmake -B ./build -DCMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS} -s TOTAL_MEMORY=64MB" && emmake make -C ./build
|
|
166
|
+
emcmake cmake -DUSEARCH_BUILD_BENCHMARK=0 -DUSEARCH_BUILD_WASM=1 -B ./build -DCMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS} -s TOTAL_MEMORY=64MB" && emmake make -C ./build
|
|
167
167
|
- name: Run tests
|
|
168
168
|
run: node ./build/test.js
|
|
169
169
|
|
package/.releaserc
CHANGED
|
@@ -11,6 +11,10 @@
|
|
|
11
11
|
{
|
|
12
12
|
"preset": "eslint",
|
|
13
13
|
"releaseRules": [
|
|
14
|
+
{
|
|
15
|
+
"tag": "Break",
|
|
16
|
+
"release": "major"
|
|
17
|
+
},
|
|
14
18
|
{
|
|
15
19
|
"tag": "Add",
|
|
16
20
|
"release": "minor"
|
|
@@ -35,6 +39,10 @@
|
|
|
35
39
|
{
|
|
36
40
|
"preset": "eslint",
|
|
37
41
|
"releaseRules": [
|
|
42
|
+
{
|
|
43
|
+
"tag": "Break",
|
|
44
|
+
"release": "major"
|
|
45
|
+
},
|
|
38
46
|
{
|
|
39
47
|
"tag": "Add",
|
|
40
48
|
"release": "minor"
|
|
@@ -69,6 +77,7 @@
|
|
|
69
77
|
"conanfile.py",
|
|
70
78
|
"package.json",
|
|
71
79
|
"Cargo.toml",
|
|
80
|
+
"wasmer.toml"
|
|
72
81
|
],
|
|
73
82
|
"message": "Build: Released ${nextRelease.version} [skip ci]\n\n${nextRelease.notes}"
|
|
74
83
|
}
|
package/.vscode/launch.json
CHANGED
|
@@ -13,21 +13,25 @@
|
|
|
13
13
|
"environment": [
|
|
14
14
|
{
|
|
15
15
|
"name": "ASAN_OPTIONS",
|
|
16
|
-
"value": "detect_leaks=0:atexit=1:strict_init_order=1:strict_string_checks=1"
|
|
16
|
+
"value": "detect_leaks=0:atexit=1:strict_init_order=1:strict_string_checks=1:abort_on_error=0"
|
|
17
|
+
},
|
|
18
|
+
{
|
|
19
|
+
"name": "LSAN_OPTIONS",
|
|
20
|
+
"value": "verbosity=1:log_threads=1"
|
|
17
21
|
}
|
|
18
22
|
],
|
|
19
23
|
"stopAtEntry": false,
|
|
20
24
|
"linux": {
|
|
21
|
-
"preLaunchTask": "Linux Build C++
|
|
25
|
+
"preLaunchTask": "Linux Build C++ Debug",
|
|
22
26
|
"MIMode": "gdb"
|
|
23
27
|
},
|
|
24
28
|
"osx": {
|
|
25
|
-
"preLaunchTask": "MacOS Build C++
|
|
29
|
+
"preLaunchTask": "MacOS Build C++ Debug",
|
|
26
30
|
"MIMode": "lldb"
|
|
27
31
|
}
|
|
28
32
|
},
|
|
29
33
|
{
|
|
30
|
-
"name": "Debug C++
|
|
34
|
+
"name": "Debug C++ on Wiki 1M @Cos",
|
|
31
35
|
"type": "cppdbg",
|
|
32
36
|
"request": "launch",
|
|
33
37
|
"program": "${workspaceFolder}/build_debug/bench",
|
|
@@ -49,7 +53,7 @@
|
|
|
49
53
|
],
|
|
50
54
|
"stopAtEntry": false,
|
|
51
55
|
"linux": {
|
|
52
|
-
"preLaunchTask": "Linux Build C++
|
|
56
|
+
"preLaunchTask": "Linux Build C++ Debug",
|
|
53
57
|
"MIMode": "gdb",
|
|
54
58
|
"environment": [
|
|
55
59
|
{
|
|
@@ -59,12 +63,12 @@
|
|
|
59
63
|
]
|
|
60
64
|
},
|
|
61
65
|
"osx": {
|
|
62
|
-
"preLaunchTask": "MacOS Build C++
|
|
66
|
+
"preLaunchTask": "MacOS Build C++ Debug",
|
|
63
67
|
"MIMode": "lldb"
|
|
64
68
|
}
|
|
65
69
|
},
|
|
66
70
|
{
|
|
67
|
-
"name": "Debug C++
|
|
71
|
+
"name": "Debug C++ on Text-to-Image 1B @Cos",
|
|
68
72
|
"type": "cppdbg",
|
|
69
73
|
"request": "launch",
|
|
70
74
|
"program": "${workspaceFolder}/build_debug/bench",
|
|
@@ -86,16 +90,16 @@
|
|
|
86
90
|
],
|
|
87
91
|
"stopAtEntry": false,
|
|
88
92
|
"linux": {
|
|
89
|
-
"preLaunchTask": "Linux Build C++
|
|
93
|
+
"preLaunchTask": "Linux Build C++ Debug",
|
|
90
94
|
"MIMode": "gdb"
|
|
91
95
|
},
|
|
92
96
|
"osx": {
|
|
93
|
-
"preLaunchTask": "MacOS Build C++
|
|
97
|
+
"preLaunchTask": "MacOS Build C++ Debug",
|
|
94
98
|
"MIMode": "lldb"
|
|
95
99
|
}
|
|
96
100
|
},
|
|
97
101
|
{
|
|
98
|
-
"name": "Run C++
|
|
102
|
+
"name": "Run C++ on Text-to-Image 1M @Cos",
|
|
99
103
|
"type": "cppdbg",
|
|
100
104
|
"request": "launch",
|
|
101
105
|
"program": "${workspaceFolder}/build_release/bench",
|
|
@@ -109,7 +113,7 @@
|
|
|
109
113
|
"datasets/t2i_1M/groundtruth.public.100K.ibin",
|
|
110
114
|
],
|
|
111
115
|
"stopAtEntry": false,
|
|
112
|
-
"preLaunchTask": "Linux Build C++
|
|
116
|
+
"preLaunchTask": "Linux Build C++ Release",
|
|
113
117
|
"MIMode": "gdb"
|
|
114
118
|
},
|
|
115
119
|
{
|
package/.vscode/settings.json
CHANGED
|
@@ -100,7 +100,9 @@
|
|
|
100
100
|
"version": "cpp",
|
|
101
101
|
"future": "cpp",
|
|
102
102
|
"__verbose_abort": "cpp",
|
|
103
|
-
"shared_mutex": "cpp"
|
|
103
|
+
"shared_mutex": "cpp",
|
|
104
|
+
"ranges": "cpp",
|
|
105
|
+
"span": "cpp"
|
|
104
106
|
},
|
|
105
107
|
"cSpell.words": [
|
|
106
108
|
"allclose",
|
|
@@ -151,6 +153,14 @@
|
|
|
151
153
|
"python.testing.pytestArgs": [
|
|
152
154
|
"python"
|
|
153
155
|
],
|
|
156
|
+
"python.analysis.diagnosticSeverityOverrides": {
|
|
157
|
+
"reportMissingImports": "none"
|
|
158
|
+
},
|
|
154
159
|
"python.testing.unittestEnabled": false,
|
|
155
|
-
"python.testing.pytestEnabled": true
|
|
160
|
+
"python.testing.pytestEnabled": true,
|
|
161
|
+
"[go]": {
|
|
162
|
+
"editor.insertSpaces": true,
|
|
163
|
+
"editor.formatOnSave": true,
|
|
164
|
+
"editor.defaultFormatter": "golang.go"
|
|
165
|
+
},
|
|
156
166
|
}
|
package/.vscode/tasks.json
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"version": "2.0.0",
|
|
3
3
|
"tasks": [
|
|
4
4
|
{
|
|
5
|
-
"label": "Linux Build C++
|
|
5
|
+
"label": "Linux Build C++ Debug",
|
|
6
6
|
"command": "cmake -DCMAKE_CXX_COMPILER=gcc-12 -DCMAKE_CXX_COMPILER=g++-12 -DCMAKE_BUILD_TYPE=Debug -B ./build_debug && make -C ./build_debug",
|
|
7
7
|
"args": [],
|
|
8
8
|
"type": "shell",
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
]
|
|
12
12
|
},
|
|
13
13
|
{
|
|
14
|
-
"label": "Linux Build C++
|
|
14
|
+
"label": "Linux Build C++ Release",
|
|
15
15
|
"command": "cmake -DCMAKE_CXX_COMPILER=gcc-12 -DCMAKE_CXX_COMPILER=g++-12 -DCMAKE_BUILD_TYPE=RelWithDebInfo -B ./build_release && make -C ./build_release",
|
|
16
16
|
"args": [],
|
|
17
17
|
"type": "shell",
|
|
@@ -20,13 +20,13 @@
|
|
|
20
20
|
]
|
|
21
21
|
},
|
|
22
22
|
{
|
|
23
|
-
"label": "MacOS Build C++
|
|
23
|
+
"label": "MacOS Build C++ Debug",
|
|
24
24
|
"command": "cmake -DCMAKE_C_COMPILER=/opt/homebrew/opt/llvm/bin/clang -DCMAKE_CXX_COMPILER=/opt/homebrew/opt/llvm/bin/clang++ -DCMAKE_BUILD_TYPE=Debug -B ./build_debug && make -C ./build_debug",
|
|
25
25
|
"args": [],
|
|
26
26
|
"type": "shell",
|
|
27
27
|
},
|
|
28
28
|
{
|
|
29
|
-
"label": "MacOS Build C++
|
|
29
|
+
"label": "MacOS Build C++ Release",
|
|
30
30
|
"command": "cmake -DCMAKE_C_COMPILER=/opt/homebrew/opt/llvm/bin/clang -DCMAKE_CXX_COMPILER=/opt/homebrew/opt/llvm/bin/clang++ -DCMAKE_BUILD_TYPE=RelWithDebInfo -B ./build_release && make -C ./build_release",
|
|
31
31
|
"args": [],
|
|
32
32
|
"type": "shell"
|
package/CMakeLists.txt
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# This CMake file is heavily inspired by following `usearch` CMake:
|
|
2
2
|
# https://github.com/nlohmann/json/blob/develop/CMakeLists.txt
|
|
3
3
|
cmake_minimum_required(VERSION 3.1)
|
|
4
|
-
project(usearch VERSION 0.6.0 LANGUAGES CXX)
|
|
4
|
+
project(usearch VERSION 0.6.0 LANGUAGES C CXX)
|
|
5
5
|
|
|
6
6
|
# Determine if USearch is built as a subproject (using `add_subdirectory`) or if it is the main project
|
|
7
7
|
set(USEARCH_IS_MAIN_PROJECT OFF)
|
|
@@ -14,6 +14,7 @@ option(USEARCH_INSTALL "Install CMake targets" OFF)
|
|
|
14
14
|
option(USEARCH_BUILD_TEST "Compile a native unit test in C++" ${USEARCH_IS_MAIN_PROJECT})
|
|
15
15
|
option(USEARCH_BUILD_BENCHMARK "Compile a native benchmark in C++" ${USEARCH_IS_MAIN_PROJECT})
|
|
16
16
|
option(USEARCH_BUILD_WOLFRAM "Compile Wolfram Language bindings" OFF)
|
|
17
|
+
option(USEARCH_BUILD_CLIB "Compile a native library for the C 99 interface" OFF)
|
|
17
18
|
|
|
18
19
|
# Includes
|
|
19
20
|
set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake ${CMAKE_MODULE_PATH})
|
|
@@ -122,3 +123,7 @@ endif()
|
|
|
122
123
|
if(${USEARCH_BUILD_WASM})
|
|
123
124
|
add_subdirectory(wasm)
|
|
124
125
|
endif()
|
|
126
|
+
|
|
127
|
+
if (${USEARCH_BUILD_CLIB})
|
|
128
|
+
add_subdirectory(c)
|
|
129
|
+
endif()
|
package/Cargo.toml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
[package]
|
|
2
2
|
name = "usearch"
|
|
3
|
-
version = "0.
|
|
4
|
-
authors = ["
|
|
3
|
+
version = "1.0.0"
|
|
4
|
+
authors = ["Ash Vardanian <1983160+ashvardanian@users.noreply.github.com>"]
|
|
5
5
|
description = "Smaller & Faster Single-File Vector Search Engine from Unum"
|
|
6
6
|
edition = "2021"
|
|
7
7
|
license = "Apache-2.0"
|
package/README.md
CHANGED
|
@@ -38,17 +38,15 @@ Linux • MacOS • Windows • Docker • WebAssembly
|
|
|
38
38
|
|
|
39
39
|
- ✅ Benchmark-topping performance.
|
|
40
40
|
- ✅ Simple and extensible [single C++11 header][usearch-header] implementation.
|
|
41
|
-
- ✅ SIMD-optimized and [user-defined metrics](#user-defined-functions) with JIT
|
|
42
|
-
- ✅ Variable dimensionality vectors for unique applications, including
|
|
43
|
-
- ✅
|
|
44
|
-
- ✅
|
|
41
|
+
- ✅ SIMD-optimized and [user-defined metrics](#user-defined-functions) with JIT compilation.
|
|
42
|
+
- ✅ Variable dimensionality vectors for unique applications, including search over compressed data.
|
|
43
|
+
- ✅ Bitwise Tanimoto and Sorensen coefficients for [Genomics and Chemistry applications](#usearch--rdkit--molecular-search).
|
|
44
|
+
- ✅ Hardware-agnostic `f16` & `i8` - [half-precision & quarter-precision support](#memory-efficiency-downcasting-and-quantization).
|
|
45
45
|
- ✅ [View large indexes from disk](#disk-based-indexes) without loading into RAM.
|
|
46
46
|
- ✅ Space-efficient point-clouds with `uint40_t`, accommodating 4B+ size.
|
|
47
|
-
- ✅
|
|
48
|
-
- ✅
|
|
49
|
-
- ✅
|
|
50
|
-
- ✅ Utilize [USearch and UForm Transformers for Semantic Search](#usearch--ai--multi-modal-semantic-search).
|
|
51
|
-
- ✅ Combine [USearch and RDKit for Molecule Search](#usearch--rdkit--molecular-search).
|
|
47
|
+
- ✅ Compatible with OpenMP and custom "executors", for fine-grained control over CPU utilization.
|
|
48
|
+
- ✅ Heterogeneous lookups, renaming/relabeling, and on-the-fly deletions.
|
|
49
|
+
- ✅ [Semantic Search](#usearch--ai--multi-modal-semantic-search) and [Joins](#joins).
|
|
52
50
|
|
|
53
51
|
[usearch-header]: https://github.com/unum-cloud/usearch/blob/main/include/usearch/index.hpp
|
|
54
52
|
[obscure-use-cases]: https://ashvardanian.com/posts/abusing-vector-search
|
|
@@ -70,7 +68,7 @@ USearch is compact and broadly compatible without sacrificing performance, with
|
|
|
70
68
|
|
|
71
69
|
[sloc]: https://en.wikipedia.org/wiki/Source_lines_of_code
|
|
72
70
|
|
|
73
|
-
Base functionality is identical to FAISS, and the interface must be familiar if you have ever investigated Approximate Nearest
|
|
71
|
+
Base functionality is identical to FAISS, and the interface must be familiar if you have ever investigated Approximate Nearest Neighbors search:
|
|
74
72
|
|
|
75
73
|
```py
|
|
76
74
|
$ pip install usearch numpy
|
|
@@ -81,7 +79,7 @@ from usearch.index import Index
|
|
|
81
79
|
index = Index(
|
|
82
80
|
ndim=3, # Define the number of dimensions in input vectors
|
|
83
81
|
metric='cos', # Choose 'l2sq', 'haversine' or other metric, default = 'ip'
|
|
84
|
-
dtype='f32', # Quantize to 'f16' or '
|
|
82
|
+
dtype='f32', # Quantize to 'f16' or 'i8' if needed, default = 'f32'
|
|
85
83
|
connectivity=16, # Optional: How frequent should the connections in the graph be
|
|
86
84
|
expansion_add=128, # Optional: Control the recall of indexing
|
|
87
85
|
expansion_search=64, # Optional: Control the quality of search
|
|
@@ -89,12 +87,12 @@ index = Index(
|
|
|
89
87
|
|
|
90
88
|
vector = np.array([0.2, 0.6, 0.4])
|
|
91
89
|
index.add(42, vector)
|
|
92
|
-
matches
|
|
90
|
+
matches: Matches = index.search(vector, 10)
|
|
93
91
|
|
|
94
92
|
assert len(index) == 1
|
|
95
|
-
assert
|
|
96
|
-
assert matches[0] == 42
|
|
97
|
-
assert
|
|
93
|
+
assert len(matches) == 1
|
|
94
|
+
assert matches[0].key == 42
|
|
95
|
+
assert matches[0].distance <= 0.001
|
|
98
96
|
assert np.allclose(index[42], vector)
|
|
99
97
|
```
|
|
100
98
|
|
|
@@ -109,7 +107,7 @@ Unlike older approaches indexing high-dimensional spaces, like KD-Trees and Loca
|
|
|
109
107
|
They only have to be comparable.
|
|
110
108
|
So you can apply it in [obscure][obscure] applications, like searching for similar sets or fuzzy text matching, using [GZip][gzip-similarity] as a distance function.
|
|
111
109
|
|
|
112
|
-
> Read more about [JIT and UDF in USearch Python SDK]().
|
|
110
|
+
> Read more about [JIT and UDF in USearch Python SDK](https://unum-cloud.github.io/usearch/python#user-defined-metrics-and-jit-in-python).
|
|
113
111
|
|
|
114
112
|
[haversine]: https://ashvardanian.com/posts/abusing-vector-search#geo-spatial-indexing
|
|
115
113
|
[obscure]: https://ashvardanian.com/posts/abusing-vector-search
|
|
@@ -123,10 +121,10 @@ Those, however, are only sometimes reliable, can significantly affect the statis
|
|
|
123
121
|

|
|
124
122
|
|
|
125
123
|
Instead, we have focused on high-precision arithmetic over low-precision downcasted vectors.
|
|
126
|
-
The same index, and `add` and `search` operations will automatically down-cast or up-cast between `f32_t`, `f16_t`, `f64_t`, and `
|
|
127
|
-
Continuing the topic of memory
|
|
124
|
+
The same index, and `add` and `search` operations will automatically down-cast or up-cast between `f32_t`, `f16_t`, `f64_t`, and `i8_t` representations, even if the hardware doesn't natively support it.
|
|
125
|
+
Continuing the topic of memory efficiency, we provide a `uint40_t` to allow collection with over 4B+ vectors without allocating 8 bytes for every neighbor reference in the proximity graph.
|
|
128
126
|
|
|
129
|
-
| | FAISS, `f32` | USearch, `f32` | USearch, `f16` | USearch, `
|
|
127
|
+
| | FAISS, `f32` | USearch, `f32` | USearch, `f16` | USearch, `i8` |
|
|
130
128
|
| :----------- | -----------: | -------------: | -------------: | ----------------: |
|
|
131
129
|
| Batch Insert | 16 K/s | 73 K/s | 100 K/s | 104 K/s **+550%** |
|
|
132
130
|
| Batch Search | 82 K/s | 103 K/s | 113 K/s | 134 K/s **+63%** |
|
|
@@ -149,7 +147,7 @@ Continuing the topic of memory-efficiency, we provide a `uint40_t` to allow coll
|
|
|
149
147
|
## Disk-based Indexes
|
|
150
148
|
|
|
151
149
|
With USearch, you can serve indexes from external memory, enabling you to optimize your server choices for indexing speed and serving costs.
|
|
152
|
-
This can result in **20x
|
|
150
|
+
This can result in **20x cost reduction** on AWS and other public clouds.
|
|
153
151
|
|
|
154
152
|
```py
|
|
155
153
|
index.save("index.usearch")
|
|
@@ -163,7 +161,7 @@ other_view.view("index.usearch")
|
|
|
163
161
|
|
|
164
162
|
## Joins
|
|
165
163
|
|
|
166
|
-
One of the big questions these days is how will AI change the world of databases and data
|
|
164
|
+
One of the big questions these days is how will AI change the world of databases and data management.
|
|
167
165
|
Most databases are still struggling to implement high-quality fuzzy search, and the only kind of joins they know are deterministic.
|
|
168
166
|
A `join` is different from searching for every entry, as it requires a one-to-one mapping, banning collisions among separate search results.
|
|
169
167
|
|
|
@@ -172,7 +170,7 @@ A `join` is different from searching for every entry, as it requires a one-to-on
|
|
|
172
170
|
| Exact Join | Fuzzy Join ? | Semantic Join ?? |
|
|
173
171
|
|
|
174
172
|
Using USearch one can implement sub-quadratic complexity approximate, fuzzy, and semantic joins.
|
|
175
|
-
This can come handy in any fuzzy-matching tasks, common to Database Management Software.
|
|
173
|
+
This can come in handy in any fuzzy-matching tasks, common to Database Management Software.
|
|
176
174
|
|
|
177
175
|
```py
|
|
178
176
|
men = Index(...)
|
|
@@ -184,17 +182,17 @@ pairs: dict = men.join(women, max_proposals=0, exact=False)
|
|
|
184
182
|
|
|
185
183
|
## Functionality
|
|
186
184
|
|
|
187
|
-
By now, core functionality is supported across all bindings.
|
|
185
|
+
By now, the core functionality is supported across all bindings.
|
|
188
186
|
Broader functionality is ported per request.
|
|
189
187
|
|
|
190
|
-
| |
|
|
191
|
-
| :---------------------- |
|
|
192
|
-
|
|
|
193
|
-
|
|
|
194
|
-
|
|
|
195
|
-
|
|
|
196
|
-
|
|
|
197
|
-
| 4B+ capacities | ✅
|
|
188
|
+
| | C++ 11 | Python 3 | C 99 | Java | JavaScript | Rust | GoLang | Swift |
|
|
189
|
+
| :---------------------- | :----: | :------: | :---: | :---: | :--------: | :---: | :----: | :---: |
|
|
190
|
+
| Add, search | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
|
|
191
|
+
| Save, load, view | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
|
|
192
|
+
| Join | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ |
|
|
193
|
+
| User-defined metrics | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ |
|
|
194
|
+
| Variable-length vectors | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
|
|
195
|
+
| 4B+ capacities | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
|
|
198
196
|
|
|
199
197
|
## Application Examples
|
|
200
198
|
|
|
@@ -216,22 +214,22 @@ model = uform.get_model('unum-cloud/uform-vl-multilingual')
|
|
|
216
214
|
index = usearch.index.Index(ndim=256)
|
|
217
215
|
|
|
218
216
|
@server
|
|
219
|
-
def add(
|
|
217
|
+
def add(key: int, photo: pil.Image.Image):
|
|
220
218
|
image = model.preprocess_image(photo)
|
|
221
219
|
vector = model.encode_image(image).detach().numpy()
|
|
222
|
-
index.add(
|
|
220
|
+
index.add(key, vector.flatten(), copy=True)
|
|
223
221
|
|
|
224
222
|
@server
|
|
225
223
|
def search(query: str) -> np.ndarray:
|
|
226
224
|
tokens = model.preprocess_text(query)
|
|
227
225
|
vector = model.encode_text(tokens).detach().numpy()
|
|
228
226
|
matches = index.search(vector.flatten(), 3)
|
|
229
|
-
return matches.
|
|
227
|
+
return matches.keys
|
|
230
228
|
|
|
231
229
|
server.run()
|
|
232
230
|
```
|
|
233
231
|
|
|
234
|
-
We have pre-processed some commonly used datasets,
|
|
232
|
+
We have pre-processed some commonly used datasets, cleaned the images, produced the vectors, and pre-built the index.
|
|
235
233
|
|
|
236
234
|
| Dataset | Modalities | Images | Download |
|
|
237
235
|
| :------------------------------------- | --------------------: | -----: | ------------------------------------: |
|
|
@@ -251,10 +249,10 @@ We have pre-processed some commonly used datasets, cleaning the images, producin
|
|
|
251
249
|
|
|
252
250
|
Comparing molecule graphs and searching for similar structures is expensive and slow.
|
|
253
251
|
It can be seen as a special case of the NP-Complete Subgraph Isomorphism problem.
|
|
254
|
-
Luckily, domain-specific approximate methods
|
|
252
|
+
Luckily, domain-specific approximate methods exist.
|
|
255
253
|
The one commonly used in Chemistry, is to generate structures from [SMILES][smiles], and later hash them into binary fingerprints.
|
|
256
|
-
The
|
|
257
|
-
Below is
|
|
254
|
+
The latter are searchable with bitwise similarity metrics, like the Tanimoto coefficient.
|
|
255
|
+
Below is an example using the RDKit package.
|
|
258
256
|
|
|
259
257
|
```python
|
|
260
258
|
from usearch.index import Index, MetricKind
|
|
@@ -270,27 +268,22 @@ fingerprints = np.vstack([encoder.GetFingerprint(x) for x in molecules])
|
|
|
270
268
|
fingerprints = np.packbits(fingerprints, axis=1)
|
|
271
269
|
|
|
272
270
|
index = Index(ndim=2048, metric=MetricKind.Tanimoto)
|
|
273
|
-
|
|
271
|
+
keys = np.arange(len(molecules))
|
|
274
272
|
|
|
275
|
-
index.add(
|
|
273
|
+
index.add(keys, fingerprints)
|
|
276
274
|
matches = index.search(fingerprints, 10)
|
|
277
275
|
```
|
|
278
276
|
|
|
279
277
|
[smiles]: https://en.wikipedia.org/wiki/Simplified_molecular-input_line-entry_system
|
|
280
278
|
[rdkit-fingerprints]: https://www.rdkit.org/docs/RDKit_Book.html#additional-information-about-the-fingerprints
|
|
281
279
|
|
|
282
|
-
## TODO
|
|
283
|
-
|
|
284
|
-
- JavaScript: Allow calling from "worker threads".
|
|
285
|
-
- Rust: Allow passing a custom thread ID.
|
|
286
|
-
- C# .NET bindings.
|
|
287
280
|
|
|
288
281
|
## Integrations
|
|
289
282
|
|
|
290
283
|
- [x] GPT-Cache.
|
|
291
284
|
- [ ] LangChain.
|
|
292
285
|
- [ ] Microsoft Semantic Kernel.
|
|
293
|
-
- [ ]
|
|
286
|
+
- [ ] ClickHouse.
|
|
294
287
|
|
|
295
288
|
## Citations
|
|
296
289
|
|
package/build.rs
CHANGED
|
@@ -13,7 +13,7 @@ fn main() {
|
|
|
13
13
|
println!("cargo:rerun-if-changed=rust/lib.rs");
|
|
14
14
|
println!("cargo:rerun-if-changed=rust/lib.cpp");
|
|
15
15
|
println!("cargo:rerun-if-changed=rust/lib.hpp");
|
|
16
|
-
println!("cargo:rerun-if-changed=include/
|
|
17
|
-
println!("cargo:rerun-if-changed=include/
|
|
16
|
+
println!("cargo:rerun-if-changed=include/index_plugins.hpp");
|
|
17
|
+
println!("cargo:rerun-if-changed=include/index_dense.hpp");
|
|
18
18
|
println!("cargo:rerun-if-changed=include/usearch/index.hpp");
|
|
19
19
|
}
|
package/c/CMakeLists.txt
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
|
|
2
|
+
|
|
3
|
+
set(USEARCH_PUNNED_INCLUDE_DIRS
|
|
4
|
+
"${CMAKE_CURRENT_SOURCE_DIR}/../include"
|
|
5
|
+
"${CMAKE_CURRENT_SOURCE_DIR}/../fp16/include"
|
|
6
|
+
"${CMAKE_CURRENT_SOURCE_DIR}/../robin-map/include"
|
|
7
|
+
"${CMAKE_CURRENT_SOURCE_DIR}/../simsimd/include"
|
|
8
|
+
"${CMAKE_CURRENT_SOURCE_DIR}/"
|
|
9
|
+
)
|
|
10
|
+
|
|
11
|
+
# This article discusses a better way to allow building either static or shared libraries
|
|
12
|
+
# https://alexreinking.com/blog/building-a-dual-shared-and-static-library-with-cmake.html
|
|
13
|
+
if (${USEARCH_BUILD_STATIC})
|
|
14
|
+
set(CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS} -static-libstdc++")
|
|
15
|
+
add_library(usearch_c STATIC lib.cpp)
|
|
16
|
+
else ()
|
|
17
|
+
add_library(usearch_c SHARED lib.cpp)
|
|
18
|
+
endif()
|
|
19
|
+
|
|
20
|
+
set_target_properties(usearch_c PROPERTIES POSITION_INDEPENDENT_CODE ON)
|
|
21
|
+
set_target_properties(usearch_c PROPERTIES CXX_STANDARD 11)
|
|
22
|
+
|
|
23
|
+
target_include_directories(usearch_c PRIVATE ${USEARCH_PUNNED_INCLUDE_DIRS})
|
|
24
|
+
set_target_properties(usearch_c PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
|
|
25
|
+
set_target_properties(usearch_c PROPERTIES ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
|
|
26
|
+
set_target_properties(usearch_c PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
target_include_directories(usearch_c PRIVATE ${USEARCH_PUNNED_INCLUDE_DIRS})
|
|
30
|
+
|
|
31
|
+
if(${USEARCH_DEBUG_BUILD_ASAN})
|
|
32
|
+
# For ASAN in debug builds
|
|
33
|
+
target_link_libraries(usearch_c PRIVATE gcov)
|
|
34
|
+
endif()
|
|
35
|
+
|
|
36
|
+
if(${USEARCH_USE_SIMSIMD})
|
|
37
|
+
target_compile_definitions(usearch_c PRIVATE USEARCH_USE_SIMSIMD=1)
|
|
38
|
+
endif()
|
|
39
|
+
if (${USEARCH_LOOKUP_LABEL})
|
|
40
|
+
target_compile_definitions(usearch_c PRIVATE USEARCH_LOOKUP_LABEL=1)
|
|
41
|
+
endif()
|
|
42
|
+
|
|
43
|
+
if(${USEARCH_BUILD_TEST})
|
|
44
|
+
add_executable(test_c test.c external_retreival_testutils.c)
|
|
45
|
+
target_link_libraries(test_c usearch_c)
|
|
46
|
+
set_target_properties(test_c PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
|
|
47
|
+
|
|
48
|
+
endif()
|
package/c/README.md
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# USearch for C
|
|
2
|
+
|
|
3
|
+
## Installation
|
|
4
|
+
|
|
5
|
+
The simplest form to integrate is to copy the contents of `usearch/c/` into your project.
|
|
6
|
+
To build the library `libusearch_c`, pass enable the `USEARCH_BUILD_CLIB` CMake option:
|
|
7
|
+
|
|
8
|
+
```bash
|
|
9
|
+
cmake -DUSEARCH_BUILD_CLIB=1 -DUSEARCH_BUILD_STATIC=0 -DUSEARCH_BUILD_TEST=0 -DUSEARCH_BUILD_BENCHMARK=0 ..
|
|
10
|
+
```
|
|
11
|
+
|
|
12
|
+
You can choose between a static and dynamic library by setting `USEARCH_BUILD_STATIC`.
|
|
13
|
+
|
|
14
|
+
## Quickstart
|
|
15
|
+
|
|
16
|
+
```c
|
|
17
|
+
usearch_error_t error = NULL;
|
|
18
|
+
usearch_init_options_t opts = create_options(dimensions);
|
|
19
|
+
usearch_index_t index = usearch_init(&opts, &error);
|
|
20
|
+
usearch_reserve(index, vectors_count, &error);
|
|
21
|
+
|
|
22
|
+
float vector[dimensions] = {...};
|
|
23
|
+
usearch_add(index, 42, &vector[0], usearch_scalar_f32_k, &error);
|
|
24
|
+
|
|
25
|
+
ASSERT(!error, error);
|
|
26
|
+
ASSERT(usearch_size(index, &error) == vectors_count, error);
|
|
27
|
+
ASSERT(usearch_capacity(index, &error) == vectors_count, error);
|
|
28
|
+
|
|
29
|
+
usearch_key_t found_keys[10];
|
|
30
|
+
float found_distances[10];
|
|
31
|
+
size_t found_count = usearch_search(index, &vector[0], usearch_scalar_f32_k, 10, &found_keys[0], &found_distances[0], &error);
|
|
32
|
+
|
|
33
|
+
usearch_free(index, &error);
|
|
34
|
+
```
|