usearch 0.19.3 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/prerelease.yml +27 -0
- package/Cargo.toml +1 -1
- package/README.md +21 -2
- package/c/test.c +238 -126
- package/conanfile.py +1 -1
- package/cpp/CMakeLists.txt +6 -2
- package/cpp/bench.cpp +41 -4
- package/cpp/test.cpp +12 -12
- package/docs/benchmarks.md +14 -12
- package/include/usearch/index.hpp +603 -100
- package/include/usearch/index_punned_dense.hpp +68 -10
- package/include/usearch/index_punned_helpers.hpp +30 -3
- package/package.json +1 -1
- package/python/lib.cpp +57 -5
- package/python/scripts/join.py +151 -0
- package/python/scripts/test.py +19 -0
- package/python/usearch/eval.py +11 -3
- package/python/usearch/index.py +119 -10
- package/python/usearch/join.py +0 -98
|
@@ -17,6 +17,33 @@ permissions:
|
|
|
17
17
|
|
|
18
18
|
jobs:
|
|
19
19
|
|
|
20
|
+
build_test:
|
|
21
|
+
name: Test USearch
|
|
22
|
+
runs-on: ubuntu-latest
|
|
23
|
+
steps:
|
|
24
|
+
- uses: actions/checkout@v3
|
|
25
|
+
with:
|
|
26
|
+
ref: main-dev
|
|
27
|
+
- run: git submodule update --init --recursive
|
|
28
|
+
- name: Prepare Environment
|
|
29
|
+
run: |
|
|
30
|
+
sudo apt update &&
|
|
31
|
+
sudo apt install -y cmake g++-12 build-essential libjemalloc-dev
|
|
32
|
+
- name: Build
|
|
33
|
+
run: |
|
|
34
|
+
cmake -B ./build_release \
|
|
35
|
+
-DCMAKE_CXX_COMPILER="g++-12" \
|
|
36
|
+
-DCMAKE_BUILD_TYPE=Release \
|
|
37
|
+
-DUSEARCH_BUILD_TEST=1 \
|
|
38
|
+
-DUSEARCH_USE_OPENMP=1 \
|
|
39
|
+
-DUSEARCH_USE_SIMSIMD=1 \
|
|
40
|
+
-DUSEARCH_USE_JEMALLOC=1 \
|
|
41
|
+
-DUSEARCH_BUILD_BENCHMARK=0 &&
|
|
42
|
+
make -C ./build_release -j
|
|
43
|
+
- name: Run tests
|
|
44
|
+
run: ./build_release/test
|
|
45
|
+
|
|
46
|
+
|
|
20
47
|
test_python_311:
|
|
21
48
|
name: Test Python
|
|
22
49
|
runs-on: ${{ matrix.os }}
|
package/Cargo.toml
CHANGED
package/README.md
CHANGED
|
@@ -167,7 +167,7 @@ index.add(/* label: */ 42, /* vector: */ {&vec[0], 3});
|
|
|
167
167
|
auto results = index.search(/* query: */ {&vec[0], 3}, 5 /* neighbors */);
|
|
168
168
|
|
|
169
169
|
for (std::size_t i = 0; i != results.size(); ++i)
|
|
170
|
-
results[i].
|
|
170
|
+
results[i].member.label, results[i].member.vector, results[i].member.id, results[i].distance;
|
|
171
171
|
```
|
|
172
172
|
|
|
173
173
|
The `add` is thread-safe for concurrent index construction.
|
|
@@ -759,9 +759,28 @@ index = Index(ndim=2048, metric=MetricKind.BitwiseTanimoto)
|
|
|
759
759
|
labels = np.arange(len(molecules))
|
|
760
760
|
|
|
761
761
|
index.add(labels, fingerprints)
|
|
762
|
-
matches = index.search(fingerprints, 10)
|
|
762
|
+
matches: Matches = index.search(fingerprints, 10)
|
|
763
763
|
```
|
|
764
764
|
|
|
765
|
+
Of if you need a bit of our SIMD superpowers - take some Chemsitry-oriented precompiled metrics from [SimSIMD][simsimd].
|
|
766
|
+
|
|
767
|
+
```python
|
|
768
|
+
import simsimd as sisi
|
|
769
|
+
|
|
770
|
+
index = Index(
|
|
771
|
+
ndim=166, # MACCS fingerprints are 166-dimensional
|
|
772
|
+
metric=CompiledMetric(
|
|
773
|
+
pointer=sisi.to_int(sisi.tanimoto_maccs_neon), # For Arm Neon
|
|
774
|
+
signature=MetricSignature.ArrayArray,
|
|
775
|
+
kind=MetricKind.Tanimoto,
|
|
776
|
+
),
|
|
777
|
+
)
|
|
778
|
+
|
|
779
|
+
index.add(42, np.packbits([1] * 166))
|
|
780
|
+
matches: Matches = index.search(np.packbits([1] * 166))
|
|
781
|
+
```
|
|
782
|
+
|
|
783
|
+
[simsimd]: https://github.com/ashvardanian/simsimd
|
|
765
784
|
[smiles]: https://en.wikipedia.org/wiki/Simplified_molecular-input_line-entry_system
|
|
766
785
|
[rdkit-fingerprints]: https://www.rdkit.org/docs/RDKit_Book.html#additional-information-about-the-fingerprints
|
|
767
786
|
|
package/c/test.c
CHANGED
|
@@ -1,182 +1,294 @@
|
|
|
1
|
-
#include <
|
|
1
|
+
#include <stdio.h>
|
|
2
2
|
#include <errno.h>
|
|
3
3
|
#include <stdio.h>
|
|
4
4
|
#include <stdlib.h>
|
|
5
|
-
#include <string.h>
|
|
6
5
|
#include <sys/stat.h>
|
|
7
|
-
#include <time.h>
|
|
8
6
|
|
|
9
7
|
#include "usearch.h"
|
|
10
8
|
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
float skip;
|
|
16
|
-
|
|
17
|
-
;
|
|
18
|
-
if (stat(filename, &st) == -1) {
|
|
19
|
-
printf("Failed to read file: %s err: %d: %s\n", filename, errno, strerror(errno));
|
|
20
|
-
return NULL;
|
|
9
|
+
#define ASSERT(must_be_true, message) \
|
|
10
|
+
if (!(must_be_true)) { \
|
|
11
|
+
printf("Assert: %s\n", message); \
|
|
12
|
+
exit(-1); \
|
|
21
13
|
}
|
|
22
14
|
|
|
23
|
-
file = fopen(filename, "rb");
|
|
24
|
-
if (file == NULL) {
|
|
25
|
-
printf("Failed to open file: %s\n", filename);
|
|
26
|
-
return NULL;
|
|
27
|
-
}
|
|
28
15
|
|
|
29
|
-
|
|
30
|
-
cnt = fread(vector_dim, sizeof(int), 1, file);
|
|
31
|
-
assert(cnt == 1);
|
|
32
|
-
if (st.st_size % ((*vector_dim + 1) * sizeof(float)) != 0) {
|
|
33
|
-
printf("File does not contain a whole number of vectors.\n Vector dim: %ld, file size: %ld\n", *vector_dim,
|
|
34
|
-
st.st_size);
|
|
35
|
-
fclose(file);
|
|
36
|
-
return NULL;
|
|
37
|
-
}
|
|
16
|
+
void parse_vectors_from_file(char const* file_path, size_t* count, size_t* dimension, float** data) {
|
|
38
17
|
|
|
39
|
-
|
|
18
|
+
// Open the file
|
|
19
|
+
struct stat st = {};
|
|
20
|
+
ASSERT(stat(file_path, &st) == 0, "Failed to get metadata of the file");
|
|
21
|
+
FILE* file = fopen(file_path, "rb");
|
|
22
|
+
ASSERT(file, "Failed to open the file");
|
|
40
23
|
|
|
41
|
-
//
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
24
|
+
// Read vectors dimension and calculate count
|
|
25
|
+
int dim = 0;
|
|
26
|
+
size_t read = fread(&dim, sizeof(int), 1, file);
|
|
27
|
+
ASSERT(read == 1, "Failed to read from the file");
|
|
28
|
+
ASSERT(st.st_size % ((dim + 1) * sizeof(float)) == 0, "File does not contain a whole number of vectors");
|
|
29
|
+
*dimension = (size_t)dim;
|
|
30
|
+
*count = st.st_size / ((dim + 1) * sizeof(float));
|
|
48
31
|
|
|
49
|
-
//
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
32
|
+
// Allocate memory for the vectors' data
|
|
33
|
+
*data = (float*)malloc(*count * *dimension * sizeof(float));
|
|
34
|
+
ASSERT(*data, "Failed to allocate memory");
|
|
35
|
+
|
|
36
|
+
// Read the data
|
|
37
|
+
for (size_t i = 0; i < *count; ++i) {
|
|
38
|
+
read = fread(*data + i * *dimension, sizeof(float), *dimension, file);
|
|
39
|
+
ASSERT(read == *dimension, "Failed to read from the file");
|
|
40
|
+
if (i == *count - 1)
|
|
54
41
|
break;
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
42
|
+
|
|
43
|
+
// Skip
|
|
44
|
+
float skip;
|
|
45
|
+
read = fread(&skip, sizeof(float), 1, file);
|
|
46
|
+
ASSERT(read == 1, "Failed to read from the file");
|
|
58
47
|
}
|
|
59
48
|
|
|
60
49
|
fclose(file);
|
|
61
|
-
return data;
|
|
62
50
|
}
|
|
63
51
|
|
|
64
|
-
|
|
65
|
-
const int LOG_GRANULARITY = 10000;
|
|
66
|
-
|
|
67
|
-
void test_index(float* data, size_t num_vectors, size_t vector_dim) {
|
|
52
|
+
usearch_init_options_t create_options(size_t vector_dimension) {
|
|
68
53
|
usearch_init_options_t opts;
|
|
69
|
-
usearch_index_t idx;
|
|
70
|
-
usearch_error_t error = NULL;
|
|
71
|
-
clock_t start;
|
|
72
54
|
opts.connectivity = 2; // 32 in faiss
|
|
73
|
-
opts.dimensions =
|
|
55
|
+
opts.dimensions = vector_dimension;
|
|
74
56
|
opts.expansion_add = 40; // 40 in faiss
|
|
75
57
|
opts.expansion_search = 16; // 10 in faiss
|
|
76
58
|
opts.metric_kind = usearch_metric_ip_k;
|
|
77
59
|
opts.metric = NULL;
|
|
78
60
|
opts.quantization = usearch_scalar_f32_k;
|
|
61
|
+
return opts;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
void test_init(size_t vectors_count, size_t vector_dimension) {
|
|
65
|
+
printf("Test: Index Initialization...\n");
|
|
66
|
+
|
|
67
|
+
// Init index
|
|
68
|
+
usearch_index_t idx = NULL;
|
|
69
|
+
usearch_error_t error = NULL;
|
|
70
|
+
usearch_init_options_t opts = create_options(vector_dimension);
|
|
71
|
+
idx = usearch_init(&opts, &error);
|
|
72
|
+
ASSERT(!error, error);
|
|
73
|
+
usearch_free(idx, &error);
|
|
74
|
+
ASSERT(!error, error);
|
|
75
|
+
|
|
76
|
+
// Init second time
|
|
77
|
+
idx = usearch_init(&opts, &error);
|
|
78
|
+
ASSERT(!error, error);
|
|
79
|
+
|
|
80
|
+
ASSERT(usearch_size(idx, &error) == 0, error);
|
|
81
|
+
ASSERT(usearch_capacity(idx, &error) == 0, error);
|
|
82
|
+
ASSERT(usearch_dimensions(idx, &error) == vector_dimension, error);
|
|
83
|
+
ASSERT(usearch_connectivity(idx, &error) == opts.connectivity, error);
|
|
84
|
+
|
|
85
|
+
// Reserve
|
|
86
|
+
usearch_reserve(idx, vectors_count, &error);
|
|
87
|
+
ASSERT(!error, error);
|
|
88
|
+
ASSERT(usearch_size(idx, &error) == 0, error);
|
|
89
|
+
ASSERT(usearch_capacity(idx, &error) == vectors_count, error);
|
|
90
|
+
ASSERT(usearch_dimensions(idx, &error) == vector_dimension, error);
|
|
91
|
+
ASSERT(usearch_connectivity(idx, &error) == opts.connectivity, error);
|
|
92
|
+
|
|
93
|
+
usearch_free(idx, &error);
|
|
94
|
+
ASSERT(!error, error);
|
|
95
|
+
|
|
96
|
+
printf("Test: Index Initialization - PASSED\n");
|
|
97
|
+
}
|
|
79
98
|
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
num_vectors /= 10;
|
|
83
|
-
usearch_label_t* labels = (usearch_label_t*)calloc(k, sizeof(usearch_label_t));
|
|
84
|
-
float* distances = (float*)calloc(k, sizeof(float));
|
|
99
|
+
void test_add_vector(size_t vectors_count, size_t vector_dimension, float const* data) {
|
|
100
|
+
printf("Test: Add Vector...\n");
|
|
85
101
|
|
|
86
|
-
|
|
87
|
-
|
|
102
|
+
usearch_index_t idx = NULL;
|
|
103
|
+
usearch_error_t error = NULL;
|
|
104
|
+
usearch_init_options_t opts = create_options(vector_dimension);
|
|
88
105
|
idx = usearch_init(&opts, &error);
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
106
|
+
usearch_reserve(idx, vectors_count, &error);
|
|
107
|
+
|
|
108
|
+
// Add vectors
|
|
109
|
+
for (size_t i = 0; i < vectors_count; ++i) {
|
|
110
|
+
usearch_label_t label = i;
|
|
111
|
+
usearch_add(idx, label, data + i * vector_dimension, usearch_scalar_f32_k, &error);
|
|
112
|
+
ASSERT(!error, error);
|
|
92
113
|
}
|
|
93
114
|
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
assert(!error);
|
|
115
|
+
ASSERT(usearch_size(idx, &error) == vectors_count, error);
|
|
116
|
+
ASSERT(usearch_capacity(idx, &error) == vectors_count, error);
|
|
97
117
|
|
|
98
|
-
|
|
99
|
-
for (size_t i = 0; i <
|
|
100
|
-
if (i % LOG_GRANULARITY == 0) {
|
|
101
|
-
fprintf(stderr, "added %ld vectors...\n", i);
|
|
102
|
-
}
|
|
118
|
+
// Check vectors in the index
|
|
119
|
+
for (size_t i = 0; i < vectors_count; ++i) {
|
|
103
120
|
usearch_label_t label = i;
|
|
104
|
-
|
|
105
|
-
assert(!error);
|
|
121
|
+
ASSERT(usearch_contains(idx, label, &error), error);
|
|
106
122
|
}
|
|
123
|
+
ASSERT(!usearch_contains(idx, -1, &error), error); // Non existing label
|
|
124
|
+
|
|
125
|
+
usearch_free(idx, &error);
|
|
126
|
+
printf("Test: Add Vector - PASSED\n");
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
void test_find_vector(size_t vectors_count, size_t vector_dimension, float const* data) {
|
|
130
|
+
printf("Test: Find Vector...\n");
|
|
107
131
|
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
132
|
+
usearch_index_t idx = NULL;
|
|
133
|
+
usearch_error_t error = NULL;
|
|
134
|
+
usearch_init_options_t opts = create_options(vector_dimension);
|
|
135
|
+
idx = usearch_init(&opts, &error);
|
|
136
|
+
usearch_reserve(idx, vectors_count, &error);
|
|
137
|
+
|
|
138
|
+
// Create result buffers
|
|
139
|
+
int results_count = 10;
|
|
140
|
+
usearch_label_t* labels = (usearch_label_t*)malloc(results_count * sizeof(usearch_label_t));
|
|
141
|
+
float* distances = (float*)malloc(results_count * sizeof(float));
|
|
142
|
+
ASSERT(labels && distances, "Failed to allocate memory");
|
|
143
|
+
|
|
144
|
+
// Add vectors
|
|
145
|
+
for (size_t i = 0; i < vectors_count; ++i) {
|
|
113
146
|
usearch_label_t label = i;
|
|
114
|
-
|
|
115
|
-
|
|
147
|
+
usearch_add(idx, label, data + i * vector_dimension, usearch_scalar_f32_k, &error);
|
|
148
|
+
ASSERT(!error, error);
|
|
116
149
|
}
|
|
117
150
|
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
fprintf(stderr, "searched %ld vectors...\n", i);
|
|
125
|
-
}
|
|
126
|
-
const void *query_vector = data + i * vector_dim;
|
|
127
|
-
usearch_search(idx, query_vector, usearch_scalar_f32_k, k, labels, distances, &error);
|
|
128
|
-
assert(!error);
|
|
151
|
+
// Find the vectors
|
|
152
|
+
for (size_t i = 0; i < vectors_count; i++) {
|
|
153
|
+
const void *query_vector = data + i * vector_dimension;
|
|
154
|
+
size_t found_count = usearch_search(idx, query_vector, usearch_scalar_f32_k, results_count, labels, distances, &error);
|
|
155
|
+
ASSERT(!error, error);
|
|
156
|
+
ASSERT(found_count = results_count, "Vector is missing");
|
|
129
157
|
}
|
|
130
158
|
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
159
|
+
usearch_free(idx, &error);
|
|
160
|
+
printf("Test: Find Vector - PASSED\n");
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
void test_remove_vector(size_t vectors_count, size_t vector_dimension, float const* data) {
|
|
164
|
+
printf("Test: Remove Vector...\n");
|
|
165
|
+
|
|
166
|
+
usearch_index_t idx = NULL;
|
|
167
|
+
usearch_error_t error = NULL;
|
|
168
|
+
usearch_init_options_t opts = create_options(vector_dimension);
|
|
169
|
+
idx = usearch_init(&opts, &error);
|
|
170
|
+
usearch_reserve(idx, vectors_count, &error);
|
|
171
|
+
|
|
172
|
+
// Add vectors
|
|
173
|
+
for (size_t i = 0; i < vectors_count; ++i) {
|
|
174
|
+
usearch_label_t label = i;
|
|
175
|
+
usearch_add(idx, label, data + i * vector_dimension, usearch_scalar_f32_k, &error);
|
|
176
|
+
ASSERT(!error, error);
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
// Remove the vectors
|
|
180
|
+
for (size_t i = 0; i < vectors_count; i++) {
|
|
181
|
+
usearch_label_t label = i;
|
|
182
|
+
usearch_remove(idx, label, &error);
|
|
183
|
+
ASSERT(error, "Currently, Remove is not supported");
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
usearch_free(idx, &error);
|
|
187
|
+
printf("Test: Remove Vector - PASSED\n");
|
|
188
|
+
}
|
|
136
189
|
|
|
137
|
-
|
|
138
|
-
|
|
190
|
+
void test_save_load(size_t vectors_count, size_t vector_dimension, float const* data) {
|
|
191
|
+
printf("Test: Save/Load...\n");
|
|
192
|
+
|
|
193
|
+
usearch_index_t idx = NULL;
|
|
194
|
+
usearch_error_t error = NULL;
|
|
195
|
+
usearch_init_options_t opts = create_options(vector_dimension);
|
|
196
|
+
idx = usearch_init(&opts, &error);
|
|
197
|
+
usearch_reserve(idx, vectors_count, &error);
|
|
198
|
+
|
|
199
|
+
// Add vectors
|
|
200
|
+
for (size_t i = 0; i < vectors_count; ++i) {
|
|
201
|
+
usearch_label_t label = i;
|
|
202
|
+
usearch_add(idx, label, data + i * vector_dimension, usearch_scalar_f32_k, &error);
|
|
203
|
+
ASSERT(!error, error);
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
// Save and free the index
|
|
207
|
+
usearch_save(idx, "usearch_index.bin", &error);
|
|
208
|
+
ASSERT(!error, error);
|
|
139
209
|
usearch_free(idx, &error);
|
|
140
|
-
|
|
141
|
-
fprintf(stderr, "destroying the index took %.2f ms\n", ((double)(clock() - start)) / CLOCKS_PER_SEC * 1000);
|
|
210
|
+
ASSERT(!error, error);
|
|
142
211
|
|
|
143
|
-
|
|
144
|
-
start = clock();
|
|
212
|
+
// Reinit
|
|
145
213
|
idx = usearch_init(&opts, &error);
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
214
|
+
ASSERT(!error, error);
|
|
215
|
+
ASSERT(usearch_size(idx, &error) == 0, error);
|
|
216
|
+
|
|
217
|
+
// Load
|
|
149
218
|
usearch_load(idx, "usearch_index.bin", &error);
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
219
|
+
ASSERT(!error, error);
|
|
220
|
+
ASSERT(usearch_size(idx, &error) == vectors_count, error);
|
|
221
|
+
ASSERT(usearch_capacity(idx, &error) == vectors_count, error);
|
|
222
|
+
ASSERT(usearch_dimensions(idx, &error) == vector_dimension, error);
|
|
223
|
+
ASSERT(usearch_connectivity(idx, &error) == opts.connectivity, error);
|
|
224
|
+
|
|
225
|
+
// Check vectors in the index
|
|
226
|
+
for (size_t i = 0; i < vectors_count; ++i) {
|
|
227
|
+
usearch_label_t label = i;
|
|
228
|
+
ASSERT(usearch_contains(idx, label, &error), error);
|
|
229
|
+
}
|
|
230
|
+
|
|
154
231
|
usearch_free(idx, &error);
|
|
232
|
+
printf("Test: Save/Load - PASSED\n");
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
void test_view(size_t vectors_count, size_t vector_dimension, float const* data) {
|
|
236
|
+
printf("Test: View...\n");
|
|
155
237
|
|
|
156
|
-
|
|
157
|
-
|
|
238
|
+
usearch_index_t idx = NULL;
|
|
239
|
+
usearch_error_t error = NULL;
|
|
240
|
+
usearch_init_options_t opts = create_options(vector_dimension);
|
|
158
241
|
idx = usearch_init(&opts, &error);
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
242
|
+
usearch_reserve(idx, vectors_count, &error);
|
|
243
|
+
|
|
244
|
+
// Add vectors
|
|
245
|
+
for (size_t i = 0; i < vectors_count; ++i) {
|
|
246
|
+
usearch_label_t label = i;
|
|
247
|
+
usearch_add(idx, label, data + i * vector_dimension, usearch_scalar_f32_k, &error);
|
|
248
|
+
ASSERT(!error, error);
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
// Save and free the index
|
|
252
|
+
usearch_save(idx, "usearch_index.bin", &error);
|
|
253
|
+
ASSERT(!error, error);
|
|
167
254
|
usearch_free(idx, &error);
|
|
255
|
+
ASSERT(!error, error);
|
|
256
|
+
|
|
257
|
+
// Reinit
|
|
258
|
+
idx = usearch_init(&opts, &error);
|
|
259
|
+
ASSERT(!error, error);
|
|
260
|
+
|
|
261
|
+
// View
|
|
262
|
+
usearch_view(idx, "usearch_index.bin", &error);
|
|
263
|
+
ASSERT(!error, error);
|
|
168
264
|
|
|
265
|
+
usearch_free(idx, &error);
|
|
266
|
+
printf("Test: View - PASSED\n");
|
|
169
267
|
}
|
|
170
268
|
|
|
171
|
-
int main() {
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
269
|
+
int main(int argc, char* argv[]) {
|
|
270
|
+
// Resolve file path
|
|
271
|
+
char const* filename = "sample.fvecs";
|
|
272
|
+
if (argc == 2)
|
|
273
|
+
filename = argv[1];
|
|
176
274
|
|
|
177
|
-
|
|
275
|
+
// Read data from the file
|
|
276
|
+
float* data = NULL;
|
|
277
|
+
size_t vectors_count = 0;
|
|
278
|
+
size_t vector_dimension = 0;
|
|
279
|
+
parse_vectors_from_file(filename, &vectors_count, &vector_dimension, &data);
|
|
280
|
+
vectors_count = vectors_count > 100000 ? 100000 : vectors_count; // Just limit vectors
|
|
281
|
+
|
|
282
|
+
// Test
|
|
283
|
+
test_init(vectors_count, vector_dimension);
|
|
284
|
+
test_add_vector(vectors_count, vector_dimension, data);
|
|
285
|
+
test_find_vector(vectors_count, vector_dimension, data);
|
|
286
|
+
test_remove_vector(vectors_count, vector_dimension, data);
|
|
287
|
+
test_save_load(vectors_count, vector_dimension, data);
|
|
288
|
+
test_view(vectors_count, vector_dimension, data);
|
|
289
|
+
|
|
290
|
+
// Free vectors' buffer
|
|
291
|
+
free(data);
|
|
178
292
|
|
|
179
|
-
free(data); // Remember to free the dynamically allocated memory
|
|
180
|
-
}
|
|
181
293
|
return 0;
|
|
182
294
|
}
|
package/conanfile.py
CHANGED
|
@@ -4,7 +4,7 @@ from conans import ConanFile
|
|
|
4
4
|
class USearchConan(ConanFile):
|
|
5
5
|
|
|
6
6
|
name = 'USearch'
|
|
7
|
-
version = '0.
|
|
7
|
+
version = '0.21.0'
|
|
8
8
|
license = 'Apache 2.0'
|
|
9
9
|
url = 'https://github.com/unum-cloud/usearch'
|
|
10
10
|
description = 'Smaller & Faster Single-File Vector Search Engine from Unum'
|
package/cpp/CMakeLists.txt
CHANGED
|
@@ -13,7 +13,9 @@ endif()
|
|
|
13
13
|
# https://cmake.org/cmake/help/latest/variable/CMAKE_LANG_COMPILER_ID.html
|
|
14
14
|
if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
|
|
15
15
|
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O3")
|
|
16
|
-
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -
|
|
16
|
+
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -g")
|
|
17
|
+
set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -O3 -g")
|
|
18
|
+
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -fsanitize=address -fsanitize=leak -fsanitize=alignment -fsanitize=undefined")
|
|
17
19
|
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Wall -Wextra -Wno-conversion -Wno-unknown-pragmas")
|
|
18
20
|
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native")
|
|
19
21
|
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fmax-errors=1")
|
|
@@ -22,7 +24,9 @@ if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
|
|
|
22
24
|
|
|
23
25
|
elseif(CMAKE_CXX_COMPILER_ID MATCHES "Clang")
|
|
24
26
|
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O3")
|
|
25
|
-
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -
|
|
27
|
+
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -g")
|
|
28
|
+
set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -O3 -g")
|
|
29
|
+
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -fsanitize=address -fsanitize=leak -fsanitize=alignment -fsanitize=undefined")
|
|
26
30
|
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Wfatal-errors")
|
|
27
31
|
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fopenmp")
|
|
28
32
|
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pedantic")
|
package/cpp/bench.cpp
CHANGED
|
@@ -24,6 +24,7 @@
|
|
|
24
24
|
|
|
25
25
|
#include <sys/stat.h> // `stat`
|
|
26
26
|
|
|
27
|
+
#include <algorithm>
|
|
27
28
|
#include <csignal>
|
|
28
29
|
#include <cstdio>
|
|
29
30
|
#include <iostream> // `std::cerr`
|
|
@@ -32,6 +33,7 @@
|
|
|
32
33
|
#include <string> // `std::to_string`
|
|
33
34
|
#include <thread> // `std::thread::hardware_concurrency()`
|
|
34
35
|
#include <variant> // `std::monostate`
|
|
36
|
+
#include <vector>
|
|
35
37
|
|
|
36
38
|
#include <clipp.h> // Command Line Interface
|
|
37
39
|
#include <omp.h> // `omp_set_num_threads()`
|
|
@@ -243,10 +245,10 @@ struct running_stats_printer_t {
|
|
|
243
245
|
std::size_t new_progress = progress.load();
|
|
244
246
|
if (new_progress - last_printed_progress < step)
|
|
245
247
|
return;
|
|
246
|
-
print(new_progress);
|
|
248
|
+
print(new_progress, total);
|
|
247
249
|
}
|
|
248
250
|
|
|
249
|
-
void print(std::size_t progress) {
|
|
251
|
+
void print(std::size_t progress, std::size_t total) {
|
|
250
252
|
|
|
251
253
|
constexpr char bars_k[] = "||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||";
|
|
252
254
|
constexpr std::size_t bars_len_k = 60;
|
|
@@ -267,6 +269,7 @@ struct running_stats_printer_t {
|
|
|
267
269
|
|
|
268
270
|
last_printed_progress = progress;
|
|
269
271
|
last_printed_time = time_new;
|
|
272
|
+
this->total = total;
|
|
270
273
|
}
|
|
271
274
|
};
|
|
272
275
|
|
|
@@ -332,6 +335,8 @@ template <typename dataset_at, typename index_at> //
|
|
|
332
335
|
static void single_shot(dataset_at& dataset, index_at& index, bool construct = true) {
|
|
333
336
|
using label_t = typename index_at::label_t;
|
|
334
337
|
using distance_t = typename index_at::distance_t;
|
|
338
|
+
using join_result_t = typename index_at::join_result_t;
|
|
339
|
+
constexpr std::size_t missing_label = std::numeric_limits<label_t>::max();
|
|
335
340
|
|
|
336
341
|
std::printf("\n");
|
|
337
342
|
std::printf("------------\n");
|
|
@@ -348,7 +353,7 @@ static void single_shot(dataset_at& dataset, index_at& index, bool construct = t
|
|
|
348
353
|
search_many(index, dataset.queries_count(), dataset.query(0), dataset.dimensions(), dataset.neighborhood_size(),
|
|
349
354
|
found_neighbors.data(), found_distances.data());
|
|
350
355
|
|
|
351
|
-
// Evaluate quality
|
|
356
|
+
// Evaluate search quality
|
|
352
357
|
std::size_t recall_at_1 = 0, recall_full = 0;
|
|
353
358
|
for (std::size_t i = 0; i != dataset.queries_count(); ++i) {
|
|
354
359
|
auto expected = dataset.neighborhood(i);
|
|
@@ -360,6 +365,38 @@ static void single_shot(dataset_at& dataset, index_at& index, bool construct = t
|
|
|
360
365
|
std::printf("Recall@1 %.2f %%\n", recall_at_1 * 100.f / dataset.queries_count());
|
|
361
366
|
std::printf("Recall %.2f %%\n", recall_full * 100.f / dataset.queries_count());
|
|
362
367
|
|
|
368
|
+
// Perform joins
|
|
369
|
+
std::vector<label_t> man_to_woman(dataset.vectors_count());
|
|
370
|
+
std::vector<label_t> woman_to_man(dataset.vectors_count());
|
|
371
|
+
std::size_t join_attempts = 0;
|
|
372
|
+
{
|
|
373
|
+
index_at& men = index;
|
|
374
|
+
index_at women = index.copy().index;
|
|
375
|
+
std::fill(man_to_woman.begin(), man_to_woman.end(), missing_label);
|
|
376
|
+
std::fill(woman_to_man.begin(), woman_to_man.end(), missing_label);
|
|
377
|
+
{
|
|
378
|
+
executor_default_t executor(index.limits().threads());
|
|
379
|
+
running_stats_printer_t printer{1, "Join"};
|
|
380
|
+
join_result_t result = index_at::join( //
|
|
381
|
+
men, women, join_config_t{executor.size()}, //
|
|
382
|
+
man_to_woman.data(), woman_to_man.data(), //
|
|
383
|
+
executor, [&](std::size_t progress, std::size_t total) {
|
|
384
|
+
if (progress % 1000 == 0)
|
|
385
|
+
printer.print(progress, total);
|
|
386
|
+
});
|
|
387
|
+
join_attempts = result.cycles;
|
|
388
|
+
}
|
|
389
|
+
}
|
|
390
|
+
// Evaluate join quality
|
|
391
|
+
std::size_t recall_join = 0, unmatched_count = 0;
|
|
392
|
+
for (std::size_t i = 0; i != index.size(); ++i) {
|
|
393
|
+
recall_join += man_to_woman[i] == i;
|
|
394
|
+
unmatched_count += man_to_woman[i] == missing_label;
|
|
395
|
+
}
|
|
396
|
+
std::printf("Recall Joins %.2f %%\n", recall_join * 100.f / index.size());
|
|
397
|
+
std::printf("Unmatched %.2f %% (%zu items)\n", unmatched_count * 100.f / index.size(), unmatched_count);
|
|
398
|
+
std::printf("Proposals %.2f / man (%zu total)\n", join_attempts * 1.f / index.size(), join_attempts);
|
|
399
|
+
|
|
363
400
|
// Paginate
|
|
364
401
|
std::vector<vector_id_t> hints(dataset.queries_count());
|
|
365
402
|
for (std::size_t i = 0; i != hints.size(); ++i)
|
|
@@ -491,7 +528,7 @@ void run_punned(dataset_at& dataset, args_t const& args, index_config_t config,
|
|
|
491
528
|
|
|
492
529
|
std::printf("Will benchmark an on-disk view\n");
|
|
493
530
|
|
|
494
|
-
index_at index_view = index.fork();
|
|
531
|
+
index_at index_view = index.fork().index;
|
|
495
532
|
index_view.view(args.path_output.c_str());
|
|
496
533
|
single_shot(dataset, index_view, false);
|
|
497
534
|
}
|