usearch 0.19.3 → 0.21.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -17,6 +17,33 @@ permissions:
17
17
 
18
18
  jobs:
19
19
 
20
+ build_test:
21
+ name: Test USearch
22
+ runs-on: ubuntu-latest
23
+ steps:
24
+ - uses: actions/checkout@v3
25
+ with:
26
+ ref: main-dev
27
+ - run: git submodule update --init --recursive
28
+ - name: Prepare Environment
29
+ run: |
30
+ sudo apt update &&
31
+ sudo apt install -y cmake g++-12 build-essential libjemalloc-dev
32
+ - name: Build
33
+ run: |
34
+ cmake -B ./build_release \
35
+ -DCMAKE_CXX_COMPILER="g++-12" \
36
+ -DCMAKE_BUILD_TYPE=Release \
37
+ -DUSEARCH_BUILD_TEST=1 \
38
+ -DUSEARCH_USE_OPENMP=1 \
39
+ -DUSEARCH_USE_SIMSIMD=1 \
40
+ -DUSEARCH_USE_JEMALLOC=1 \
41
+ -DUSEARCH_BUILD_BENCHMARK=0 &&
42
+ make -C ./build_release -j
43
+ - name: Run tests
44
+ run: ./build_release/test
45
+
46
+
20
47
  test_python_311:
21
48
  name: Test Python
22
49
  runs-on: ${{ matrix.os }}
package/Cargo.toml CHANGED
@@ -1,6 +1,6 @@
1
1
  [package]
2
2
  name = "usearch"
3
- version = "0.19.3"
3
+ version = "0.21.0"
4
4
  authors = ["Ashot Vardanian <1983160+ashvardanian@users.noreply.github.com>"]
5
5
  description = "Smaller & Faster Single-File Vector Search Engine from Unum"
6
6
  edition = "2021"
package/README.md CHANGED
@@ -167,7 +167,7 @@ index.add(/* label: */ 42, /* vector: */ {&vec[0], 3});
167
167
  auto results = index.search(/* query: */ {&vec[0], 3}, 5 /* neighbors */);
168
168
 
169
169
  for (std::size_t i = 0; i != results.size(); ++i)
170
- results[i].element.label, results[i].element.vector, results[i].distance;
170
+ results[i].member.label, results[i].member.vector, results[i].member.id, results[i].distance;
171
171
  ```
172
172
 
173
173
  The `add` is thread-safe for concurrent index construction.
@@ -759,9 +759,28 @@ index = Index(ndim=2048, metric=MetricKind.BitwiseTanimoto)
759
759
  labels = np.arange(len(molecules))
760
760
 
761
761
  index.add(labels, fingerprints)
762
- matches = index.search(fingerprints, 10)
762
+ matches: Matches = index.search(fingerprints, 10)
763
763
  ```
764
764
 
765
+ Of if you need a bit of our SIMD superpowers - take some Chemsitry-oriented precompiled metrics from [SimSIMD][simsimd].
766
+
767
+ ```python
768
+ import simsimd as sisi
769
+
770
+ index = Index(
771
+ ndim=166, # MACCS fingerprints are 166-dimensional
772
+ metric=CompiledMetric(
773
+ pointer=sisi.to_int(sisi.tanimoto_maccs_neon), # For Arm Neon
774
+ signature=MetricSignature.ArrayArray,
775
+ kind=MetricKind.Tanimoto,
776
+ ),
777
+ )
778
+
779
+ index.add(42, np.packbits([1] * 166))
780
+ matches: Matches = index.search(np.packbits([1] * 166))
781
+ ```
782
+
783
+ [simsimd]: https://github.com/ashvardanian/simsimd
765
784
  [smiles]: https://en.wikipedia.org/wiki/Simplified_molecular-input_line-entry_system
766
785
  [rdkit-fingerprints]: https://www.rdkit.org/docs/RDKit_Book.html#additional-information-about-the-fingerprints
767
786
 
package/c/test.c CHANGED
@@ -1,182 +1,294 @@
1
- #include <assert.h>
1
+ #include <stdio.h>
2
2
  #include <errno.h>
3
3
  #include <stdio.h>
4
4
  #include <stdlib.h>
5
- #include <string.h>
6
5
  #include <sys/stat.h>
7
- #include <time.h>
8
6
 
9
7
  #include "usearch.h"
10
8
 
11
- float* read_fvecs_file(const char* filename, size_t* num_vectors, size_t* vector_dim) {
12
- struct stat st;
13
- FILE* file;
14
- size_t cnt;
15
- float skip;
16
-
17
- ;
18
- if (stat(filename, &st) == -1) {
19
- printf("Failed to read file: %s err: %d: %s\n", filename, errno, strerror(errno));
20
- return NULL;
9
+ #define ASSERT(must_be_true, message) \
10
+ if (!(must_be_true)) { \
11
+ printf("Assert: %s\n", message); \
12
+ exit(-1); \
21
13
  }
22
14
 
23
- file = fopen(filename, "rb");
24
- if (file == NULL) {
25
- printf("Failed to open file: %s\n", filename);
26
- return NULL;
27
- }
28
15
 
29
- // Read the number of vectors and vector dimension from the file
30
- cnt = fread(vector_dim, sizeof(int), 1, file);
31
- assert(cnt == 1);
32
- if (st.st_size % ((*vector_dim + 1) * sizeof(float)) != 0) {
33
- printf("File does not contain a whole number of vectors.\n Vector dim: %ld, file size: %ld\n", *vector_dim,
34
- st.st_size);
35
- fclose(file);
36
- return NULL;
37
- }
16
+ void parse_vectors_from_file(char const* file_path, size_t* count, size_t* dimension, float** data) {
38
17
 
39
- *num_vectors = st.st_size / ((*vector_dim + 1) * sizeof(float));
18
+ // Open the file
19
+ struct stat st = {};
20
+ ASSERT(stat(file_path, &st) == 0, "Failed to get metadata of the file");
21
+ FILE* file = fopen(file_path, "rb");
22
+ ASSERT(file, "Failed to open the file");
40
23
 
41
- // Allocate memory for the array
42
- float* data = (float*)malloc(*num_vectors * *vector_dim * sizeof(float));
43
- if (data == NULL) {
44
- printf("Memory allocation failed.\n");
45
- fclose(file);
46
- return NULL;
47
- }
24
+ // Read vectors dimension and calculate count
25
+ int dim = 0;
26
+ size_t read = fread(&dim, sizeof(int), 1, file);
27
+ ASSERT(read == 1, "Failed to read from the file");
28
+ ASSERT(st.st_size % ((dim + 1) * sizeof(float)) == 0, "File does not contain a whole number of vectors");
29
+ *dimension = (size_t)dim;
30
+ *count = st.st_size / ((dim + 1) * sizeof(float));
48
31
 
49
- // Read the data from the file into the array
50
- for (size_t i = 0; i < *num_vectors; i++) {
51
- cnt = fread(data + i * *vector_dim, sizeof(float), *vector_dim, file);
52
- assert(cnt == *vector_dim);
53
- if (i == *num_vectors - 1) {
32
+ // Allocate memory for the vectors' data
33
+ *data = (float*)malloc(*count * *dimension * sizeof(float));
34
+ ASSERT(*data, "Failed to allocate memory");
35
+
36
+ // Read the data
37
+ for (size_t i = 0; i < *count; ++i) {
38
+ read = fread(*data + i * *dimension, sizeof(float), *dimension, file);
39
+ ASSERT(read == *dimension, "Failed to read from the file");
40
+ if (i == *count - 1)
54
41
  break;
55
- }
56
- cnt = fread(&skip, sizeof(float), 1, file);
57
- assert(cnt == 1);
42
+
43
+ // Skip
44
+ float skip;
45
+ read = fread(&skip, sizeof(float), 1, file);
46
+ ASSERT(read == 1, "Failed to read from the file");
58
47
  }
59
48
 
60
49
  fclose(file);
61
- return data;
62
50
  }
63
51
 
64
- // log every this many iterations in test loops
65
- const int LOG_GRANULARITY = 10000;
66
-
67
- void test_index(float* data, size_t num_vectors, size_t vector_dim) {
52
+ usearch_init_options_t create_options(size_t vector_dimension) {
68
53
  usearch_init_options_t opts;
69
- usearch_index_t idx;
70
- usearch_error_t error = NULL;
71
- clock_t start;
72
54
  opts.connectivity = 2; // 32 in faiss
73
- opts.dimensions = vector_dim;
55
+ opts.dimensions = vector_dimension;
74
56
  opts.expansion_add = 40; // 40 in faiss
75
57
  opts.expansion_search = 16; // 10 in faiss
76
58
  opts.metric_kind = usearch_metric_ip_k;
77
59
  opts.metric = NULL;
78
60
  opts.quantization = usearch_scalar_f32_k;
61
+ return opts;
62
+ }
63
+
64
+ void test_init(size_t vectors_count, size_t vector_dimension) {
65
+ printf("Test: Index Initialization...\n");
66
+
67
+ // Init index
68
+ usearch_index_t idx = NULL;
69
+ usearch_error_t error = NULL;
70
+ usearch_init_options_t opts = create_options(vector_dimension);
71
+ idx = usearch_init(&opts, &error);
72
+ ASSERT(!error, error);
73
+ usearch_free(idx, &error);
74
+ ASSERT(!error, error);
75
+
76
+ // Init second time
77
+ idx = usearch_init(&opts, &error);
78
+ ASSERT(!error, error);
79
+
80
+ ASSERT(usearch_size(idx, &error) == 0, error);
81
+ ASSERT(usearch_capacity(idx, &error) == 0, error);
82
+ ASSERT(usearch_dimensions(idx, &error) == vector_dimension, error);
83
+ ASSERT(usearch_connectivity(idx, &error) == opts.connectivity, error);
84
+
85
+ // Reserve
86
+ usearch_reserve(idx, vectors_count, &error);
87
+ ASSERT(!error, error);
88
+ ASSERT(usearch_size(idx, &error) == 0, error);
89
+ ASSERT(usearch_capacity(idx, &error) == vectors_count, error);
90
+ ASSERT(usearch_dimensions(idx, &error) == vector_dimension, error);
91
+ ASSERT(usearch_connectivity(idx, &error) == opts.connectivity, error);
92
+
93
+ usearch_free(idx, &error);
94
+ ASSERT(!error, error);
95
+
96
+ printf("Test: Index Initialization - PASSED\n");
97
+ }
79
98
 
80
- // results arrays
81
- int k = 10;
82
- num_vectors /= 10;
83
- usearch_label_t* labels = (usearch_label_t*)calloc(k, sizeof(usearch_label_t));
84
- float* distances = (float*)calloc(k, sizeof(float));
99
+ void test_add_vector(size_t vectors_count, size_t vector_dimension, float const* data) {
100
+ printf("Test: Add Vector...\n");
85
101
 
86
- fprintf(stderr, "read %ld vectors of dimension %ld\n", num_vectors, vector_dim);
87
- fprintf(stderr, "initializing usearch...\n");
102
+ usearch_index_t idx = NULL;
103
+ usearch_error_t error = NULL;
104
+ usearch_init_options_t opts = create_options(vector_dimension);
88
105
  idx = usearch_init(&opts, &error);
89
- if (error != NULL) {
90
- fprintf(stderr, "usearch error: %s\n", error);
91
- assert(false);
106
+ usearch_reserve(idx, vectors_count, &error);
107
+
108
+ // Add vectors
109
+ for (size_t i = 0; i < vectors_count; ++i) {
110
+ usearch_label_t label = i;
111
+ usearch_add(idx, label, data + i * vector_dimension, usearch_scalar_f32_k, &error);
112
+ ASSERT(!error, error);
92
113
  }
93
114
 
94
- fprintf(stderr, "reserving capacity...\n");
95
- usearch_reserve(idx, num_vectors, &error);
96
- assert(!error);
115
+ ASSERT(usearch_size(idx, &error) == vectors_count, error);
116
+ ASSERT(usearch_capacity(idx, &error) == vectors_count, error);
97
117
 
98
- fprintf(stderr, "adding vectors...\n");
99
- for (size_t i = 0; i < num_vectors; i++) {
100
- if (i % LOG_GRANULARITY == 0) {
101
- fprintf(stderr, "added %ld vectors...\n", i);
102
- }
118
+ // Check vectors in the index
119
+ for (size_t i = 0; i < vectors_count; ++i) {
103
120
  usearch_label_t label = i;
104
- usearch_add(idx, label, data + i * vector_dim, usearch_scalar_f32_k, &error);
105
- assert(!error);
121
+ ASSERT(usearch_contains(idx, label, &error), error);
106
122
  }
123
+ ASSERT(!usearch_contains(idx, -1, &error), error); // Non existing label
124
+
125
+ usearch_free(idx, &error);
126
+ printf("Test: Add Vector - PASSED\n");
127
+ }
128
+
129
+ void test_find_vector(size_t vectors_count, size_t vector_dimension, float const* data) {
130
+ printf("Test: Find Vector...\n");
107
131
 
108
- fprintf(stderr, "checking containment of vectors...\n");
109
- for (size_t i = 0; i < num_vectors; i++) {
110
- if (i % LOG_GRANULARITY == 0) {
111
- fprintf(stderr, "checked %ld vectors...\n", i);
112
- }
132
+ usearch_index_t idx = NULL;
133
+ usearch_error_t error = NULL;
134
+ usearch_init_options_t opts = create_options(vector_dimension);
135
+ idx = usearch_init(&opts, &error);
136
+ usearch_reserve(idx, vectors_count, &error);
137
+
138
+ // Create result buffers
139
+ int results_count = 10;
140
+ usearch_label_t* labels = (usearch_label_t*)malloc(results_count * sizeof(usearch_label_t));
141
+ float* distances = (float*)malloc(results_count * sizeof(float));
142
+ ASSERT(labels && distances, "Failed to allocate memory");
143
+
144
+ // Add vectors
145
+ for (size_t i = 0; i < vectors_count; ++i) {
113
146
  usearch_label_t label = i;
114
- assert(usearch_contains(idx, label, &error));
115
- assert(!error);
147
+ usearch_add(idx, label, data + i * vector_dimension, usearch_scalar_f32_k, &error);
148
+ ASSERT(!error, error);
116
149
  }
117
150
 
118
- assert(!usearch_contains(idx, -1, &error));
119
- assert(!error);
120
-
121
- fprintf(stderr, "searching vectors...\n");
122
- for (size_t i = 0; i < num_vectors; i++) {
123
- if (i % LOG_GRANULARITY == 0) {
124
- fprintf(stderr, "searched %ld vectors...\n", i);
125
- }
126
- const void *query_vector = data + i * vector_dim;
127
- usearch_search(idx, query_vector, usearch_scalar_f32_k, k, labels, distances, &error);
128
- assert(!error);
151
+ // Find the vectors
152
+ for (size_t i = 0; i < vectors_count; i++) {
153
+ const void *query_vector = data + i * vector_dimension;
154
+ size_t found_count = usearch_search(idx, query_vector, usearch_scalar_f32_k, results_count, labels, distances, &error);
155
+ ASSERT(!error, error);
156
+ ASSERT(found_count = results_count, "Vector is missing");
129
157
  }
130
158
 
131
- fprintf(stderr, "saving the index...\n");
132
- start = clock();
133
- usearch_save(idx, "usearch_index.bin", &error);
134
- assert(!error);
135
- fprintf(stderr, "saving took %.2f ms\n", ((double)(clock() - start)) / CLOCKS_PER_SEC * 1000);
159
+ usearch_free(idx, &error);
160
+ printf("Test: Find Vector - PASSED\n");
161
+ }
162
+
163
+ void test_remove_vector(size_t vectors_count, size_t vector_dimension, float const* data) {
164
+ printf("Test: Remove Vector...\n");
165
+
166
+ usearch_index_t idx = NULL;
167
+ usearch_error_t error = NULL;
168
+ usearch_init_options_t opts = create_options(vector_dimension);
169
+ idx = usearch_init(&opts, &error);
170
+ usearch_reserve(idx, vectors_count, &error);
171
+
172
+ // Add vectors
173
+ for (size_t i = 0; i < vectors_count; ++i) {
174
+ usearch_label_t label = i;
175
+ usearch_add(idx, label, data + i * vector_dimension, usearch_scalar_f32_k, &error);
176
+ ASSERT(!error, error);
177
+ }
178
+
179
+ // Remove the vectors
180
+ for (size_t i = 0; i < vectors_count; i++) {
181
+ usearch_label_t label = i;
182
+ usearch_remove(idx, label, &error);
183
+ ASSERT(error, "Currently, Remove is not supported");
184
+ }
185
+
186
+ usearch_free(idx, &error);
187
+ printf("Test: Remove Vector - PASSED\n");
188
+ }
136
189
 
137
- fprintf(stderr, "destroying the index...\n");
138
- start = clock();
190
+ void test_save_load(size_t vectors_count, size_t vector_dimension, float const* data) {
191
+ printf("Test: Save/Load...\n");
192
+
193
+ usearch_index_t idx = NULL;
194
+ usearch_error_t error = NULL;
195
+ usearch_init_options_t opts = create_options(vector_dimension);
196
+ idx = usearch_init(&opts, &error);
197
+ usearch_reserve(idx, vectors_count, &error);
198
+
199
+ // Add vectors
200
+ for (size_t i = 0; i < vectors_count; ++i) {
201
+ usearch_label_t label = i;
202
+ usearch_add(idx, label, data + i * vector_dimension, usearch_scalar_f32_k, &error);
203
+ ASSERT(!error, error);
204
+ }
205
+
206
+ // Save and free the index
207
+ usearch_save(idx, "usearch_index.bin", &error);
208
+ ASSERT(!error, error);
139
209
  usearch_free(idx, &error);
140
- assert(!error);
141
- fprintf(stderr, "destroying the index took %.2f ms\n", ((double)(clock() - start)) / CLOCKS_PER_SEC * 1000);
210
+ ASSERT(!error, error);
142
211
 
143
- fprintf(stderr, "loading the index...\n");
144
- start = clock();
212
+ // Reinit
145
213
  idx = usearch_init(&opts, &error);
146
- assert(!error);
147
- assert(usearch_size(idx, &error) == 0);
148
- assert(!error);
214
+ ASSERT(!error, error);
215
+ ASSERT(usearch_size(idx, &error) == 0, error);
216
+
217
+ // Load
149
218
  usearch_load(idx, "usearch_index.bin", &error);
150
- assert(!error);
151
- fprintf(stderr, "loading the index took %.2f ms\n", ((double)(clock() - start)) / CLOCKS_PER_SEC * 1000);
152
- assert(usearch_size(idx, &error) == num_vectors);
153
- assert(!error);
219
+ ASSERT(!error, error);
220
+ ASSERT(usearch_size(idx, &error) == vectors_count, error);
221
+ ASSERT(usearch_capacity(idx, &error) == vectors_count, error);
222
+ ASSERT(usearch_dimensions(idx, &error) == vector_dimension, error);
223
+ ASSERT(usearch_connectivity(idx, &error) == opts.connectivity, error);
224
+
225
+ // Check vectors in the index
226
+ for (size_t i = 0; i < vectors_count; ++i) {
227
+ usearch_label_t label = i;
228
+ ASSERT(usearch_contains(idx, label, &error), error);
229
+ }
230
+
154
231
  usearch_free(idx, &error);
232
+ printf("Test: Save/Load - PASSED\n");
233
+ }
234
+
235
+ void test_view(size_t vectors_count, size_t vector_dimension, float const* data) {
236
+ printf("Test: View...\n");
155
237
 
156
- fprintf(stderr, "viewing the index...\n");
157
- start = clock();
238
+ usearch_index_t idx = NULL;
239
+ usearch_error_t error = NULL;
240
+ usearch_init_options_t opts = create_options(vector_dimension);
158
241
  idx = usearch_init(&opts, &error);
159
- assert(!error);
160
- assert(usearch_size(idx, &error) == 0);
161
- assert(!error);
162
- usearch_view(idx, "usearch_index.bin", &error);
163
- assert(!error);
164
- fprintf(stderr, "viewing the index took %.2f ms\n", ((double)(clock() - start)) / CLOCKS_PER_SEC * 1000);
165
- assert(usearch_size(idx, &error) == num_vectors);
166
- assert(!error);
242
+ usearch_reserve(idx, vectors_count, &error);
243
+
244
+ // Add vectors
245
+ for (size_t i = 0; i < vectors_count; ++i) {
246
+ usearch_label_t label = i;
247
+ usearch_add(idx, label, data + i * vector_dimension, usearch_scalar_f32_k, &error);
248
+ ASSERT(!error, error);
249
+ }
250
+
251
+ // Save and free the index
252
+ usearch_save(idx, "usearch_index.bin", &error);
253
+ ASSERT(!error, error);
167
254
  usearch_free(idx, &error);
255
+ ASSERT(!error, error);
256
+
257
+ // Reinit
258
+ idx = usearch_init(&opts, &error);
259
+ ASSERT(!error, error);
260
+
261
+ // View
262
+ usearch_view(idx, "usearch_index.bin", &error);
263
+ ASSERT(!error, error);
168
264
 
265
+ usearch_free(idx, &error);
266
+ printf("Test: View - PASSED\n");
169
267
  }
170
268
 
171
- int main() {
172
- const char* filename = "sift/sift_base.fvecs";
173
- size_t num_vectors = 0, vector_dim = 0;
174
- float* data = read_fvecs_file(filename, &num_vectors, &vector_dim);
175
- if (data != NULL) {
269
+ int main(int argc, char* argv[]) {
270
+ // Resolve file path
271
+ char const* filename = "sample.fvecs";
272
+ if (argc == 2)
273
+ filename = argv[1];
176
274
 
177
- test_index(data, num_vectors, vector_dim);
275
+ // Read data from the file
276
+ float* data = NULL;
277
+ size_t vectors_count = 0;
278
+ size_t vector_dimension = 0;
279
+ parse_vectors_from_file(filename, &vectors_count, &vector_dimension, &data);
280
+ vectors_count = vectors_count > 100000 ? 100000 : vectors_count; // Just limit vectors
281
+
282
+ // Test
283
+ test_init(vectors_count, vector_dimension);
284
+ test_add_vector(vectors_count, vector_dimension, data);
285
+ test_find_vector(vectors_count, vector_dimension, data);
286
+ test_remove_vector(vectors_count, vector_dimension, data);
287
+ test_save_load(vectors_count, vector_dimension, data);
288
+ test_view(vectors_count, vector_dimension, data);
289
+
290
+ // Free vectors' buffer
291
+ free(data);
178
292
 
179
- free(data); // Remember to free the dynamically allocated memory
180
- }
181
293
  return 0;
182
294
  }
package/conanfile.py CHANGED
@@ -4,7 +4,7 @@ from conans import ConanFile
4
4
  class USearchConan(ConanFile):
5
5
 
6
6
  name = 'USearch'
7
- version = '0.19.3'
7
+ version = '0.21.0'
8
8
  license = 'Apache 2.0'
9
9
  url = 'https://github.com/unum-cloud/usearch'
10
10
  description = 'Smaller & Faster Single-File Vector Search Engine from Unum'
@@ -13,7 +13,9 @@ endif()
13
13
  # https://cmake.org/cmake/help/latest/variable/CMAKE_LANG_COMPILER_ID.html
14
14
  if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
15
15
  set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O3")
16
- set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -fsanitize=address")
16
+ set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -g")
17
+ set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -O3 -g")
18
+ set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -fsanitize=address -fsanitize=leak -fsanitize=alignment -fsanitize=undefined")
17
19
  set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Wall -Wextra -Wno-conversion -Wno-unknown-pragmas")
18
20
  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native")
19
21
  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fmax-errors=1")
@@ -22,7 +24,9 @@ if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
22
24
 
23
25
  elseif(CMAKE_CXX_COMPILER_ID MATCHES "Clang")
24
26
  set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O3")
25
- set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -fsanitize=address")
27
+ set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -g")
28
+ set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -O3 -g")
29
+ set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -fsanitize=address -fsanitize=leak -fsanitize=alignment -fsanitize=undefined")
26
30
  set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Wfatal-errors")
27
31
  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fopenmp")
28
32
  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pedantic")
package/cpp/bench.cpp CHANGED
@@ -24,6 +24,7 @@
24
24
 
25
25
  #include <sys/stat.h> // `stat`
26
26
 
27
+ #include <algorithm>
27
28
  #include <csignal>
28
29
  #include <cstdio>
29
30
  #include <iostream> // `std::cerr`
@@ -32,6 +33,7 @@
32
33
  #include <string> // `std::to_string`
33
34
  #include <thread> // `std::thread::hardware_concurrency()`
34
35
  #include <variant> // `std::monostate`
36
+ #include <vector>
35
37
 
36
38
  #include <clipp.h> // Command Line Interface
37
39
  #include <omp.h> // `omp_set_num_threads()`
@@ -243,10 +245,10 @@ struct running_stats_printer_t {
243
245
  std::size_t new_progress = progress.load();
244
246
  if (new_progress - last_printed_progress < step)
245
247
  return;
246
- print(new_progress);
248
+ print(new_progress, total);
247
249
  }
248
250
 
249
- void print(std::size_t progress) {
251
+ void print(std::size_t progress, std::size_t total) {
250
252
 
251
253
  constexpr char bars_k[] = "||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||";
252
254
  constexpr std::size_t bars_len_k = 60;
@@ -267,6 +269,7 @@ struct running_stats_printer_t {
267
269
 
268
270
  last_printed_progress = progress;
269
271
  last_printed_time = time_new;
272
+ this->total = total;
270
273
  }
271
274
  };
272
275
 
@@ -332,6 +335,8 @@ template <typename dataset_at, typename index_at> //
332
335
  static void single_shot(dataset_at& dataset, index_at& index, bool construct = true) {
333
336
  using label_t = typename index_at::label_t;
334
337
  using distance_t = typename index_at::distance_t;
338
+ using join_result_t = typename index_at::join_result_t;
339
+ constexpr std::size_t missing_label = std::numeric_limits<label_t>::max();
335
340
 
336
341
  std::printf("\n");
337
342
  std::printf("------------\n");
@@ -348,7 +353,7 @@ static void single_shot(dataset_at& dataset, index_at& index, bool construct = t
348
353
  search_many(index, dataset.queries_count(), dataset.query(0), dataset.dimensions(), dataset.neighborhood_size(),
349
354
  found_neighbors.data(), found_distances.data());
350
355
 
351
- // Evaluate quality
356
+ // Evaluate search quality
352
357
  std::size_t recall_at_1 = 0, recall_full = 0;
353
358
  for (std::size_t i = 0; i != dataset.queries_count(); ++i) {
354
359
  auto expected = dataset.neighborhood(i);
@@ -360,6 +365,38 @@ static void single_shot(dataset_at& dataset, index_at& index, bool construct = t
360
365
  std::printf("Recall@1 %.2f %%\n", recall_at_1 * 100.f / dataset.queries_count());
361
366
  std::printf("Recall %.2f %%\n", recall_full * 100.f / dataset.queries_count());
362
367
 
368
+ // Perform joins
369
+ std::vector<label_t> man_to_woman(dataset.vectors_count());
370
+ std::vector<label_t> woman_to_man(dataset.vectors_count());
371
+ std::size_t join_attempts = 0;
372
+ {
373
+ index_at& men = index;
374
+ index_at women = index.copy().index;
375
+ std::fill(man_to_woman.begin(), man_to_woman.end(), missing_label);
376
+ std::fill(woman_to_man.begin(), woman_to_man.end(), missing_label);
377
+ {
378
+ executor_default_t executor(index.limits().threads());
379
+ running_stats_printer_t printer{1, "Join"};
380
+ join_result_t result = index_at::join( //
381
+ men, women, join_config_t{executor.size()}, //
382
+ man_to_woman.data(), woman_to_man.data(), //
383
+ executor, [&](std::size_t progress, std::size_t total) {
384
+ if (progress % 1000 == 0)
385
+ printer.print(progress, total);
386
+ });
387
+ join_attempts = result.cycles;
388
+ }
389
+ }
390
+ // Evaluate join quality
391
+ std::size_t recall_join = 0, unmatched_count = 0;
392
+ for (std::size_t i = 0; i != index.size(); ++i) {
393
+ recall_join += man_to_woman[i] == i;
394
+ unmatched_count += man_to_woman[i] == missing_label;
395
+ }
396
+ std::printf("Recall Joins %.2f %%\n", recall_join * 100.f / index.size());
397
+ std::printf("Unmatched %.2f %% (%zu items)\n", unmatched_count * 100.f / index.size(), unmatched_count);
398
+ std::printf("Proposals %.2f / man (%zu total)\n", join_attempts * 1.f / index.size(), join_attempts);
399
+
363
400
  // Paginate
364
401
  std::vector<vector_id_t> hints(dataset.queries_count());
365
402
  for (std::size_t i = 0; i != hints.size(); ++i)
@@ -491,7 +528,7 @@ void run_punned(dataset_at& dataset, args_t const& args, index_config_t config,
491
528
 
492
529
  std::printf("Will benchmark an on-disk view\n");
493
530
 
494
- index_at index_view = index.fork();
531
+ index_at index_view = index.fork().index;
495
532
  index_view.view(args.path_output.c_str());
496
533
  single_shot(dataset, index_view, false);
497
534
  }