usearch 2.11.1 → 2.13.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. package/README.md +100 -30
  2. package/binding.gyp +11 -12
  3. package/c/README.md +3 -0
  4. package/c/lib.cpp +50 -13
  5. package/c/test.c +60 -55
  6. package/c/usearch.h +16 -0
  7. package/cpp/CMakeLists.txt +6 -0
  8. package/cpp/README.md +34 -13
  9. package/cpp/bench.cpp +99 -47
  10. package/cpp/test.cpp +947 -132
  11. package/include/usearch/index.hpp +894 -272
  12. package/include/usearch/index_dense.hpp +418 -156
  13. package/include/usearch/index_plugins.hpp +99 -49
  14. package/javascript/dist/cjs/usearch.d.ts +39 -1
  15. package/javascript/dist/cjs/usearch.js +11 -1
  16. package/javascript/dist/esm/usearch.d.ts +39 -1
  17. package/javascript/dist/esm/usearch.js +11 -1
  18. package/javascript/tsconfig-esm.json +3 -2
  19. package/javascript/usearch.ts +18 -7
  20. package/package.json +2 -2
  21. package/prebuilds/darwin-arm64+x64/usearch.node +0 -0
  22. package/prebuilds/linux-arm64/usearch.node +0 -0
  23. package/prebuilds/linux-x64/usearch.node +0 -0
  24. package/simsimd/.cmake-format.py +19 -0
  25. package/simsimd/CONTRIBUTING.md +49 -8
  26. package/simsimd/Cargo.lock +1 -1
  27. package/simsimd/Cargo.toml +10 -5
  28. package/simsimd/Package.swift +16 -0
  29. package/simsimd/README.md +69 -15
  30. package/simsimd/binding.gyp +14 -19
  31. package/simsimd/build.rs +2 -0
  32. package/simsimd/c/lib.c +18 -0
  33. package/simsimd/include/module.modulemap +4 -0
  34. package/simsimd/include/simsimd/binary.h +2 -0
  35. package/simsimd/include/simsimd/dot.h +466 -32
  36. package/simsimd/include/simsimd/probability.h +20 -10
  37. package/simsimd/include/simsimd/simsimd.h +444 -98
  38. package/simsimd/include/simsimd/spatial.h +480 -44
  39. package/simsimd/include/simsimd/types.h +161 -39
  40. package/simsimd/javascript/simsimd.ts +26 -2
  41. package/simsimd/package.json +1 -1
  42. package/simsimd/rust/benches/cosine.rs +8 -2
  43. package/simsimd/rust/benches/native.rs +240 -2
  44. package/simsimd/rust/benches/sqeuclidean.rs +8 -2
  45. package/simsimd/rust/lib.rs +50 -13
  46. package/simsimd/swift/source/SimSIMD.swift +123 -0
  47. package/simsimd/swift/tests/SimSIMDTests.swift +92 -0
  48. package/prebuilds/win32-ia32/usearch.node +0 -0
  49. package/prebuilds/win32-x64/usearch.node +0 -0
  50. package/simsimd/.releaserc +0 -87
package/c/test.c CHANGED
@@ -5,11 +5,15 @@
5
5
 
6
6
  #include "usearch.h"
7
7
 
8
- #define ASSERT(must_be_true, message) \
9
- if (!(must_be_true)) { \
10
- printf("Assert: %s\n", message); \
11
- exit(-1); \
12
- }
8
+ void expect(bool must_be_true, char const* message) {
9
+ if (must_be_true)
10
+ return;
11
+ message = message ? message : "C unit test failed";
12
+ printf("Assert: %s\n", message);
13
+ exit(-1);
14
+ }
15
+
16
+ #define expect_eq(a, b, message) expect(a == b, message)
13
17
 
14
18
  /**
15
19
  * @brief Creates and initializes vectors with random float values.
@@ -20,7 +24,7 @@
20
24
  */
21
25
  float* create_vectors(size_t const count, size_t const dimensions) {
22
26
  float* data = (float*)malloc(count * dimensions * sizeof(float));
23
- ASSERT(data, "Failed to allocate memory");
27
+ expect(data, "Failed to allocate memory");
24
28
  for (size_t index = 0; index < count * dimensions; ++index)
25
29
  data[index] = (float)rand() / (float)RAND_MAX;
26
30
  return data;
@@ -52,31 +56,31 @@ void test_init(size_t const collection_size, size_t const dimensions) {
52
56
  usearch_error_t error = NULL;
53
57
  usearch_init_options_t opts = create_options(dimensions);
54
58
  usearch_index_t index = usearch_init(&opts, &error);
55
- ASSERT(!error, error);
59
+ expect(!error, error);
56
60
  usearch_free(index, &error);
57
- ASSERT(!error, error);
61
+ expect(!error, error);
58
62
 
59
63
  // Init second time
60
64
  index = usearch_init(&opts, &error);
61
- ASSERT(!error, error);
65
+ expect(!error, error);
62
66
 
63
- ASSERT(usearch_size(index, &error) == 0, error);
64
- ASSERT(usearch_capacity(index, &error) == 0, error);
65
- ASSERT(usearch_dimensions(index, &error) == dimensions, error);
66
- ASSERT(usearch_connectivity(index, &error) == opts.connectivity, error);
67
+ expect_eq(usearch_size(index, &error), 0, error);
68
+ expect_eq(usearch_capacity(index, &error), 0, error);
69
+ expect_eq(usearch_dimensions(index, &error), dimensions, error);
70
+ expect_eq(usearch_connectivity(index, &error), opts.connectivity, error);
67
71
 
68
72
  // Reserve
69
73
  usearch_reserve(index, collection_size, &error);
70
- ASSERT(!error, error);
71
- ASSERT(usearch_size(index, &error) == 0, error);
72
- ASSERT(usearch_capacity(index, &error) == collection_size, error);
73
- ASSERT(usearch_dimensions(index, &error) == dimensions, error);
74
- ASSERT(usearch_connectivity(index, &error) == opts.connectivity, error);
75
- ASSERT(usearch_hardware_acceleration(index, &error), error);
76
- ASSERT(usearch_memory_usage(index, &error), error);
74
+ expect(!error, error);
75
+ expect_eq(usearch_size(index, &error), 0, error);
76
+ expect(usearch_capacity(index, &error) >= collection_size, error);
77
+ expect_eq(usearch_dimensions(index, &error), dimensions, error);
78
+ expect_eq(usearch_connectivity(index, &error), opts.connectivity, error);
79
+ expect(usearch_hardware_acceleration(index, &error), error);
80
+ expect(usearch_memory_usage(index, &error), error);
77
81
 
78
82
  usearch_free(index, &error);
79
- ASSERT(!error, error);
83
+ expect(!error, error);
80
84
 
81
85
  printf("Test: Index Initialization - PASSED\n");
82
86
  }
@@ -99,18 +103,18 @@ void test_add_vector(size_t const collection_size, size_t const dimensions) {
99
103
  for (size_t i = 0; i < collection_size; ++i) {
100
104
  usearch_key_t key = i;
101
105
  usearch_add(index, key, data + i * dimensions, usearch_scalar_f32_k, &error);
102
- ASSERT(!error, error);
106
+ expect(!error, error);
103
107
  }
104
108
 
105
- ASSERT(usearch_size(index, &error) == collection_size, error);
106
- ASSERT(usearch_capacity(index, &error) == collection_size, error);
109
+ expect_eq(usearch_size(index, &error), collection_size, error);
110
+ expect(usearch_capacity(index, &error) >= collection_size, error);
107
111
 
108
112
  // Check vectors in the index
109
113
  for (size_t i = 0; i < collection_size; ++i) {
110
114
  usearch_key_t key = i;
111
- ASSERT(usearch_contains(index, key, &error), error);
115
+ expect(usearch_contains(index, key, &error), error);
112
116
  }
113
- ASSERT(!usearch_contains(index, -1, &error), error); // Non existing key
117
+ expect(!usearch_contains(index, -1, &error), error); // Non existing key
114
118
 
115
119
  free(data);
116
120
  usearch_free(index, &error);
@@ -133,22 +137,22 @@ void test_find_vector(size_t const collection_size, size_t const dimensions) {
133
137
  // Create result buffers
134
138
  usearch_key_t* keys = (usearch_key_t*)malloc(collection_size * sizeof(usearch_key_t));
135
139
  float* distances = (float*)malloc(collection_size * sizeof(float));
136
- ASSERT(keys && distances, "Failed to allocate memory");
140
+ expect(keys && distances, "Failed to allocate memory");
137
141
 
138
142
  // Add vectors
139
143
  float* data = create_vectors(collection_size, dimensions);
140
144
  for (size_t i = 0; i < collection_size; ++i) {
141
145
  usearch_key_t key = i;
142
146
  usearch_add(index, key, data + i * dimensions, usearch_scalar_f32_k, &error);
143
- ASSERT(!error, error);
147
+ expect(!error, error);
144
148
  }
145
149
 
146
150
  // Find the vectors
147
151
  for (size_t i = 0; i < collection_size; i++) {
148
152
  size_t found_count = usearch_search(index, data + i * dimensions, usearch_scalar_f32_k, collection_size, keys,
149
153
  distances, &error);
150
- ASSERT(!error, error);
151
- ASSERT(found_count >= 1 && found_count <= collection_size, "Vector is missing");
154
+ expect(!error, error);
155
+ expect(found_count >= 1 && found_count <= collection_size, "Vector is missing");
152
156
  }
153
157
 
154
158
  free(data);
@@ -174,19 +178,19 @@ void test_get_vector(size_t const collection_size, size_t const dimensions) {
174
178
 
175
179
  // Create result buffers
176
180
  float* vectors = (float*)malloc(collection_size * dimensions * sizeof(float));
177
- ASSERT(vectors, "Failed to allocate memory");
181
+ expect(vectors, "Failed to allocate memory");
178
182
 
179
183
  // Add multiple vectors with SAME key
180
184
  usearch_key_t const key = 1;
181
185
  float* data = create_vectors(collection_size, dimensions);
182
186
  for (size_t i = 0; i < collection_size; i++) {
183
187
  usearch_add(index, key, data + i * dimensions, usearch_scalar_f32_k, &error);
184
- ASSERT(!error, error);
188
+ expect(!error, error);
185
189
  }
186
190
 
187
191
  // Retrieve vectors from index
188
192
  size_t found_count = usearch_get(index, key, collection_size, vectors, usearch_scalar_f32_k, &error);
189
- ASSERT(found_count == collection_size, "Vector is missing");
193
+ expect_eq(found_count, collection_size, "Vector is missing");
190
194
 
191
195
  free(vectors);
192
196
  free(data);
@@ -213,14 +217,14 @@ void test_remove_vector(size_t const collection_size, size_t const dimensions) {
213
217
  for (size_t i = 0; i < collection_size; ++i) {
214
218
  usearch_key_t key = i;
215
219
  usearch_add(index, key, data + i * dimensions, usearch_scalar_f32_k, &error);
216
- ASSERT(!error, error);
220
+ expect(!error, error);
217
221
  }
218
222
 
219
223
  // Remove the vectors
220
224
  for (size_t i = 0; i < collection_size; i++) {
221
225
  usearch_key_t key = i;
222
226
  usearch_remove(index, key, &error);
223
- ASSERT(!error, "Currently, Remove is not supported");
227
+ expect(!error, "Currently, Remove is not supported");
224
228
  }
225
229
 
226
230
  free(data);
@@ -256,14 +260,14 @@ void test_save_load(size_t const collection_size, size_t const dimensions) {
256
260
  for (size_t i = 0; i < collection_size; ++i) {
257
261
  usearch_key_t key = i;
258
262
  usearch_add(index, key, data + i * dimensions, usearch_scalar_f32_k, &error);
259
- ASSERT(!error, error);
263
+ expect(!error, error);
260
264
  }
261
265
 
262
266
  // Save and free the index
263
267
  usearch_save(index, "tmp.usearch", &error);
264
- ASSERT(!error, error);
268
+ expect(!error, error);
265
269
  usearch_free(index, &error);
266
- ASSERT(!error, error);
270
+ expect(!error, error);
267
271
  }
268
272
 
269
273
  // Reset the options
@@ -278,35 +282,36 @@ void test_save_load(size_t const collection_size, size_t const dimensions) {
278
282
  // Reinit
279
283
  {
280
284
 
281
- usearch_index_t index = usearch_init(&opts, &error);
282
- ASSERT(!error, error);
283
- ASSERT(usearch_size(index, &error) == 0, error);
285
+ usearch_index_t index = usearch_init(NULL, &error);
286
+ expect(!error, error);
287
+ // expect(usearch_size(index, &error) == 0, error);
284
288
 
285
289
  // Load
286
290
  usearch_load(index, "tmp.usearch", &error);
287
- ASSERT(!error, error);
288
- ASSERT(usearch_size(index, &error) == collection_size, error);
289
- ASSERT(usearch_capacity(index, &error) == collection_size, error);
290
- ASSERT(usearch_dimensions(index, &error) == dimensions, error);
291
- ASSERT(usearch_connectivity(index, &error) == weird_ops.connectivity, error);
291
+ expect(!error, error);
292
+ expect(usearch_size(index, &error) == collection_size, error);
293
+ expect(usearch_capacity(index, &error) == collection_size, error);
294
+ expect(usearch_dimensions(index, &error) == dimensions, error);
295
+ expect(usearch_connectivity(index, &error) == weird_ops.connectivity, error);
292
296
 
293
297
  // Check vectors in the index
294
298
  for (size_t i = 0; i < collection_size; ++i) {
295
299
  usearch_key_t key = i;
296
- ASSERT(usearch_contains(index, key, &error), error);
300
+ expect(usearch_contains(index, key, &error), error);
297
301
  }
298
302
 
299
303
  // Create result buffers
300
304
  usearch_key_t* keys = (usearch_key_t*)malloc(collection_size * sizeof(usearch_key_t));
301
305
  float* distances = (float*)malloc(collection_size * sizeof(float));
302
- ASSERT(keys && distances, "Failed to allocate memory");
306
+ expect(keys && distances, "Failed to allocate memory");
303
307
 
304
308
  // Find the vectors
309
+ usearch_change_threads_search(index, 1, &error);
305
310
  for (size_t i = 0; i < collection_size; i++) {
306
311
  size_t found_count = usearch_search(index, data + i * dimensions, usearch_scalar_f32_k, collection_size,
307
312
  keys, distances, &error);
308
- ASSERT(!error, error);
309
- ASSERT(found_count >= 1 && found_count <= collection_size, "Vector is missing");
313
+ expect(!error, error);
314
+ expect(found_count >= 1 && found_count <= collection_size, "Vector is missing");
310
315
  }
311
316
 
312
317
  free(keys);
@@ -340,22 +345,22 @@ void test_view(size_t const collection_size, size_t const dimensions) {
340
345
  for (size_t i = 0; i < collection_size; ++i) {
341
346
  usearch_key_t key = i;
342
347
  usearch_add(index, key, data + i * dimensions, usearch_scalar_f32_k, &error);
343
- ASSERT(!error, error);
348
+ expect(!error, error);
344
349
  }
345
350
 
346
351
  // Save and free the index
347
352
  usearch_save(index, "tmp.usearch", &error);
348
- ASSERT(!error, error);
353
+ expect(!error, error);
349
354
  usearch_free(index, &error);
350
- ASSERT(!error, error);
355
+ expect(!error, error);
351
356
 
352
357
  // Reinit
353
358
  index = usearch_init(&opts, &error);
354
- ASSERT(!error, error);
359
+ expect(!error, error);
355
360
 
356
361
  // View
357
362
  usearch_view(index, "tmp.usearch", &error);
358
- ASSERT(!error, error);
363
+ expect(!error, error);
359
364
 
360
365
  free(data);
361
366
  usearch_free(index, &error);
package/c/usearch.h CHANGED
@@ -281,6 +281,22 @@ USEARCH_EXPORT void usearch_change_expansion_add(usearch_index_t index, size_t e
281
281
  */
282
282
  USEARCH_EXPORT void usearch_change_expansion_search(usearch_index_t index, size_t expansion, usearch_error_t* error);
283
283
 
284
+ /**
285
+ * @brief Updates the number of threads that would be used to construct the index. Rarely used.
286
+ * @param[in] index The handle to the USearch index to be queried.
287
+ * @param[in] threads The new limit for the number of concurrent threads.
288
+ * @param[out] error Pointer to a string where the error message will be stored, if an error occurs.
289
+ */
290
+ USEARCH_EXPORT void usearch_change_threads_add(usearch_index_t index, size_t threads, usearch_error_t* error);
291
+
292
+ /**
293
+ * @brief Updates the number of threads that will be performing concurrent traversals. Rarely used.
294
+ * @param[in] index The handle to the USearch index to be queried.
295
+ * @param[in] threads The new limit for the number of concurrent threads.
296
+ * @param[out] error Pointer to a string where the error message will be stored, if an error occurs.
297
+ */
298
+ USEARCH_EXPORT void usearch_change_threads_search(usearch_index_t index, size_t threads, usearch_error_t* error);
299
+
284
300
  /**
285
301
  * @brief Updates the metric kind used for distance calculation between vectors.
286
302
  * @param[in] index The handle to the USearch index to be queried.
@@ -4,6 +4,12 @@ if (USEARCH_BUILD_TEST_CPP)
4
4
  include(CTest)
5
5
  enable_testing()
6
6
  add_test(NAME test_cpp COMMAND test_cpp)
7
+
8
+ target_include_directories(test_cpp PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/../stringzilla/include)
9
+
10
+ if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU" OR CMAKE_CXX_COMPILER_ID MATCHES "Clang")
11
+ target_compile_options(test_cpp PRIVATE -Wno-vla -Wno-unused-function -Wno-cast-function-type)
12
+ endif ()
7
13
  endif ()
8
14
 
9
15
  if (USEARCH_BUILD_BENCH_CPP)
package/cpp/README.md CHANGED
@@ -16,21 +16,29 @@ Once included, the high-level C++11 interface is as simple as it gets: `reserve(
16
16
  This covers 90% of use cases.
17
17
 
18
18
  ```cpp
19
- using namespace unum::usearch;
20
-
21
- metric_punned_t metric(256, metric_kind_t::l2sq_k, scalar_kind_t::f32_k);
22
-
23
- // If you plan to store more than 4 Billion entries - use `index_dense_big_t`.
24
- // Or directly instantiate the template variant you need - `index_dense_gt<vector_key_t, internal_id_t>`.
25
- index_dense_t index = index_dense_t::make(metric);
26
- float vec[3] = {0.1, 0.3, 0.2};
19
+ #include <usearch/index.hpp>
20
+ #include <usearch/index_dense.hpp>
27
21
 
28
- index.reserve(10); // Pre-allocate memory for 10 vectors
29
- index.add(42, &vec[0]); // Pass a key and a vector
30
- auto results = index.search(&vec[0], 5); // Pass a query and limit number of results
22
+ using namespace unum::usearch;
31
23
 
32
- for (std::size_t i = 0; i != results.size(); ++i)
33
- results[i].element.key, results[i].element.vector, results[i].distance;
24
+ int main(int argc, char **argv) {
25
+ metric_punned_t metric(3, metric_kind_t::l2sq_k, scalar_kind_t::f32_k);
26
+
27
+ // If you plan to store more than 4 Billion entries - use `index_dense_big_t`.
28
+ // Or directly instantiate the template variant you need - `index_dense_gt<vector_key_t, internal_id_t>`.
29
+ index_dense_t index = index_dense_t::make(metric);
30
+ float vec[3] = {0.1, 0.3, 0.2};
31
+
32
+ index.reserve(10); // Pre-allocate memory for 10 vectors
33
+ index.add(42, &vec[0]); // Pass a key and a vector
34
+ auto results = index.search(&vec[0], 5); // Pass a query and limit number of results
35
+
36
+ for (std::size_t i = 0; i != results.size(); ++i)
37
+ // You can access the following properties of every match:
38
+ // results[i].element.key, results[i].element.vector, results[i].distance;
39
+ std::printf("Found matching key: %zu", results[i].element.key);
40
+ return 0;
41
+ }
34
42
  ```
35
43
 
36
44
  Here we:
@@ -89,6 +97,19 @@ Aside from the `executor_default_t`, you can take advantage of one of the provid
89
97
  - `executor_stl_t`, that will spawn `std::thread` instances.
90
98
  - `dummy_executor_t`, that will run everything sequentially.
91
99
 
100
+ ## Error Handling
101
+
102
+ Unlike most open-source C++ libraries USearch doesn't use exceptions, as they often lead to corrupted states in concurrent data-structures.
103
+ They are also not suited for systems that always operate on the edge of available memory, and would end up raising `std::bad_alloc` exceptions all the time.
104
+ For that reason, most operations return a "result" object, that can be checked for success or failure.
105
+
106
+ ```cpp
107
+ bool success;
108
+ success = (bool)index.try_reserve(10); // Recommended over `reserve()`
109
+ success = (bool)index.add(42, &vec[0]); // Explicitly convert `add_result_t`
110
+ success = (bool)index.search(&vec[0], 5); // Explicitly convert `search_result_t`
111
+ ```
112
+
92
113
  ## Clustering
93
114
 
94
115
  Aside from basic Create-Read-Update-Delete (CRUD) operations and search, USearch also supports clustering.
package/cpp/bench.cpp CHANGED
@@ -72,6 +72,8 @@ struct alignas(32) persisted_matrix_gt {
72
72
  std::uint32_t cols{};
73
73
  scalar_t const* scalars{};
74
74
 
75
+ persisted_matrix_gt() {}
76
+
75
77
  persisted_matrix_gt(char const* path) noexcept(false) {
76
78
  if (!path || !std::strlen(path))
77
79
  throw std::invalid_argument("The file path is empty");
@@ -133,6 +135,11 @@ struct alignas(32) persisted_matrix_gt {
133
135
  std::size_t size_bytes() const noexcept { return rows * row_size_bytes(); }
134
136
  };
135
137
 
138
+ /**
139
+ * @brief A view of a large dataset in external memory, that may or may not contain
140
+ * ground-truth queries and their optimal search results. In the second case,
141
+ * self-recall is measured.
142
+ */
136
143
  template <typename scalar_at, typename vector_id_at> //
137
144
  struct persisted_dataset_gt {
138
145
  using scalar_t = scalar_at;
@@ -140,12 +147,21 @@ struct persisted_dataset_gt {
140
147
  persisted_matrix_gt<scalar_t> vectors_;
141
148
  persisted_matrix_gt<scalar_t> queries_;
142
149
  persisted_matrix_gt<compressed_slot_t> neighborhoods_;
150
+ std::vector<compressed_slot_t> neighborhoods_iota_{};
143
151
  std::size_t vectors_to_skip_{};
144
152
  std::size_t vectors_to_take_{};
145
153
 
154
+ persisted_dataset_gt(char const* path_vectors, std::size_t vectors_to_skip = 0,
155
+ std::size_t vectors_to_take = 0) noexcept(false)
156
+ : vectors_(path_vectors), queries_(), neighborhoods_(), vectors_to_skip_(vectors_to_skip),
157
+ vectors_to_take_(vectors_to_take) {
158
+ neighborhoods_iota_.resize(vectors_.rows);
159
+ std::iota(neighborhoods_iota_.begin(), neighborhoods_iota_.end(), 0);
160
+ }
161
+
146
162
  persisted_dataset_gt(char const* path_vectors, char const* path_queries, char const* path_neighbors,
147
163
  std::size_t vectors_to_skip = 0, std::size_t vectors_to_take = 0) noexcept(false)
148
- : vectors_(path_vectors), queries_(path_queries), neighborhoods_(path_neighbors),
164
+ : vectors_(path_vectors), queries_(path_queries), neighborhoods_(path_neighbors), neighborhoods_iota_(),
149
165
  vectors_to_skip_(vectors_to_skip), vectors_to_take_(vectors_to_take) {
150
166
 
151
167
  if (vectors_.cols != queries_.cols)
@@ -154,13 +170,18 @@ struct persisted_dataset_gt {
154
170
  throw std::invalid_argument("Number of ground-truth neighborhoods doesn't match number of queries");
155
171
  }
156
172
 
173
+ bool search_itself() const noexcept { return vectors_count() && !queries_.rows; }
174
+
157
175
  std::size_t dimensions() const noexcept { return vectors_.cols; }
158
- std::size_t queries_count() const noexcept { return queries_.rows; }
159
- std::size_t neighborhood_size() const noexcept { return neighborhoods_.cols; }
176
+ std::size_t queries_count() const noexcept { return search_itself() ? vectors_count() : queries_.rows; }
177
+ std::size_t neighborhood_size() const noexcept { return search_itself() ? 1 : neighborhoods_.cols; }
160
178
  scalar_t const* vector(std::size_t i) const noexcept { return vectors_.row(i + vectors_to_skip_); }
161
- scalar_t const* query(std::size_t i) const noexcept { return queries_.row(i); }
162
- compressed_slot_t const* neighborhood(std::size_t i) const noexcept { return neighborhoods_.row(i); }
163
-
179
+ scalar_t const* query(std::size_t i) const noexcept {
180
+ return search_itself() ? vectors_view().at(i) : queries_.row(i);
181
+ }
182
+ compressed_slot_t const* neighborhood(std::size_t i) const noexcept {
183
+ return search_itself() ? neighborhoods_iota_.data() + i : neighborhoods_.row(i);
184
+ }
164
185
  std::size_t vectors_count() const noexcept {
165
186
  return vectors_to_take_ ? vectors_to_take_ : (vectors_.rows - vectors_to_skip_);
166
187
  }
@@ -238,6 +259,8 @@ struct running_stats_printer_t {
238
259
  print(new_progress, total);
239
260
  }
240
261
 
262
+ void print() { print(progress.load(), total); }
263
+
241
264
  void print(std::size_t progress, std::size_t total) {
242
265
 
243
266
  constexpr char bars_k[] = "||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||";
@@ -263,8 +286,8 @@ struct running_stats_printer_t {
263
286
  }
264
287
  };
265
288
 
266
- template <typename index_at, typename vector_id_at, typename real_at>
267
- void index_many(index_at& index, std::size_t n, vector_id_at const* ids, real_at const* vectors, std::size_t dims) {
289
+ template <typename index_at, typename vector_id_at, typename scalar_at>
290
+ void index_many(index_at& index, std::size_t n, vector_id_at const* ids, scalar_at const* vectors, std::size_t dims) {
268
291
 
269
292
  running_stats_printer_t printer{n, "Indexing"};
270
293
 
@@ -276,18 +299,21 @@ void index_many(index_at& index, std::size_t n, vector_id_at const* ids, real_at
276
299
  #if USEARCH_USE_OPENMP
277
300
  config.thread = omp_get_thread_num();
278
301
  #endif
279
- float_span_t vector{vectors + dims * i, dims};
302
+ span_gt<scalar_at const> vector{vectors + dims * i, dims};
280
303
  index.add(ids[i], vector, config.thread);
281
304
  printer.progress++;
282
305
  if (config.thread == 0)
283
306
  printer.refresh();
284
307
  }
308
+
309
+ // Refresh once again to show 100% completion
310
+ printer.print();
285
311
  }
286
312
 
287
- template <typename index_at, typename vector_id_at, typename real_at>
313
+ template <typename index_at, typename vector_id_at, typename scalar_at, typename distance_at>
288
314
  void search_many( //
289
- index_at& index, std::size_t n, real_at const* vectors, std::size_t dims, std::size_t wanted, vector_id_at* ids,
290
- real_at* distances) {
315
+ index_at& index, std::size_t n, scalar_at const* vectors, std::size_t dims, std::size_t wanted, vector_id_at* ids,
316
+ distance_at* distances) {
291
317
 
292
318
  std::string name = "Search " + std::to_string(wanted);
293
319
  running_stats_printer_t printer{n, name.c_str()};
@@ -300,12 +326,15 @@ void search_many( //
300
326
  #if USEARCH_USE_OPENMP
301
327
  config.thread = omp_get_thread_num();
302
328
  #endif
303
- float_span_t vector{vectors + dims * i, dims};
329
+ span_gt<scalar_at const> vector{vectors + dims * i, dims};
304
330
  index.search(vector, wanted, config.thread).dump_to(ids + wanted * i, distances + wanted * i);
305
331
  printer.progress++;
306
332
  if (config.thread == 0)
307
333
  printer.refresh();
308
334
  }
335
+
336
+ // Refresh once again to show 100% completion
337
+ printer.print();
309
338
  }
310
339
 
311
340
  template <typename dataset_at, typename index_at> //
@@ -346,7 +375,7 @@ static void single_shot(dataset_at& dataset, index_at& index, bool construct = t
346
375
  std::size_t join_attempts = 0;
347
376
  {
348
377
  index_at& men = index;
349
- index_at women = index.copy().index;
378
+ index_at women = index.copy();
350
379
  std::fill(man_to_woman.begin(), man_to_woman.end(), missing_key);
351
380
  std::fill(woman_to_man.begin(), woman_to_man.end(), missing_key);
352
381
  {
@@ -360,6 +389,8 @@ static void single_shot(dataset_at& dataset, index_at& index, bool construct = t
360
389
  printer.print(progress, total);
361
390
  return true;
362
391
  });
392
+ // Refresh once again to show 100% completion
393
+ printer.print();
363
394
  join_attempts = result.visited_members;
364
395
  }
365
396
  }
@@ -499,7 +530,7 @@ void run_punned(dataset_at& dataset, args_t const& args, index_config_t config,
499
530
 
500
531
  std::printf("Will benchmark an on-disk view\n");
501
532
 
502
- index_at index_view = index.fork().index;
533
+ index_at index_view = index.fork();
503
534
  index_view.view(args.path_output.c_str());
504
535
  single_shot(dataset, index_view, false);
505
536
  }
@@ -521,6 +552,44 @@ void run_typed(dataset_at& dataset, args_t const& args, index_config_t config, i
521
552
  single_shot(dataset, index_view, false);
522
553
  }
523
554
 
555
+ template <typename dataset_scalar_at> void bench_with_args(args_t const& args) {
556
+ using dataset_t = persisted_dataset_gt<dataset_scalar_at, compressed_slot_t>;
557
+ dataset_t dataset = args.path_queries.empty() && args.path_neighbors.empty()
558
+ ? dataset_t(args.path_vectors.c_str(), //
559
+ args.vectors_to_skip, //
560
+ args.vectors_to_take //
561
+ )
562
+ : dataset_t(args.path_vectors.c_str(), //
563
+ args.path_queries.c_str(), //
564
+ args.path_neighbors.c_str(), //
565
+ args.vectors_to_skip, //
566
+ args.vectors_to_take //
567
+ );
568
+ std::printf("-- Dimensions: %zu\n", dataset.dimensions());
569
+ std::printf("-- Vectors count: %zu\n", dataset.vectors_count());
570
+ std::printf("-- Queries count: %zu\n", dataset.queries_count());
571
+ std::printf("-- Neighbors per query: %zu\n", dataset.neighborhood_size());
572
+
573
+ index_dense_config_t config(args.connectivity, args.expansion_add, args.expansion_search);
574
+ index_limits_t limits;
575
+ limits.threads_add = limits.threads_search = args.threads;
576
+ limits.members = dataset.vectors_count();
577
+
578
+ std::printf("- Index: \n");
579
+ std::printf("-- Connectivity: %zu\n", config.connectivity);
580
+ std::printf("-- Expansion @ Add: %zu\n", config.expansion_add);
581
+ std::printf("-- Expansion @ Search: %zu\n", config.expansion_search);
582
+
583
+ if (args.big)
584
+ #ifdef USEARCH_64BIT_ENV
585
+ run_punned<index_dense_gt<default_key_t, uint40_t>>(dataset, args, config, limits);
586
+ #else
587
+ std::printf("Error: Don't use 40 bit identifiers in 32bit environment\n");
588
+ #endif
589
+ else
590
+ run_punned<index_dense_gt<default_key_t, std::uint32_t>>(dataset, args, config, limits);
591
+ }
592
+
524
593
  int main(int argc, char** argv) {
525
594
 
526
595
  // Print backtrace if something goes wrong.
@@ -530,8 +599,8 @@ int main(int argc, char** argv) {
530
599
 
531
600
  auto args = args_t{};
532
601
  auto cli = ( //
533
- (option("--vectors") & value("path", args.path_vectors)).doc(".fbin file path to construct the index"),
534
- (option("--queries") & value("path", args.path_queries)).doc(".fbin file path to query the index"),
602
+ (option("--vectors") & value("path", args.path_vectors)).doc(".[fhbd]bin file path to construct the index"),
603
+ (option("--queries") & value("path", args.path_queries)).doc(".[fhbd]bin file path to query the index"),
535
604
  (option("--neighbors") & value("path", args.path_neighbors)).doc(".ibin file path with ground truth"),
536
605
  (option("-o", "--output") & value("path", args.path_output)).doc(".usearch output file path"),
537
606
  (option("-b", "--big").set(args.big)).doc("Will switch to uint40_t for neighbors lists with over 4B entries"),
@@ -578,39 +647,22 @@ int main(int argc, char** argv) {
578
647
  std::printf("-- Query vectors path: %s\n", args.path_queries.c_str());
579
648
  std::printf("-- Ground truth neighbors path: %s\n", args.path_neighbors.c_str());
580
649
 
581
- persisted_dataset_gt<float, compressed_slot_t> dataset{
582
- args.path_vectors.c_str(), //
583
- args.path_queries.c_str(), //
584
- args.path_neighbors.c_str(), //
585
- args.vectors_to_skip, //
586
- args.vectors_to_take, //
650
+ auto ends_with = [](std::string_view stack, std::string_view needle) -> bool {
651
+ if (needle.empty())
652
+ return false;
653
+ return stack.find(needle, stack.size() - needle.size()) != std::string_view::npos;
587
654
  };
588
- std::printf("-- Dimensions: %zu\n", dataset.dimensions());
589
- std::printf("-- Vectors count: %zu\n", dataset.vectors_count());
590
- std::printf("-- Queries count: %zu\n", dataset.queries_count());
591
- std::printf("-- Neighbors per query: %zu\n", dataset.neighborhood_size());
592
-
593
- // report_alternative_setups();
594
- // report_expected_losses(dataset);
595
-
596
- index_dense_config_t config(args.connectivity, args.expansion_add, args.expansion_search);
597
- index_limits_t limits;
598
- limits.threads_add = limits.threads_search = args.threads;
599
- limits.members = dataset.vectors_count();
600
-
601
- std::printf("- Index: \n");
602
- std::printf("-- Connectivity: %zu\n", config.connectivity);
603
- std::printf("-- Expansion @ Add: %zu\n", config.expansion_add);
604
- std::printf("-- Expansion @ Search: %zu\n", config.expansion_search);
605
655
 
606
- if (args.big)
607
- #ifdef USEARCH_64BIT_ENV
608
- run_punned<index_dense_gt<default_key_t, uint40_t>>(dataset, args, config, limits);
609
- #else
610
- std::printf("Error: Don't use 40 bit identifiers in 32bit environment\n");
611
- #endif
656
+ if (ends_with(args.path_vectors, ".fbin"))
657
+ bench_with_args<f32_t>(args);
658
+ else if (ends_with(args.path_vectors, ".dbin"))
659
+ bench_with_args<f64_t>(args);
660
+ else if (ends_with(args.path_vectors, ".hbin"))
661
+ bench_with_args<f16_t>(args);
662
+ else if (ends_with(args.path_vectors, ".bbin"))
663
+ bench_with_args<b1x8_t>(args);
612
664
  else
613
- run_punned<index_dense_gt<default_key_t, std::uint32_t>>(dataset, args, config, limits);
665
+ throw std::runtime_error("Unknown input file path");
614
666
 
615
667
  return 0;
616
668
  }