usearch 2.9.1 → 2.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (169) hide show
  1. package/README.md +50 -52
  2. package/javascript/{types → dist/cjs}/usearch.d.ts +90 -115
  3. package/javascript/dist/cjs/usearch.js +531 -0
  4. package/javascript/dist/esm/usearch.d.ts +212 -0
  5. package/javascript/{usearch.js → dist/esm/usearch.js} +202 -153
  6. package/package.json +25 -23
  7. package/prebuilds/darwin-arm64+x64/usearch.node +0 -0
  8. package/prebuilds/linux-arm64/usearch.node +0 -0
  9. package/prebuilds/linux-x64/usearch.node +0 -0
  10. package/prebuilds/win32-ia32/usearch.node +0 -0
  11. package/prebuilds/win32-x64/usearch.node +0 -0
  12. package/.clang-format +0 -10
  13. package/.clang-tidy +0 -60
  14. package/.cmake-format.py +0 -19
  15. package/.dockerignore +0 -93
  16. package/.gitattributes +0 -9
  17. package/.github/ISSUE_TEMPLATE/bug_report.yml +0 -90
  18. package/.github/ISSUE_TEMPLATE/config.yml +0 -5
  19. package/.github/ISSUE_TEMPLATE/feature_request.yml +0 -64
  20. package/.github/workflows/Dockerfile.libs +0 -36
  21. package/.github/workflows/prerelease.yml +0 -331
  22. package/.github/workflows/release.yml +0 -722
  23. package/.github/workflows/update_version.sh +0 -15
  24. package/.gitmodules +0 -9
  25. package/.releaserc +0 -90
  26. package/CITATION.cff +0 -11
  27. package/CMakeLists.txt +0 -324
  28. package/CODE_OF_CONDUCT.md +0 -128
  29. package/CONTRIBUTING.md +0 -353
  30. package/Cargo.toml +0 -19
  31. package/Dockerfile +0 -16
  32. package/VERSION +0 -1
  33. package/assets/unum.png +0 -0
  34. package/assets/usearch-approaches-transparent.png +0 -0
  35. package/assets/usearch-approaches-white.png +0 -0
  36. package/assets/usearch-neighbor-types.png +0 -0
  37. package/binding.gyp +0 -87
  38. package/build.gradle +0 -204
  39. package/build.rs +0 -18
  40. package/c/CMakeLists.txt +0 -25
  41. package/c/README.md +0 -34
  42. package/c/lib.cpp +0 -358
  43. package/c/test.c +0 -374
  44. package/c/usearch.h +0 -311
  45. package/cmake/clipp.cmake +0 -7
  46. package/cmake/config.cmake.in +0 -15
  47. package/cmake/emscripten.cmake +0 -8
  48. package/cmake/jemalloc.cmake +0 -62
  49. package/cmake/pkg-config.pc.in +0 -4
  50. package/cmake/usearchConfigVersion.cmake.in +0 -20
  51. package/conanfile.py +0 -26
  52. package/cpp/CMakeLists.txt +0 -14
  53. package/cpp/README.md +0 -161
  54. package/cpp/bench.cpp +0 -618
  55. package/cpp/test.cpp +0 -323
  56. package/fp16/CMakeLists.txt +0 -176
  57. package/fp16/LICENSE +0 -11
  58. package/fp16/README.md +0 -20
  59. package/fp16/bench/alt-element.cc +0 -89
  60. package/fp16/bench/from-alt-array.cc +0 -175
  61. package/fp16/bench/from-ieee-array.cc +0 -399
  62. package/fp16/bench/ieee-element.cc +0 -272
  63. package/fp16/bench/to-alt-array.cc +0 -91
  64. package/fp16/bench/to-ieee-array.cc +0 -303
  65. package/fp16/cmake/DownloadGoogleBenchmark.cmake +0 -15
  66. package/fp16/cmake/DownloadGoogleTest.cmake +0 -15
  67. package/fp16/cmake/DownloadPSimd.cmake +0 -15
  68. package/fp16/configure.py +0 -65
  69. package/fp16/confu.yaml +0 -8
  70. package/fp16/include/fp16/__init__.py +0 -0
  71. package/fp16/include/fp16/avx.py +0 -50
  72. package/fp16/include/fp16/avx2.py +0 -53
  73. package/fp16/include/fp16/bitcasts.h +0 -92
  74. package/fp16/include/fp16/fp16.h +0 -451
  75. package/fp16/include/fp16/psimd.h +0 -131
  76. package/fp16/include/fp16.h +0 -11
  77. package/fp16/jni/Android.mk +0 -6
  78. package/fp16/jni/Application.mk +0 -4
  79. package/fp16/test/alt-from-fp32-value.cc +0 -498
  80. package/fp16/test/alt-to-fp32-bits.cc +0 -263
  81. package/fp16/test/alt-to-fp32-psimd.cc +0 -145
  82. package/fp16/test/alt-to-fp32-value.cc +0 -348
  83. package/fp16/test/alt-to-fp32x2-psimd.cc +0 -245
  84. package/fp16/test/bitcasts.cc +0 -56
  85. package/fp16/test/ieee-from-fp32-value.cc +0 -523
  86. package/fp16/test/ieee-to-fp32-bits.cc +0 -333
  87. package/fp16/test/ieee-to-fp32-psimd.cc +0 -312
  88. package/fp16/test/ieee-to-fp32-value.cc +0 -420
  89. package/fp16/test/ieee-to-fp32x2-psimd.cc +0 -541
  90. package/fp16/test/peachpy/alt-xmm-to-fp32-xmm-avx.cc +0 -96
  91. package/fp16/test/peachpy/alt-xmm-to-fp32-ymm-avx2.cc +0 -96
  92. package/fp16/test/peachpy/stubs.py +0 -38
  93. package/fp16/test/tables.cc +0 -530
  94. package/fp16/test/tables.h +0 -13
  95. package/fp16/third-party/THHalf.h +0 -92
  96. package/fp16/third-party/eigen-half.h +0 -170
  97. package/fp16/third-party/float16-compressor.h +0 -80
  98. package/fp16/third-party/half.hpp +0 -2916
  99. package/fp16/third-party/npy-halffloat.h +0 -328
  100. package/include/usearch/index.hpp +0 -3836
  101. package/include/usearch/index_dense.hpp +0 -2004
  102. package/include/usearch/index_plugins.hpp +0 -2203
  103. package/javascript/lib.cpp +0 -341
  104. package/javascript/types/usearch.d.ts.map +0 -1
  105. package/javascript/usearch.test.js +0 -74
  106. package/package-ci.json +0 -10
  107. package/pyproject.toml +0 -51
  108. package/setup.py +0 -201
  109. package/simsimd/.releaserc +0 -86
  110. package/simsimd/Cargo.lock +0 -708
  111. package/simsimd/Cargo.toml +0 -36
  112. package/simsimd/LICENSE +0 -201
  113. package/simsimd/README.md +0 -275
  114. package/simsimd/binding.gyp +0 -36
  115. package/simsimd/build.rs +0 -20
  116. package/simsimd/include/simsimd/binary.h +0 -201
  117. package/simsimd/include/simsimd/probability.h +0 -548
  118. package/simsimd/include/simsimd/simsimd.h +0 -456
  119. package/simsimd/include/simsimd/spatial.h +0 -1175
  120. package/simsimd/include/simsimd/types.h +0 -207
  121. package/simsimd/javascript/bench.js +0 -127
  122. package/simsimd/javascript/fallback.ts +0 -193
  123. package/simsimd/javascript/lib.c +0 -102
  124. package/simsimd/javascript/node-gyp-build.d.ts +0 -1
  125. package/simsimd/javascript/simsimd.ts +0 -124
  126. package/simsimd/javascript/test.mjs +0 -153
  127. package/simsimd/javascript/tsconfig-base.json +0 -25
  128. package/simsimd/javascript/tsconfig-cjs.json +0 -8
  129. package/simsimd/javascript/tsconfig-esm.json +0 -8
  130. package/simsimd/package-lock.json +0 -1688
  131. package/simsimd/package.json +0 -61
  132. package/simsimd/rust/benches/cosine.rs +0 -27
  133. package/simsimd/rust/benches/native.rs +0 -27
  134. package/simsimd/rust/benches/sqeuclidean.rs +0 -27
  135. package/simsimd/rust/lib.c +0 -68
  136. package/simsimd/rust/lib.rs +0 -232
  137. package/sqlite/README.md +0 -178
  138. package/sqlite/lib.cpp +0 -325
  139. package/stringzilla/CONTRIBUTING.md +0 -429
  140. package/stringzilla/Cargo.lock +0 -25
  141. package/stringzilla/Cargo.toml +0 -26
  142. package/stringzilla/LICENSE +0 -201
  143. package/stringzilla/README.md +0 -1281
  144. package/stringzilla/VERSION +0 -1
  145. package/stringzilla/assets/cover-strinzilla.jpeg +0 -0
  146. package/stringzilla/assets/meme-stringzilla-v2.jpeg +0 -0
  147. package/stringzilla/assets/meme-stringzilla-v3.jpeg +0 -0
  148. package/stringzilla/binding.gyp +0 -16
  149. package/stringzilla/build.rs +0 -14
  150. package/stringzilla/c/lib.c +0 -304
  151. package/stringzilla/cli/README.md +0 -48
  152. package/stringzilla/cli/split.py +0 -72
  153. package/stringzilla/cli/wc.py +0 -37
  154. package/stringzilla/include/stringzilla/experimental.h +0 -724
  155. package/stringzilla/include/stringzilla/module.modulemap +0 -9
  156. package/stringzilla/include/stringzilla/stringzilla.h +0 -5240
  157. package/stringzilla/include/stringzilla/stringzilla.hpp +0 -3830
  158. package/stringzilla/javascript/lib.c +0 -125
  159. package/stringzilla/javascript/stringzilla.js +0 -22
  160. package/stringzilla/package-ci.json +0 -10
  161. package/stringzilla/package.json +0 -32
  162. package/stringzilla/rust/lib.rs +0 -362
  163. package/stringzilla/swift/StringProtocol+StringZilla.swift +0 -280
  164. package/stringzilla/swift/Test.swift +0 -60
  165. package/tsconfig.json +0 -21
  166. package/wasmer.toml +0 -12
  167. package/winlibinstaller.bat +0 -42
  168. /package/{simsimd/javascript/dist-package-cjs.json → javascript/dist/cjs/package.json} +0 -0
  169. /package/{simsimd/javascript/dist-package-esm.json → javascript/dist/esm/package.json} +0 -0
package/cpp/bench.cpp DELETED
@@ -1,618 +0,0 @@
1
- /**
2
- * @brief A benchmark for the construction speed of the USearch index
3
- * and the resulting quantization (recall) of the Approximate Nearest Neighbors
4
- * Search queries.
5
- */
6
-
7
- #if defined(WIN32) || defined(_WIN32) || defined(__WIN32__) || defined(__NT__)
8
- #define NOMINMAX // define this macro to prevent the definition of min/max macros in Windows.h
9
- #define _USE_MATH_DEFINES
10
-
11
- #include <Windows.h>
12
-
13
- #include <DbgHelp.h>
14
- #pragma comment(lib, "Dbghelp.lib")
15
-
16
- #define STDERR_FILENO HANDLE(2)
17
- #else
18
- #if defined(__linux__)
19
- #include <execinfo.h> // `backtrace`
20
- #endif
21
- #include <fcntl.h> // `open`
22
- #include <stdlib.h> // `getenv`
23
- #include <sys/mman.h> // `mmap`
24
- #include <unistd.h>
25
- #endif
26
-
27
- #include <sys/stat.h> // `stat`
28
-
29
- #include <algorithm>
30
- #include <csignal>
31
- #include <cstdio>
32
- #include <iostream> // `std::cerr`
33
- #include <numeric> // `std::iota`
34
- #include <stdexcept> // `std::invalid_argument`
35
- #include <string> // `std::to_string`
36
- #include <thread> // `std::thread::hardware_concurrency()`
37
- #include <variant> // `std::monostate`
38
- #include <vector>
39
-
40
- #include <clipp.h> // Command Line Interface
41
- #if USEARCH_USE_OPENMP
42
- #include <omp.h> // `omp_set_num_threads()`
43
- #endif
44
-
45
- #include <simsimd/simsimd.h>
46
-
47
- #include <usearch/index_dense.hpp>
48
-
49
- using namespace unum::usearch;
50
- using namespace unum;
51
-
52
- using compressed_slot_t = std::uint32_t;
53
- using float_span_t = span_gt<float const>;
54
-
55
- template <typename element_at>
56
- std::size_t offset_of(element_at const* begin, element_at const* end, element_at v) noexcept {
57
- auto iterator = begin;
58
- for (; iterator != end; ++iterator)
59
- if (*iterator == v)
60
- break;
61
- return iterator - begin;
62
- }
63
-
64
- template <typename element_at> bool contains(element_at const* begin, element_at const* end, element_at v) noexcept {
65
- return offset_of(begin, end, v) != static_cast<std::size_t>(end - begin);
66
- }
67
-
68
- template <typename scalar_at> //
69
- struct alignas(32) persisted_matrix_gt {
70
- using scalar_t = scalar_at;
71
- std::uint8_t const* raw_handle{};
72
- std::size_t raw_length{};
73
- std::uint32_t rows{};
74
- std::uint32_t cols{};
75
- scalar_t const* scalars{};
76
-
77
- persisted_matrix_gt(char const* path) noexcept(false) {
78
- if (!path || !std::strlen(path))
79
- throw std::invalid_argument("The file path is empty");
80
- #if defined(USEARCH_DEFINED_WINDOWS)
81
-
82
- HANDLE file_handle =
83
- CreateFileA(path, GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr);
84
-
85
- if (file_handle == INVALID_HANDLE_VALUE)
86
- throw std::invalid_argument("Couldn't open provided file path");
87
-
88
- LARGE_INTEGER file_size;
89
- if (!GetFileSizeEx(file_handle, &file_size))
90
- throw std::invalid_argument("Couldn't obtain file stats");
91
-
92
- raw_length = file_size.QuadPart;
93
- HANDLE mapping_handle = CreateFileMapping(file_handle, nullptr, PAGE_READONLY, 0, 0, nullptr);
94
-
95
- if (mapping_handle == nullptr)
96
- throw std::invalid_argument("Couldn't create file mapping");
97
-
98
- raw_handle = (std::uint8_t*)MapViewOfFile(mapping_handle, FILE_MAP_READ, 0, 0, raw_length);
99
-
100
- if (raw_handle == nullptr)
101
- throw std::invalid_argument("Couldn't memory-map the file");
102
-
103
- std::memcpy(&rows, raw_handle, sizeof(rows));
104
- std::memcpy(&cols, raw_handle + sizeof(rows), sizeof(cols));
105
- scalars = (scalar_t*)(raw_handle + sizeof(rows) + sizeof(cols));
106
-
107
- #else
108
- auto file_descriptor = open(path, O_RDONLY | O_CLOEXEC);
109
- if (file_descriptor == -1)
110
- throw std::invalid_argument("Couldn't open provided file path");
111
- struct stat stat_vectors;
112
- if (fstat(file_descriptor, &stat_vectors) == -1)
113
- throw std::invalid_argument("Couldn't obtain file stats");
114
- raw_length = stat_vectors.st_size;
115
- raw_handle = (std::uint8_t*)mmap(NULL, raw_length, PROT_READ, MAP_PRIVATE, file_descriptor, 0);
116
- if (raw_handle == nullptr)
117
- throw std::invalid_argument("Couldn't memory-map the file");
118
- std::memcpy(&rows, raw_handle, sizeof(rows));
119
- std::memcpy(&cols, raw_handle + sizeof(rows), sizeof(cols));
120
- scalars = (scalar_t*)(raw_handle + sizeof(rows) + sizeof(cols));
121
- #endif // WINDOWS
122
- }
123
-
124
- ~persisted_matrix_gt() {
125
- if (raw_handle != nullptr)
126
- #if defined(USEARCH_DEFINED_WINDOWS)
127
- UnmapViewOfFile(raw_handle);
128
- #else
129
- munmap((void*)raw_handle, raw_length);
130
- #endif // WINDOWS
131
- }
132
-
133
- scalar_t const* row(std::size_t i) const noexcept { return scalars + i * cols; }
134
- std::size_t row_size_bytes() const noexcept { return cols * sizeof(scalar_t); }
135
- std::size_t size_bytes() const noexcept { return rows * row_size_bytes(); }
136
- };
137
-
138
- template <typename scalar_at, typename vector_id_at> //
139
- struct persisted_dataset_gt {
140
- using scalar_t = scalar_at;
141
- using compressed_slot_t = vector_id_at;
142
- persisted_matrix_gt<scalar_t> vectors_;
143
- persisted_matrix_gt<scalar_t> queries_;
144
- persisted_matrix_gt<compressed_slot_t> neighborhoods_;
145
- std::size_t vectors_to_skip_{};
146
- std::size_t vectors_to_take_{};
147
-
148
- persisted_dataset_gt(char const* path_vectors, char const* path_queries, char const* path_neighbors,
149
- std::size_t vectors_to_skip = 0, std::size_t vectors_to_take = 0) noexcept(false)
150
- : vectors_(path_vectors), queries_(path_queries), neighborhoods_(path_neighbors),
151
- vectors_to_skip_(vectors_to_skip), vectors_to_take_(vectors_to_take) {
152
-
153
- if (vectors_.cols != queries_.cols)
154
- throw std::invalid_argument("Contents and queries have different dimensionality");
155
- if (queries_.rows != neighborhoods_.rows)
156
- throw std::invalid_argument("Number of ground-truth neighborhoods doesn't match number of queries");
157
- }
158
-
159
- std::size_t dimensions() const noexcept { return vectors_.cols; }
160
- std::size_t queries_count() const noexcept { return queries_.rows; }
161
- std::size_t neighborhood_size() const noexcept { return neighborhoods_.cols; }
162
- scalar_t const* vector(std::size_t i) const noexcept { return vectors_.row(i + vectors_to_skip_); }
163
- scalar_t const* query(std::size_t i) const noexcept { return queries_.row(i); }
164
- compressed_slot_t const* neighborhood(std::size_t i) const noexcept { return neighborhoods_.row(i); }
165
-
166
- std::size_t vectors_count() const noexcept {
167
- return vectors_to_take_ ? vectors_to_take_ : (vectors_.rows - vectors_to_skip_);
168
- }
169
- vectors_view_gt<scalar_t> vectors_view() const noexcept {
170
- return {vector(vectors_to_skip_), vectors_count(), dimensions()};
171
- }
172
- };
173
-
174
- template <typename scalar_at, typename vector_id_at> //
175
- struct in_memory_dataset_gt {
176
- using scalar_t = scalar_at;
177
- using compressed_slot_t = vector_id_at;
178
-
179
- std::vector<scalar_t> vectors_{};
180
- std::vector<scalar_t> queries_{};
181
- std::vector<compressed_slot_t> neighborhoods_{};
182
- std::size_t dimensions_{};
183
- std::size_t vectors_count_{};
184
- std::size_t neighborhood_size_{};
185
- std::size_t queries_count_{};
186
-
187
- in_memory_dataset_gt( //
188
- std::size_t dimensions, std::size_t vectors_count, std::size_t queries_count,
189
- std::size_t neighborhood_size) noexcept(false)
190
- : vectors_(vectors_count * dimensions), queries_(queries_count * dimensions),
191
- neighborhoods_(queries_count * neighborhood_size), dimensions_(dimensions), vectors_count_(vectors_count),
192
- queries_count_(queries_count), neighborhood_size_(neighborhood_size) {}
193
-
194
- std::size_t dimensions() const noexcept { return dimensions_; }
195
- std::size_t vectors_count() const noexcept { return vectors_count_; }
196
- std::size_t queries_count() const noexcept { return vectors_count(); }
197
- std::size_t neighborhood_size() const noexcept { return 1; }
198
- scalar_t const* vector(std::size_t i) const noexcept { return vectors_.data() + i * dimensions_; }
199
- scalar_t const* query(std::size_t i) const noexcept { return queries_.data() + i * dimensions_; }
200
- compressed_slot_t const* neighborhood(std::size_t i) const noexcept {
201
- return neighborhoods_.data() + i * neighborhood_size_;
202
- }
203
-
204
- scalar_t* vector(std::size_t i) noexcept { return vectors_.data() + i * dimensions_; }
205
- scalar_t* query(std::size_t i) noexcept { return queries_.data() + i * dimensions_; }
206
- compressed_slot_t* neighborhood(std::size_t i) noexcept { return neighborhoods_.data() + i * neighborhood_size_; }
207
-
208
- vectors_view_gt<scalar_t> vectors_view() const noexcept { return {vector(0), vectors_count(), dimensions()}; }
209
- };
210
-
211
- char const* getenv_or(char const* name, char const* default_) { return getenv(name) ? getenv(name) : default_; }
212
-
213
- using timestamp_t = std::chrono::time_point<std::chrono::high_resolution_clock>;
214
-
215
- struct running_stats_printer_t {
216
- std::size_t total{};
217
- std::atomic<std::size_t> progress{};
218
- std::size_t last_printed_progress{};
219
- timestamp_t last_printed_time{};
220
- timestamp_t start_time{};
221
-
222
- running_stats_printer_t(std::size_t n, char const* msg) {
223
- std::printf("%s. %zu items\n", msg, n);
224
- total = n;
225
- last_printed_time = start_time = std::chrono::high_resolution_clock::now();
226
- }
227
-
228
- ~running_stats_printer_t() {
229
- std::size_t count = progress.load();
230
- timestamp_t time = std::chrono::high_resolution_clock::now();
231
- std::size_t duration = std::chrono::duration_cast<std::chrono::nanoseconds>(time - start_time).count();
232
- float vectors_per_second = count * 1e9 / duration;
233
- std::printf("\r\33[2K100 %% completed, %.0f vectors/s\n", vectors_per_second);
234
- }
235
-
236
- void refresh(std::size_t step = 1024 * 32) {
237
- std::size_t new_progress = progress.load();
238
- if (new_progress - last_printed_progress < step)
239
- return;
240
- print(new_progress, total);
241
- }
242
-
243
- void print(std::size_t progress, std::size_t total) {
244
-
245
- constexpr char bars_k[] = "||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||";
246
- constexpr std::size_t bars_len_k = 60;
247
-
248
- float percentage = progress * 1.f / total;
249
- int left_pad = (int)(percentage * bars_len_k);
250
- int right_pad = bars_len_k - left_pad;
251
-
252
- std::size_t count_new = progress - last_printed_progress;
253
- timestamp_t time_new = std::chrono::high_resolution_clock::now();
254
- std::size_t duration =
255
- std::chrono::duration_cast<std::chrono::nanoseconds>(time_new - last_printed_time).count();
256
- float vectors_per_second = count_new * 1e9 / duration;
257
-
258
- std::printf("\r%3.3f%% [%.*s%*s] %.0f vectors/s, finished %zu/%zu", percentage * 100.f, left_pad, bars_k,
259
- right_pad, "", vectors_per_second, progress, total);
260
- std::fflush(stdout);
261
-
262
- last_printed_progress = progress;
263
- last_printed_time = time_new;
264
- this->total = total;
265
- }
266
- };
267
-
268
- template <typename index_at, typename vector_id_at, typename real_at>
269
- void index_many(index_at& index, std::size_t n, vector_id_at const* ids, real_at const* vectors, std::size_t dims) {
270
-
271
- running_stats_printer_t printer{n, "Indexing"};
272
-
273
- #if USEARCH_USE_OPENMP
274
- #pragma omp parallel for schedule(static, 32)
275
- #endif
276
- for (std::size_t i = 0; i < n; ++i) {
277
- index_update_config_t config;
278
- #if USEARCH_USE_OPENMP
279
- config.thread = omp_get_thread_num();
280
- #endif
281
- float_span_t vector{vectors + dims * i, dims};
282
- index.add(ids[i], vector, config.thread);
283
- printer.progress++;
284
- if (config.thread == 0)
285
- printer.refresh();
286
- }
287
- }
288
-
289
- template <typename index_at, typename vector_id_at, typename real_at>
290
- void search_many( //
291
- index_at& index, std::size_t n, real_at const* vectors, std::size_t dims, std::size_t wanted, vector_id_at* ids,
292
- real_at* distances) {
293
-
294
- std::string name = "Search " + std::to_string(wanted);
295
- running_stats_printer_t printer{n, name.c_str()};
296
-
297
- #if USEARCH_USE_OPENMP
298
- #pragma omp parallel for schedule(static, 32)
299
- #endif
300
- for (std::size_t i = 0; i < n; ++i) {
301
- index_search_config_t config;
302
- #if USEARCH_USE_OPENMP
303
- config.thread = omp_get_thread_num();
304
- #endif
305
- float_span_t vector{vectors + dims * i, dims};
306
- index.search(vector, wanted, config.thread).dump_to(ids + wanted * i, distances + wanted * i);
307
- printer.progress++;
308
- if (config.thread == 0)
309
- printer.refresh();
310
- }
311
- }
312
-
313
- template <typename dataset_at, typename index_at> //
314
- static void single_shot(dataset_at& dataset, index_at& index, bool construct = true) {
315
- using distance_t = typename index_at::distance_t;
316
- constexpr default_key_t missing_key = std::numeric_limits<default_key_t>::max();
317
-
318
- std::printf("\n");
319
- std::printf("------------\n");
320
- if (construct) {
321
- // Perform insertions, evaluate speed
322
- std::vector<default_key_t> ids(dataset.vectors_count());
323
- std::iota(ids.begin(), ids.end(), 0);
324
- index_many(index, dataset.vectors_count(), ids.data(), dataset.vector(0), dataset.dimensions());
325
- }
326
-
327
- // Perform search, evaluate speed
328
- std::vector<default_key_t> found_neighbors(dataset.queries_count() * dataset.neighborhood_size());
329
- std::vector<distance_t> found_distances(dataset.queries_count() * dataset.neighborhood_size());
330
- search_many(index, dataset.queries_count(), dataset.query(0), dataset.dimensions(), dataset.neighborhood_size(),
331
- found_neighbors.data(), found_distances.data());
332
-
333
- // Evaluate search quality
334
- std::size_t recall_at_1 = 0, recall_full = 0;
335
- for (std::size_t i = 0; i != dataset.queries_count(); ++i) {
336
- auto expected = dataset.neighborhood(i);
337
- auto received = found_neighbors.data() + i * dataset.neighborhood_size();
338
- recall_at_1 += expected[0] == received[0];
339
- recall_full += contains(received, received + dataset.neighborhood_size(), default_key_t{expected[0]});
340
- }
341
-
342
- std::printf("Recall@1 %.2f %%\n", recall_at_1 * 100.f / dataset.queries_count());
343
- std::printf("Recall %.2f %%\n", recall_full * 100.f / dataset.queries_count());
344
-
345
- // Perform joins
346
- std::vector<default_key_t> man_to_woman(dataset.vectors_count());
347
- std::vector<default_key_t> woman_to_man(dataset.vectors_count());
348
- std::size_t join_attempts = 0;
349
- {
350
- index_at& men = index;
351
- index_at women = index.copy().index;
352
- std::fill(man_to_woman.begin(), man_to_woman.end(), missing_key);
353
- std::fill(woman_to_man.begin(), woman_to_man.end(), missing_key);
354
- {
355
- executor_default_t executor(index.limits().threads());
356
- running_stats_printer_t printer{1, "Join"};
357
- join_result_t result = join( //
358
- men, women, index_join_config_t{executor.size()}, //
359
- man_to_woman.data(), woman_to_man.data(), //
360
- executor, [&](std::size_t progress, std::size_t total) {
361
- if (progress % 1000 == 0)
362
- printer.print(progress, total);
363
- return true;
364
- });
365
- join_attempts = result.visited_members;
366
- }
367
- }
368
- // Evaluate join quality
369
- std::size_t recall_join = 0, unmatched_count = 0;
370
- for (std::size_t i = 0; i != index.size(); ++i) {
371
- recall_join += man_to_woman[i] == static_cast<default_key_t>(i);
372
- unmatched_count += man_to_woman[i] == missing_key;
373
- }
374
- std::printf("Recall Joins %.2f %%\n", recall_join * 100.f / index.size());
375
- std::printf("Unmatched %.2f %% (%zu items)\n", unmatched_count * 100.f / index.size(), unmatched_count);
376
- std::printf("Proposals %.2f / man (%zu total)\n", join_attempts * 1.f / index.size(), join_attempts);
377
-
378
- std::printf("------------\n");
379
- std::printf("\n");
380
- }
381
-
382
- void handler(int sig) {
383
- void* array[10];
384
- size_t size;
385
-
386
- // get void*'s for all entries on the stack
387
- #if defined(USEARCH_DEFINED_WINDOWS)
388
- size = CaptureStackBackTrace(0, 10, array, NULL);
389
- #elif defined(USEARCH_DEFINED_LINUX)
390
- size = backtrace(array, 10);
391
- #endif // WINDOWS
392
-
393
- // print out all the frames to stderr
394
- fprintf(stderr, "Error: signal %d:\n", sig);
395
-
396
- #if defined(USEARCH_DEFINED_WINDOWS)
397
- SYMBOL_INFO* symbol = (SYMBOL_INFO*)calloc(sizeof(SYMBOL_INFO) + 256 * sizeof(char), 1);
398
- symbol->MaxNameLen = 255;
399
- symbol->SizeOfStruct = sizeof(SYMBOL_INFO);
400
- for (int i = 0; i < size; i++) {
401
- SymFromAddr(GetCurrentProcess(), (DWORD64)(array[i]), 0, symbol);
402
- const char* name = symbol->Name;
403
- if (name == NULL) {
404
- name = "<unknown>";
405
- }
406
- DWORD bytes_written;
407
- WriteFile(STDERR_FILENO, name, strlen(name), &bytes_written, NULL);
408
- WriteFile(STDERR_FILENO, "\n", 1, &bytes_written, NULL);
409
- }
410
- free(symbol);
411
- #elif defined(USEARCH_DEFINED_LINUX)
412
- backtrace_symbols_fd(array, size, STDERR_FILENO);
413
- #endif // WINDOWS
414
-
415
- exit(1);
416
- }
417
-
418
- bool ends_with(std::string const& value, std::string const& ending) {
419
- if (ending.size() > value.size())
420
- return false;
421
- return std::equal(ending.rbegin(), ending.rend(), value.rbegin());
422
- }
423
-
424
- struct args_t {
425
- std::string path_vectors;
426
- std::string path_queries;
427
- std::string path_neighbors;
428
- std::string path_output = "last.usearch";
429
-
430
- std::size_t connectivity = default_connectivity();
431
- std::size_t expansion_add = default_expansion_add();
432
- std::size_t expansion_search = default_expansion_search();
433
- std::size_t threads = std::thread::hardware_concurrency();
434
-
435
- std::size_t vectors_to_skip = 0;
436
- std::size_t vectors_to_take = 0;
437
-
438
- bool help = false;
439
-
440
- bool big = false;
441
-
442
- bool quantize_f16 = false;
443
- bool quantize_i8 = false;
444
- bool quantize_b1 = false;
445
-
446
- bool metric_ip = false;
447
- bool metric_l2 = false;
448
- bool metric_cos = false;
449
- bool metric_haversine = false;
450
- bool metric_divergence = false;
451
- bool metric_hamming = false;
452
- bool metric_tanimoto = false;
453
- bool metric_sorensen = false;
454
-
455
- metric_kind_t metric() const noexcept {
456
- if (metric_l2)
457
- return metric_kind_t::l2sq_k;
458
- if (metric_cos)
459
- return metric_kind_t::cos_k;
460
- if (metric_haversine)
461
- return metric_kind_t::haversine_k;
462
- if (metric_divergence)
463
- return metric_kind_t::divergence_k;
464
- if (metric_hamming)
465
- return metric_kind_t::hamming_k;
466
- if (metric_tanimoto)
467
- return metric_kind_t::tanimoto_k;
468
- if (metric_sorensen)
469
- return metric_kind_t::sorensen_k;
470
- return metric_kind_t::ip_k;
471
- }
472
-
473
- scalar_kind_t quantization() const noexcept {
474
- if (quantize_f16)
475
- return scalar_kind_t::f16_k;
476
- if (quantize_i8)
477
- return scalar_kind_t::i8_k;
478
- if (quantize_b1)
479
- return scalar_kind_t::b1x8_k;
480
- return scalar_kind_t::f32_k;
481
- }
482
- };
483
-
484
- template <typename index_at, typename dataset_at> //
485
- void run_punned(dataset_at& dataset, args_t const& args, index_config_t config, index_limits_t limits) {
486
-
487
- scalar_kind_t quantization = args.quantization();
488
- std::printf("-- Quantization: %s\n", scalar_kind_name(quantization));
489
-
490
- metric_kind_t kind = args.metric();
491
- std::printf("-- Metric: %s\n", metric_kind_name(kind));
492
-
493
- metric_punned_t metric(dataset.dimensions(), kind, quantization);
494
- index_at index = index_at::make(metric, config);
495
- index.reserve(limits);
496
- std::printf("-- Hardware acceleration: %s\n", index.metric().isa_name());
497
- std::printf("Will benchmark in-memory\n");
498
-
499
- single_shot(dataset, index, true);
500
- index.save(args.path_output.c_str());
501
-
502
- std::printf("Will benchmark an on-disk view\n");
503
-
504
- index_at index_view = index.fork().index;
505
- index_view.view(args.path_output.c_str());
506
- single_shot(dataset, index_view, false);
507
- }
508
-
509
- template <typename index_at, typename dataset_at> //
510
- void run_typed(dataset_at& dataset, args_t const& args, index_config_t config, index_limits_t limits) {
511
-
512
- index_at index(config);
513
- index.reserve(limits);
514
- std::printf("Will benchmark in-memory\n");
515
-
516
- single_shot(dataset, index, true);
517
- index.save(args.path_output.c_str());
518
-
519
- std::printf("Will benchmark an on-disk view\n");
520
-
521
- index_at index_view = index.fork();
522
- index_view.view(args.path_output.c_str());
523
- single_shot(dataset, index_view, false);
524
- }
525
-
526
- int main(int argc, char** argv) {
527
-
528
- // Print backtrace if something goes wrong.
529
- signal(SIGSEGV, handler);
530
-
531
- using namespace clipp;
532
-
533
- auto args = args_t{};
534
- auto cli = ( //
535
- (option("--vectors") & value("path", args.path_vectors)).doc(".fbin file path to construct the index"),
536
- (option("--queries") & value("path", args.path_queries)).doc(".fbin file path to query the index"),
537
- (option("--neighbors") & value("path", args.path_neighbors)).doc(".ibin file path with ground truth"),
538
- (option("-o", "--output") & value("path", args.path_output)).doc(".usearch output file path"),
539
- (option("-b", "--big").set(args.big)).doc("Will switch to uint40_t for neighbors lists with over 4B entries"),
540
- (option("-j", "--threads") & value("integer", args.threads)).doc("Uses all available cores by default"),
541
- (option("-c", "--connectivity") & value("integer", args.connectivity)).doc("Index granularity"),
542
- (option("--expansion-add") & value("integer", args.expansion_add)).doc("Affects indexing depth"),
543
- (option("--expansion-search") & value("integer", args.expansion_search)).doc("Affects search depth"),
544
- (option("--rows-skip") & value("integer", args.vectors_to_skip)).doc("Number of vectors to skip"),
545
- (option("--rows-take") & value("integer", args.vectors_to_take)).doc("Number of vectors to take"),
546
- ( //
547
- option("-f16", "--f16quant").set(args.quantize_f16).doc("Enable `f16_t` quantization") |
548
- option("-i8", "--i8quant").set(args.quantize_i8).doc("Enable `i8_t` quantization") |
549
- option("-b1", "--b1quant").set(args.quantize_b1).doc("Enable `b1x8_t` quantization")),
550
- ( //
551
- option("--ip").set(args.metric_ip).doc("Choose Inner Product metric") |
552
- option("--l2sq").set(args.metric_l2).doc("Choose L2 Euclidean metric") |
553
- option("--cos").set(args.metric_cos).doc("Choose Angular metric") |
554
- option("--hamming").set(args.metric_hamming).doc("Choose Hamming metric") |
555
- option("--tanimoto").set(args.metric_tanimoto).doc("Choose Tanimoto metric") |
556
- option("--sorensen").set(args.metric_sorensen).doc("Choose Sorensen metric") |
557
- option("--haversine").set(args.metric_haversine).doc("Choose Haversine metric")),
558
- option("-h", "--help").set(args.help).doc("Print this help information on this tool and exit"));
559
-
560
- if (!parse(argc, argv, cli)) {
561
- std::cerr << make_man_page(cli, argv[0]);
562
- exit(1);
563
- }
564
- if (args.help) {
565
- std::cout << make_man_page(cli, argv[0]);
566
- exit(0);
567
- }
568
-
569
- #if USEARCH_USE_OPENMP
570
- // Instead of relying on `multithreaded` from "index_dense.hpp" we will use OpenMP
571
- // to better estimate statistics between tasks batches, without having to recreate
572
- // the threads.
573
- omp_set_dynamic(true);
574
- omp_set_num_threads(args.threads);
575
- std::printf("- OpenMP threads: %d\n", omp_get_max_threads());
576
- #endif
577
-
578
- std::printf("- Dataset: \n");
579
- std::printf("-- Base vectors path: %s\n", args.path_vectors.c_str());
580
- std::printf("-- Query vectors path: %s\n", args.path_queries.c_str());
581
- std::printf("-- Ground truth neighbors path: %s\n", args.path_neighbors.c_str());
582
-
583
- persisted_dataset_gt<float, compressed_slot_t> dataset{
584
- args.path_vectors.c_str(), //
585
- args.path_queries.c_str(), //
586
- args.path_neighbors.c_str(), //
587
- args.vectors_to_skip, //
588
- args.vectors_to_take, //
589
- };
590
- std::printf("-- Dimensions: %zu\n", dataset.dimensions());
591
- std::printf("-- Vectors count: %zu\n", dataset.vectors_count());
592
- std::printf("-- Queries count: %zu\n", dataset.queries_count());
593
- std::printf("-- Neighbors per query: %zu\n", dataset.neighborhood_size());
594
-
595
- // report_alternative_setups();
596
- // report_expected_losses(dataset);
597
-
598
- index_dense_config_t config(args.connectivity, args.expansion_add, args.expansion_search);
599
- index_limits_t limits;
600
- limits.threads_add = limits.threads_search = args.threads;
601
- limits.members = dataset.vectors_count();
602
-
603
- std::printf("- Index: \n");
604
- std::printf("-- Connectivity: %zu\n", config.connectivity);
605
- std::printf("-- Expansion @ Add: %zu\n", config.expansion_add);
606
- std::printf("-- Expansion @ Search: %zu\n", config.expansion_search);
607
-
608
- if (args.big)
609
- #ifdef USEARCH_64BIT_ENV
610
- run_punned<index_dense_gt<default_key_t, uint40_t>>(dataset, args, config, limits);
611
- #else
612
- std::printf("Error: Don't use 40 bit identifiers in 32bit environment\n");
613
- #endif
614
- else
615
- run_punned<index_dense_gt<default_key_t, std::uint32_t>>(dataset, args, config, limits);
616
-
617
- return 0;
618
- }