usearch 0.2.4 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.clang-tidy +60 -0
- package/.github/ISSUE_TEMPLATE/bug_report.yml +90 -0
- package/.github/ISSUE_TEMPLATE/config.yml +5 -0
- package/.github/ISSUE_TEMPLATE/feature_request.yml +64 -0
- package/.github/workflows/release.yml +47 -20
- package/.github/workflows/update_version.sh +1 -0
- package/.vscode/launch.json +22 -0
- package/.vscode/settings.json +9 -1
- package/.vscode/tasks.json +4 -4
- package/CITATION.cff +11 -0
- package/CMakeLists.txt +108 -95
- package/CODE_OF_CONDUCT.md +128 -0
- package/CONTRIBUTING.md +12 -0
- package/Cargo.toml +1 -1
- package/Dockerfile +14 -0
- package/README.md +21 -12
- package/VERSION +1 -1
- package/binding.gyp +13 -9
- package/build.rs +2 -1
- package/cmake/config.cmake.in +15 -0
- package/cmake/pkg-config.pc.in +4 -0
- package/cmake/usearchConfigVersion.cmake.in +20 -0
- package/conanfile.py +1 -1
- package/docs/{Benchmarking.md → benchmarks.md} +11 -2
- package/docs/benchmarks.rst +5 -0
- package/docs/{Compilation.md → compilation.md} +54 -27
- package/docs/compilation.rst +5 -0
- package/docs/conf.dox +2668 -0
- package/docs/index.rst +7 -0
- package/include/usearch/usearch.hpp +833 -439
- package/java/cloud/unum/usearch/cloud_unum_usearch_Index.cpp +17 -16
- package/javascript/lib.cpp +94 -20
- package/javascript/test.js +1 -1
- package/objc/USearchObjective.mm +47 -30
- package/objc/include/USearchObjective.h +18 -26
- package/package.json +1 -1
- package/pyproject.toml +3 -2
- package/python/lib.cpp +216 -130
- package/python/{bench.py → scripts/bench.py} +6 -1
- package/python/scripts/bench_params.py +112 -0
- package/python/{test.py → scripts/test.py} +32 -13
- package/python/usearch/client.py +100 -14
- package/python/usearch/server.py +104 -23
- package/rust/lib.cpp +20 -15
- package/rust/lib.hpp +5 -5
- package/rust/lib.rs +3 -3
- package/setup.py +7 -0
- package/src/CMakeLists.txt +107 -0
- package/src/bench.cpp +54 -19
- package/src/{advanced.hpp → punned.hpp} +235 -203
- package/src/test.cpp +62 -0
- package/swift/Index+Sugar.swift +1 -0
- package/swift/Test.swift +2 -1
- package/wolfram/lib.cpp +14 -14
|
@@ -9,51 +9,84 @@
|
|
|
9
9
|
#ifndef UNUM_USEARCH_H
|
|
10
10
|
#define UNUM_USEARCH_H
|
|
11
11
|
|
|
12
|
+
// Inferring C++ version
|
|
13
|
+
#if ((defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) || __cplusplus >= 201703L)
|
|
14
|
+
#define USEARCH_IS_CPP17
|
|
15
|
+
#endif
|
|
16
|
+
|
|
17
|
+
// Inferring target OS
|
|
12
18
|
#if defined(WIN32) || defined(_WIN32) || defined(__WIN32__) || defined(__NT__)
|
|
13
|
-
#define
|
|
19
|
+
#define USEARCH_IS_WINDOWS
|
|
20
|
+
#elif defined(__APPLE__) && defined(__MACH__)
|
|
21
|
+
#define USEARCH_IS_APPLE
|
|
22
|
+
#elif defined(__linux__)
|
|
23
|
+
#define USEARCH_IS_LINUX
|
|
24
|
+
#endif
|
|
14
25
|
|
|
15
|
-
|
|
26
|
+
// Inferring the compiler
|
|
27
|
+
#if defined(__clang__)
|
|
28
|
+
#define USEARCH_IS_CLANG
|
|
29
|
+
#elif defined(__GNUC__)
|
|
30
|
+
#define USEARCH_IS_GCC
|
|
31
|
+
#endif
|
|
16
32
|
|
|
17
|
-
|
|
18
|
-
#
|
|
19
|
-
#define
|
|
33
|
+
// Inferring hardware architecture: x86 vs Arm
|
|
34
|
+
#if defined(__x86_64__)
|
|
35
|
+
#define USEARCH_IS_X86
|
|
36
|
+
#elif defined(__aarch64__)
|
|
37
|
+
#define USEARCH_IS_ARM
|
|
38
|
+
#endif
|
|
20
39
|
|
|
40
|
+
// OS-specific includes
|
|
41
|
+
#if defined(USEARCH_IS_WINDOWS)
|
|
42
|
+
#define _USE_MATH_DEFINES
|
|
43
|
+
#include <Windows.h>
|
|
44
|
+
#include <sys/stat.h> // `fstat` for file size
|
|
21
45
|
#else
|
|
22
46
|
#include <fcntl.h> // `fallocate`
|
|
23
47
|
#include <stdlib.h> // `posix_memalign`
|
|
24
48
|
#include <sys/mman.h> // `mmap`
|
|
49
|
+
#include <sys/stat.h> // `fstat` for file size
|
|
25
50
|
#include <unistd.h> // `open`, `close`
|
|
26
|
-
|
|
27
|
-
#define usearch_pack_m __attribute__((packed))
|
|
28
|
-
#define usearch_align_m __attribute__((aligned(64)))
|
|
29
51
|
#endif
|
|
30
52
|
|
|
31
|
-
|
|
32
|
-
|
|
53
|
+
// STL includes
|
|
33
54
|
#include <algorithm> // `std::sort_heap`
|
|
34
55
|
#include <atomic> // `std::atomic`
|
|
35
56
|
#include <bitset> // `std::bitset`
|
|
36
57
|
#include <climits> // `CHAR_BIT`
|
|
37
58
|
#include <cmath> // `std::sqrt`
|
|
38
59
|
#include <cstring> // `std::memset`
|
|
60
|
+
#include <iterator> // `std::reverse_iterator`
|
|
39
61
|
#include <mutex> // `std::unique_lock` - replacement candidate
|
|
40
62
|
#include <random> // `std::default_random_engine` - replacement candidate
|
|
41
63
|
#include <stdexcept> // `std::runtime_exception`
|
|
42
|
-
#include <utility> // `std::exchange`
|
|
64
|
+
#include <utility> // `std::exchange`, `std::pair`
|
|
43
65
|
#include <vector> // `std::vector`
|
|
44
66
|
|
|
45
|
-
|
|
67
|
+
// Prefetching
|
|
68
|
+
#if defined(USEARCH_IS_GCC)
|
|
46
69
|
// https://gcc.gnu.org/onlinedocs/gcc/Other-Builtins.html
|
|
47
70
|
// Zero means we are only going to read from that memory.
|
|
48
71
|
// Three means high temporal locality and suggests to keep
|
|
49
72
|
// the data in all layers of cache.
|
|
50
73
|
#define prefetch_m(ptr) __builtin_prefetch((void*)(ptr), 0, 3)
|
|
51
|
-
#elif defined(
|
|
74
|
+
#elif defined(USEARCH_IS_X86)
|
|
52
75
|
#define prefetch_m(ptr) _mm_prefetch((void*)(ptr), _MM_HINT_T0)
|
|
53
76
|
#else
|
|
54
77
|
#define prefetch_m(ptr)
|
|
55
78
|
#endif
|
|
56
79
|
|
|
80
|
+
// Alignment
|
|
81
|
+
#if defined(USEARCH_IS_WINDOWS)
|
|
82
|
+
#define usearch_pack_m
|
|
83
|
+
#define usearch_align_m __declspec(align(64))
|
|
84
|
+
#else
|
|
85
|
+
#define usearch_pack_m __attribute__((packed))
|
|
86
|
+
#define usearch_align_m __attribute__((aligned(64)))
|
|
87
|
+
#endif
|
|
88
|
+
|
|
89
|
+
// Debugging
|
|
57
90
|
#if defined(NDEBUG)
|
|
58
91
|
#define assert_m(must_be_true, message)
|
|
59
92
|
#else
|
|
@@ -68,11 +101,16 @@ namespace usearch {
|
|
|
68
101
|
|
|
69
102
|
using f64_t = double;
|
|
70
103
|
using f32_t = float;
|
|
104
|
+
using byte_t = char;
|
|
71
105
|
|
|
72
106
|
template <typename at> at angle_to_radians(at angle) noexcept { return angle * M_PI / 180.f; }
|
|
73
107
|
|
|
74
108
|
template <typename at> at square(at value) noexcept { return value * value; }
|
|
75
109
|
|
|
110
|
+
template <std::size_t multiple_ak> inline std::size_t divide_round_up(std::size_t num) noexcept {
|
|
111
|
+
return (num + multiple_ak - 1) / multiple_ak;
|
|
112
|
+
}
|
|
113
|
+
|
|
76
114
|
inline std::size_t ceil2(std::size_t v) noexcept {
|
|
77
115
|
v--;
|
|
78
116
|
v |= v >> 1;
|
|
@@ -85,6 +123,71 @@ inline std::size_t ceil2(std::size_t v) noexcept {
|
|
|
85
123
|
return v;
|
|
86
124
|
}
|
|
87
125
|
|
|
126
|
+
template <typename at> void misaligned_store(void* ptr, at v) noexcept { std::memcpy(ptr, &v, sizeof(v)); }
|
|
127
|
+
|
|
128
|
+
template <typename at> at misaligned_load(void* ptr) noexcept {
|
|
129
|
+
at v;
|
|
130
|
+
std::memcpy(&v, ptr, sizeof(v));
|
|
131
|
+
return v;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
template <typename at> class misaligned_ref_gt {
|
|
135
|
+
byte_t* ptr_;
|
|
136
|
+
|
|
137
|
+
public:
|
|
138
|
+
misaligned_ref_gt(byte_t* ptr) noexcept : ptr_(ptr) {}
|
|
139
|
+
operator at() const noexcept { return misaligned_load<at>(ptr_); }
|
|
140
|
+
misaligned_ref_gt& operator=(at const& v) noexcept {
|
|
141
|
+
misaligned_store<at>(ptr_, v);
|
|
142
|
+
return *this;
|
|
143
|
+
}
|
|
144
|
+
misaligned_ref_gt& operator=(at v) noexcept {
|
|
145
|
+
misaligned_store<at>(ptr_, v);
|
|
146
|
+
return *this;
|
|
147
|
+
}
|
|
148
|
+
};
|
|
149
|
+
|
|
150
|
+
template <typename at> class misaligned_ptr_gt {
|
|
151
|
+
byte_t* ptr_;
|
|
152
|
+
|
|
153
|
+
public:
|
|
154
|
+
using iterator_category = std::random_access_iterator_tag;
|
|
155
|
+
using value_type = at;
|
|
156
|
+
using difference_type = std::ptrdiff_t;
|
|
157
|
+
using pointer = misaligned_ptr_gt<at>;
|
|
158
|
+
using reference = misaligned_ref_gt<at>;
|
|
159
|
+
|
|
160
|
+
reference operator*() const noexcept { return {ptr_}; }
|
|
161
|
+
|
|
162
|
+
misaligned_ptr_gt(byte_t* ptr) noexcept : ptr_(ptr) {}
|
|
163
|
+
misaligned_ptr_gt operator++(int) noexcept { return misaligned_ptr_gt(ptr_ + sizeof(at)); }
|
|
164
|
+
misaligned_ptr_gt operator--(int) noexcept { return misaligned_ptr_gt(ptr_ - sizeof(at)); }
|
|
165
|
+
misaligned_ptr_gt operator+(difference_type d) noexcept { return misaligned_ptr_gt(ptr_ + d * sizeof(at)); }
|
|
166
|
+
misaligned_ptr_gt operator-(difference_type d) noexcept { return misaligned_ptr_gt(ptr_ - d * sizeof(at)); }
|
|
167
|
+
|
|
168
|
+
misaligned_ptr_gt& operator++() noexcept {
|
|
169
|
+
ptr_ += sizeof(at);
|
|
170
|
+
return *this;
|
|
171
|
+
}
|
|
172
|
+
misaligned_ptr_gt& operator--() noexcept {
|
|
173
|
+
ptr_ -= sizeof(at);
|
|
174
|
+
return *this;
|
|
175
|
+
}
|
|
176
|
+
misaligned_ptr_gt& operator+=(difference_type d) noexcept {
|
|
177
|
+
ptr_ += d * sizeof(at);
|
|
178
|
+
return *this;
|
|
179
|
+
}
|
|
180
|
+
misaligned_ptr_gt& operator-=(difference_type d) noexcept {
|
|
181
|
+
ptr_ -= d * sizeof(at);
|
|
182
|
+
return *this;
|
|
183
|
+
}
|
|
184
|
+
bool operator==(misaligned_ptr_gt const& other) noexcept { return ptr_ == other.ptr_; }
|
|
185
|
+
bool operator!=(misaligned_ptr_gt const& other) noexcept { return ptr_ != other.ptr_; }
|
|
186
|
+
};
|
|
187
|
+
|
|
188
|
+
/**
|
|
189
|
+
* @brief Inner (Dot) Product distance.
|
|
190
|
+
*/
|
|
88
191
|
template <typename scalar_at, typename result_at = scalar_at> struct ip_gt {
|
|
89
192
|
using scalar_t = scalar_at;
|
|
90
193
|
using result_t = result_at;
|
|
@@ -92,12 +195,12 @@ template <typename scalar_at, typename result_at = scalar_at> struct ip_gt {
|
|
|
92
195
|
|
|
93
196
|
inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t dim, std::size_t = 0) const noexcept {
|
|
94
197
|
result_type ab{};
|
|
95
|
-
#if defined(
|
|
96
|
-
#pragma GCC ivdep
|
|
97
|
-
#elif defined(__clang__)
|
|
98
|
-
#pragma clang loop vectorize(enable)
|
|
99
|
-
#elif defined(_OPENMP)
|
|
198
|
+
#if defined(USEARCH_USE_OPENMP)
|
|
100
199
|
#pragma omp simd reduction(+ : ab)
|
|
200
|
+
#elif defined(USEARCH_IS_CLANG)
|
|
201
|
+
#pragma clang loop vectorize(enable)
|
|
202
|
+
#elif defined(USEARCH_IS_GCC)
|
|
203
|
+
#pragma GCC ivdep
|
|
101
204
|
#endif
|
|
102
205
|
for (std::size_t i = 0; i != dim; ++i)
|
|
103
206
|
ab += result_t(a[i]) * result_t(b[i]);
|
|
@@ -105,6 +208,12 @@ template <typename scalar_at, typename result_at = scalar_at> struct ip_gt {
|
|
|
105
208
|
}
|
|
106
209
|
};
|
|
107
210
|
|
|
211
|
+
/**
|
|
212
|
+
* @brief Cosine (Angular) distance.
|
|
213
|
+
* Identical to the Inner Product of normalized vectors.
|
|
214
|
+
* Unless you are running on an tiny embedded platform, this metric
|
|
215
|
+
* is recommended over `::ip_gt` for low-precision scalars.
|
|
216
|
+
*/
|
|
108
217
|
template <typename scalar_at, typename result_at = scalar_at> struct cos_gt {
|
|
109
218
|
using scalar_t = scalar_at;
|
|
110
219
|
using result_t = result_at;
|
|
@@ -112,21 +221,25 @@ template <typename scalar_at, typename result_at = scalar_at> struct cos_gt {
|
|
|
112
221
|
|
|
113
222
|
inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t dim, std::size_t = 0) const noexcept {
|
|
114
223
|
result_t ab{}, a2{}, b2{};
|
|
115
|
-
#if defined(
|
|
116
|
-
#pragma GCC ivdep
|
|
117
|
-
#elif defined(__clang__)
|
|
118
|
-
#pragma clang loop vectorize(enable)
|
|
119
|
-
#elif defined(_OPENMP)
|
|
224
|
+
#if defined(USEARCH_USE_OPENMP)
|
|
120
225
|
#pragma omp simd reduction(+ : ab, a2, b2)
|
|
226
|
+
#elif defined(USEARCH_IS_CLANG)
|
|
227
|
+
#pragma clang loop vectorize(enable)
|
|
228
|
+
#elif defined(USEARCH_IS_GCC)
|
|
229
|
+
#pragma GCC ivdep
|
|
121
230
|
#endif
|
|
122
231
|
for (std::size_t i = 0; i != dim; ++i)
|
|
123
232
|
ab += result_t(a[i]) * result_t(b[i]), //
|
|
124
233
|
a2 += square<result_t>(a[i]), //
|
|
125
234
|
b2 += square<result_t>(b[i]);
|
|
126
|
-
return 1 - ab / (std::sqrt(a2) * std::sqrt(b2));
|
|
235
|
+
return ab ? (1 - ab / (std::sqrt(a2) * std::sqrt(b2))) : 0;
|
|
127
236
|
}
|
|
128
237
|
};
|
|
129
238
|
|
|
239
|
+
/**
|
|
240
|
+
* @brief Squared Euclidean (L2) distance.
|
|
241
|
+
* Square root is avoided at the end, as it won't affect the ordering.
|
|
242
|
+
*/
|
|
130
243
|
template <typename scalar_at, typename result_at = scalar_at> struct l2sq_gt {
|
|
131
244
|
using scalar_t = scalar_at;
|
|
132
245
|
using result_t = result_at;
|
|
@@ -134,12 +247,12 @@ template <typename scalar_at, typename result_at = scalar_at> struct l2sq_gt {
|
|
|
134
247
|
|
|
135
248
|
inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t dim, std::size_t = 0) const noexcept {
|
|
136
249
|
result_t ab_deltas_sq{};
|
|
137
|
-
#if defined(
|
|
138
|
-
#pragma GCC ivdep
|
|
139
|
-
#elif defined(__clang__)
|
|
140
|
-
#pragma clang loop vectorize(enable)
|
|
141
|
-
#elif defined(_OPENMP)
|
|
250
|
+
#if defined(USEARCH_USE_OPENMP)
|
|
142
251
|
#pragma omp simd reduction(+ : ab_deltas_sq)
|
|
252
|
+
#elif defined(USEARCH_IS_CLANG)
|
|
253
|
+
#pragma clang loop vectorize(enable)
|
|
254
|
+
#elif defined(USEARCH_IS_GCC)
|
|
255
|
+
#pragma GCC ivdep
|
|
143
256
|
#endif
|
|
144
257
|
for (std::size_t i = 0; i != dim; ++i)
|
|
145
258
|
ab_deltas_sq += square(result_t(a[i]) - result_t(b[i]));
|
|
@@ -156,17 +269,16 @@ template <typename scalar_at, typename result_at = std::size_t> struct hamming_g
|
|
|
156
269
|
using result_t = result_at;
|
|
157
270
|
using result_type = result_t;
|
|
158
271
|
|
|
159
|
-
inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t
|
|
160
|
-
std::size_t = 0) const noexcept {
|
|
272
|
+
inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t dim, std::size_t = 0) const noexcept {
|
|
161
273
|
result_t matches{};
|
|
162
|
-
#if defined(
|
|
163
|
-
#pragma GCC ivdep
|
|
164
|
-
#elif defined(__clang__)
|
|
165
|
-
#pragma clang loop vectorize(enable)
|
|
166
|
-
#elif defined(_OPENMP)
|
|
274
|
+
#if defined(USEARCH_USE_OPENMP)
|
|
167
275
|
#pragma omp simd reduction(+ : matches)
|
|
276
|
+
#elif defined(USEARCH_IS_CLANG)
|
|
277
|
+
#pragma clang loop vectorize(enable)
|
|
278
|
+
#elif defined(USEARCH_IS_GCC)
|
|
279
|
+
#pragma GCC ivdep
|
|
168
280
|
#endif
|
|
169
|
-
for (std::size_t i = 0; i !=
|
|
281
|
+
for (std::size_t i = 0; i != dim; ++i)
|
|
170
282
|
matches += a[i] != b[i];
|
|
171
283
|
return matches;
|
|
172
284
|
}
|
|
@@ -182,17 +294,17 @@ template <typename scalar_at, typename result_at = std::size_t> struct bit_hammi
|
|
|
182
294
|
using result_t = result_at;
|
|
183
295
|
using result_type = result_t;
|
|
184
296
|
static_assert(std::is_unsigned<scalar_t>::value, "Hamming distance requires unsigned integral words");
|
|
185
|
-
static constexpr std::size_t bits_per_word_k = sizeof(scalar_t) * CHAR_BIT;
|
|
186
297
|
|
|
187
298
|
inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t words,
|
|
188
299
|
std::size_t = 0) const noexcept {
|
|
300
|
+
constexpr std::size_t bits_per_word_k = sizeof(scalar_t) * CHAR_BIT;
|
|
189
301
|
result_t matches{};
|
|
190
|
-
#if defined(
|
|
191
|
-
#pragma GCC ivdep
|
|
192
|
-
#elif defined(__clang__)
|
|
193
|
-
#pragma clang loop vectorize(enable)
|
|
194
|
-
#elif defined(_OPENMP)
|
|
302
|
+
#if defined(USEARCH_USE_OPENMP)
|
|
195
303
|
#pragma omp simd reduction(+ : matches)
|
|
304
|
+
#elif defined(USEARCH_IS_CLANG)
|
|
305
|
+
#pragma clang loop vectorize(enable)
|
|
306
|
+
#elif defined(USEARCH_IS_GCC)
|
|
307
|
+
#pragma GCC ivdep
|
|
196
308
|
#endif
|
|
197
309
|
for (std::size_t i = 0; i != words; ++i)
|
|
198
310
|
matches += std::bitset<bits_per_word_k>(a[i] ^ b[i]).count();
|
|
@@ -239,12 +351,12 @@ template <typename scalar_at, typename result_at = scalar_at> struct pearson_cor
|
|
|
239
351
|
scalar_t const* a, scalar_t const* b, std::size_t dim, std::size_t = 0) const noexcept {
|
|
240
352
|
result_t a_sum{}, b_sum{}, ab_sum{};
|
|
241
353
|
result_t a_sq_sum{}, b_sq_sum{};
|
|
242
|
-
#if defined(
|
|
243
|
-
#pragma GCC ivdep
|
|
244
|
-
#elif defined(__clang__)
|
|
245
|
-
#pragma clang loop vectorize(enable)
|
|
246
|
-
#elif defined(_OPENMP)
|
|
354
|
+
#if defined(USEARCH_USE_OPENMP)
|
|
247
355
|
#pragma omp simd reduction(+ : a_sum, b_sum, ab_sum, a_sq_sum, b_sq_sum)
|
|
356
|
+
#elif defined(USEARCH_IS_CLANG)
|
|
357
|
+
#pragma clang loop vectorize(enable)
|
|
358
|
+
#elif defined(USEARCH_IS_GCC)
|
|
359
|
+
#pragma GCC ivdep
|
|
248
360
|
#endif
|
|
249
361
|
for (std::size_t i = 0; i != dim; ++i) {
|
|
250
362
|
a_sum += a[i];
|
|
@@ -287,16 +399,21 @@ template <typename scalar_at, typename result_at = scalar_at> struct haversine_g
|
|
|
287
399
|
}
|
|
288
400
|
};
|
|
289
401
|
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
402
|
+
/**
|
|
403
|
+
* @brief Light-weight bitset implementation to track visited nodes during graph traversal.
|
|
404
|
+
* Extends basic functionality with atomic operations.
|
|
405
|
+
*/
|
|
294
406
|
template <typename allocator_at = std::allocator<char>> class visits_bitset_gt {
|
|
295
407
|
using allocator_t = allocator_at;
|
|
296
408
|
using byte_t = typename allocator_t::value_type;
|
|
297
409
|
static_assert(sizeof(byte_t) == 1, "Allocator must allocate separate addressable bytes");
|
|
298
410
|
|
|
411
|
+
#if defined(USEARCH_IS_WINDOWS)
|
|
412
|
+
using slot_t = LONG volatile;
|
|
413
|
+
static_assert(sizeof(slot_t) == 8);
|
|
414
|
+
#else
|
|
299
415
|
using slot_t = std::uint64_t;
|
|
416
|
+
#endif
|
|
300
417
|
|
|
301
418
|
std::uint64_t* u64s_{};
|
|
302
419
|
std::size_t slots_{};
|
|
@@ -306,6 +423,7 @@ template <typename allocator_at = std::allocator<char>> class visits_bitset_gt {
|
|
|
306
423
|
visits_bitset_gt(std::size_t capacity) noexcept {
|
|
307
424
|
slots_ = divide_round_up<64>(capacity);
|
|
308
425
|
u64s_ = (slot_t*)allocator_t{}.allocate(slots_ * sizeof(slot_t));
|
|
426
|
+
clear();
|
|
309
427
|
}
|
|
310
428
|
~visits_bitset_gt() noexcept { allocator_t{}.deallocate((byte_t*)u64s_, slots_ * sizeof(slot_t)), u64s_ = nullptr; }
|
|
311
429
|
void clear() noexcept { std::memset(u64s_, 0, slots_ * sizeof(slot_t)); }
|
|
@@ -324,6 +442,42 @@ template <typename allocator_at = std::allocator<char>> class visits_bitset_gt {
|
|
|
324
442
|
std::swap(slots_, other.slots_);
|
|
325
443
|
return *this;
|
|
326
444
|
}
|
|
445
|
+
|
|
446
|
+
#if defined(USEARCH_IS_WINDOWS)
|
|
447
|
+
|
|
448
|
+
inline bool atomic_test(std::size_t i) noexcept {
|
|
449
|
+
slot_t mask{1ul << (i & 63ul)};
|
|
450
|
+
return InterlockedOr(&u64s_[i / 64], 0) & mask;
|
|
451
|
+
}
|
|
452
|
+
|
|
453
|
+
inline bool atomic_set(std::size_t i) noexcept {
|
|
454
|
+
slot_t mask{1ul << (i & 63ul)};
|
|
455
|
+
return InterLockedOr(&u64s_[i / 64], mask) & mask;
|
|
456
|
+
}
|
|
457
|
+
|
|
458
|
+
inline void atomic_reset(std::size_t i) noexcept {
|
|
459
|
+
slot_t mask{1ul << (i & 63ul)};
|
|
460
|
+
InterLockedAnd((LONG volatile*)&u64s_[i / 64], ~mask);
|
|
461
|
+
}
|
|
462
|
+
|
|
463
|
+
#else
|
|
464
|
+
|
|
465
|
+
inline bool atomic_test(std::size_t i) noexcept {
|
|
466
|
+
slot_t mask{1ul << (i & 63ul)};
|
|
467
|
+
return __atomic_load_n(&u64s_[i / 64], __ATOMIC_RELAXED) & mask;
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
inline bool atomic_set(std::size_t i) noexcept {
|
|
471
|
+
slot_t mask{1ul << (i & 63ul)};
|
|
472
|
+
return __atomic_fetch_or(&u64s_[i / 64], mask, __ATOMIC_ACQUIRE) & mask;
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
inline void atomic_reset(std::size_t i) noexcept {
|
|
476
|
+
slot_t mask{1ul << (i & 63ul)};
|
|
477
|
+
__atomic_fetch_and(&u64s_[i / 64], ~mask, __ATOMIC_RELEASE);
|
|
478
|
+
}
|
|
479
|
+
|
|
480
|
+
#endif
|
|
327
481
|
};
|
|
328
482
|
|
|
329
483
|
using visits_bitset_t = visits_bitset_gt<>;
|
|
@@ -564,63 +718,13 @@ class sorted_buffer_gt {
|
|
|
564
718
|
static bool less(element_t const& a, element_t const& b) noexcept { return comparator_t{}(a, b); }
|
|
565
719
|
};
|
|
566
720
|
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
*/
|
|
570
|
-
class mutex_t {
|
|
571
|
-
#if defined(WINDOWS)
|
|
572
|
-
using slot_t = volatile LONG;
|
|
573
|
-
#else
|
|
574
|
-
using slot_t = std::int32_t;
|
|
575
|
-
#endif // WINDOWS
|
|
576
|
-
|
|
577
|
-
slot_t flag_;
|
|
578
|
-
|
|
579
|
-
public:
|
|
580
|
-
inline mutex_t(slot_t flag = 0) noexcept : flag_(flag) {}
|
|
581
|
-
inline ~mutex_t() noexcept {}
|
|
582
|
-
|
|
583
|
-
inline bool try_lock() noexcept {
|
|
584
|
-
slot_t raw = 0;
|
|
585
|
-
#if defined(WINDOWS)
|
|
586
|
-
return InterlockedCompareExchange(&flag_, 1, raw);
|
|
587
|
-
#else
|
|
588
|
-
return __atomic_compare_exchange_n(&flag_, &raw, 1, true, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
|
|
589
|
-
#endif // WINDOWS
|
|
590
|
-
}
|
|
591
|
-
|
|
592
|
-
inline void lock() noexcept {
|
|
593
|
-
slot_t raw = 0;
|
|
594
|
-
#if defined(WINDOWS)
|
|
595
|
-
InterlockedCompareExchange(&flag_, 1, raw);
|
|
596
|
-
#else
|
|
597
|
-
lock_again:
|
|
598
|
-
raw = 0;
|
|
599
|
-
if (!__atomic_compare_exchange_n(&flag_, &raw, 1, true, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED))
|
|
600
|
-
goto lock_again;
|
|
601
|
-
#endif // WINDOWS
|
|
602
|
-
}
|
|
603
|
-
|
|
604
|
-
inline void unlock() noexcept {
|
|
605
|
-
#if defined(WINDOWS)
|
|
606
|
-
InterlockedExchange(&flag_, 0);
|
|
607
|
-
#else
|
|
608
|
-
__atomic_store_n(&flag_, 0, __ATOMIC_RELEASE);
|
|
721
|
+
#if defined(USEARCH_IS_WINDOWS)
|
|
722
|
+
#pragma pack(push, 1) // Pack struct members on 1-byte alignment
|
|
609
723
|
#endif
|
|
610
|
-
}
|
|
611
|
-
};
|
|
612
|
-
|
|
613
|
-
static_assert(sizeof(mutex_t) == sizeof(std::int32_t), "Mutex is larger than expected");
|
|
614
|
-
|
|
615
|
-
using lock_t = std::unique_lock<mutex_t>;
|
|
616
724
|
|
|
617
725
|
/**
|
|
618
726
|
* @brief Five-byte integer type to address node clouds with over 4B entries.
|
|
619
727
|
*/
|
|
620
|
-
#if defined(WINDOWS)
|
|
621
|
-
#pragma pack(push, 1) // Pack struct members on 1-byte alignment
|
|
622
|
-
#endif
|
|
623
|
-
|
|
624
728
|
class usearch_pack_m uint40_t {
|
|
625
729
|
unsigned char octets[5];
|
|
626
730
|
|
|
@@ -628,7 +732,7 @@ class usearch_pack_m uint40_t {
|
|
|
628
732
|
inline uint40_t() noexcept { std::memset(octets, 0, 5); }
|
|
629
733
|
inline uint40_t(std::uint32_t n) noexcept { std::memcpy(octets + 1, (char*)&n, 4), octets[0] = 0; }
|
|
630
734
|
inline uint40_t(std::uint64_t n) noexcept { std::memcpy(octets, (char*)&n + 3, 5); }
|
|
631
|
-
#if defined(
|
|
735
|
+
#if defined(USEARCH_IS_CLANG)
|
|
632
736
|
inline uint40_t(std::size_t n) noexcept { std::memcpy(octets, (char*)&n + 3, 5); }
|
|
633
737
|
#endif
|
|
634
738
|
|
|
@@ -658,12 +762,16 @@ class usearch_pack_m uint40_t {
|
|
|
658
762
|
return old;
|
|
659
763
|
}
|
|
660
764
|
};
|
|
661
|
-
|
|
765
|
+
|
|
766
|
+
#if defined(USEARCH_IS_WINDOWS)
|
|
662
767
|
#pragma pack(pop) // Reset alignment to default
|
|
663
768
|
#endif
|
|
664
769
|
|
|
665
770
|
static_assert(sizeof(uint40_t) == 5, "uint40_t must be exactly 5 bytes");
|
|
666
771
|
|
|
772
|
+
/**
|
|
773
|
+
* @brief Non-owning memory range view, similar to `std::span`, but for C++11.
|
|
774
|
+
*/
|
|
667
775
|
template <typename scalar_at> class span_gt {
|
|
668
776
|
scalar_at* data_;
|
|
669
777
|
std::size_t size_;
|
|
@@ -676,48 +784,61 @@ template <typename scalar_at> class span_gt {
|
|
|
676
784
|
operator scalar_at*() const noexcept { return data(); }
|
|
677
785
|
};
|
|
678
786
|
|
|
787
|
+
constexpr std::size_t default_connectivity() { return 16; }
|
|
788
|
+
constexpr std::size_t default_expansion_add() { return 128; }
|
|
789
|
+
constexpr std::size_t default_expansion_search() { return 64; }
|
|
790
|
+
|
|
679
791
|
/**
|
|
680
|
-
* @brief
|
|
681
|
-
*
|
|
682
|
-
*
|
|
683
|
-
* and two expansion factors - for construction and search.
|
|
792
|
+
* @brief Configuration settings for the index construction.
|
|
793
|
+
* Includes the main `::connectivity` parameter (`M` in the paper)
|
|
794
|
+
* and two expansion factors - for construction and search.
|
|
684
795
|
*/
|
|
685
796
|
struct config_t {
|
|
686
797
|
|
|
687
798
|
/// @brief Number of neighbors per graph node.
|
|
688
799
|
/// Defaults to 32 in FAISS and 16 in hnswlib.
|
|
689
800
|
/// > It is called `M` in the paper.
|
|
690
|
-
|
|
801
|
+
std::size_t connectivity = default_connectivity();
|
|
691
802
|
|
|
692
803
|
/// @brief Hyper-parameter controlling the quality of indexing.
|
|
693
804
|
/// Defaults to 40 in FAISS and 200 in hnswlib.
|
|
694
805
|
/// > It is called `efConstruction` in the paper.
|
|
695
|
-
|
|
806
|
+
std::size_t expansion_add = default_expansion_add();
|
|
696
807
|
|
|
697
808
|
/// @brief Hyper-parameter controlling the quality of search.
|
|
698
809
|
/// Defaults to 16 in FAISS and 10 in hnswlib.
|
|
699
810
|
/// > It is called `ef` in the paper.
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
std::size_t connectivity = connectivity_default_k;
|
|
703
|
-
std::size_t expansion_add = expansion_add_default_k;
|
|
704
|
-
std::size_t expansion_search = expansion_search_default_k;
|
|
811
|
+
std::size_t expansion_search = default_expansion_search();
|
|
705
812
|
|
|
706
813
|
///
|
|
707
814
|
std::size_t max_elements = 0;
|
|
708
|
-
std::size_t max_threads_add =
|
|
709
|
-
std::size_t max_threads_search =
|
|
815
|
+
std::size_t max_threads_add = 1;
|
|
816
|
+
std::size_t max_threads_search = 1;
|
|
817
|
+
|
|
818
|
+
std::size_t max_threads() const noexcept { return (std::max)(max_threads_add, max_threads_search); }
|
|
819
|
+
std::size_t concurrency() const noexcept { return (std::min)(max_threads_add, max_threads_search); }
|
|
820
|
+
};
|
|
821
|
+
|
|
822
|
+
struct add_config_t {
|
|
823
|
+
/// @brief Optional thread identifier for multi-threaded construction.
|
|
824
|
+
std::size_t thread{};
|
|
825
|
+
/// @brief Don't copy the ::vector, if it's persisted elsewhere.
|
|
826
|
+
bool store_vector{true};
|
|
827
|
+
};
|
|
828
|
+
|
|
829
|
+
struct search_config_t {
|
|
830
|
+
std::size_t thread{};
|
|
831
|
+
bool exact{};
|
|
710
832
|
};
|
|
711
833
|
|
|
712
834
|
/**
|
|
713
|
-
* @brief
|
|
714
|
-
*
|
|
715
|
-
* Hierarchical Navigable Small World graph algorithm.
|
|
835
|
+
* @brief Approximate Nearest Neighbors Search index using the
|
|
836
|
+
* Hierarchical Navigable Small World graph algorithm.
|
|
716
837
|
*
|
|
717
838
|
* @section Features
|
|
718
|
-
* -
|
|
719
|
-
* -
|
|
720
|
-
* -
|
|
839
|
+
* - Thread-safe for concurrent construction.
|
|
840
|
+
* - Doesn't allocate new threads, and reuses the ones its called from.
|
|
841
|
+
* - Search for vectors of different dimensionality, if ::`metric_at` supports that.
|
|
721
842
|
*
|
|
722
843
|
* @tparam metric_at
|
|
723
844
|
* A function vector responsible for computing the distance between two vectors.
|
|
@@ -733,6 +854,40 @@ struct config_t {
|
|
|
733
854
|
*
|
|
734
855
|
* @tparam allocator_at
|
|
735
856
|
* Dynamic memory allocator.
|
|
857
|
+
*
|
|
858
|
+
*
|
|
859
|
+
* @section Implementation Details
|
|
860
|
+
*
|
|
861
|
+
* Like every HNSW implementation, USearch builds levels of "Proximity Graphs".
|
|
862
|
+
* Every added vector forms a node in one or more levels of the graph.
|
|
863
|
+
* Every node is present in the base level. Every following level contains a smaller
|
|
864
|
+
* fraction of nodes. During search, the operation starts with the smaller levels
|
|
865
|
+
* and zooms-in on every following iteration of larger graph traversals.
|
|
866
|
+
*
|
|
867
|
+
* Just one memory allocation is performed regardless of the number of levels.
|
|
868
|
+
* The adjacency lists across all levels are concatenated into that single buffer.
|
|
869
|
+
* That buffer starts with a "head", that stores the metadata, such as the
|
|
870
|
+
* tallest "level" of the graph that it belongs to, the external "label", and the
|
|
871
|
+
* number of "dimensions" in the vector.
|
|
872
|
+
*
|
|
873
|
+
* @subsection Colocated Nodes
|
|
874
|
+
*
|
|
875
|
+
* Every node contains some graph-related data, as well as the original vector.
|
|
876
|
+
* Assuming the vectors can get large, we allow separating the index from the vectors.
|
|
877
|
+
* They can either be stored in a single continuous block of memory, or in two separate
|
|
878
|
+
* locations. Vector storage can also be detached from `index_gt` and managed by the user.
|
|
879
|
+
* To properly serialize and deserialize, use the right `save()`, `load()`, and `view()`
|
|
880
|
+
* overloads.
|
|
881
|
+
*
|
|
882
|
+
* @subsection Missing Functionality
|
|
883
|
+
*
|
|
884
|
+
* To simplify the implementation, the `index_gt` lacks endpoints to remove existing
|
|
885
|
+
* vectors. That, however, is solved by `punned_gt`, which also adds automatic casting.
|
|
886
|
+
*
|
|
887
|
+
* @subsection Ugly Implementation Parts
|
|
888
|
+
*
|
|
889
|
+
* Internally, a number of `std::vector`-s is created, often `mutable`.
|
|
890
|
+
*
|
|
736
891
|
*/
|
|
737
892
|
template <typename metric_at = ip_gt<float>, //
|
|
738
893
|
typename label_at = std::size_t, //
|
|
@@ -747,18 +902,104 @@ class index_gt {
|
|
|
747
902
|
using id_t = id_at;
|
|
748
903
|
using allocator_t = allocator_at;
|
|
749
904
|
|
|
905
|
+
/// C++17 and newer version deprecate the `std::result_of`
|
|
750
906
|
using distance_t =
|
|
907
|
+
#if defined(USEARCH_IS_CPP17)
|
|
908
|
+
typename std::invoke_result<metric_t, scalar_t const*, scalar_t const*, std::size_t, std::size_t>::type;
|
|
909
|
+
#else
|
|
751
910
|
typename std::result_of<metric_t(scalar_t const*, scalar_t const*, std::size_t, std::size_t)>::type;
|
|
911
|
+
#endif
|
|
912
|
+
|
|
913
|
+
using vector_view_t = span_gt<scalar_t const>;
|
|
914
|
+
|
|
915
|
+
struct member_ref_t {
|
|
916
|
+
misaligned_ref_gt<label_t> label;
|
|
917
|
+
vector_view_t vector;
|
|
918
|
+
};
|
|
919
|
+
|
|
920
|
+
struct member_cref_t {
|
|
921
|
+
label_t label;
|
|
922
|
+
vector_view_t vector;
|
|
923
|
+
};
|
|
924
|
+
|
|
925
|
+
template <typename ref_at> class member_iterator_gt {
|
|
926
|
+
using ref_t = ref_at;
|
|
927
|
+
index_gt& index_;
|
|
928
|
+
std::size_t offset_;
|
|
929
|
+
|
|
930
|
+
member_iterator_gt() noexcept {}
|
|
931
|
+
member_iterator_gt(index_gt& index, std::size_t offset) noexcept : index_(index), offset_(offset) {}
|
|
932
|
+
|
|
933
|
+
public:
|
|
934
|
+
using iterator_category = std::random_access_iterator_tag;
|
|
935
|
+
using value_type = ref_t;
|
|
936
|
+
using difference_type = std::ptrdiff_t;
|
|
937
|
+
using pointer = void;
|
|
938
|
+
using reference = ref_t;
|
|
939
|
+
|
|
940
|
+
reference operator*() const noexcept { return index_.node_with_id_(offset_); }
|
|
941
|
+
member_iterator_gt operator++(int) noexcept { return member_iterator_gt(index_, offset_ + 1); }
|
|
942
|
+
member_iterator_gt operator--(int) noexcept { return member_iterator_gt(index_, offset_ - 1); }
|
|
943
|
+
member_iterator_gt operator+(difference_type d) noexcept { return member_iterator_gt(index_, offset_ + d); }
|
|
944
|
+
member_iterator_gt operator-(difference_type d) noexcept { return member_iterator_gt(index_, offset_ - d); }
|
|
945
|
+
|
|
946
|
+
member_iterator_gt& operator++() noexcept {
|
|
947
|
+
offset_ += 1;
|
|
948
|
+
return *this;
|
|
949
|
+
}
|
|
950
|
+
member_iterator_gt& operator--() noexcept {
|
|
951
|
+
offset_ -= 1;
|
|
952
|
+
return *this;
|
|
953
|
+
}
|
|
954
|
+
member_iterator_gt& operator+=(difference_type d) noexcept {
|
|
955
|
+
offset_ += d;
|
|
956
|
+
return *this;
|
|
957
|
+
}
|
|
958
|
+
member_iterator_gt& operator-=(difference_type d) noexcept {
|
|
959
|
+
offset_ -= d;
|
|
960
|
+
return *this;
|
|
961
|
+
}
|
|
962
|
+
bool operator!=(member_iterator_gt const& other) noexcept {
|
|
963
|
+
return &index_ != &other.index_ || offset_ != other.offset_;
|
|
964
|
+
}
|
|
965
|
+
};
|
|
966
|
+
|
|
967
|
+
using member_iterator_t = member_iterator_gt<member_ref_t>;
|
|
968
|
+
using member_citerator_t = member_iterator_gt<member_cref_t>;
|
|
969
|
+
|
|
970
|
+
// STL compatibility:
|
|
971
|
+
using value_type = std::pair<label_t, distance_t>;
|
|
972
|
+
using allocator_type = allocator_t;
|
|
973
|
+
using size_type = std::size_t;
|
|
974
|
+
using difference_type = std::ptrdiff_t;
|
|
975
|
+
using reference = member_ref_t;
|
|
976
|
+
using const_reference = member_cref_t;
|
|
977
|
+
using pointer = void;
|
|
978
|
+
using const_pointer = void;
|
|
979
|
+
using iterator = member_iterator_t;
|
|
980
|
+
using const_iterator = member_citerator_t;
|
|
981
|
+
using reverse_iterator = std::reverse_iterator<member_iterator_t>;
|
|
982
|
+
using reverse_const_iterator = std::reverse_iterator<member_citerator_t>;
|
|
752
983
|
|
|
753
984
|
private:
|
|
754
|
-
using neighbors_count_t =
|
|
985
|
+
using neighbors_count_t = std::uint32_t;
|
|
755
986
|
using dim_t = std::uint32_t;
|
|
756
987
|
using level_t = std::int32_t;
|
|
757
988
|
|
|
758
989
|
using allocator_traits_t = std::allocator_traits<allocator_t>;
|
|
759
990
|
using byte_t = typename allocator_t::value_type;
|
|
760
991
|
static_assert(sizeof(byte_t) == 1, "Allocator must allocate separate addressable bytes");
|
|
761
|
-
|
|
992
|
+
|
|
993
|
+
/**
|
|
994
|
+
* @brief How much larger (number of neighbors per node) will the base level be
|
|
995
|
+
* comapred to other levels.
|
|
996
|
+
*/
|
|
997
|
+
static constexpr std::size_t base_level_multiple_() { return 2; }
|
|
998
|
+
|
|
999
|
+
/**
|
|
1000
|
+
* @brief How many bytes of memory are needed to form the "head" of the node.
|
|
1001
|
+
*/
|
|
1002
|
+
static constexpr std::size_t node_head_bytes_() { return sizeof(label_t) + sizeof(dim_t) + sizeof(level_t); }
|
|
762
1003
|
|
|
763
1004
|
using visits_bitset_t = visits_bitset_gt<allocator_t>;
|
|
764
1005
|
|
|
@@ -767,14 +1008,13 @@ class index_gt {
|
|
|
767
1008
|
std::size_t connectivity_max_base{};
|
|
768
1009
|
std::size_t neighbors_bytes{};
|
|
769
1010
|
std::size_t neighbors_base_bytes{};
|
|
770
|
-
std::size_t mutex_bytes{};
|
|
771
1011
|
};
|
|
772
1012
|
struct candidate_t {
|
|
773
|
-
distance_t
|
|
774
|
-
id_t
|
|
1013
|
+
distance_t distance;
|
|
1014
|
+
id_t id;
|
|
775
1015
|
};
|
|
776
1016
|
struct compare_by_distance_t {
|
|
777
|
-
inline bool operator()(candidate_t a, candidate_t b) const noexcept { return a.
|
|
1017
|
+
inline bool operator()(candidate_t a, candidate_t b) const noexcept { return a.distance < b.distance; }
|
|
778
1018
|
};
|
|
779
1019
|
|
|
780
1020
|
using candidates_view_t = span_gt<candidate_t const>;
|
|
@@ -782,69 +1022,85 @@ class index_gt {
|
|
|
782
1022
|
using top_candidates_t = sorted_buffer_gt<candidate_t, compare_by_distance_t, candidates_allocator_t>;
|
|
783
1023
|
using next_candidates_t = max_heap_gt<candidate_t, compare_by_distance_t, candidates_allocator_t>;
|
|
784
1024
|
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
inline id_t* end() noexcept { return neighbors_ + count_; }
|
|
793
|
-
inline id_t const* begin() const noexcept { return neighbors_; }
|
|
794
|
-
inline id_t const* end() const noexcept { return neighbors_ + count_; }
|
|
795
|
-
inline id_t& operator[](std::size_t i) noexcept { return neighbors_[i]; }
|
|
796
|
-
inline id_t operator[](std::size_t i) const noexcept { return neighbors_[i]; }
|
|
797
|
-
inline std::size_t size() const noexcept { return count_; }
|
|
798
|
-
inline void clear() noexcept { count_ = 0u; }
|
|
799
|
-
inline void push_back(id_t id) noexcept { neighbors_[count_] = id, count_++; }
|
|
800
|
-
};
|
|
801
|
-
|
|
802
|
-
#if defined(WINDOWS)
|
|
803
|
-
#pragma pack(push, 1) // Pack struct members on 1-byte alignment
|
|
804
|
-
#endif
|
|
805
|
-
struct usearch_pack_m node_head_t {
|
|
806
|
-
label_t label;
|
|
807
|
-
dim_t dim;
|
|
808
|
-
level_t level;
|
|
809
|
-
// Variable length array, that has multiple similarly structured segments.
|
|
810
|
-
// Each starts with a `neighbors_count_t` and is followed by such number of `id_t`s.
|
|
811
|
-
byte_t neighbors[1];
|
|
812
|
-
};
|
|
813
|
-
#if defined(WINDOWS)
|
|
814
|
-
#pragma pack(pop) // Reset alignment to default
|
|
815
|
-
#endif
|
|
816
|
-
|
|
817
|
-
static constexpr std::size_t head_bytes_k = sizeof(label_t) + sizeof(dim_t) + sizeof(level_t);
|
|
818
|
-
|
|
819
|
-
struct node_t {
|
|
1025
|
+
/**
|
|
1026
|
+
* @brief A loosely-structured handle for every node. One such node is created for every vector.
|
|
1027
|
+
* To minimize memory usage and maximize the number of entries per cache-line, it only
|
|
1028
|
+
* stores to pointers. To reinterpret those into a structured `node_t` more
|
|
1029
|
+
* information is needed from the parent `index_gt`.
|
|
1030
|
+
*/
|
|
1031
|
+
class node_t {
|
|
820
1032
|
byte_t* tape_{};
|
|
821
1033
|
scalar_t* vector_{};
|
|
822
1034
|
|
|
1035
|
+
public:
|
|
823
1036
|
explicit node_t(byte_t* tape, scalar_t* vector) noexcept : tape_(tape), vector_(vector) {}
|
|
1037
|
+
byte_t* tape() const noexcept { return tape_; }
|
|
1038
|
+
scalar_t* vector() const noexcept { return vector_; }
|
|
1039
|
+
byte_t* neighbors_tape() const noexcept { return tape_ + node_head_bytes_(); }
|
|
824
1040
|
|
|
825
1041
|
node_t() = default;
|
|
826
1042
|
node_t(node_t const&) = default;
|
|
827
1043
|
node_t& operator=(node_t const&) = default;
|
|
1044
|
+
|
|
1045
|
+
label_t label() const noexcept { return misaligned_load<label_t>(tape_); }
|
|
1046
|
+
dim_t dim() const noexcept { return misaligned_load<dim_t>(tape_ + sizeof(label_t)); }
|
|
1047
|
+
level_t level() const noexcept { return misaligned_load<level_t>(tape_ + sizeof(label_t) + sizeof(dim_t)); }
|
|
1048
|
+
|
|
1049
|
+
void label(label_t v) noexcept { return misaligned_store<label_t>(tape_, v); }
|
|
1050
|
+
void dim(dim_t v) noexcept { return misaligned_store<dim_t>(tape_ + sizeof(label_t), v); }
|
|
1051
|
+
void level(level_t v) noexcept { return misaligned_store<level_t>(tape_ + sizeof(label_t) + sizeof(dim_t), v); }
|
|
1052
|
+
|
|
1053
|
+
operator vector_view_t() const noexcept { return {vector(), dim()}; }
|
|
1054
|
+
vector_view_t vector_view() const noexcept { return {vector(), dim()}; }
|
|
1055
|
+
member_ref_t ref() noexcept { return {label(), vector_view()}; }
|
|
1056
|
+
member_cref_t cref() const noexcept { return {label(), vector_view()}; }
|
|
828
1057
|
};
|
|
829
1058
|
|
|
830
|
-
|
|
831
|
-
|
|
1059
|
+
/**
|
|
1060
|
+
* @brief A slice of the node's tape, containing a the list of neighbors
|
|
1061
|
+
* for a node in a single graph level. It's pre-allocated to fit
|
|
1062
|
+
* as many neighbors IDs, as may be needed at the target level,
|
|
1063
|
+
* and starts with a single integer `neighbors_count_t` counter.
|
|
1064
|
+
*/
|
|
1065
|
+
class neighbors_ref_t {
|
|
1066
|
+
byte_t* tape_;
|
|
1067
|
+
|
|
1068
|
+
static constexpr std::size_t shift(std::size_t i = 0) { return sizeof(neighbors_count_t) + sizeof(id_t) * i; }
|
|
832
1069
|
|
|
833
1070
|
public:
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
1071
|
+
neighbors_ref_t(byte_t* tape) noexcept : tape_(tape) {}
|
|
1072
|
+
misaligned_ptr_gt<id_t> begin() noexcept { return tape_ + shift(); }
|
|
1073
|
+
misaligned_ptr_gt<id_t> end() noexcept { return begin() + size(); }
|
|
1074
|
+
misaligned_ptr_gt<id_t const> begin() const noexcept { return tape_ + shift(); }
|
|
1075
|
+
misaligned_ptr_gt<id_t const> end() const noexcept { return begin() + size(); }
|
|
1076
|
+
id_t operator[](std::size_t i) const noexcept { return misaligned_load<id_t>(tape_ + shift(i)); }
|
|
1077
|
+
std::size_t size() const noexcept { return misaligned_load<neighbors_count_t>(tape_); }
|
|
1078
|
+
void clear() noexcept { misaligned_store<neighbors_count_t>(tape_, 0); }
|
|
1079
|
+
void push_back(id_t id) noexcept {
|
|
1080
|
+
neighbors_count_t n = misaligned_load<neighbors_count_t>(tape_);
|
|
1081
|
+
misaligned_store<id_t>(tape_ + shift(n), id);
|
|
1082
|
+
misaligned_store<neighbors_count_t>(tape_, n + 1);
|
|
1083
|
+
}
|
|
840
1084
|
};
|
|
841
1085
|
|
|
1086
|
+
/**
|
|
1087
|
+
* @brief A package of all kinds of temporary data-structures, that the threads
|
|
1088
|
+
* would reuse to process requests. Similar to having all of those as
|
|
1089
|
+
* separate `thread_local` global variables.
|
|
1090
|
+
*/
|
|
842
1091
|
struct usearch_align_m thread_context_t {
|
|
843
|
-
top_candidates_t top_candidates;
|
|
844
|
-
next_candidates_t next_candidates;
|
|
845
|
-
visits_bitset_t visits;
|
|
846
|
-
std::default_random_engine level_generator;
|
|
847
|
-
metric_t metric;
|
|
1092
|
+
top_candidates_t top_candidates{};
|
|
1093
|
+
next_candidates_t next_candidates{};
|
|
1094
|
+
visits_bitset_t visits{};
|
|
1095
|
+
std::default_random_engine level_generator{};
|
|
1096
|
+
metric_t metric{};
|
|
1097
|
+
std::size_t iteration_cycles{};
|
|
1098
|
+
std::size_t measurements_count{};
|
|
1099
|
+
|
|
1100
|
+
inline distance_t measure(vector_view_t a, vector_view_t b) noexcept {
|
|
1101
|
+
measurements_count++;
|
|
1102
|
+
return metric(a.data(), b.data(), a.size(), b.size());
|
|
1103
|
+
}
|
|
848
1104
|
};
|
|
849
1105
|
|
|
850
1106
|
config_t config_{};
|
|
@@ -856,12 +1112,15 @@ class index_gt {
|
|
|
856
1112
|
usearch_align_m mutable std::atomic<std::size_t> capacity_{};
|
|
857
1113
|
usearch_align_m mutable std::atomic<std::size_t> size_{};
|
|
858
1114
|
|
|
859
|
-
|
|
1115
|
+
/// @brief Controls access to `max_level_` and `entry_id_`.
|
|
1116
|
+
/// If any thread is updating those values, no other threads can `add()` or `search()`.
|
|
1117
|
+
std::mutex global_mutex_{};
|
|
860
1118
|
level_t max_level_{};
|
|
861
1119
|
id_t entry_id_{};
|
|
862
1120
|
|
|
863
1121
|
using node_allocator_t = typename allocator_traits_t::template rebind_alloc<node_t>;
|
|
864
1122
|
std::vector<node_t, node_allocator_t> nodes_{};
|
|
1123
|
+
mutable visits_bitset_t nodes_mutexes_{};
|
|
865
1124
|
|
|
866
1125
|
using thread_context_allocator_t = typename allocator_traits_t::template rebind_alloc<thread_context_t>;
|
|
867
1126
|
mutable std::vector<thread_context_t, thread_context_allocator_t> thread_contexts_{};
|
|
@@ -870,16 +1129,15 @@ class index_gt {
|
|
|
870
1129
|
std::size_t connectivity() const noexcept { return config_.connectivity; }
|
|
871
1130
|
std::size_t capacity() const noexcept { return capacity_; }
|
|
872
1131
|
std::size_t size() const noexcept { return size_; }
|
|
873
|
-
std::size_t
|
|
1132
|
+
std::size_t max_level() const noexcept { return static_cast<std::size_t>(max_level_); }
|
|
1133
|
+
config_t const& config() const { return config_; }
|
|
874
1134
|
bool is_immutable() const noexcept { return viewed_file_descriptor_ != 0; }
|
|
875
|
-
bool synchronize() const noexcept { return config_.max_threads_add > 1; }
|
|
876
1135
|
|
|
877
1136
|
index_gt(config_t config = {}, metric_t metric = {}, allocator_t allocator = {}) noexcept(false)
|
|
878
|
-
: config_(config), metric_(metric), allocator_(allocator) {
|
|
1137
|
+
: config_(normalize(config)), metric_(metric), allocator_(allocator) {
|
|
879
1138
|
|
|
880
1139
|
// Externally defined hyper-parameters:
|
|
881
|
-
|
|
882
|
-
pre_ = precompute(config);
|
|
1140
|
+
pre_ = precompute_(config);
|
|
883
1141
|
|
|
884
1142
|
// Configure initial empty state:
|
|
885
1143
|
size_ = 0u;
|
|
@@ -888,12 +1146,15 @@ class index_gt {
|
|
|
888
1146
|
viewed_file_descriptor_ = 0;
|
|
889
1147
|
|
|
890
1148
|
// Dynamic memory:
|
|
891
|
-
thread_contexts_.resize(
|
|
1149
|
+
thread_contexts_.resize(config.max_threads());
|
|
892
1150
|
for (thread_context_t& context : thread_contexts_)
|
|
893
1151
|
context.metric = metric;
|
|
894
1152
|
reserve(config.max_elements);
|
|
895
1153
|
}
|
|
896
1154
|
|
|
1155
|
+
/**
|
|
1156
|
+
* @brief Clones the structure with the same hyper-parameters, but without contents.
|
|
1157
|
+
*/
|
|
897
1158
|
index_gt fork() noexcept(false) { return {config_, metric_, allocator_}; }
|
|
898
1159
|
|
|
899
1160
|
~index_gt() noexcept { clear(); }
|
|
@@ -907,15 +1168,22 @@ class index_gt {
|
|
|
907
1168
|
|
|
908
1169
|
#pragma region Adjusting Configuration
|
|
909
1170
|
|
|
1171
|
+
/**
|
|
1172
|
+
* @brief Erases all the vectors from the index.
|
|
1173
|
+
* Will change `size()` to zero, but will keep the same `capacity()`.
|
|
1174
|
+
*/
|
|
910
1175
|
void clear() noexcept {
|
|
911
1176
|
std::size_t n = size_;
|
|
912
1177
|
for (std::size_t i = 0; i != n; ++i)
|
|
913
|
-
|
|
1178
|
+
node_free_(i);
|
|
914
1179
|
size_ = 0;
|
|
915
1180
|
max_level_ = -1;
|
|
916
1181
|
entry_id_ = 0u;
|
|
917
1182
|
}
|
|
918
1183
|
|
|
1184
|
+
/**
|
|
1185
|
+
* @brief Swaps the underlying memory buffers and thread contexts.
|
|
1186
|
+
*/
|
|
919
1187
|
void swap(index_gt& other) noexcept {
|
|
920
1188
|
std::swap(config_, other.config_);
|
|
921
1189
|
std::swap(metric_, other.metric_);
|
|
@@ -936,123 +1204,259 @@ class index_gt {
|
|
|
936
1204
|
other.size_ = size;
|
|
937
1205
|
}
|
|
938
1206
|
|
|
1207
|
+
/**
|
|
1208
|
+
* @brief Increases the `capacity()` of the index to allow adding more vectors.
|
|
1209
|
+
*/
|
|
939
1210
|
void reserve(std::size_t new_capacity) noexcept(false) {
|
|
940
1211
|
|
|
941
1212
|
assert_m(new_capacity >= size_, "Can't drop existing values");
|
|
942
1213
|
nodes_.resize(new_capacity);
|
|
1214
|
+
nodes_mutexes_ = visits_bitset_t(new_capacity);
|
|
943
1215
|
for (thread_context_t& context : thread_contexts_)
|
|
944
1216
|
context.visits = visits_bitset_t(new_capacity);
|
|
945
1217
|
|
|
946
1218
|
capacity_ = new_capacity;
|
|
947
1219
|
}
|
|
948
1220
|
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
1221
|
+
/**
|
|
1222
|
+
* @brief Optimizes configuration options to fit the maximum number
|
|
1223
|
+
* of neighbors in CPU-cache-aligned buffers.
|
|
1224
|
+
*/
|
|
1225
|
+
static config_t optimize(config_t config) noexcept {
|
|
1226
|
+
precomputed_constants_t pre = precompute_(config);
|
|
1227
|
+
std::size_t bytes_per_node_base = node_head_bytes_() + pre.neighbors_base_bytes;
|
|
952
1228
|
std::size_t rounded_size = divide_round_up<64>(bytes_per_node_base) * 64;
|
|
953
1229
|
std::size_t added_connections = (rounded_size - rounded_size) / sizeof(id_t);
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
1230
|
+
config.connectivity = config.connectivity + added_connections / base_level_multiple_();
|
|
1231
|
+
return config;
|
|
1232
|
+
}
|
|
1233
|
+
|
|
1234
|
+
/**
|
|
1235
|
+
* @brief Validates/normalizes the configuration options and throws an exception
|
|
1236
|
+
* if those are incorrect.
|
|
1237
|
+
*/
|
|
1238
|
+
static config_t normalize(config_t config) noexcept {
|
|
1239
|
+
config.expansion_add = (std::max<std::size_t>)(config.expansion_add, config.connectivity);
|
|
1240
|
+
config.max_threads_add = (std::max<std::size_t>)(1u, config.max_threads_add);
|
|
1241
|
+
config.max_threads_search = (std::max<std::size_t>)(1u, config.max_threads_search);
|
|
1242
|
+
return config;
|
|
957
1243
|
}
|
|
958
1244
|
|
|
959
1245
|
#pragma endregion
|
|
960
1246
|
|
|
961
1247
|
#pragma region Construction and Search
|
|
962
1248
|
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
std::size_t
|
|
1249
|
+
struct add_result_t {
|
|
1250
|
+
std::size_t new_size{};
|
|
1251
|
+
std::size_t cycles{};
|
|
1252
|
+
std::size_t measurements{};
|
|
1253
|
+
id_t id{};
|
|
1254
|
+
};
|
|
1255
|
+
|
|
1256
|
+
struct search_result_t {
|
|
1257
|
+
member_cref_t member;
|
|
1258
|
+
distance_t distance;
|
|
1259
|
+
};
|
|
1260
|
+
|
|
1261
|
+
class search_results_t {
|
|
1262
|
+
index_gt const& index_;
|
|
1263
|
+
top_candidates_t& top_;
|
|
1264
|
+
|
|
1265
|
+
friend class index_gt;
|
|
1266
|
+
inline search_results_t(index_gt const& index, top_candidates_t& top) noexcept : index_(index), top_(top) {}
|
|
1267
|
+
|
|
1268
|
+
public:
|
|
1269
|
+
std::size_t count{};
|
|
1270
|
+
std::size_t cycles{};
|
|
1271
|
+
std::size_t measurements{};
|
|
1272
|
+
|
|
1273
|
+
inline search_results_t(search_results_t&&) = default;
|
|
1274
|
+
inline search_results_t& operator=(search_results_t&&) = default;
|
|
1275
|
+
|
|
1276
|
+
inline operator std::size_t() const noexcept { return count; }
|
|
1277
|
+
inline std::size_t size() const noexcept { return count; }
|
|
1278
|
+
inline search_result_t operator[](std::size_t i) const noexcept {
|
|
1279
|
+
candidate_t const* top_ordered = top_.data();
|
|
1280
|
+
return {index_.node_with_id_(top_ordered[i].id).cref(), top_ordered[i].distance};
|
|
1281
|
+
}
|
|
1282
|
+
inline std::size_t dump_to(label_t* labels, distance_t* distances) const noexcept {
|
|
1283
|
+
for (std::size_t i = 0; i != count; ++i) {
|
|
1284
|
+
search_result_t result = operator[](i);
|
|
1285
|
+
labels[i] = result.member.label;
|
|
1286
|
+
distances[i] = result.distance;
|
|
1287
|
+
}
|
|
1288
|
+
return count;
|
|
1289
|
+
}
|
|
1290
|
+
};
|
|
1291
|
+
|
|
1292
|
+
/**
|
|
1293
|
+
* @brief Inserts a new vector into the index. Thread-safe.
|
|
1294
|
+
*
|
|
1295
|
+
* @param[in] label External identifier/name/descriptor for the vector.
|
|
1296
|
+
* @param[in] vector Contiguous range of scalars forming a vector view.
|
|
1297
|
+
* @param[in] config Configuration options for this specific operation.
|
|
1298
|
+
*/
|
|
1299
|
+
add_result_t add(label_t label, vector_view_t vector, add_config_t config = {}) noexcept(false) {
|
|
966
1300
|
|
|
967
1301
|
assert_m(!is_immutable(), "Can't add to an immutable index");
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
// Determining how much memory to allocate depends on the target level
|
|
973
|
-
|
|
974
|
-
level_t max_level = max_level_;
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
1302
|
+
add_result_t result;
|
|
1303
|
+
result.new_size = size_.fetch_add(1);
|
|
1304
|
+
id_t new_id = result.id = static_cast<id_t>(result.new_size);
|
|
1305
|
+
|
|
1306
|
+
// Determining how much memory to allocate depends on the target level
|
|
1307
|
+
std::unique_lock<std::mutex> new_level_lock(global_mutex_);
|
|
1308
|
+
level_t max_level = max_level_; // Copy under lock
|
|
1309
|
+
id_t entry_id = entry_id_; // Copy under lock
|
|
1310
|
+
thread_context_t& context = thread_contexts_[config.thread];
|
|
1311
|
+
level_t target_level = choose_random_level_(context.level_generator);
|
|
1312
|
+
if (target_level <= max_level)
|
|
978
1313
|
new_level_lock.unlock();
|
|
979
1314
|
|
|
980
1315
|
// Allocate the neighbors
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
1316
|
+
nodes_[result.new_size] = node_malloc_(label, vector, target_level, config.store_vector);
|
|
1317
|
+
node_lock_t new_lock = node_lock_(new_id);
|
|
1318
|
+
result.new_size++;
|
|
984
1319
|
|
|
985
1320
|
// Do nothing for the first element
|
|
986
1321
|
if (!new_id) {
|
|
987
|
-
|
|
988
|
-
|
|
1322
|
+
entry_id_ = new_id;
|
|
1323
|
+
max_level_ = target_level;
|
|
1324
|
+
return result;
|
|
989
1325
|
}
|
|
990
1326
|
|
|
1327
|
+
// Pull stats
|
|
1328
|
+
result.measurements = context.measurements_count;
|
|
1329
|
+
result.cycles = context.iteration_cycles;
|
|
1330
|
+
|
|
991
1331
|
// Go down the level, tracking only the closest match
|
|
992
|
-
id_t closest_id =
|
|
1332
|
+
id_t closest_id = search_for_one_(entry_id, vector, max_level, target_level, context);
|
|
993
1333
|
|
|
994
|
-
// From `
|
|
995
|
-
for (level_t level = (std::min)(
|
|
996
|
-
|
|
997
|
-
closest_id =
|
|
1334
|
+
// From `target_level` down perform proper extensive search
|
|
1335
|
+
for (level_t level = (std::min)(target_level, max_level); level >= 0; level--) {
|
|
1336
|
+
search_to_insert_(closest_id, vector, level, context);
|
|
1337
|
+
closest_id = connect_new_node_(new_id, level, context);
|
|
998
1338
|
}
|
|
999
1339
|
|
|
1000
|
-
//
|
|
1001
|
-
|
|
1340
|
+
// Normalize stats
|
|
1341
|
+
result.measurements = context.measurements_count - result.measurements;
|
|
1342
|
+
result.cycles = context.iteration_cycles - result.cycles;
|
|
1343
|
+
|
|
1344
|
+
// Updating the entry point if needed
|
|
1345
|
+
if (target_level > max_level) {
|
|
1002
1346
|
entry_id_ = new_id;
|
|
1003
|
-
max_level_ =
|
|
1347
|
+
max_level_ = target_level;
|
|
1004
1348
|
}
|
|
1005
|
-
return
|
|
1349
|
+
return result;
|
|
1006
1350
|
}
|
|
1007
1351
|
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1352
|
+
/**
|
|
1353
|
+
* @brief Searches for the closest members to the given ::query. Thread-safe.
|
|
1354
|
+
*
|
|
1355
|
+
* @param[in] query Contiguous range of scalars forming a vector view.
|
|
1356
|
+
* @param[in] wanted The upper bound for the number of results to return.
|
|
1357
|
+
* @param[in] config Configuration options for this specific operation.
|
|
1358
|
+
* @return Smart object referencing temporary memory. Valid until next `search()` or `add()`.
|
|
1359
|
+
*/
|
|
1360
|
+
search_results_t search( //
|
|
1361
|
+
vector_view_t query, std::size_t wanted, //
|
|
1362
|
+
search_config_t config = {}) const noexcept(false) {
|
|
1012
1363
|
|
|
1364
|
+
thread_context_t& context = thread_contexts_[config.thread];
|
|
1365
|
+
top_candidates_t& top = context.top_candidates;
|
|
1366
|
+
search_results_t result{*this, top};
|
|
1013
1367
|
if (!size_)
|
|
1014
|
-
return;
|
|
1368
|
+
return result;
|
|
1015
1369
|
|
|
1016
|
-
|
|
1017
|
-
|
|
1370
|
+
// Go down the level, tracking only the closest match
|
|
1371
|
+
result.measurements = context.measurements_count;
|
|
1372
|
+
result.cycles = context.iteration_cycles;
|
|
1373
|
+
|
|
1374
|
+
if (config.exact) {
|
|
1375
|
+
search_exact_(query, wanted, context);
|
|
1376
|
+
} else {
|
|
1377
|
+
id_t closest_id = search_for_one_(entry_id_, query, max_level_, 0, context);
|
|
1378
|
+
std::size_t expansion = (std::max)(config_.expansion_search, wanted);
|
|
1379
|
+
// For bottom layer we need a more optimized procedure
|
|
1380
|
+
search_to_find_in_base_(closest_id, query, expansion, context);
|
|
1381
|
+
}
|
|
1382
|
+
|
|
1383
|
+
top.sort_ascending();
|
|
1384
|
+
top.shrink(wanted);
|
|
1385
|
+
|
|
1386
|
+
// Normalize stats
|
|
1387
|
+
result.measurements = context.measurements_count - result.measurements;
|
|
1388
|
+
result.cycles = context.iteration_cycles - result.cycles;
|
|
1389
|
+
result.count = top.size();
|
|
1390
|
+
return result;
|
|
1391
|
+
}
|
|
1392
|
+
|
|
1393
|
+
search_results_t search_around( //
|
|
1394
|
+
id_t hint, vector_view_t query, std::size_t wanted, //
|
|
1395
|
+
search_config_t config = {}) const noexcept(false) {
|
|
1396
|
+
|
|
1397
|
+
thread_context_t& context = thread_contexts_[config.thread];
|
|
1398
|
+
top_candidates_t& top = context.top_candidates;
|
|
1399
|
+
search_results_t result{*this, top};
|
|
1400
|
+
if (!size_)
|
|
1401
|
+
return result;
|
|
1018
1402
|
|
|
1019
1403
|
// Go down the level, tracking only the closest match
|
|
1020
|
-
|
|
1021
|
-
|
|
1404
|
+
result.measurements = context.measurements_count;
|
|
1405
|
+
result.cycles = context.iteration_cycles;
|
|
1022
1406
|
|
|
1023
|
-
// For bottom layer we need a more optimized procedure
|
|
1024
1407
|
std::size_t expansion = (std::max)(config_.expansion_search, wanted);
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
std::size_t
|
|
1038
|
-
|
|
1039
|
-
std::size_t
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1408
|
+
search_to_find_in_base_(hint, query, expansion, context);
|
|
1409
|
+
top.sort_ascending();
|
|
1410
|
+
top.shrink(wanted);
|
|
1411
|
+
|
|
1412
|
+
// Normalize stats
|
|
1413
|
+
result.measurements = context.measurements_count - result.measurements;
|
|
1414
|
+
result.cycles = context.iteration_cycles - result.cycles;
|
|
1415
|
+
result.count = top.size();
|
|
1416
|
+
return result;
|
|
1417
|
+
}
|
|
1418
|
+
|
|
1419
|
+
struct stats_t {
|
|
1420
|
+
std::size_t nodes;
|
|
1421
|
+
std::size_t edges;
|
|
1422
|
+
std::size_t max_edged;
|
|
1423
|
+
};
|
|
1424
|
+
|
|
1425
|
+
stats_t stats() const noexcept {
|
|
1426
|
+
stats_t result{};
|
|
1427
|
+
result.nodes = nodes_.size();
|
|
1428
|
+
for (std::size_t i = 0; i != result.nodes; ++i) {
|
|
1429
|
+
node_t node = node_with_id_(i);
|
|
1430
|
+
std::size_t max_edges = node.level() * config_.connectivity + base_level_multiple_() * config_.connectivity;
|
|
1431
|
+
std::size_t edges = 0;
|
|
1432
|
+
for (level_t level = 0; level != node.level(); ++level)
|
|
1433
|
+
edges += neighbors_(node, level).size();
|
|
1434
|
+
|
|
1435
|
+
result.edges += edges;
|
|
1436
|
+
result.max_edges += max_edges;
|
|
1437
|
+
}
|
|
1438
|
+
return result;
|
|
1439
|
+
}
|
|
1440
|
+
|
|
1441
|
+
stats_t stats(std::size_t level) const noexcept {
|
|
1442
|
+
stats_t result{};
|
|
1443
|
+
result.nodes = nodes_.size();
|
|
1444
|
+
for (std::size_t i = 0; i != result.nodes; ++i) {
|
|
1445
|
+
node_t node = node_with_id_(i);
|
|
1446
|
+
if (node.level() >= level)
|
|
1447
|
+
result.edges += neighbors_(node, level).size();
|
|
1448
|
+
}
|
|
1449
|
+
std::size_t max_edges_per_node = level ? config_.connectivity : base_level_multiple_() * config_.connectivity;
|
|
1450
|
+
result.max_edges = result.nodes * max_edges_per_node;
|
|
1451
|
+
return result;
|
|
1049
1452
|
}
|
|
1050
1453
|
|
|
1051
1454
|
#pragma endregion
|
|
1052
1455
|
|
|
1053
1456
|
#pragma region Serialization
|
|
1054
1457
|
|
|
1055
|
-
|
|
1458
|
+
private:
|
|
1459
|
+
struct serialized_state_t {
|
|
1056
1460
|
// Check compatibility
|
|
1057
1461
|
std::uint64_t bytes_per_label{};
|
|
1058
1462
|
std::uint64_t bytes_per_id{};
|
|
@@ -1064,12 +1468,15 @@ class index_gt {
|
|
|
1064
1468
|
std::uint64_t max_level{};
|
|
1065
1469
|
};
|
|
1066
1470
|
|
|
1471
|
+
public:
|
|
1067
1472
|
/**
|
|
1068
|
-
*
|
|
1473
|
+
* @brief Saves serialized binary index representation to disk,
|
|
1474
|
+
* co-locating vectors and neighbors lists.
|
|
1475
|
+
* Available on Linux, MacOS, Windows.
|
|
1069
1476
|
*/
|
|
1070
1477
|
void save(char const* file_path) const noexcept(false) {
|
|
1071
1478
|
|
|
1072
|
-
|
|
1479
|
+
serialized_state_t state;
|
|
1073
1480
|
// Check compatibility
|
|
1074
1481
|
state.bytes_per_label = sizeof(label_t);
|
|
1075
1482
|
state.bytes_per_id = sizeof(id_t);
|
|
@@ -1083,53 +1490,52 @@ class index_gt {
|
|
|
1083
1490
|
if (!file)
|
|
1084
1491
|
throw std::runtime_error(std::strerror(errno));
|
|
1085
1492
|
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
std::size_t written = std::fwrite(&state, sizeof(state), 1, file);
|
|
1493
|
+
auto write_chunk = [&](void* begin, std::size_t length) {
|
|
1494
|
+
std::size_t written = std::fwrite(begin, length, 1, file);
|
|
1089
1495
|
if (!written) {
|
|
1090
1496
|
std::fclose(file);
|
|
1091
1497
|
throw std::runtime_error(std::strerror(errno));
|
|
1092
1498
|
}
|
|
1093
|
-
}
|
|
1499
|
+
};
|
|
1500
|
+
|
|
1501
|
+
// Write the header
|
|
1502
|
+
write_chunk(&state, sizeof(state));
|
|
1094
1503
|
|
|
1095
1504
|
// Serialize nodes one by one
|
|
1096
1505
|
for (std::size_t i = 0; i != state.size; ++i) {
|
|
1097
|
-
|
|
1098
|
-
std::size_t
|
|
1099
|
-
std::size_t
|
|
1100
|
-
// Dump
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
std::fclose(file);
|
|
1104
|
-
throw std::runtime_error(std::strerror(errno));
|
|
1105
|
-
}
|
|
1106
|
-
// Dump the vector
|
|
1107
|
-
written = std::fwrite(node_ref.vector, bytes_in_vec, 1, file);
|
|
1108
|
-
if (!written) {
|
|
1109
|
-
std::fclose(file);
|
|
1110
|
-
throw std::runtime_error(std::strerror(errno));
|
|
1111
|
-
}
|
|
1506
|
+
node_t node = node_with_id_(i);
|
|
1507
|
+
std::size_t node_bytes = node_bytes_(node);
|
|
1508
|
+
std::size_t node_vector_bytes = node_vector_bytes_(node);
|
|
1509
|
+
// Dump neighbors and vectors, as vectors may be in a disjoint location
|
|
1510
|
+
write_chunk(node.tape(), node_bytes - node_vector_bytes);
|
|
1511
|
+
write_chunk(node.vector(), node_vector_bytes);
|
|
1112
1512
|
}
|
|
1113
1513
|
|
|
1114
1514
|
std::fclose(file);
|
|
1115
1515
|
}
|
|
1116
1516
|
|
|
1117
1517
|
/**
|
|
1118
|
-
*
|
|
1518
|
+
* @brief Loads the serialized binary index representation from disk,
|
|
1519
|
+
* copying both vectors and neighbors lists into RAM.
|
|
1520
|
+
* Available on Linux, MacOS, Windows.
|
|
1119
1521
|
*/
|
|
1120
1522
|
void load(char const* file_path) noexcept(false) {
|
|
1121
|
-
|
|
1523
|
+
serialized_state_t state;
|
|
1122
1524
|
std::FILE* file = std::fopen(file_path, "r");
|
|
1123
1525
|
if (!file)
|
|
1124
1526
|
throw std::runtime_error(std::strerror(errno));
|
|
1125
1527
|
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
std::size_t read = std::fread(&state, sizeof(state), 1, file);
|
|
1528
|
+
auto read_chunk = [&](void* begin, std::size_t length) {
|
|
1529
|
+
std::size_t read = std::fread(begin, length, 1, file);
|
|
1129
1530
|
if (!read) {
|
|
1130
1531
|
std::fclose(file);
|
|
1131
1532
|
throw std::runtime_error(std::strerror(errno));
|
|
1132
1533
|
}
|
|
1534
|
+
};
|
|
1535
|
+
|
|
1536
|
+
// Read the header
|
|
1537
|
+
{
|
|
1538
|
+
read_chunk(&state, sizeof(state));
|
|
1133
1539
|
if (state.bytes_per_label != sizeof(label_t)) {
|
|
1134
1540
|
std::fclose(file);
|
|
1135
1541
|
throw std::runtime_error("Incompatible label type!");
|
|
@@ -1141,7 +1547,7 @@ class index_gt {
|
|
|
1141
1547
|
|
|
1142
1548
|
config_.connectivity = state.connectivity;
|
|
1143
1549
|
config_.max_elements = state.size;
|
|
1144
|
-
pre_ =
|
|
1550
|
+
pre_ = precompute_(config_);
|
|
1145
1551
|
reserve(state.size);
|
|
1146
1552
|
size_ = state.size;
|
|
1147
1553
|
max_level_ = static_cast<level_t>(state.max_level);
|
|
@@ -1150,22 +1556,17 @@ class index_gt {
|
|
|
1150
1556
|
|
|
1151
1557
|
// Load nodes one by one
|
|
1152
1558
|
for (std::size_t i = 0; i != state.size; ++i) {
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
std::size_t
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
std::fclose(file);
|
|
1165
|
-
throw std::runtime_error(std::strerror(errno));
|
|
1166
|
-
}
|
|
1167
|
-
|
|
1168
|
-
nodes_[i] = node_ref;
|
|
1559
|
+
label_t label;
|
|
1560
|
+
dim_t dim;
|
|
1561
|
+
level_t level;
|
|
1562
|
+
read_chunk(&label, sizeof(label));
|
|
1563
|
+
read_chunk(&dim, sizeof(dim));
|
|
1564
|
+
read_chunk(&level, sizeof(level));
|
|
1565
|
+
|
|
1566
|
+
std::size_t node_bytes = node_bytes_(dim, level);
|
|
1567
|
+
node_t node = node_malloc_(label, {nullptr, dim}, level, true);
|
|
1568
|
+
read_chunk(node.tape() + node_head_bytes_(), node_bytes - node_head_bytes_());
|
|
1569
|
+
nodes_[i] = node;
|
|
1169
1570
|
}
|
|
1170
1571
|
|
|
1171
1572
|
std::fclose(file);
|
|
@@ -1173,13 +1574,15 @@ class index_gt {
|
|
|
1173
1574
|
}
|
|
1174
1575
|
|
|
1175
1576
|
/**
|
|
1176
|
-
*
|
|
1577
|
+
* @brief Memory-maps the serialized binary index representation from disk,
|
|
1578
|
+
* @b without copying the vectors and neighbors lists into RAM.
|
|
1579
|
+
* Available on Linux, MacOS, but @b not on Windows.
|
|
1177
1580
|
*/
|
|
1178
1581
|
void view(char const* file_path) noexcept(false) {
|
|
1179
|
-
#if defined(
|
|
1582
|
+
#if defined(USEARCH_IS_WINDOWS)
|
|
1180
1583
|
throw std::logic_error("Memory-mapping is not yet available for Windows");
|
|
1181
1584
|
#else
|
|
1182
|
-
|
|
1585
|
+
serialized_state_t state;
|
|
1183
1586
|
int open_flags = O_RDONLY;
|
|
1184
1587
|
#if __linux__
|
|
1185
1588
|
open_flags |= O_NOATIME;
|
|
@@ -1216,7 +1619,7 @@ class index_gt {
|
|
|
1216
1619
|
config_.connectivity = state.connectivity;
|
|
1217
1620
|
config_.max_elements = state.size;
|
|
1218
1621
|
config_.max_threads_add = 0;
|
|
1219
|
-
pre_ =
|
|
1622
|
+
pre_ = precompute_(config_);
|
|
1220
1623
|
reserve(state.size);
|
|
1221
1624
|
size_ = state.size;
|
|
1222
1625
|
max_level_ = static_cast<level_t>(state.max_level);
|
|
@@ -1226,13 +1629,15 @@ class index_gt {
|
|
|
1226
1629
|
// Locate every node packed into file
|
|
1227
1630
|
std::size_t progress = sizeof(state);
|
|
1228
1631
|
for (std::size_t i = 0; i != state.size; ++i) {
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1632
|
+
byte_t* tape = (byte_t*)(file + progress);
|
|
1633
|
+
dim_t dim = misaligned_load<dim_t>(tape + sizeof(label_t));
|
|
1634
|
+
level_t level = misaligned_load<level_t>(tape + sizeof(label_t) + sizeof(dim_t));
|
|
1635
|
+
|
|
1636
|
+
std::size_t node_bytes = node_bytes_(dim, level);
|
|
1637
|
+
std::size_t node_vector_bytes = dim * sizeof(scalar_t);
|
|
1638
|
+
nodes_[i] = node_t{tape, (scalar_t*)(tape + node_bytes - node_vector_bytes)};
|
|
1639
|
+
progress += node_bytes;
|
|
1640
|
+
max_level_ = (std::max)(max_level_, level);
|
|
1236
1641
|
}
|
|
1237
1642
|
|
|
1238
1643
|
viewed_file_descriptor_ = descriptor;
|
|
@@ -1242,124 +1647,114 @@ class index_gt {
|
|
|
1242
1647
|
#pragma endregion
|
|
1243
1648
|
|
|
1244
1649
|
private:
|
|
1245
|
-
inline static precomputed_constants_t
|
|
1650
|
+
inline static precomputed_constants_t precompute_(config_t const& config) noexcept {
|
|
1246
1651
|
precomputed_constants_t pre;
|
|
1247
|
-
pre.connectivity_max_base = config.connectivity *
|
|
1652
|
+
pre.connectivity_max_base = config.connectivity * base_level_multiple_();
|
|
1248
1653
|
pre.inverse_log_connectivity = 1.0 / std::log(static_cast<double>(config.connectivity));
|
|
1249
1654
|
pre.neighbors_bytes = config.connectivity * sizeof(id_t) + sizeof(neighbors_count_t);
|
|
1250
1655
|
pre.neighbors_base_bytes = pre.connectivity_max_base * sizeof(id_t) + sizeof(neighbors_count_t);
|
|
1251
|
-
pre.mutex_bytes = sizeof(mutex_t) * (config.max_threads_add > 1);
|
|
1252
1656
|
return pre;
|
|
1253
1657
|
}
|
|
1254
1658
|
|
|
1255
|
-
inline std::size_t
|
|
1256
|
-
|
|
1659
|
+
inline std::size_t node_bytes_(node_t node) const noexcept { return node_bytes_(node.dim(), node.level()); }
|
|
1660
|
+
inline std::size_t node_bytes_(dim_t dim, level_t level) const noexcept {
|
|
1661
|
+
return node_head_bytes_() + pre_.neighbors_base_bytes + pre_.neighbors_bytes * level + sizeof(scalar_t) * dim;
|
|
1257
1662
|
}
|
|
1258
1663
|
|
|
1259
|
-
|
|
1664
|
+
inline bool node_stored_(node_t node) const noexcept {
|
|
1665
|
+
std::size_t levels_bytes = pre_.neighbors_base_bytes + pre_.neighbors_bytes * node.level();
|
|
1666
|
+
return (node.tape() + node_head_bytes_() + levels_bytes) == (byte_t*)node.vector();
|
|
1667
|
+
}
|
|
1668
|
+
|
|
1669
|
+
inline std::size_t node_vector_bytes_(dim_t dim) const noexcept { return dim * sizeof(scalar_t); }
|
|
1670
|
+
inline std::size_t node_vector_bytes_(node_t node) const noexcept { return node_vector_bytes_(node.dim()); }
|
|
1671
|
+
|
|
1672
|
+
void node_free_(std::size_t id) noexcept {
|
|
1260
1673
|
|
|
1261
1674
|
if (viewed_file_descriptor_ != 0)
|
|
1262
1675
|
return;
|
|
1263
1676
|
|
|
1264
|
-
// This function is rarely called and can be as expensive as needed for higher space-efficiency.
|
|
1265
1677
|
node_t& node = nodes_[id];
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
node_head_t const& head = *(node_head_t const*)(node.tape_ + pre_.mutex_bytes);
|
|
1270
|
-
std::size_t levels_bytes = pre_.neighbors_base_bytes + pre_.neighbors_bytes * head.level;
|
|
1271
|
-
bool store_vector = (byte_t*)(node.tape_ + pre_.mutex_bytes + head_bytes_k + levels_bytes) == //
|
|
1272
|
-
(byte_t*)(node.vector_);
|
|
1273
|
-
std::size_t node_bytes = //
|
|
1274
|
-
pre_.mutex_bytes + // Optional concurrency-control
|
|
1275
|
-
head_bytes_k + levels_bytes + // Obligatory neighborhood index
|
|
1276
|
-
head.dim * store_vector // Optional vector copy
|
|
1277
|
-
;
|
|
1278
|
-
|
|
1279
|
-
allocator_t{}.deallocate(node.tape_, node_bytes);
|
|
1678
|
+
std::size_t node_bytes = node_bytes_(node) - node_vector_bytes_(node) * !node_stored_(node);
|
|
1679
|
+
allocator_t{}.deallocate(node.tape(), node_bytes);
|
|
1280
1680
|
node = node_t{};
|
|
1281
1681
|
}
|
|
1282
1682
|
|
|
1283
|
-
|
|
1284
|
-
label_t label, scalar_t const* vector, std::size_t dim, //
|
|
1285
|
-
level_t level, bool store_vector = true) noexcept(false) {
|
|
1683
|
+
node_t node_malloc_(label_t label, vector_view_t vector, level_t level, bool store_vector) noexcept(false) {
|
|
1286
1684
|
|
|
1287
|
-
|
|
1288
|
-
std::size_t
|
|
1289
|
-
std::size_t node_bytes =
|
|
1290
|
-
pre_.mutex_bytes + // Optional concurrency-control
|
|
1291
|
-
head_bytes_k + levels_bytes + // Obligatory neighborhood index
|
|
1292
|
-
sizeof(scalar_t) * dim * store_vector // Optional vector copy
|
|
1293
|
-
;
|
|
1685
|
+
std::size_t dim = vector.size();
|
|
1686
|
+
std::size_t stored_vector_bytes = node_vector_bytes_(dim) * store_vector;
|
|
1687
|
+
std::size_t node_bytes = node_bytes_(dim, level) - node_vector_bytes_(dim) * !store_vector;
|
|
1294
1688
|
|
|
1295
1689
|
byte_t* data = (byte_t*)allocator_t{}.allocate(node_bytes);
|
|
1296
1690
|
assert_m(data, "Not enough memory for links");
|
|
1297
|
-
|
|
1298
|
-
mutex_t* mutex = synchronize() ? (mutex_t*)data : nullptr;
|
|
1299
|
-
scalar_t* scalars = store_vector //
|
|
1300
|
-
? (scalar_t*)(data + pre_.mutex_bytes + head_bytes_k + levels_bytes)
|
|
1301
|
-
: (scalar_t*)(vector);
|
|
1302
|
-
|
|
1303
1691
|
std::memset(data, 0, node_bytes);
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
node_head_t& head = *(node_head_t*)(data + pre_.mutex_bytes);
|
|
1307
|
-
head.label = label;
|
|
1308
|
-
head.dim = static_cast<dim_t>(dim);
|
|
1309
|
-
head.level = level;
|
|
1692
|
+
if (vector.data())
|
|
1693
|
+
std::memcpy(data + node_bytes - stored_vector_bytes, vector.data(), stored_vector_bytes);
|
|
1310
1694
|
|
|
1311
|
-
|
|
1695
|
+
scalar_t* scalars = store_vector //
|
|
1696
|
+
? (scalar_t*)(data + node_bytes - stored_vector_bytes)
|
|
1697
|
+
: (scalar_t*)(vector.data());
|
|
1698
|
+
|
|
1699
|
+
node_t node{data, scalars};
|
|
1700
|
+
node.label(label);
|
|
1701
|
+
node.dim(static_cast<dim_t>(dim));
|
|
1702
|
+
node.level(level);
|
|
1703
|
+
return node;
|
|
1312
1704
|
}
|
|
1313
1705
|
|
|
1314
|
-
inline
|
|
1706
|
+
inline node_t node_with_id_(std::size_t idx) const noexcept { return nodes_[idx]; }
|
|
1707
|
+
inline neighbors_ref_t neighbors_base_(node_t node) const noexcept { return {node.neighbors_tape()}; }
|
|
1315
1708
|
|
|
1316
|
-
inline
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
node_head_t& head = *(node_head_t*)(data + pre_.mutex_bytes);
|
|
1320
|
-
scalar_t* scalars = node.vector_;
|
|
1709
|
+
inline neighbors_ref_t neighbors_non_base_(node_t node, level_t level) const noexcept {
|
|
1710
|
+
return {node.neighbors_tape() + pre_.neighbors_base_bytes + (level - 1) * pre_.neighbors_bytes};
|
|
1711
|
+
}
|
|
1321
1712
|
|
|
1322
|
-
|
|
1713
|
+
inline neighbors_ref_t neighbors_(node_t node, level_t level) const noexcept {
|
|
1714
|
+
return level ? neighbors_non_base_(node, level) : neighbors_base_(node);
|
|
1323
1715
|
}
|
|
1324
1716
|
|
|
1325
|
-
|
|
1717
|
+
struct node_lock_t {
|
|
1718
|
+
visits_bitset_t& bitset;
|
|
1719
|
+
std::size_t idx;
|
|
1326
1720
|
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
}
|
|
1721
|
+
inline ~node_lock_t() noexcept { bitset.atomic_reset(idx); }
|
|
1722
|
+
};
|
|
1330
1723
|
|
|
1331
|
-
inline
|
|
1332
|
-
|
|
1724
|
+
inline node_lock_t node_lock_(std::size_t idx) const noexcept {
|
|
1725
|
+
while (nodes_mutexes_.atomic_set(idx))
|
|
1726
|
+
;
|
|
1727
|
+
return {nodes_mutexes_, idx};
|
|
1333
1728
|
}
|
|
1334
1729
|
|
|
1335
|
-
id_t
|
|
1730
|
+
id_t connect_new_node_(id_t new_id, level_t level, thread_context_t& context) noexcept(false) {
|
|
1336
1731
|
|
|
1337
|
-
|
|
1732
|
+
node_t new_node = node_with_id_(new_id);
|
|
1338
1733
|
top_candidates_t& top = context.top_candidates;
|
|
1339
1734
|
std::size_t const connectivity_max = level ? config_.connectivity : pre_.connectivity_max_base;
|
|
1340
1735
|
|
|
1341
1736
|
// Outgoing links from `new_id`:
|
|
1342
|
-
neighbors_ref_t new_neighbors =
|
|
1737
|
+
neighbors_ref_t new_neighbors = neighbors_(new_node, level);
|
|
1343
1738
|
{
|
|
1344
1739
|
assert_m(!new_neighbors.size(), "The newly inserted element should have blank link list");
|
|
1345
|
-
candidates_view_t top_view =
|
|
1740
|
+
candidates_view_t top_view = refine_(top, config_.connectivity, context);
|
|
1346
1741
|
|
|
1347
1742
|
for (std::size_t idx = 0; idx != top_view.size(); idx++) {
|
|
1348
1743
|
assert_m(!new_neighbors[idx], "Possible memory corruption");
|
|
1349
|
-
assert_m(level <=
|
|
1350
|
-
new_neighbors.push_back(top_view[idx].
|
|
1744
|
+
assert_m(level <= node_with_id_(top_view[idx].id).level(), "Linking to missing level");
|
|
1745
|
+
new_neighbors.push_back(top_view[idx].id);
|
|
1351
1746
|
}
|
|
1352
1747
|
}
|
|
1353
1748
|
|
|
1354
1749
|
// Reverse links from the neighbors:
|
|
1355
1750
|
for (id_t close_id : new_neighbors) {
|
|
1356
|
-
|
|
1357
|
-
|
|
1751
|
+
node_t close_node = node_with_id_(close_id);
|
|
1752
|
+
node_lock_t close_lock = node_lock_(close_id);
|
|
1358
1753
|
|
|
1359
|
-
neighbors_ref_t close_header =
|
|
1754
|
+
neighbors_ref_t close_header = neighbors_(close_node, level);
|
|
1360
1755
|
assert_m(close_header.size() <= connectivity_max, "Possible corruption");
|
|
1361
1756
|
assert_m(close_id != new_id, "Self-loops are impossible");
|
|
1362
|
-
assert_m(level <= close_node.
|
|
1757
|
+
assert_m(level <= close_node.level(), "Linking to missing level");
|
|
1363
1758
|
|
|
1364
1759
|
// If `new_id` is already present in the neighboring connections of `close_id`
|
|
1365
1760
|
// then no need to modify any connections or run the heuristics.
|
|
@@ -1371,66 +1766,56 @@ class index_gt {
|
|
|
1371
1766
|
// To fit a new connection we need to drop an existing one.
|
|
1372
1767
|
top.clear();
|
|
1373
1768
|
top.reserve(close_header.size() + 1);
|
|
1374
|
-
top.emplace_reserved(
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
new_id);
|
|
1378
|
-
for (id_t successor_id : close_header) {
|
|
1379
|
-
node_ref_t successor_node = node(successor_id);
|
|
1380
|
-
top.emplace_reserved( //
|
|
1381
|
-
context.metric( //
|
|
1382
|
-
successor_node.vector, close_node.vector, successor_node.head.dim, close_node.head.dim),
|
|
1383
|
-
successor_id);
|
|
1384
|
-
}
|
|
1769
|
+
top.emplace_reserved(context.measure(new_node, close_node), new_id);
|
|
1770
|
+
for (id_t successor_id : close_header)
|
|
1771
|
+
top.emplace_reserved(context.measure(node_with_id_(successor_id), close_node), successor_id);
|
|
1385
1772
|
|
|
1386
1773
|
// Export the results:
|
|
1387
1774
|
close_header.clear();
|
|
1388
|
-
candidates_view_t top_view =
|
|
1775
|
+
candidates_view_t top_view = refine_(top, connectivity_max, context);
|
|
1389
1776
|
for (std::size_t idx = 0; idx != top_view.size(); idx++)
|
|
1390
|
-
close_header.push_back(top_view[idx].
|
|
1777
|
+
close_header.push_back(top_view[idx].id);
|
|
1391
1778
|
}
|
|
1392
1779
|
|
|
1393
1780
|
return new_neighbors[0];
|
|
1394
1781
|
}
|
|
1395
1782
|
|
|
1396
|
-
level_t
|
|
1783
|
+
level_t choose_random_level_(std::default_random_engine& level_generator) const noexcept {
|
|
1397
1784
|
std::uniform_real_distribution<double> distribution(0.0, 1.0);
|
|
1398
1785
|
double r = -std::log(distribution(level_generator)) * pre_.inverse_log_connectivity;
|
|
1399
1786
|
return (level_t)r;
|
|
1400
1787
|
}
|
|
1401
1788
|
|
|
1402
|
-
id_t
|
|
1403
|
-
id_t
|
|
1404
|
-
level_t begin_level, level_t end_level,
|
|
1789
|
+
id_t search_for_one_( //
|
|
1790
|
+
id_t closest_id, vector_view_t query, //
|
|
1791
|
+
level_t begin_level, level_t end_level, //
|
|
1792
|
+
thread_context_t& context) const noexcept {
|
|
1405
1793
|
|
|
1406
|
-
|
|
1407
|
-
distance_t closest_dist =
|
|
1408
|
-
context.metric(query_vec, node(closest_id).vector, query_dim, node(closest_id).head.dim);
|
|
1794
|
+
distance_t closest_dist = context.measure(query, node_with_id_(closest_id));
|
|
1409
1795
|
for (level_t level = begin_level; level > end_level; level--) {
|
|
1410
1796
|
bool changed;
|
|
1411
1797
|
do {
|
|
1412
1798
|
changed = false;
|
|
1413
|
-
|
|
1414
|
-
|
|
1415
|
-
neighbors_ref_t
|
|
1416
|
-
for (id_t candidate_id :
|
|
1417
|
-
|
|
1418
|
-
distance_t candidate_dist =
|
|
1419
|
-
context.metric(query_vec, candidate_node.vector, query_dim, candidate_node.head.dim);
|
|
1799
|
+
node_t closest_node = node_with_id_(closest_id);
|
|
1800
|
+
node_lock_t closest_lock = node_lock_(closest_id);
|
|
1801
|
+
neighbors_ref_t closest_neighbors = neighbors_non_base_(closest_node, level);
|
|
1802
|
+
for (id_t candidate_id : closest_neighbors) {
|
|
1803
|
+
distance_t candidate_dist = context.measure(query, node_with_id_(candidate_id));
|
|
1420
1804
|
if (candidate_dist < closest_dist) {
|
|
1421
1805
|
closest_dist = candidate_dist;
|
|
1422
1806
|
closest_id = candidate_id;
|
|
1423
1807
|
changed = true;
|
|
1424
1808
|
}
|
|
1425
1809
|
}
|
|
1810
|
+
context.iteration_cycles++;
|
|
1426
1811
|
} while (changed);
|
|
1427
1812
|
}
|
|
1428
1813
|
return closest_id;
|
|
1429
1814
|
}
|
|
1430
1815
|
|
|
1431
|
-
void
|
|
1432
|
-
id_t start_id,
|
|
1433
|
-
|
|
1816
|
+
void search_to_insert_( //
|
|
1817
|
+
id_t start_id, vector_view_t query, level_t level, //
|
|
1818
|
+
thread_context_t& context) noexcept(false) {
|
|
1434
1819
|
|
|
1435
1820
|
visits_bitset_t& visits = context.visits;
|
|
1436
1821
|
next_candidates_t& candidates = context.next_candidates; // pop min, push
|
|
@@ -1442,7 +1827,7 @@ class index_gt {
|
|
|
1442
1827
|
top.clear();
|
|
1443
1828
|
top.reserve(top_limit + 1);
|
|
1444
1829
|
|
|
1445
|
-
distance_t radius = context.
|
|
1830
|
+
distance_t radius = context.measure(query, node_with_id_(start_id));
|
|
1446
1831
|
candidates.emplace(-radius, start_id);
|
|
1447
1832
|
top.emplace_reserved(radius, start_id);
|
|
1448
1833
|
visits.set(start_id);
|
|
@@ -1450,38 +1835,39 @@ class index_gt {
|
|
|
1450
1835
|
while (!candidates.empty()) {
|
|
1451
1836
|
|
|
1452
1837
|
candidate_t candidacy = candidates.top();
|
|
1453
|
-
if ((-candidacy.
|
|
1838
|
+
if ((-candidacy.distance) > radius && top.size() == top_limit)
|
|
1454
1839
|
break;
|
|
1455
1840
|
|
|
1456
1841
|
candidates.pop();
|
|
1457
|
-
|
|
1458
|
-
node_ref_t candidate_node = node(candidate_id);
|
|
1459
|
-
lock_t candidate_lock = candidate_node.lock();
|
|
1460
|
-
neighbors_ref_t candidate_header = neighbors(candidate_node, level);
|
|
1842
|
+
context.iteration_cycles++;
|
|
1461
1843
|
|
|
1462
|
-
|
|
1463
|
-
|
|
1844
|
+
id_t candidate_id = candidacy.id;
|
|
1845
|
+
node_t candidate_ref = node_with_id_(candidate_id);
|
|
1846
|
+
node_lock_t candidate_lock = node_lock_(candidate_id);
|
|
1847
|
+
neighbors_ref_t candidate_neighbors = neighbors_(candidate_ref, level);
|
|
1848
|
+
|
|
1849
|
+
prefetch_neighbors_(candidate_neighbors, visits);
|
|
1850
|
+
for (id_t successor_id : candidate_neighbors) {
|
|
1464
1851
|
if (visits.test(successor_id))
|
|
1465
1852
|
continue;
|
|
1466
1853
|
|
|
1467
1854
|
visits.set(successor_id);
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
context.metric(query_vec, successor_node.vector, query_dim, successor_node.head.dim);
|
|
1855
|
+
distance_t successor_dist = context.measure(query, node_with_id_(successor_id));
|
|
1856
|
+
|
|
1471
1857
|
if (top.size() < top_limit || successor_dist < radius) {
|
|
1472
1858
|
candidates.emplace(-successor_dist, successor_id);
|
|
1473
1859
|
top.emplace_reserved(successor_dist, successor_id);
|
|
1474
1860
|
if (top.size() > top_limit)
|
|
1475
1861
|
top.pop();
|
|
1476
|
-
radius = top.top().
|
|
1862
|
+
radius = top.top().distance;
|
|
1477
1863
|
}
|
|
1478
1864
|
}
|
|
1479
1865
|
}
|
|
1480
1866
|
}
|
|
1481
1867
|
|
|
1482
|
-
void
|
|
1483
|
-
id_t start_id,
|
|
1484
|
-
|
|
1868
|
+
void search_to_find_in_base_( //
|
|
1869
|
+
id_t start_id, vector_view_t query, std::size_t expansion, //
|
|
1870
|
+
thread_context_t& context) const noexcept(false) {
|
|
1485
1871
|
|
|
1486
1872
|
visits_bitset_t& visits = context.visits;
|
|
1487
1873
|
next_candidates_t& candidates = context.next_candidates; // pop min, push
|
|
@@ -1493,7 +1879,7 @@ class index_gt {
|
|
|
1493
1879
|
top.clear();
|
|
1494
1880
|
top.reserve(top_limit + 1);
|
|
1495
1881
|
|
|
1496
|
-
distance_t radius = context.
|
|
1882
|
+
distance_t radius = context.measure(query, node_with_id_(start_id));
|
|
1497
1883
|
candidates.emplace(-radius, start_id);
|
|
1498
1884
|
top.emplace_reserved(radius, start_id);
|
|
1499
1885
|
visits.set(start_id);
|
|
@@ -1501,43 +1887,53 @@ class index_gt {
|
|
|
1501
1887
|
while (!candidates.empty()) {
|
|
1502
1888
|
|
|
1503
1889
|
candidate_t candidate = candidates.top();
|
|
1504
|
-
if ((-candidate.
|
|
1890
|
+
if ((-candidate.distance) > radius)
|
|
1505
1891
|
break;
|
|
1506
1892
|
|
|
1507
1893
|
candidates.pop();
|
|
1894
|
+
context.iteration_cycles++;
|
|
1508
1895
|
|
|
1509
|
-
id_t candidate_id = candidate.
|
|
1510
|
-
neighbors_ref_t
|
|
1896
|
+
id_t candidate_id = candidate.id;
|
|
1897
|
+
neighbors_ref_t candidate_neighbors = neighbors_base_(node_with_id_(candidate_id));
|
|
1511
1898
|
|
|
1512
|
-
|
|
1513
|
-
for (id_t successor_id :
|
|
1899
|
+
prefetch_neighbors_(candidate_neighbors, visits);
|
|
1900
|
+
for (id_t successor_id : candidate_neighbors) {
|
|
1514
1901
|
if (visits.test(successor_id))
|
|
1515
1902
|
continue;
|
|
1516
1903
|
|
|
1517
1904
|
visits.set(successor_id);
|
|
1518
|
-
|
|
1519
|
-
distance_t successor_dist =
|
|
1520
|
-
context.metric(query_vec, successor_node.vector, query_dim, successor_node.head.dim);
|
|
1905
|
+
distance_t successor_dist = context.measure(query, node_with_id_(successor_id));
|
|
1521
1906
|
|
|
1522
1907
|
if (top.size() < top_limit || successor_dist < radius) {
|
|
1523
1908
|
candidates.emplace(-successor_dist, successor_id);
|
|
1524
1909
|
top.emplace_reserved(successor_dist, successor_id);
|
|
1525
1910
|
if (top.size() > top_limit)
|
|
1526
1911
|
top.pop();
|
|
1527
|
-
radius = top.top().
|
|
1912
|
+
radius = top.top().distance;
|
|
1528
1913
|
}
|
|
1529
1914
|
}
|
|
1530
1915
|
}
|
|
1531
1916
|
}
|
|
1532
1917
|
|
|
1533
|
-
void
|
|
1918
|
+
void search_exact_(vector_view_t query, std::size_t count, thread_context_t& context) const noexcept(false) {
|
|
1919
|
+
top_candidates_t& top = context.top_candidates;
|
|
1920
|
+
top.clear();
|
|
1921
|
+
top.reserve(count);
|
|
1922
|
+
for (std::size_t i = 0; i != nodes_.size(); ++i)
|
|
1923
|
+
top.emplace_reserved(context.measure(query, node_with_id_(i)), static_cast<id_t>(i));
|
|
1924
|
+
}
|
|
1925
|
+
|
|
1926
|
+
void prefetch_neighbors_(neighbors_ref_t, visits_bitset_t const&) const noexcept {}
|
|
1534
1927
|
|
|
1535
1928
|
/**
|
|
1536
1929
|
* @brief This algorithm from the original paper implements a heuristic,
|
|
1537
1930
|
* that massively reduces the number of connections a point has,
|
|
1538
1931
|
* to keep only the neighbors, that are from each other.
|
|
1539
1932
|
*/
|
|
1540
|
-
|
|
1933
|
+
|
|
1934
|
+
candidates_view_t refine_( //
|
|
1935
|
+
top_candidates_t& top, std::size_t needed, //
|
|
1936
|
+
thread_context_t& context) const noexcept {
|
|
1541
1937
|
|
|
1542
1938
|
top.sort_ascending();
|
|
1543
1939
|
candidate_t* top_data = top.data();
|
|
@@ -1549,15 +1945,13 @@ class index_gt {
|
|
|
1549
1945
|
std::size_t consumed_count = 1; /// Always equal or greater than `submitted_count`.
|
|
1550
1946
|
while (submitted_count < needed && consumed_count < top_count) {
|
|
1551
1947
|
candidate_t candidate = top_data[consumed_count];
|
|
1552
|
-
|
|
1553
|
-
distance_t candidate_dist = candidate.
|
|
1948
|
+
node_t candidate_ref = node_with_id_(candidate.id);
|
|
1949
|
+
distance_t candidate_dist = candidate.distance;
|
|
1554
1950
|
bool good = true;
|
|
1555
1951
|
for (std::size_t idx = 0; idx < submitted_count; idx++) {
|
|
1556
1952
|
candidate_t submitted = top_data[idx];
|
|
1557
|
-
|
|
1558
|
-
distance_t inter_result_dist = context.
|
|
1559
|
-
submitted_node.vector, candidate_node.vector, //
|
|
1560
|
-
submitted_node.head.dim, candidate_node.head.dim);
|
|
1953
|
+
node_t submitted_node = node_with_id_(submitted.id);
|
|
1954
|
+
distance_t inter_result_dist = context.measure(submitted_node, candidate_ref);
|
|
1561
1955
|
if (inter_result_dist < candidate_dist) {
|
|
1562
1956
|
good = false;
|
|
1563
1957
|
break;
|