usearch 2.11.0 → 2.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +126 -37
- package/binding.gyp +11 -12
- package/c/README.md +9 -3
- package/c/lib.cpp +50 -13
- package/c/test.c +60 -55
- package/c/usearch.h +16 -0
- package/cpp/CMakeLists.txt +6 -0
- package/cpp/README.md +46 -14
- package/cpp/bench.cpp +99 -47
- package/cpp/test.cpp +948 -132
- package/include/usearch/index.hpp +894 -272
- package/include/usearch/index_dense.hpp +419 -152
- package/include/usearch/index_plugins.hpp +99 -49
- package/javascript/dist/cjs/usearch.d.ts +39 -1
- package/javascript/dist/cjs/usearch.js +11 -1
- package/javascript/dist/esm/usearch.d.ts +39 -1
- package/javascript/dist/esm/usearch.js +11 -1
- package/javascript/tsconfig-esm.json +3 -2
- package/javascript/usearch.ts +18 -7
- package/package.json +2 -2
- package/prebuilds/darwin-arm64+x64/usearch.node +0 -0
- package/prebuilds/linux-arm64/usearch.node +0 -0
- package/prebuilds/linux-x64/usearch.node +0 -0
- package/simsimd/.cmake-format.py +19 -0
- package/simsimd/CONTRIBUTING.md +49 -8
- package/simsimd/Cargo.lock +1 -1
- package/simsimd/Cargo.toml +10 -5
- package/simsimd/Package.swift +16 -0
- package/simsimd/README.md +69 -15
- package/simsimd/binding.gyp +14 -19
- package/simsimd/build.rs +2 -0
- package/simsimd/c/lib.c +18 -0
- package/simsimd/include/module.modulemap +4 -0
- package/simsimd/include/simsimd/binary.h +2 -0
- package/simsimd/include/simsimd/dot.h +466 -32
- package/simsimd/include/simsimd/probability.h +20 -10
- package/simsimd/include/simsimd/simsimd.h +444 -98
- package/simsimd/include/simsimd/spatial.h +480 -44
- package/simsimd/include/simsimd/types.h +161 -39
- package/simsimd/javascript/simsimd.ts +26 -2
- package/simsimd/package.json +1 -1
- package/simsimd/rust/benches/cosine.rs +8 -2
- package/simsimd/rust/benches/native.rs +240 -2
- package/simsimd/rust/benches/sqeuclidean.rs +8 -2
- package/simsimd/rust/lib.rs +50 -13
- package/simsimd/swift/source/SimSIMD.swift +123 -0
- package/simsimd/swift/tests/SimSIMDTests.swift +92 -0
- package/prebuilds/win32-ia32/usearch.node +0 -0
- package/prebuilds/win32-x64/usearch.node +0 -0
- package/simsimd/.releaserc +0 -87
package/cpp/test.cpp
CHANGED
|
@@ -1,12 +1,26 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
2
|
+
* @file test.cpp
|
|
3
|
+
* @author Ash Vardanian
|
|
4
|
+
* @brief Unit-testing vector-search functionality.
|
|
5
|
+
* @date June 10, 2023
|
|
6
|
+
*
|
|
7
|
+
*
|
|
8
|
+
* Key and slot types:
|
|
9
|
+
* - 64-bit `std::int64_t` keys and `std::uint32_t` slots are most popular.
|
|
10
|
+
* - 64-bit `std::uint64_t` keys and `std::uint40_t` are most space-efficient for
|
|
11
|
+
* point clouds 4B+ in size.
|
|
12
|
+
* - 128-bit `uuid_t` keys and `enum slot64_t : std::uint64_t` make most sense for
|
|
13
|
+
* for database users, implementing portable, concurrent systems.
|
|
3
14
|
*/
|
|
4
|
-
#include <algorithm>
|
|
5
|
-
#include <cassert>
|
|
6
|
-
#include <random>
|
|
7
|
-
#include <stdexcept>
|
|
8
|
-
#include <unordered_map>
|
|
9
|
-
#include <vector>
|
|
15
|
+
#include <algorithm> // `std::shuffle`
|
|
16
|
+
#include <cassert> // `assert`
|
|
17
|
+
#include <random> // `std::default_random_engine`
|
|
18
|
+
#include <stdexcept> // `std::terminate`
|
|
19
|
+
#include <unordered_map> // `std::unordered_map`
|
|
20
|
+
#include <vector> // `std::vector`
|
|
21
|
+
|
|
22
|
+
#define SZ_USE_X86_AVX512 0 // Sanitizers hate AVX512
|
|
23
|
+
#include <stringzilla/stringzilla.hpp> // Levenshtein distance implementation
|
|
10
24
|
|
|
11
25
|
#include <usearch/index.hpp>
|
|
12
26
|
#include <usearch/index_dense.hpp>
|
|
@@ -15,63 +29,230 @@
|
|
|
15
29
|
using namespace unum::usearch;
|
|
16
30
|
using namespace unum;
|
|
17
31
|
|
|
18
|
-
void
|
|
19
|
-
if (
|
|
20
|
-
|
|
32
|
+
void __expect(bool must_be_true, char const* file, int line, char const* message = nullptr) {
|
|
33
|
+
if (must_be_true)
|
|
34
|
+
return;
|
|
35
|
+
message = message ? message : "C++ unit test failed";
|
|
36
|
+
char buffer[512];
|
|
37
|
+
std::snprintf(buffer, sizeof(buffer), "%s at %s:%d", message, file, line);
|
|
38
|
+
__usearch_raise_runtime_error(buffer);
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
template <typename value_at>
|
|
42
|
+
void __expect_eq(value_at a, value_at b, char const* file, int line, char const* message = nullptr) {
|
|
43
|
+
__expect(a == b, file, line, message);
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
#define expect(cond) __expect((bool)(cond), __FILE__, __LINE__)
|
|
47
|
+
#define expect_eq(a, b) __expect_eq<decltype(a)>((a), (b), __FILE__, __LINE__)
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* Less error-prone type definition to differentiate `std::uint32_t` and other native
|
|
51
|
+
* types in logs and avoid implicit conversions.
|
|
52
|
+
*/
|
|
53
|
+
enum slot32_t : std::uint32_t {};
|
|
54
|
+
template <> struct unum::usearch::hash_gt<slot32_t> : public unum::usearch::hash_gt<std::uint32_t> {};
|
|
55
|
+
template <> struct unum::usearch::default_free_value_gt<slot32_t> {
|
|
56
|
+
static slot32_t value() noexcept { return static_cast<slot32_t>(std::numeric_limits<std::uint32_t>::max()); }
|
|
57
|
+
};
|
|
58
|
+
|
|
59
|
+
/*
|
|
60
|
+
* Let's instantiate several templates to make all of their symbols available for testing.
|
|
61
|
+
* https://dhashe.com/how-to-build-highly-debuggable-c-binaries.html
|
|
62
|
+
*/
|
|
63
|
+
template class unum::usearch::index_gt<float, std::int64_t, slot32_t>;
|
|
64
|
+
template class unum::usearch::index_gt<float, std::int64_t, uint40_t>;
|
|
65
|
+
template class unum::usearch::index_dense_gt<std::int64_t, slot32_t>;
|
|
66
|
+
template class unum::usearch::index_dense_gt<std::int64_t, uint40_t>;
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* @brief Convenience wrapper combining combined allocation and construction of an index.
|
|
70
|
+
*/
|
|
71
|
+
template <typename index_at> struct aligned_wrapper_gt {
|
|
72
|
+
using index_t = index_at;
|
|
73
|
+
using alloc_t = aligned_allocator_gt<index_t, 64>;
|
|
74
|
+
|
|
75
|
+
alloc_t alloc;
|
|
76
|
+
index_t* index = nullptr;
|
|
77
|
+
|
|
78
|
+
template <typename... args_at> aligned_wrapper_gt(args_at&&... args) {
|
|
79
|
+
|
|
80
|
+
alloc_t index_alloc;
|
|
81
|
+
index_t* index_typed = index_alloc.allocate(1);
|
|
82
|
+
expect(index_typed != nullptr);
|
|
83
|
+
expect(((unsigned long long)(index_typed) % 64ull) == 0ull);
|
|
84
|
+
|
|
85
|
+
new (index_typed) index_t(std::forward<args_at>(args)...);
|
|
86
|
+
index = index_typed;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
~aligned_wrapper_gt() {
|
|
90
|
+
if (index != nullptr) {
|
|
91
|
+
index->~index_t();
|
|
92
|
+
alloc.deallocate(index, 1);
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
};
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* Tests the functionality of the custom uint40_t type ensuring consistent
|
|
99
|
+
* behavior across various constructors from uint32_t, uint64_t, and size_t types,
|
|
100
|
+
* and validates the relational ordering between various values.
|
|
101
|
+
*/
|
|
102
|
+
void test_uint40() {
|
|
103
|
+
|
|
104
|
+
union uint64_octets_t {
|
|
105
|
+
std::uint64_t value;
|
|
106
|
+
std::uint8_t octets[8];
|
|
107
|
+
};
|
|
108
|
+
|
|
109
|
+
// Constants for tests
|
|
110
|
+
std::uint64_t max_uint40_k = (1ULL << 40) - 1;
|
|
111
|
+
|
|
112
|
+
// Set of test numbers
|
|
113
|
+
std::vector<std::uint64_t> test_numbers = {
|
|
114
|
+
42ull, // Typical small number
|
|
115
|
+
4242ull, // Larger number still within uint40 range
|
|
116
|
+
(1ull << 39), // A high number within range
|
|
117
|
+
(1ull << 40) - 1, // Maximum value representable in uint40
|
|
118
|
+
1ull << 40, // Exactly at the boundary of uint40
|
|
119
|
+
(1ull << 40) + 1, // Just beyond the boundary of uint40
|
|
120
|
+
1ull << 63 // Well beyond the uint40 boundary, tests masking
|
|
121
|
+
};
|
|
122
|
+
|
|
123
|
+
for (std::uint64_t input_u64 : test_numbers) {
|
|
124
|
+
std::uint32_t input_u32 = static_cast<std::uint32_t>(input_u64);
|
|
125
|
+
std::size_t input_size = static_cast<std::size_t>(input_u64);
|
|
126
|
+
|
|
127
|
+
// Create uint40_t instances from different types
|
|
128
|
+
uint40_t u40_from_u32(input_u32);
|
|
129
|
+
uint40_t u40_from_u64(input_u64);
|
|
130
|
+
uint40_t u40_from_size(input_size);
|
|
131
|
+
|
|
132
|
+
// Expected value after masking
|
|
133
|
+
uint64_octets_t input_clamped;
|
|
134
|
+
input_clamped.value = input_u64 & max_uint40_k;
|
|
135
|
+
|
|
136
|
+
// Check if all conversions are equal to the masked value
|
|
137
|
+
expect_eq(u40_from_u32, input_clamped.value & 0xFFFFFFFF);
|
|
138
|
+
expect_eq(u40_from_u64, input_clamped.value);
|
|
139
|
+
expect_eq(u40_from_size, input_clamped.value);
|
|
140
|
+
|
|
141
|
+
// Check relative ordering against all other test numbers
|
|
142
|
+
for (std::uint64_t other_u64 : test_numbers) {
|
|
143
|
+
uint64_octets_t other_clamped;
|
|
144
|
+
other_clamped.value = other_u64 & max_uint40_k;
|
|
145
|
+
uint40_t other_u40(other_clamped.value);
|
|
146
|
+
|
|
147
|
+
// Check < and >
|
|
148
|
+
expect_eq(input_clamped.value < other_clamped.value, u40_from_u64 < other_u40);
|
|
149
|
+
expect_eq(input_clamped.value > other_clamped.value, u40_from_u64 > other_u40);
|
|
150
|
+
|
|
151
|
+
// Check <= and >=
|
|
152
|
+
expect_eq(input_clamped.value <= other_clamped.value, u40_from_u64 <= other_u40);
|
|
153
|
+
expect_eq(input_clamped.value >= other_clamped.value, u40_from_u64 >= other_u40);
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
// Test equality and inequality operators
|
|
158
|
+
for (std::uint64_t input_u64 : test_numbers) {
|
|
159
|
+
uint64_octets_t input_clamped;
|
|
160
|
+
input_clamped.value = input_u64 & max_uint40_k;
|
|
161
|
+
uint40_t u40(input_clamped.value);
|
|
162
|
+
|
|
163
|
+
expect_eq(u40 == uint40_t(input_clamped.value), true);
|
|
164
|
+
expect_eq(u40 != uint40_t(input_clamped.value + 1), true);
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
// Test min and max functions
|
|
168
|
+
expect_eq(uint40_t::min(), uint40_t(0u));
|
|
169
|
+
expect_eq(uint40_t::max(), uint40_t(max_uint40_k));
|
|
170
|
+
|
|
171
|
+
// Test copy and move semantics
|
|
172
|
+
for (std::uint64_t input_u64 : test_numbers) {
|
|
173
|
+
uint40_t u40_orig(input_u64);
|
|
174
|
+
|
|
175
|
+
uint40_t u40_copy(u40_orig);
|
|
176
|
+
expect_eq(u40_orig, u40_copy);
|
|
177
|
+
|
|
178
|
+
uint40_t u40_move(std::move(u40_orig));
|
|
179
|
+
expect_eq(u40_copy, u40_move);
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
// Test default constructor (zero initialization)
|
|
183
|
+
uint40_t u40_default;
|
|
184
|
+
expect_eq(u40_default, uint40_t(0u));
|
|
21
185
|
}
|
|
22
186
|
|
|
187
|
+
/**
|
|
188
|
+
* @brief Tests the functionality of the custom float16_t type ensuring consistent.
|
|
189
|
+
*/
|
|
190
|
+
void test_float16() {}
|
|
191
|
+
|
|
192
|
+
/**
|
|
193
|
+
* The goal of this test is to invoke as many different interfaces as possible, making sure that all code-paths compile.
|
|
194
|
+
* For that it only uses a tiny set of 3 predefined vectors.
|
|
195
|
+
*
|
|
196
|
+
* @param index Reference to the index where vectors will be stored and searched.
|
|
197
|
+
* @param vectors A collection of vectors to be tested.
|
|
198
|
+
* @param args Additional arguments for configuring search or index operations.
|
|
199
|
+
* @tparam punned_ak Template parameter that determines specific behaviors or checks in the test based on its value.
|
|
200
|
+
* @tparam index_at Type of the index being tested.
|
|
201
|
+
* @tparam scalar_at Data type of the elements in the vectors.
|
|
202
|
+
* @tparam extra_args_at Variadic template parameter types for additional configuration.
|
|
203
|
+
*/
|
|
23
204
|
template <bool punned_ak, typename index_at, typename scalar_at, typename... extra_args_at>
|
|
24
|
-
void
|
|
205
|
+
void test_minimal_three_vectors(index_at& index, //
|
|
206
|
+
typename index_at::vector_key_t key_first, std::vector<scalar_at> const& vector_first,
|
|
207
|
+
typename index_at::vector_key_t key_second, std::vector<scalar_at> const& vector_second,
|
|
208
|
+
typename index_at::vector_key_t key_third, std::vector<scalar_at> const& vector_third,
|
|
209
|
+
extra_args_at&&... args) {
|
|
25
210
|
|
|
26
211
|
using scalar_t = scalar_at;
|
|
27
212
|
using index_t = index_at;
|
|
28
213
|
using vector_key_t = typename index_t::vector_key_t;
|
|
29
214
|
using distance_t = typename index_t::distance_t;
|
|
30
215
|
|
|
31
|
-
// Generate some keys starting from end,
|
|
32
|
-
// for three vectors from the dataset
|
|
33
|
-
vector_key_t const key_max = default_free_value<vector_key_t>() - 1;
|
|
34
|
-
vector_key_t const key_first = key_max - 0;
|
|
35
|
-
vector_key_t const key_second = key_max - 1;
|
|
36
|
-
vector_key_t const key_third = key_max - 2;
|
|
37
|
-
scalar_t const* vector_first = vectors[0].data();
|
|
38
|
-
scalar_t const* vector_second = vectors[1].data();
|
|
39
|
-
scalar_t const* vector_third = vectors[2].data();
|
|
40
|
-
std::size_t dimensions = vectors[0].size();
|
|
41
|
-
|
|
42
216
|
// Try checking the empty state
|
|
43
217
|
if constexpr (punned_ak) {
|
|
44
|
-
|
|
45
|
-
|
|
218
|
+
if (index.config().enable_key_lookups) {
|
|
219
|
+
expect(!index.contains(key_first));
|
|
220
|
+
expect(!index.get(key_first, (f32_t*)nullptr, 1));
|
|
221
|
+
}
|
|
46
222
|
}
|
|
47
223
|
|
|
48
224
|
// Add data
|
|
49
|
-
index.
|
|
50
|
-
index.add(key_first, vector_first, args...);
|
|
225
|
+
expect(index.try_reserve(10));
|
|
226
|
+
expect(index.add(key_first, vector_first.data(), args...));
|
|
51
227
|
|
|
52
228
|
// Default approximate search
|
|
53
229
|
vector_key_t matched_keys[10] = {0};
|
|
54
230
|
distance_t matched_distances[10] = {0};
|
|
55
|
-
std::size_t matched_count = index.search(vector_first, 5, args...).dump_to(matched_keys, matched_distances);
|
|
231
|
+
std::size_t matched_count = index.search(vector_first.data(), 5, args...).dump_to(matched_keys, matched_distances);
|
|
56
232
|
|
|
57
233
|
expect(matched_count == 1);
|
|
58
234
|
expect(matched_keys[0] == key_first);
|
|
59
235
|
expect(std::abs(matched_distances[0]) < 0.01);
|
|
60
236
|
|
|
61
237
|
// Add more entries
|
|
62
|
-
index.add(key_second, vector_second, args...);
|
|
63
|
-
index.add(key_third, vector_third, args...);
|
|
238
|
+
index.add(key_second, vector_second.data(), args...);
|
|
239
|
+
index.add(key_third, vector_third.data(), args...);
|
|
64
240
|
expect(index.size() == 3);
|
|
65
241
|
|
|
66
|
-
// Perform
|
|
67
|
-
|
|
68
|
-
|
|
242
|
+
// Perform single entry search
|
|
243
|
+
{
|
|
244
|
+
auto search_result = index.search(vector_first.data(), 5, args...);
|
|
245
|
+
expect(search_result);
|
|
246
|
+
matched_count = search_result.dump_to(matched_keys, matched_distances);
|
|
247
|
+
expect(matched_count != 0);
|
|
248
|
+
}
|
|
69
249
|
|
|
70
250
|
// Perform filtered exact search, keeping only odd values
|
|
71
251
|
if constexpr (punned_ak) {
|
|
72
252
|
auto is_odd = [](vector_key_t key) -> bool { return (key & 1) != 0; };
|
|
73
|
-
|
|
74
|
-
|
|
253
|
+
auto search_result = index.filtered_search(vector_first.data(), 5, is_odd, args...);
|
|
254
|
+
expect(search_result);
|
|
255
|
+
matched_count = search_result.dump_to(matched_keys, matched_distances);
|
|
75
256
|
expect(matched_count != 0);
|
|
76
257
|
for (std::size_t i = 0; i < matched_count; i++)
|
|
77
258
|
expect(is_odd(matched_keys[i]));
|
|
@@ -81,79 +262,317 @@ void test_cosine(index_at& index, std::vector<std::vector<scalar_at>> const& vec
|
|
|
81
262
|
std::size_t count = 0;
|
|
82
263
|
for (auto member : index) {
|
|
83
264
|
vector_key_t id = member.key;
|
|
84
|
-
expect(id
|
|
265
|
+
expect(id >= key_first && id <= key_third);
|
|
85
266
|
count++;
|
|
86
267
|
}
|
|
87
268
|
expect((count == 3));
|
|
88
269
|
expect((index.stats(0).nodes == 3));
|
|
89
270
|
|
|
90
271
|
// Check if clustering endpoint compiles
|
|
91
|
-
index.cluster(vector_first, 0, args...);
|
|
272
|
+
index.cluster(vector_first.data(), 0, args...);
|
|
92
273
|
|
|
93
274
|
// Try removals and replacements
|
|
94
275
|
if constexpr (punned_ak) {
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
276
|
+
if (index.config().enable_key_lookups) {
|
|
277
|
+
using labeling_result_t = typename index_t::labeling_result_t;
|
|
278
|
+
labeling_result_t result = index.remove(key_third);
|
|
279
|
+
expect(result);
|
|
280
|
+
expect(index.size() == 2);
|
|
281
|
+
index.add(key_third, vector_third.data(), args...);
|
|
282
|
+
expect(index.size() == 3);
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
expect(index.save("tmp.usearch"));
|
|
287
|
+
|
|
288
|
+
// Perform content and scan validations for a copy
|
|
289
|
+
{
|
|
290
|
+
auto copy_result = index.copy();
|
|
291
|
+
expect(copy_result);
|
|
292
|
+
index_at copied_index = std::move(copy_result.index);
|
|
293
|
+
|
|
294
|
+
// Perform single entry search
|
|
295
|
+
auto search_result = copied_index.search(vector_first.data(), 5, args...);
|
|
296
|
+
expect(search_result);
|
|
297
|
+
matched_count = search_result.dump_to(matched_keys, matched_distances);
|
|
298
|
+
expect(matched_count != 0);
|
|
299
|
+
|
|
300
|
+
// Validate scans
|
|
301
|
+
std::size_t count = 0;
|
|
302
|
+
for (auto member : copied_index) {
|
|
303
|
+
vector_key_t id = member.key;
|
|
304
|
+
expect(id >= key_first && id <= key_third);
|
|
305
|
+
count++;
|
|
306
|
+
}
|
|
307
|
+
expect_eq(count, 3);
|
|
308
|
+
expect_eq(copied_index.stats(0).nodes, 3);
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
// Perform content and scan validations for a moved
|
|
312
|
+
{
|
|
313
|
+
index_at moved_index(std::move(index));
|
|
314
|
+
|
|
315
|
+
// Perform single entry search
|
|
316
|
+
auto search_result = moved_index.search(vector_first.data(), 5, args...);
|
|
317
|
+
expect(search_result);
|
|
318
|
+
matched_count = search_result.dump_to(matched_keys, matched_distances);
|
|
319
|
+
expect(matched_count != 0);
|
|
320
|
+
|
|
321
|
+
// Validate scans
|
|
322
|
+
std::size_t count = 0;
|
|
323
|
+
for (auto member : moved_index) {
|
|
324
|
+
vector_key_t id = member.key;
|
|
325
|
+
expect(id >= key_first && id <= key_third);
|
|
326
|
+
count++;
|
|
327
|
+
}
|
|
328
|
+
expect_eq(count, 3);
|
|
329
|
+
expect_eq(moved_index.stats(0).nodes, 3);
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
// Check if metadata is retrieved correctly
|
|
333
|
+
if constexpr (punned_ak) {
|
|
334
|
+
auto head_result = index_dense_metadata_from_path("tmp.usearch");
|
|
335
|
+
expect(head_result);
|
|
336
|
+
expect_eq(3ull, head_result.head.count_present);
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
// Try loading and move assignment
|
|
340
|
+
{
|
|
341
|
+
index_at loaded_index;
|
|
342
|
+
auto load_result = loaded_index.load("tmp.usearch");
|
|
343
|
+
expect(load_result);
|
|
344
|
+
index = std::move(loaded_index);
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
// Check the copy of the restored index
|
|
348
|
+
{
|
|
349
|
+
auto copy_result = index.copy();
|
|
350
|
+
expect(copy_result);
|
|
351
|
+
index_at copied_index = std::move(copy_result.index);
|
|
352
|
+
expect_eq(copied_index.size(), 3);
|
|
101
353
|
}
|
|
102
354
|
|
|
103
355
|
// Search again over reconstructed index
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
356
|
+
{
|
|
357
|
+
matched_count = index.search(vector_first.data(), 5, args...).dump_to(matched_keys, matched_distances);
|
|
358
|
+
expect_eq(matched_count, 3);
|
|
359
|
+
expect_eq(matched_keys[0], key_first);
|
|
360
|
+
expect(std::abs(matched_distances[0]) < 0.01);
|
|
361
|
+
}
|
|
110
362
|
|
|
363
|
+
// Try retrieving a vector from a deserialized index
|
|
111
364
|
if constexpr (punned_ak) {
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
365
|
+
if (index.config().enable_key_lookups) {
|
|
366
|
+
std::size_t dimensions = vector_first.size();
|
|
367
|
+
std::vector<scalar_t> vector_reloaded(dimensions);
|
|
368
|
+
expect(index.get(key_second, vector_reloaded.data()));
|
|
369
|
+
expect(std::equal(vector_second.data(), vector_second.data() + dimensions, vector_reloaded.data()));
|
|
370
|
+
}
|
|
115
371
|
}
|
|
372
|
+
}
|
|
116
373
|
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
374
|
+
/**
|
|
375
|
+
* @brief Tests value removals, by repeatedly adding and removing a couple of vectors.
|
|
376
|
+
*
|
|
377
|
+
* @param index Reference to the index where vectors will be stored and searched.
|
|
378
|
+
* @param vector_first First vector to be tested.
|
|
379
|
+
* @param vector_second Second vector to be tested.
|
|
380
|
+
* @param args Additional arguments for configuring search or index operations.
|
|
381
|
+
*/
|
|
382
|
+
template <typename index_at, typename scalar_at>
|
|
383
|
+
void test_punned_add_remove_vector( //
|
|
384
|
+
index_at& index, //
|
|
385
|
+
std::vector<scalar_at> const& vector_first, //
|
|
386
|
+
std::vector<scalar_at> const& vector_second) {
|
|
387
|
+
|
|
388
|
+
using index_t = index_at;
|
|
389
|
+
using vector_key_t = typename index_t::vector_key_t;
|
|
390
|
+
|
|
391
|
+
// Creating the index
|
|
392
|
+
expect(index.try_reserve(10));
|
|
393
|
+
expect(index.capacity() >= 10);
|
|
394
|
+
|
|
395
|
+
// Max 64-bit unsigned key value is: 18446744073709551615
|
|
396
|
+
vector_key_t key_first = 483367403120493160;
|
|
397
|
+
vector_key_t key_second = 483367403120558696;
|
|
398
|
+
vector_key_t key_third = 483367403120624232;
|
|
399
|
+
vector_key_t key_fourth = 483367403120624233;
|
|
400
|
+
|
|
401
|
+
// Adding, getting, and removing vectors
|
|
402
|
+
expect(index.add(key_first, vector_first.data()));
|
|
403
|
+
std::vector<float> found_slice(vector_first.size(), 0.0f);
|
|
404
|
+
expect_eq(index.get(key_first, found_slice.data()), 1);
|
|
405
|
+
expect(index.remove(key_first));
|
|
406
|
+
|
|
407
|
+
expect(index.add(key_second, vector_second.data()));
|
|
408
|
+
expect_eq(index.get(key_second, found_slice.data()), 1);
|
|
409
|
+
expect(index.remove(key_second));
|
|
410
|
+
|
|
411
|
+
expect(index.add(key_third, vector_second.data()));
|
|
412
|
+
expect_eq(index.get(key_third, found_slice.data()), 1);
|
|
413
|
+
expect(index.remove(key_third));
|
|
414
|
+
|
|
415
|
+
expect(index.add(key_fourth, vector_second.data()));
|
|
416
|
+
expect_eq(index.get(key_fourth, found_slice.data()), 1);
|
|
417
|
+
expect(index.remove(key_fourth));
|
|
418
|
+
|
|
419
|
+
expect_eq(index.size(), 0);
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
/**
|
|
423
|
+
* Tests the normal operational mode of the library, dealing with a variable length collection
|
|
424
|
+
* of `vectors` with monotonically increasing keys starting from `start_key`.
|
|
425
|
+
*
|
|
426
|
+
* @param index Reference to the index where vectors will be stored and searched.
|
|
427
|
+
* @param start_key The key for the first `vector`, others are generated with increments.
|
|
428
|
+
* @param vectors A collection of vectors to be tested.
|
|
429
|
+
* @param args Additional arguments for configuring search or index operations.
|
|
430
|
+
* @tparam punned_ak Template parameter that determines specific behaviors or checks in the test based on its value.
|
|
431
|
+
* @tparam index_at Type of the index being tested.
|
|
432
|
+
* @tparam scalar_at Data type of the elements in the vectors.
|
|
433
|
+
* @tparam extra_args_at Variadic template parameter types for additional configuration.
|
|
434
|
+
*/
|
|
435
|
+
template <bool punned_ak, typename index_at, typename scalar_at, typename... extra_args_at>
|
|
436
|
+
void test_collection(index_at& index, typename index_at::vector_key_t const start_key,
|
|
437
|
+
std::vector<std::vector<scalar_at>> const& vectors, extra_args_at&&... args) {
|
|
438
|
+
|
|
439
|
+
using scalar_t = scalar_at;
|
|
440
|
+
using index_t = index_at;
|
|
441
|
+
using vector_key_t = typename index_t::vector_key_t;
|
|
442
|
+
using distance_t = typename index_t::distance_t;
|
|
443
|
+
using index_add_result_t = typename index_t::add_result_t;
|
|
444
|
+
using index_search_result_t = typename index_t::search_result_t;
|
|
445
|
+
|
|
446
|
+
// Generate some keys starting from end,
|
|
447
|
+
// for three vectors from the dataset
|
|
448
|
+
vector_key_t const key_first = start_key;
|
|
449
|
+
std::vector<scalar_at> const& vector_first = vectors[0];
|
|
450
|
+
std::size_t dimensions = vector_first.size();
|
|
451
|
+
|
|
452
|
+
// Try batch requests, heavily over-subscribing the CPU cores
|
|
453
|
+
std::size_t executor_threads = std::thread::hardware_concurrency();
|
|
454
|
+
executor_default_t executor(executor_threads);
|
|
455
|
+
expect(index.try_reserve({vectors.size(), executor.size()}));
|
|
456
|
+
executor.fixed(vectors.size(), [&](std::size_t thread, std::size_t task) {
|
|
121
457
|
if constexpr (punned_ak) {
|
|
122
|
-
index.add(
|
|
458
|
+
index_add_result_t result = index.add(start_key + task, vectors[task].data(), args...);
|
|
459
|
+
expect(result);
|
|
123
460
|
} else {
|
|
124
461
|
index_update_config_t config;
|
|
125
462
|
config.thread = thread;
|
|
126
|
-
index.add(
|
|
463
|
+
index_add_result_t result = index.add(start_key + task, vectors[task].data(), args..., config);
|
|
464
|
+
expect(result);
|
|
465
|
+
}
|
|
466
|
+
});
|
|
467
|
+
|
|
468
|
+
// Make sure we didn't lose parallelism settings after reload
|
|
469
|
+
expect(index.limits().threads_search >= executor.size());
|
|
470
|
+
if constexpr (punned_ak)
|
|
471
|
+
expect(index.currently_available_threads() >= executor.size());
|
|
472
|
+
|
|
473
|
+
// Parallel search over the same vectors
|
|
474
|
+
executor.fixed(vectors.size(), [&](std::size_t thread, std::size_t task) {
|
|
475
|
+
std::size_t max_possible_matches = vectors.size();
|
|
476
|
+
std::size_t count_requested = max_possible_matches;
|
|
477
|
+
std::vector<vector_key_t> matched_keys(count_requested);
|
|
478
|
+
std::vector<distance_t> matched_distances(count_requested);
|
|
479
|
+
std::size_t matched_count = 0;
|
|
480
|
+
|
|
481
|
+
// Invoke the search kernel
|
|
482
|
+
if constexpr (punned_ak) {
|
|
483
|
+
index_search_result_t result = index.search(vectors[task].data(), count_requested, args...);
|
|
484
|
+
expect(result);
|
|
485
|
+
matched_count = result.dump_to(matched_keys.data(), matched_distances.data());
|
|
486
|
+
} else {
|
|
487
|
+
index_search_config_t config;
|
|
488
|
+
config.thread = thread;
|
|
489
|
+
index_search_result_t result = index.search(vectors[task].data(), count_requested, args..., config);
|
|
490
|
+
expect(result);
|
|
491
|
+
matched_count = result.dump_to(matched_keys.data(), matched_distances.data());
|
|
127
492
|
}
|
|
493
|
+
|
|
494
|
+
// In approximate search we can't always expect the right answer to be found
|
|
495
|
+
// expect_eq(matched_count, max_possible_matches);
|
|
496
|
+
// expect_eq(matched_keys[0], start_key + task);
|
|
497
|
+
// expect(std::abs(matched_distances[0]) < 0.01);
|
|
498
|
+
expect(matched_count <= max_possible_matches);
|
|
499
|
+
|
|
500
|
+
// Check that all the distance are monotonically rising
|
|
501
|
+
for (std::size_t i = 1; i < matched_count; i++)
|
|
502
|
+
expect(matched_distances[i - 1] <= matched_distances[i]);
|
|
128
503
|
});
|
|
129
504
|
|
|
505
|
+
// Search again over mapped index
|
|
506
|
+
expect(index.save("tmp.usearch"));
|
|
507
|
+
|
|
508
|
+
{
|
|
509
|
+
auto copy_result = index.copy();
|
|
510
|
+
expect(copy_result);
|
|
511
|
+
index_at copied_index = std::move(copy_result.index);
|
|
512
|
+
expect_eq(copied_index.size(), vectors.size());
|
|
513
|
+
}
|
|
514
|
+
|
|
130
515
|
// Check for duplicates
|
|
131
516
|
if constexpr (punned_ak) {
|
|
132
|
-
index.
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
517
|
+
if (index.config().enable_key_lookups) {
|
|
518
|
+
expect(index.try_reserve({vectors.size() + 1u, executor.size()}));
|
|
519
|
+
index_add_result_t result = index.add(key_first, vector_first.data(), args...);
|
|
520
|
+
expect_eq(!!result, index.multi());
|
|
521
|
+
result.error.release();
|
|
136
522
|
|
|
137
|
-
|
|
138
|
-
|
|
523
|
+
std::size_t first_key_count = index.count(key_first);
|
|
524
|
+
expect_eq(first_key_count, 1ul + index.multi());
|
|
525
|
+
}
|
|
139
526
|
}
|
|
140
527
|
|
|
141
|
-
//
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
528
|
+
// Recover the state before the duplicate insertion
|
|
529
|
+
expect(index.view("tmp.usearch"));
|
|
530
|
+
|
|
531
|
+
// Parallel search over the same vectors
|
|
532
|
+
executor.fixed(vectors.size(), [&](std::size_t thread, std::size_t task) {
|
|
533
|
+
// Check over-sampling beyond the size of the collection
|
|
534
|
+
std::size_t max_possible_matches = vectors.size();
|
|
535
|
+
std::size_t count_requested = max_possible_matches * 10;
|
|
536
|
+
std::vector<vector_key_t> matched_keys(count_requested);
|
|
537
|
+
std::vector<distance_t> matched_distances(count_requested);
|
|
538
|
+
std::size_t matched_count = 0;
|
|
149
539
|
|
|
540
|
+
// Invoke the search kernel
|
|
541
|
+
if constexpr (punned_ak) {
|
|
542
|
+
index_search_result_t result = index.search(vectors[task].data(), count_requested, args...);
|
|
543
|
+
expect(result);
|
|
544
|
+
matched_count = result.dump_to(matched_keys.data(), matched_distances.data());
|
|
545
|
+
} else {
|
|
546
|
+
index_search_config_t config;
|
|
547
|
+
config.thread = thread;
|
|
548
|
+
index_search_result_t result = index.search(vectors[task].data(), count_requested, args..., config);
|
|
549
|
+
expect(result);
|
|
550
|
+
matched_count = result.dump_to(matched_keys.data(), matched_distances.data());
|
|
551
|
+
}
|
|
552
|
+
|
|
553
|
+
// In approximate search we can't always expect the right answer to be found
|
|
554
|
+
// expect_eq(matched_count, max_possible_matches);
|
|
555
|
+
// expect_eq(matched_keys[0], start_key + task);
|
|
556
|
+
// expect(std::abs(matched_distances[0]) < 0.01);
|
|
557
|
+
expect(matched_count <= max_possible_matches);
|
|
558
|
+
|
|
559
|
+
// Check that all the distance are monotonically rising
|
|
560
|
+
for (std::size_t i = 1; i < matched_count; i++)
|
|
561
|
+
expect(matched_distances[i - 1] <= matched_distances[i]);
|
|
562
|
+
});
|
|
563
|
+
|
|
564
|
+
// Try retrieving a vector from a deserialized index
|
|
150
565
|
if constexpr (punned_ak) {
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
566
|
+
if (index.config().enable_key_lookups) {
|
|
567
|
+
expect(index.contains(key_first));
|
|
568
|
+
expect_eq(index.count(key_first), 1);
|
|
569
|
+
std::vector<scalar_t> vector_reloaded(dimensions);
|
|
570
|
+
index.get(key_first, vector_reloaded.data());
|
|
571
|
+
expect(std::equal(vector_first.data(), vector_first.data() + dimensions, vector_reloaded.data()));
|
|
572
|
+
}
|
|
154
573
|
|
|
155
574
|
auto compaction_result = index.compact();
|
|
156
|
-
expect(
|
|
575
|
+
expect(compaction_result);
|
|
157
576
|
}
|
|
158
577
|
|
|
159
578
|
expect(index.memory_usage() > 0);
|
|
@@ -162,10 +581,73 @@ void test_cosine(index_at& index, std::vector<std::vector<scalar_at>> const& vec
|
|
|
162
581
|
// Check metadata
|
|
163
582
|
if constexpr (punned_ak) {
|
|
164
583
|
index_dense_metadata_result_t meta = index_dense_metadata_from_path("tmp.usearch");
|
|
165
|
-
expect(
|
|
584
|
+
expect(meta);
|
|
166
585
|
}
|
|
167
586
|
}
|
|
168
587
|
|
|
588
|
+
/**
|
|
589
|
+
* Stress-tests the behavior of the type-punned higher-level index under heavy concurrent insertions,
|
|
590
|
+
* removals and updates.
|
|
591
|
+
*
|
|
592
|
+
* @param index Reference to the index where vectors will be stored and searched.
|
|
593
|
+
* @param start_key The key for the first `vector`, others are generated with increments.
|
|
594
|
+
* @param vectors A collection of vectors to be tested.
|
|
595
|
+
* @param executor_threads Number of threads to be used for concurrent operations.
|
|
596
|
+
* @tparam punned_ak Template parameter that determines specific behaviors or checks in the test based on its value.
|
|
597
|
+
* @tparam index_at Type of the index being tested.
|
|
598
|
+
* @tparam scalar_at Data type of the elements in the vectors.
|
|
599
|
+
* @tparam extra_args_at Variadic template parameter types for additional configuration.
|
|
600
|
+
*/
|
|
601
|
+
template <typename index_at, typename scalar_at, typename... extra_args_at>
|
|
602
|
+
void test_punned_concurrent_updates(index_at& index, typename index_at::vector_key_t const start_key,
|
|
603
|
+
std::vector<std::vector<scalar_at>> const& vectors, std::size_t executor_threads) {
|
|
604
|
+
|
|
605
|
+
using index_t = index_at;
|
|
606
|
+
|
|
607
|
+
// Try batch requests, heavily oversubscribing the CPU cores
|
|
608
|
+
executor_default_t executor(executor_threads);
|
|
609
|
+
expect(index.try_reserve({vectors.size(), executor.size()}));
|
|
610
|
+
executor.fixed(vectors.size(), [&](std::size_t, std::size_t task) {
|
|
611
|
+
using add_result_t = typename index_t::add_result_t;
|
|
612
|
+
add_result_t result = index.add(start_key + task, vectors[task].data());
|
|
613
|
+
expect(result);
|
|
614
|
+
});
|
|
615
|
+
expect_eq(index.size(), vectors.size());
|
|
616
|
+
|
|
617
|
+
// Without key lookups we can't do much more
|
|
618
|
+
if (!index.config().enable_key_lookups)
|
|
619
|
+
return;
|
|
620
|
+
|
|
621
|
+
// Remove all the keys
|
|
622
|
+
executor.fixed(vectors.size(), [&](std::size_t, std::size_t task) {
|
|
623
|
+
using labeling_result_t = typename index_t::labeling_result_t;
|
|
624
|
+
labeling_result_t result = index.remove(start_key + task);
|
|
625
|
+
expect(result);
|
|
626
|
+
});
|
|
627
|
+
expect_eq(index.size(), 0);
|
|
628
|
+
|
|
629
|
+
// Add them back, which under the hood will trigger the `update`
|
|
630
|
+
executor.fixed(vectors.size(), [&](std::size_t, std::size_t task) {
|
|
631
|
+
using add_result_t = typename index_t::add_result_t;
|
|
632
|
+
add_result_t result = index.add(start_key + task, vectors[task].data());
|
|
633
|
+
expect(result);
|
|
634
|
+
});
|
|
635
|
+
expect_eq(index.size(), vectors.size());
|
|
636
|
+
}
|
|
637
|
+
|
|
638
|
+
/**
|
|
639
|
+
* Overloaded function to test cosine similarity index functionality using specific scalar, key, and slot types.
|
|
640
|
+
*
|
|
641
|
+
* This function initializes vectors and an index instance to test cosine similarity calculations and index operations.
|
|
642
|
+
* It involves creating vectors with random values, constructing an index, and verifying that the index operations
|
|
643
|
+
* like search work correctly with respect to the cosine similarity metric.
|
|
644
|
+
*
|
|
645
|
+
* @param collection_size Number of vectors to be included in the test.
|
|
646
|
+
* @param dimensions Number of dimensions each vector should have.
|
|
647
|
+
* @tparam scalar_at Data type of the elements in the vectors.
|
|
648
|
+
* @tparam key_at Data type used for the keys in the index.
|
|
649
|
+
* @tparam slot_at Data type used for slots in the index.
|
|
650
|
+
*/
|
|
169
651
|
template <typename scalar_at, typename key_at, typename slot_at> //
|
|
170
652
|
void test_cosine(std::size_t collection_size, std::size_t dimensions) {
|
|
171
653
|
|
|
@@ -205,35 +687,95 @@ void test_cosine(std::size_t collection_size, std::size_t dimensions) {
|
|
|
205
687
|
};
|
|
206
688
|
|
|
207
689
|
// Template:
|
|
208
|
-
|
|
209
|
-
std::printf("
|
|
690
|
+
auto run_templated = [&](std::size_t connectivity) {
|
|
691
|
+
std::printf("-- templates with connectivity %zu \n", connectivity);
|
|
210
692
|
metric_t metric{&vector_of_vectors, dimensions};
|
|
211
693
|
index_config_t config(connectivity);
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
694
|
+
|
|
695
|
+
// Toy example
|
|
696
|
+
if (vector_of_vectors.size() >= 3) {
|
|
697
|
+
aligned_wrapper_gt<index_typed_t> aligned_index(config);
|
|
698
|
+
test_minimal_three_vectors<false, index_typed_t>( //
|
|
699
|
+
*aligned_index.index, //
|
|
700
|
+
42, vector_of_vectors[0], //
|
|
701
|
+
43, vector_of_vectors[1], //
|
|
702
|
+
44, vector_of_vectors[2], metric);
|
|
703
|
+
}
|
|
704
|
+
// Larger collection
|
|
705
|
+
{
|
|
706
|
+
aligned_wrapper_gt<index_typed_t> aligned_index(config);
|
|
707
|
+
test_collection<false, index_typed_t>(*aligned_index.index, 42, vector_of_vectors, metric);
|
|
708
|
+
}
|
|
709
|
+
};
|
|
710
|
+
for (std::size_t connectivity : {3, 13, 50})
|
|
711
|
+
run_templated(connectivity);
|
|
215
712
|
|
|
216
713
|
// Type-punned:
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
714
|
+
auto run_punned = [&](bool multi, bool enable_key_lookups, std::size_t connectivity) {
|
|
715
|
+
std::printf("-- punned with connectivity %zu, multi: %s, lookups: %s \n", connectivity, multi ? "yes" : "no",
|
|
716
|
+
enable_key_lookups ? "yes" : "no");
|
|
717
|
+
using index_t = index_dense_gt<vector_key_t, slot_t>;
|
|
718
|
+
using index_result_t = typename index_t::state_result_t;
|
|
719
|
+
metric_punned_t metric(dimensions, metric_kind_t::cos_k, scalar_kind<scalar_at>());
|
|
720
|
+
index_dense_config_t config(connectivity);
|
|
721
|
+
config.multi = multi;
|
|
722
|
+
config.enable_key_lookups = enable_key_lookups;
|
|
723
|
+
|
|
724
|
+
// Toy example
|
|
725
|
+
if (vector_of_vectors.size() >= 3) {
|
|
726
|
+
index_result_t index_result = index_t::make(metric, config);
|
|
727
|
+
test_minimal_three_vectors<true>( //
|
|
728
|
+
index_result.index, //
|
|
729
|
+
42, vector_of_vectors[0], //
|
|
730
|
+
43, vector_of_vectors[1], //
|
|
731
|
+
44, vector_of_vectors[2]);
|
|
233
732
|
}
|
|
234
|
-
|
|
733
|
+
if (vector_of_vectors.size() >= 3 && enable_key_lookups) {
|
|
734
|
+
index_result_t index_result = index_t::make(metric, config);
|
|
735
|
+
test_punned_add_remove_vector( //
|
|
736
|
+
index_result.index, //
|
|
737
|
+
vector_of_vectors[0], //
|
|
738
|
+
vector_of_vectors[1]);
|
|
739
|
+
}
|
|
740
|
+
// Larger collection
|
|
741
|
+
{
|
|
742
|
+
index_result_t index_result = index_t::make(metric, config);
|
|
743
|
+
test_collection<true>(index_result.index, 42, vector_of_vectors);
|
|
744
|
+
}
|
|
745
|
+
|
|
746
|
+
// Try running benchmarks with a different number of threads
|
|
747
|
+
for (std::size_t threads : {
|
|
748
|
+
static_cast<std::size_t>(1),
|
|
749
|
+
// TODO: Multithreaded updates should word differently and may involve a search first
|
|
750
|
+
// static_cast<std::size_t>(2),
|
|
751
|
+
// static_cast<std::size_t>(std::thread::hardware_concurrency()),
|
|
752
|
+
// static_cast<std::size_t>(std::thread::hardware_concurrency() * 4),
|
|
753
|
+
// static_cast<std::size_t>(vector_of_vectors.size()),
|
|
754
|
+
}) {
|
|
755
|
+
index_result_t index_result = index_t::make(metric, config);
|
|
756
|
+
index_t& index = index_result.index;
|
|
757
|
+
test_punned_concurrent_updates(index, 42, vector_of_vectors, threads);
|
|
758
|
+
}
|
|
759
|
+
};
|
|
760
|
+
|
|
761
|
+
for (bool multi : {false, true})
|
|
762
|
+
for (bool enable_key_lookups : {true, false})
|
|
763
|
+
for (std::size_t connectivity : {3, 13, 50})
|
|
764
|
+
run_punned(multi, enable_key_lookups, connectivity);
|
|
235
765
|
}
|
|
236
766
|
|
|
767
|
+
/**
|
|
768
|
+
* Tests the functionality of the Tanimoto coefficient calculation and indexing.
|
|
769
|
+
*
|
|
770
|
+
* Initializes a dense index configured for Tanimoto similarity and fills it with randomly generated binary vectors.
|
|
771
|
+
* It performs concurrent additions of these vectors to the index to ensure thread safety and correctness of concurrent
|
|
772
|
+
* operations.
|
|
773
|
+
*
|
|
774
|
+
* @param dimensions Number of dimensions for the binary vectors.
|
|
775
|
+
* @param connectivity The degree of connectivity for the index configuration.
|
|
776
|
+
* @tparam key_at Data type used for the keys in the index.
|
|
777
|
+
* @tparam slot_at Data type used for slots in the index.
|
|
778
|
+
*/
|
|
237
779
|
template <typename key_at, typename slot_at> void test_tanimoto(std::size_t dimensions, std::size_t connectivity) {
|
|
238
780
|
|
|
239
781
|
using vector_key_t = key_at;
|
|
@@ -243,21 +785,96 @@ template <typename key_at, typename slot_at> void test_tanimoto(std::size_t dime
|
|
|
243
785
|
std::size_t words = divide_round_up<CHAR_BIT>(dimensions);
|
|
244
786
|
metric_punned_t metric(words, metric_kind_t::tanimoto_k, scalar_kind_t::b1x8_k);
|
|
245
787
|
index_config_t config(connectivity);
|
|
246
|
-
|
|
788
|
+
auto index_result = index_punned_t::make(metric, config);
|
|
789
|
+
expect(index_result);
|
|
790
|
+
index_punned_t& index = index_result.index;
|
|
247
791
|
|
|
248
792
|
executor_default_t executor;
|
|
249
793
|
std::size_t batch_size = 1000;
|
|
250
794
|
std::vector<b1x8_t> scalars(batch_size * index.scalar_words());
|
|
251
795
|
std::generate(scalars.begin(), scalars.end(), [] { return static_cast<b1x8_t>(std::rand()); });
|
|
252
796
|
|
|
253
|
-
index.
|
|
797
|
+
index.try_reserve({batch_size + index.size(), executor.size()});
|
|
254
798
|
executor.fixed(batch_size, [&](std::size_t thread, std::size_t task) {
|
|
255
799
|
index.add(task + 25000, scalars.data() + index.scalar_words() * task, thread);
|
|
256
800
|
});
|
|
257
801
|
}
|
|
258
802
|
|
|
803
|
+
/**
|
|
804
|
+
* Performs a unit test on the index with a ridiculous variety of configurations and parameters.
|
|
805
|
+
*
|
|
806
|
+
* This test aims to evaluate the index under extreme conditions, including small and potentially invalid parameters for
|
|
807
|
+
* connectivity, dimensions, and other configurations. It tests both the addition of vectors and their retrieval in
|
|
808
|
+
* these edge cases to ensure stability and error handling.
|
|
809
|
+
*
|
|
810
|
+
* @param dimensions Number of dimensions for the vectors.
|
|
811
|
+
* @param connectivity Index connectivity configuration.
|
|
812
|
+
* @param expansion_add Expansion factor during addition operations.
|
|
813
|
+
* @param expansion_search Expansion factor during search operations.
|
|
814
|
+
* @param count_vectors Number of vectors to add to the index.
|
|
815
|
+
* @param count_wanted Number of results wanted from search operations.
|
|
816
|
+
* @tparam key_at Data type used for the keys in the index.
|
|
817
|
+
* @tparam slot_at Data type used for slots in the index.
|
|
818
|
+
*/
|
|
819
|
+
template <typename key_at, typename slot_at>
|
|
820
|
+
void test_absurd(std::size_t dimensions, std::size_t connectivity, std::size_t expansion_add,
|
|
821
|
+
std::size_t expansion_search, std::size_t count_vectors, std::size_t count_wanted) {
|
|
822
|
+
|
|
823
|
+
using vector_key_t = key_at;
|
|
824
|
+
using slot_t = slot_at;
|
|
825
|
+
|
|
826
|
+
using index_punned_t = index_dense_gt<vector_key_t, slot_t>;
|
|
827
|
+
metric_punned_t metric(dimensions, metric_kind_t::cos_k, scalar_kind_t::f32_k);
|
|
828
|
+
index_dense_config_t config(connectivity, expansion_add, expansion_search);
|
|
829
|
+
auto index_result = index_punned_t::make(metric, config);
|
|
830
|
+
expect(index_result);
|
|
831
|
+
index_punned_t& index = index_result.index;
|
|
832
|
+
|
|
833
|
+
std::size_t count_max = (std::max)(count_vectors, count_wanted);
|
|
834
|
+
std::size_t needed_scalars = count_max * dimensions;
|
|
835
|
+
std::vector<f32_t> scalars(needed_scalars);
|
|
836
|
+
std::generate(scalars.begin(), scalars.end(), [] { return static_cast<f32_t>(std::rand()); });
|
|
837
|
+
|
|
838
|
+
expect(index.try_reserve({count_vectors, count_max}));
|
|
839
|
+
index.change_expansion_add(expansion_add);
|
|
840
|
+
index.change_expansion_search(expansion_search);
|
|
841
|
+
|
|
842
|
+
// Parallel construction
|
|
843
|
+
{
|
|
844
|
+
executor_default_t executor(count_vectors);
|
|
845
|
+
executor.fixed(count_vectors, [&](std::size_t thread, std::size_t task) {
|
|
846
|
+
expect(index.add(task + 25000, scalars.data() + index.scalar_words() * task, thread));
|
|
847
|
+
});
|
|
848
|
+
}
|
|
849
|
+
|
|
850
|
+
// Parallel search
|
|
851
|
+
{
|
|
852
|
+
executor_default_t executor(count_max);
|
|
853
|
+
executor.fixed(count_max, [&](std::size_t thread, std::size_t task) {
|
|
854
|
+
std::vector<vector_key_t> keys(count_wanted);
|
|
855
|
+
std::vector<f32_t> distances(count_wanted);
|
|
856
|
+
auto results = index.search(scalars.data() + index.scalar_words() * task, count_wanted, thread);
|
|
857
|
+
expect(results);
|
|
858
|
+
auto count_found = results.dump_to(keys.data(), distances.data());
|
|
859
|
+
expect(count_found <= count_wanted);
|
|
860
|
+
if (count_vectors && count_wanted)
|
|
861
|
+
expect(count_found > 0);
|
|
862
|
+
});
|
|
863
|
+
}
|
|
864
|
+
}
|
|
865
|
+
|
|
866
|
+
/**
|
|
867
|
+
* Tests the exact search functionality over a dataset of vectors, @b without constructing the index.
|
|
868
|
+
*
|
|
869
|
+
* Generates a dataset of vectors and performs exact search queries to verify that the search results are correct.
|
|
870
|
+
* This function mainly validates the basic functionality of exact searches using a given similarity metric.
|
|
871
|
+
*
|
|
872
|
+
* @param dataset_count Number of vectors in the dataset.
|
|
873
|
+
* @param queries_count Number of query vectors.
|
|
874
|
+
* @param wanted_count Number of top matches required from each query.
|
|
875
|
+
*/
|
|
259
876
|
void test_exact_search(std::size_t dataset_count, std::size_t queries_count, std::size_t wanted_count) {
|
|
260
|
-
std::size_t dimensions =
|
|
877
|
+
std::size_t dimensions = 32;
|
|
261
878
|
metric_punned_t metric(dimensions, metric_kind_t::cos_k);
|
|
262
879
|
|
|
263
880
|
std::random_device rd;
|
|
@@ -276,60 +893,259 @@ void test_exact_search(std::size_t dataset_count, std::size_t queries_count, std
|
|
|
276
893
|
assert(results.at(i)[0].offset == i); // Validate the top match
|
|
277
894
|
}
|
|
278
895
|
|
|
279
|
-
|
|
896
|
+
/**
|
|
897
|
+
* Tests handling of variable length sets (group of sorted unique integers), as opposed to @b equi-dimensional vectors.
|
|
898
|
+
*
|
|
899
|
+
* Adds a predefined number of vectors to an index and checks if the size of the index is updated correctly.
|
|
900
|
+
* It serves as a simple verification and showcase of how the same index can be used to handle strings and other types.
|
|
901
|
+
*
|
|
902
|
+
* @param index A reference to the index instance to be tested.
|
|
903
|
+
* @tparam index_at Type of the index being tested.
|
|
904
|
+
*/
|
|
905
|
+
template <typename key_at, typename slot_at>
|
|
906
|
+
void test_sets(std::size_t collection_size, std::size_t min_set_length, std::size_t max_set_length) {
|
|
280
907
|
|
|
281
|
-
|
|
282
|
-
using
|
|
283
|
-
|
|
908
|
+
/// Type of set elements, should support strong ordering
|
|
909
|
+
using set_member_t = std::uint32_t;
|
|
910
|
+
/// Jaccard is a fraction, so let's use a some float
|
|
911
|
+
using set_distance_t = double;
|
|
284
912
|
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
913
|
+
// Aliases for the index overload
|
|
914
|
+
using vector_key_t = key_at;
|
|
915
|
+
using slot_t = slot_at;
|
|
916
|
+
using index_t = index_gt<set_distance_t, vector_key_t, slot_t>;
|
|
288
917
|
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
918
|
+
// Let's allocate some data for indexing
|
|
919
|
+
using set_view_t = span_gt<set_member_t const>;
|
|
920
|
+
using sets_t = std::vector<std::vector<set_member_t>>;
|
|
921
|
+
sets_t sets(collection_size);
|
|
922
|
+
for (auto& set : sets) {
|
|
923
|
+
std::size_t set_size = min_set_length + std::rand() % (max_set_length - min_set_length);
|
|
924
|
+
set.resize(set_size);
|
|
925
|
+
std::size_t upper_bound = (max_set_length - min_set_length) * 3;
|
|
926
|
+
std::generate(set.begin(), set.end(), [=] { return static_cast<set_member_t>(std::rand() % upper_bound); });
|
|
927
|
+
std::sort(set.begin(), set.end());
|
|
928
|
+
}
|
|
293
929
|
|
|
294
|
-
|
|
295
|
-
|
|
930
|
+
// Wrap the data into a proxy object
|
|
931
|
+
struct metric_t {
|
|
932
|
+
using member_cref_t = typename index_t::member_cref_t;
|
|
933
|
+
using member_citerator_t = typename index_t::member_citerator_t;
|
|
296
934
|
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
935
|
+
sets_t const* sets_ptr = nullptr;
|
|
936
|
+
|
|
937
|
+
set_view_t set_at(std::size_t i) const noexcept { return {(*sets_ptr)[i].data(), (*sets_ptr)[i].size()}; }
|
|
938
|
+
set_distance_t between(set_view_t a, set_view_t b) const {
|
|
939
|
+
return metric_jaccard_gt<set_member_t, set_distance_t>{}(a.data(), b.data(), a.size(), b.size());
|
|
940
|
+
}
|
|
941
|
+
|
|
942
|
+
set_distance_t operator()(member_cref_t const& a, member_cref_t const& b) const {
|
|
943
|
+
return between(set_at(get_slot(b)), set_at(get_slot(a)));
|
|
944
|
+
}
|
|
945
|
+
set_distance_t operator()(set_view_t some_vector, member_cref_t const& member) const {
|
|
946
|
+
return between(some_vector, set_at(get_slot(member)));
|
|
947
|
+
}
|
|
948
|
+
set_distance_t operator()(member_citerator_t const& a, member_citerator_t const& b) const {
|
|
949
|
+
return between(set_at(get_slot(b)), set_at(get_slot(a)));
|
|
950
|
+
}
|
|
951
|
+
set_distance_t operator()(set_view_t some_vector, member_citerator_t const& member) const {
|
|
952
|
+
return between(some_vector, set_at(get_slot(member)));
|
|
953
|
+
}
|
|
954
|
+
};
|
|
955
|
+
|
|
956
|
+
// Perform indexing
|
|
957
|
+
aligned_wrapper_gt<index_t> aligned_index;
|
|
958
|
+
aligned_index.index->reserve(sets.size());
|
|
959
|
+
for (std::size_t i = 0; i < sets.size(); i++)
|
|
960
|
+
aligned_index.index->add(i, set_view_t{sets[i].data(), sets[i].size()}, metric_t{&sets});
|
|
961
|
+
expect(aligned_index.index->size() == sets.size());
|
|
962
|
+
|
|
963
|
+
// Perform the search queries
|
|
964
|
+
for (std::size_t i = 0; i < sets.size(); i++) {
|
|
965
|
+
auto results = aligned_index.index->search(set_view_t{sets[i].data(), sets[i].size()}, 5, metric_t{&sets});
|
|
966
|
+
expect(results.size() > 0);
|
|
306
967
|
}
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
968
|
+
}
|
|
969
|
+
|
|
970
|
+
/**
|
|
971
|
+
* Tests similarity search over strings using Levenshtein distances
|
|
972
|
+
* implementation from StringZilla.
|
|
973
|
+
*
|
|
974
|
+
* Adds a predefined number of long strings, comparing them.
|
|
975
|
+
*
|
|
976
|
+
* @param index A reference to the index instance to be tested.
|
|
977
|
+
* @tparam index_at Type of the index being tested.
|
|
978
|
+
*/
|
|
979
|
+
template <typename key_at, typename slot_at> void test_strings() {
|
|
980
|
+
|
|
981
|
+
namespace sz = ashvardanian::stringzilla;
|
|
982
|
+
|
|
983
|
+
/// Levenshtein distance is an integer
|
|
984
|
+
using levenshtein_distance_t = std::int64_t;
|
|
985
|
+
|
|
986
|
+
// Aliases for the index overload
|
|
987
|
+
using vector_key_t = key_at;
|
|
988
|
+
using slot_t = slot_at;
|
|
989
|
+
using index_t = index_gt<levenshtein_distance_t, vector_key_t, slot_t>;
|
|
990
|
+
|
|
991
|
+
std::string_view str0 = "ACGTACGTACGTACGTACGTACGTACGTACGTACGT";
|
|
992
|
+
std::string_view str1 = "ACG_ACTC_TAC-TACGTA_GTACACG_ACGT";
|
|
993
|
+
std::string_view str2 = "A_GTACTACGTA-GTAC_TACGTACGTA-GTAGT";
|
|
994
|
+
std::string_view str3 = "GTACGTAGT-ACGTACGACGTACGTACG-TACGTAC";
|
|
995
|
+
std::vector<std::string_view> strings({str0, str1, str2, str3});
|
|
996
|
+
|
|
997
|
+
// Wrap the data into a proxy object
|
|
998
|
+
struct metric_t {
|
|
999
|
+
using member_cref_t = typename index_t::member_cref_t;
|
|
1000
|
+
using member_citerator_t = typename index_t::member_citerator_t;
|
|
1001
|
+
|
|
1002
|
+
std::vector<std::string_view> const* strings_ptr = nullptr;
|
|
1003
|
+
|
|
1004
|
+
std::string_view str_at(std::size_t i) const noexcept { return (*strings_ptr)[i]; }
|
|
1005
|
+
levenshtein_distance_t between(std::string_view a, std::string_view b) const {
|
|
1006
|
+
sz::string_view asz{a.data(), a.size()};
|
|
1007
|
+
sz::string_view bsz{b.data(), b.size()};
|
|
1008
|
+
return sz::edit_distance<char const>(asz, bsz);
|
|
1009
|
+
}
|
|
1010
|
+
|
|
1011
|
+
levenshtein_distance_t operator()(member_cref_t const& a, member_cref_t const& b) const {
|
|
1012
|
+
return between(str_at(get_slot(b)), str_at(get_slot(a)));
|
|
1013
|
+
}
|
|
1014
|
+
levenshtein_distance_t operator()(std::string_view some_vector, member_cref_t const& member) const {
|
|
1015
|
+
return between(some_vector, str_at(get_slot(member)));
|
|
1016
|
+
}
|
|
1017
|
+
levenshtein_distance_t operator()(member_citerator_t const& a, member_citerator_t const& b) const {
|
|
1018
|
+
return between(str_at(get_slot(*b)), str_at(get_slot(*a)));
|
|
1019
|
+
}
|
|
1020
|
+
levenshtein_distance_t operator()(std::string_view some_vector, member_citerator_t const& member) const {
|
|
1021
|
+
return between(some_vector, str_at(get_slot(*member)));
|
|
1022
|
+
}
|
|
1023
|
+
};
|
|
1024
|
+
|
|
1025
|
+
// Perform indexing
|
|
1026
|
+
aligned_wrapper_gt<index_t> aligned_index;
|
|
1027
|
+
aligned_index.index->reserve(strings.size());
|
|
1028
|
+
for (std::size_t i = 0; i < strings.size(); i++)
|
|
1029
|
+
aligned_index.index->add(i, strings[i], metric_t{&strings});
|
|
1030
|
+
expect(aligned_index.index->size() == strings.size());
|
|
1031
|
+
|
|
1032
|
+
// Perform the search queries
|
|
1033
|
+
for (std::size_t i = 0; i < strings.size(); i++) {
|
|
1034
|
+
auto results = aligned_index.index->search(strings[i], 5, metric_t{&strings});
|
|
1035
|
+
expect(results.size() > 0);
|
|
312
1036
|
}
|
|
313
1037
|
}
|
|
314
1038
|
|
|
1039
|
+
/**
|
|
1040
|
+
* @brief Tests replacing and updating entries in index_dense_gt to ensure consistency after modifications.
|
|
1041
|
+
*/
|
|
1042
|
+
template <typename key_at, typename slot_at> void test_replacing_update() {
|
|
1043
|
+
|
|
1044
|
+
using vector_key_t = key_at;
|
|
1045
|
+
using slot_t = slot_at;
|
|
1046
|
+
|
|
1047
|
+
using index_punned_t = index_dense_gt<vector_key_t, slot_t>;
|
|
1048
|
+
metric_punned_t metric(1, metric_kind_t::l2sq_k, scalar_kind_t::f32_k);
|
|
1049
|
+
auto index_result = index_punned_t::make(metric);
|
|
1050
|
+
expect(index_result);
|
|
1051
|
+
index_punned_t& index = index_result.index;
|
|
1052
|
+
|
|
1053
|
+
// Reserve space for 3 entries
|
|
1054
|
+
index.try_reserve(3);
|
|
1055
|
+
auto as_ptr = [](float v) {
|
|
1056
|
+
static float value;
|
|
1057
|
+
value = v;
|
|
1058
|
+
return &value;
|
|
1059
|
+
};
|
|
1060
|
+
|
|
1061
|
+
// Add 3 entries
|
|
1062
|
+
index.add(42, as_ptr(1.1f));
|
|
1063
|
+
index.add(43, as_ptr(2.1f));
|
|
1064
|
+
index.add(44, as_ptr(3.1f));
|
|
1065
|
+
expect_eq(index.size(), 3);
|
|
1066
|
+
|
|
1067
|
+
// Assert initial state
|
|
1068
|
+
auto initial_search = index.search(as_ptr(1.0f), 3);
|
|
1069
|
+
expect_eq(initial_search.size(), 3);
|
|
1070
|
+
expect_eq(initial_search[0].member.key, 42);
|
|
1071
|
+
expect_eq(initial_search[1].member.key, 43);
|
|
1072
|
+
expect_eq(initial_search[2].member.key, 44);
|
|
1073
|
+
|
|
1074
|
+
// Replace the second entry
|
|
1075
|
+
index.remove(43);
|
|
1076
|
+
index.add(43, as_ptr(2.2f));
|
|
1077
|
+
expect_eq(index.size(), 3);
|
|
1078
|
+
|
|
1079
|
+
// Assert state after replacing second entry
|
|
1080
|
+
auto post_second_replacement = index.search(as_ptr(1.0f), 3);
|
|
1081
|
+
expect_eq(post_second_replacement.size(), 3);
|
|
1082
|
+
expect_eq(post_second_replacement[0].member.key, 42);
|
|
1083
|
+
expect_eq(post_second_replacement[1].member.key, 43);
|
|
1084
|
+
expect_eq(post_second_replacement[2].member.key, 44);
|
|
1085
|
+
|
|
1086
|
+
// Replace the first entry
|
|
1087
|
+
index.remove(42);
|
|
1088
|
+
index.add(42, as_ptr(1.2f));
|
|
1089
|
+
expect_eq(index.size(), 3);
|
|
1090
|
+
|
|
1091
|
+
// Assert state after replacing first entry
|
|
1092
|
+
auto final_search = index.search(as_ptr(1.0f), 3, 0);
|
|
1093
|
+
expect_eq(final_search.size(), 3);
|
|
1094
|
+
expect_eq(final_search[0].member.key, 42);
|
|
1095
|
+
expect_eq(final_search[1].member.key, 43);
|
|
1096
|
+
expect_eq(final_search[2].member.key, 44);
|
|
1097
|
+
}
|
|
1098
|
+
|
|
315
1099
|
int main(int, char**) {
|
|
1100
|
+
test_uint40();
|
|
1101
|
+
test_cosine<float, std::int64_t, uint40_t>(10, 10);
|
|
316
1102
|
|
|
1103
|
+
// Exact search without constructing indexes.
|
|
1104
|
+
// Great for validating the distance functions.
|
|
1105
|
+
std::printf("Testing exact search\n");
|
|
317
1106
|
for (std::size_t dataset_count : {10, 100})
|
|
318
1107
|
for (std::size_t queries_count : {1, 10})
|
|
319
1108
|
for (std::size_t wanted_count : {1, 5})
|
|
320
1109
|
test_exact_search(dataset_count, queries_count, wanted_count);
|
|
321
1110
|
|
|
1111
|
+
// Make sure the initializers and the algorithms can work with inadequately small values.
|
|
1112
|
+
// Be warned - this combinatorial explosion of tests produces close to __500'000__ tests!
|
|
1113
|
+
std::printf("Testing allowed, but absurd index configs\n");
|
|
1114
|
+
for (std::size_t connectivity : {2, 3}) // ! Zero maps to default, one degenerates
|
|
1115
|
+
for (std::size_t dimensions : {1, 3}) // ! Zero will raise
|
|
1116
|
+
for (std::size_t expansion : {0, 1, 3})
|
|
1117
|
+
for (std::size_t count_vectors : {0, 1, 2, 17})
|
|
1118
|
+
for (std::size_t count_wanted : {0, 1, 3, 19}) {
|
|
1119
|
+
test_absurd<std::int64_t, slot32_t>(dimensions, connectivity, expansion, expansion,
|
|
1120
|
+
count_vectors, count_wanted);
|
|
1121
|
+
test_absurd<uint40_t, uint40_t>(dimensions, connectivity, expansion, expansion, count_vectors,
|
|
1122
|
+
count_wanted);
|
|
1123
|
+
}
|
|
1124
|
+
|
|
1125
|
+
// TODO: Test absurd configs that are banned
|
|
1126
|
+
// for (metric_kind_t metric_kind : {metric_kind_t::cos_k, metric_kind_t::unknown_k, metric_kind_t::haversine_k}) {}
|
|
1127
|
+
|
|
1128
|
+
// Test with cosine metric - the most common use case
|
|
1129
|
+
std::printf("Testing common cases\n");
|
|
322
1130
|
for (std::size_t collection_size : {10, 500})
|
|
323
1131
|
for (std::size_t dimensions : {97, 256}) {
|
|
324
|
-
std::printf("Indexing %zu vectors with cos: <float, std::int64_t,
|
|
325
|
-
test_cosine<float, std::int64_t,
|
|
326
|
-
std::printf("Indexing %zu vectors with cos: <float, std::int64_t, uint40_t> \n", collection_size);
|
|
1132
|
+
std::printf("- Indexing %zu vectors with cos: <float, std::int64_t, slot32_t> \n", collection_size);
|
|
1133
|
+
test_cosine<float, std::int64_t, slot32_t>(collection_size, dimensions);
|
|
1134
|
+
std::printf("- Indexing %zu vectors with cos: <float, std::int64_t, uint40_t> \n", collection_size);
|
|
327
1135
|
test_cosine<float, std::int64_t, uint40_t>(collection_size, dimensions);
|
|
328
1136
|
}
|
|
329
1137
|
|
|
1138
|
+
// Test with binary vectors
|
|
1139
|
+
std::printf("Testing binary vectors\n");
|
|
330
1140
|
for (std::size_t connectivity : {3, 13, 50})
|
|
331
1141
|
for (std::size_t dimensions : {97, 256})
|
|
332
|
-
test_tanimoto<std::int64_t,
|
|
1142
|
+
test_tanimoto<std::int64_t, slot32_t>(dimensions, connectivity);
|
|
1143
|
+
|
|
1144
|
+
// Beyond dense equi-dimensional vectors - integer sets
|
|
1145
|
+
std::printf("Testing sparse vectors, strings, and sets\n");
|
|
1146
|
+
for (std::size_t set_size : {1, 100, 1000})
|
|
1147
|
+
test_sets<std::int64_t, slot32_t>(set_size, 20, 30);
|
|
1148
|
+
test_strings<std::int64_t, slot32_t>();
|
|
333
1149
|
|
|
334
1150
|
return 0;
|
|
335
1151
|
}
|