usearch 0.19.0 → 0.19.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Cargo.toml +1 -1
- package/README.md +11 -34
- package/c/lib.cpp +71 -51
- package/c/test.c +182 -0
- package/c/usearch.h +11 -5
- package/conanfile.py +1 -1
- package/cpp/bench.cpp +4 -4
- package/docs/compilation.md +1 -0
- package/include/usearch/index.hpp +22 -5
- package/include/usearch/index_punned_dense.hpp +5 -3
- package/include/usearch/index_punned_helpers.hpp +2 -0
- package/objc/USearchObjective.mm +4 -0
- package/objc/include/USearchObjective.h +5 -0
- package/package.json +2 -2
- package/python/lib.cpp +125 -166
- package/python/scripts/test.py +51 -11
- package/python/usearch/index.py +152 -65
- package/python/usearch/io.py +2 -2
- package/setup.py +56 -59
- package/swift/Test.swift +13 -9
package/Cargo.toml
CHANGED
package/README.md
CHANGED
|
@@ -387,6 +387,10 @@ index = Index(ndim=ndim, metric=CompiledMetric(
|
|
|
387
387
|
))
|
|
388
388
|
```
|
|
389
389
|
|
|
390
|
+
```sh
|
|
391
|
+
conda install -c conda-forge cppyy
|
|
392
|
+
```
|
|
393
|
+
|
|
390
394
|
##### [PeachPy][peachpy]
|
|
391
395
|
|
|
392
396
|
We have covered JIT-ing Python with Numba and C++ with Cppyy and Cling.
|
|
@@ -641,6 +645,7 @@ https://github.com/unum-cloud/usearch
|
|
|
641
645
|
let index = Index.l2sq(dimensions: 3, connectivity: 8)
|
|
642
646
|
let vectorA: [Float32] = [0.3, 0.5, 1.2]
|
|
643
647
|
let vectorB: [Float32] = [0.4, 0.2, 1.2]
|
|
648
|
+
index.reserve(2)
|
|
644
649
|
index.add(label: 42, vector: vectorA[...])
|
|
645
650
|
index.add(label: 43, vector: vectorB[...])
|
|
646
651
|
|
|
@@ -717,22 +722,16 @@ server.run()
|
|
|
717
722
|
|
|
718
723
|
We have pre-processed some commonly used datasets, cleaning the images, producing the vectors, and pre-building the index.
|
|
719
724
|
|
|
720
|
-
| Dataset
|
|
721
|
-
|
|
|
722
|
-
| [Unsplash 25K][unsplash-25k-origin]
|
|
723
|
-
| [
|
|
724
|
-
| [LAION 400M][laion-400m-origin] | - | 400 M | [HF][laion-400m-hf] |
|
|
725
|
-
| [LAION 5B][laion-5b-origin] | - | 5 B | [HF][laion-5b-hf] |
|
|
725
|
+
| Dataset | Size | Images | Preprocessed |
|
|
726
|
+
| :----------------------------------- | ---: | -----: | --------------------: |
|
|
727
|
+
| [Unsplash 25K][unsplash-25k-origin] | - | 25 K | [HF][unsplash-25k-hf] |
|
|
728
|
+
| [Createve Captions 3M][cc-3m-origin] | - | 3 M | [HF][cc-3m-hf] |
|
|
726
729
|
|
|
727
730
|
|
|
728
731
|
[unsplash-25k-origin]: https://unum.cloud
|
|
729
|
-
[
|
|
730
|
-
[laion-400m-origin]: https://unum.cloud
|
|
731
|
-
[laion-5b-origin]: https://unum.cloud
|
|
732
|
+
[cc-3m-origin]: https://unum.cloud
|
|
732
733
|
[unsplash-25k-hf]: https://unum.cloud
|
|
733
|
-
[
|
|
734
|
-
[laion-400m-hf]: https://unum.cloud
|
|
735
|
-
[laion-5b-hf]: https://unum.cloud
|
|
734
|
+
[cc-3m-hf]: https://unum.cloud
|
|
736
735
|
|
|
737
736
|
### USearch + RDKit = Molecular Search
|
|
738
737
|
|
|
@@ -763,31 +762,9 @@ index.add(labels, fingerprints)
|
|
|
763
762
|
matches = index.search(fingerprints, 10)
|
|
764
763
|
```
|
|
765
764
|
|
|
766
|
-
RDKit [provides][rdkit-fingerprints] following fingerprinting techniques:
|
|
767
|
-
|
|
768
|
-
- Atom-Pair,
|
|
769
|
-
- Topological Torsion,
|
|
770
|
-
- Morgan,
|
|
771
|
-
- Layered Fingerprints.
|
|
772
|
-
|
|
773
|
-
We have preprocessed some of the most commonly used datasets, and made it available for free on the HuggingFace portal, together with visual interface.
|
|
774
|
-
|
|
775
|
-
| Dataset | Size | Molecules | Preprocessed |
|
|
776
|
-
| :------------------------ | -------: | ----------: | ---------------: |
|
|
777
|
-
| [PubChem][pubchem-origin] | 8 GB | 115'034'339 | [HF][pubchem-hf] |
|
|
778
|
-
| [GDB 13][gdb13-origin] | 2.3 GB | 977'468'301 | [HF][gdb13-hf] |
|
|
779
|
-
| [REAL][real-origin] | > 100 GB | 6 B | [HF][real-hf] |
|
|
780
|
-
|
|
781
765
|
[smiles]: https://en.wikipedia.org/wiki/Simplified_molecular-input_line-entry_system
|
|
782
766
|
[rdkit-fingerprints]: https://www.rdkit.org/docs/RDKit_Book.html#additional-information-about-the-fingerprints
|
|
783
767
|
|
|
784
|
-
[gdb13-origin]: https://zenodo.org/record/5172018/files/gdb13.tgz?download=1
|
|
785
|
-
[pubchem-origin]: ftp://ftp.ncbi.nlm.nih.gov/pubchem/Compound/Extras/CID-SMILES.gz
|
|
786
|
-
[real-origin]: https://enamine.net/compound-collections/real-compounds/real-database
|
|
787
|
-
[pubchem-hf]: https://unum.cloud
|
|
788
|
-
[gdb13-hf]: https://unum.cloud
|
|
789
|
-
[real-hf]: https://unum.cloud
|
|
790
|
-
|
|
791
768
|
## TODO
|
|
792
769
|
|
|
793
770
|
- JavaScript: Allow calling from "worker threads".
|
package/c/lib.cpp
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
|
-
#include
|
|
1
|
+
#include <cassert>
|
|
2
|
+
|
|
3
|
+
#include <usearch/index_punned_dense.hpp>
|
|
2
4
|
|
|
3
5
|
extern "C" {
|
|
4
6
|
#include "usearch.h"
|
|
@@ -12,10 +14,11 @@ using distance_t = usearch_distance_t;
|
|
|
12
14
|
using index_t = index_punned_dense_gt<label_t>;
|
|
13
15
|
using add_result_t = index_t::add_result_t;
|
|
14
16
|
using search_result_t = index_t::search_result_t;
|
|
15
|
-
using
|
|
17
|
+
using serialization_result_t = index_t::serialization_result_t;
|
|
16
18
|
|
|
17
19
|
using span_t = span_gt<float>;
|
|
18
20
|
|
|
21
|
+
// helper functions that are not part of the C ABI
|
|
19
22
|
metric_kind_t to_native_metric(usearch_metric_kind_t kind) {
|
|
20
23
|
switch (kind) {
|
|
21
24
|
case usearch_metric_ip_k: return metric_kind_t::ip_k;
|
|
@@ -37,30 +40,76 @@ scalar_kind_t to_native_scalar(usearch_scalar_kind_t kind) {
|
|
|
37
40
|
case usearch_scalar_f64_k: return scalar_kind_t::f64_k;
|
|
38
41
|
case usearch_scalar_f16_k: return scalar_kind_t::f16_k;
|
|
39
42
|
case usearch_scalar_f8_k: return scalar_kind_t::f8_k;
|
|
40
|
-
case usearch_scalar_b1_k: return scalar_kind_t::
|
|
43
|
+
case usearch_scalar_b1_k: return scalar_kind_t::b1x8_k;
|
|
41
44
|
default: return scalar_kind_t::unknown_k;
|
|
42
45
|
}
|
|
43
46
|
}
|
|
44
47
|
|
|
45
|
-
|
|
48
|
+
add_result_t add_(index_t* index, usearch_label_t label, void const* vector, scalar_kind_t kind) {
|
|
49
|
+
switch (kind) {
|
|
50
|
+
case scalar_kind_t::f32_k: return index->add(label, (f32_t const*)vector);
|
|
51
|
+
case scalar_kind_t::f64_k: return index->add(label, (f64_t const*)vector);
|
|
52
|
+
case scalar_kind_t::f16_k: return index->add(label, (f16_t const*)vector);
|
|
53
|
+
case scalar_kind_t::f8_k: return index->add(label, (f8_bits_t const*)vector);
|
|
54
|
+
case scalar_kind_t::b1x8_k: return index->add(label, (b1x8_t const*)vector);
|
|
55
|
+
default: return add_result_t{}.failed("Unknown scalar kind!");
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
bool get_(index_t* index, label_t label, void* vector, scalar_kind_t kind) {
|
|
60
|
+
switch (kind) {
|
|
61
|
+
case scalar_kind_t::f32_k: return index->get(label, (f32_t*)vector);
|
|
62
|
+
case scalar_kind_t::f64_k: return index->get(label, (f64_t*)vector);
|
|
63
|
+
case scalar_kind_t::f16_k: return index->get(label, (f16_t*)vector);
|
|
64
|
+
case scalar_kind_t::f8_k: return index->get(label, (f8_bits_t*)vector);
|
|
65
|
+
case scalar_kind_t::b1x8_k: return index->get(label, (b1x8_t*)vector);
|
|
66
|
+
default: return index->empty_search_result().failed("Unknown scalar kind!");
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
search_result_t search_(index_t* index, void const* vector, scalar_kind_t kind, size_t n) {
|
|
71
|
+
switch (kind) {
|
|
72
|
+
case scalar_kind_t::f32_k: return index->search((f32_t const*)vector, n);
|
|
73
|
+
case scalar_kind_t::f64_k: return index->search((f64_t const*)vector, n);
|
|
74
|
+
case scalar_kind_t::f16_k: return index->search((f16_t const*)vector, n);
|
|
75
|
+
case scalar_kind_t::f8_k: return index->search((f8_bits_t const*)vector, n);
|
|
76
|
+
case scalar_kind_t::b1x8_k: return index->search((b1x8_t const*)vector, n);
|
|
77
|
+
default: return index->empty_search_result().failed("Unknown scalar kind!");
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
index_punned_dense_metric_t udf(metric_kind_t kind, usearch_metric_t raw_ptr) {
|
|
82
|
+
index_punned_dense_metric_t result;
|
|
83
|
+
result.kind_ = kind;
|
|
84
|
+
result.func_ = [raw_ptr](punned_vector_view_t a, punned_vector_view_t b) -> distance_t {
|
|
85
|
+
return raw_ptr((void const*)a.data(), (void const*)b.data());
|
|
86
|
+
};
|
|
87
|
+
return result;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
extern "C" {
|
|
46
91
|
|
|
47
|
-
|
|
48
|
-
config.expansion_add = static_cast<std::size_t>(expansion_add);
|
|
49
|
-
config.expansion_search = static_cast<std::size_t>(expansion_search);
|
|
50
|
-
config.connectivity = static_cast<std::size_t>(connectivity);
|
|
51
|
-
config.max_elements = static_cast<std::size_t>(capacity);
|
|
52
|
-
config.max_threads_add = std::thread::hardware_concurrency();
|
|
53
|
-
config.max_threads_search = std::thread::hardware_concurrency();
|
|
92
|
+
usearch_index_t usearch_init(usearch_init_options_t* options, usearch_error_t* error) {
|
|
54
93
|
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
94
|
+
assert(options && error);
|
|
95
|
+
|
|
96
|
+
index_config_t config;
|
|
97
|
+
config.connectivity = options->connectivity;
|
|
98
|
+
index_t index = //
|
|
99
|
+
options->metric ? //
|
|
100
|
+
index_t::make( //
|
|
101
|
+
options->dimensions, udf(to_native_metric(options->metric_kind), options->metric), config,
|
|
102
|
+
to_native_scalar(options->quantization), options->expansion_add, options->expansion_search)
|
|
103
|
+
: //
|
|
104
|
+
index_t::make( //
|
|
105
|
+
options->dimensions, to_native_metric(options->metric_kind), config,
|
|
106
|
+
to_native_scalar(options->quantization), options->expansion_add, options->expansion_search);
|
|
58
107
|
|
|
59
108
|
index_t* result_ptr = new index_t(std::move(index));
|
|
60
109
|
return result_ptr;
|
|
61
110
|
}
|
|
62
111
|
|
|
63
|
-
void usearch_free(
|
|
112
|
+
void usearch_free(usearch_index_t index, usearch_error_t*) { delete reinterpret_cast<index_t*>(index); }
|
|
64
113
|
|
|
65
114
|
void usearch_save(usearch_index_t index, char const* path, usearch_error_t* error) {
|
|
66
115
|
serialization_result_t result = reinterpret_cast<index_t*>(index)->save(path);
|
|
@@ -101,17 +150,6 @@ void usearch_reserve(usearch_index_t index, size_t capacity, usearch_error_t*) {
|
|
|
101
150
|
reinterpret_cast<index_t*>(index)->reserve(capacity);
|
|
102
151
|
}
|
|
103
152
|
|
|
104
|
-
add_result_t add_(index_t* index, usearch_label_t label, void const* vector, scalar_kind_t kind) {
|
|
105
|
-
switch (kind) {
|
|
106
|
-
case scalar_kind_t::f32_k: return index->add(label, (f32_t const*)vector);
|
|
107
|
-
case scalar_kind_t::f64_k: return index->add(label, (f64_t const*)vector);
|
|
108
|
-
case scalar_kind_t::f16_k: return index->add(label, (f16_t const*)vector);
|
|
109
|
-
case scalar_kind_t::f8_k: return index->add(label, (f8_bits_t const*)vector);
|
|
110
|
-
case scalar_kind_t::b1x8_k: return index->add(label, (b1x8_t const*)vector);
|
|
111
|
-
default: return add_result_t{}.failed("Unknown scalar kind!");
|
|
112
|
-
}
|
|
113
|
-
}
|
|
114
|
-
|
|
115
153
|
void usearch_add( //
|
|
116
154
|
usearch_index_t index, usearch_label_t label, void const* vector, usearch_scalar_kind_t kind, //
|
|
117
155
|
usearch_error_t* error) {
|
|
@@ -121,18 +159,7 @@ void usearch_add(
|
|
|
121
159
|
}
|
|
122
160
|
|
|
123
161
|
bool usearch_contains(usearch_index_t index, usearch_label_t label, usearch_error_t*) {
|
|
124
|
-
reinterpret_cast<index_t*>(index)->contains(label);
|
|
125
|
-
}
|
|
126
|
-
|
|
127
|
-
search_result_t search_(index_t* index, void const* vector, scalar_kind_t kind, size_t n) {
|
|
128
|
-
switch (kind) {
|
|
129
|
-
case scalar_kind_t::f32_k: return index->search((f32_t const*)vector, n);
|
|
130
|
-
case scalar_kind_t::f64_k: return index->search((f64_t const*)vector, n);
|
|
131
|
-
case scalar_kind_t::f16_k: return index->search((f16_t const*)vector, n);
|
|
132
|
-
case scalar_kind_t::f8_k: return index->search((f8_bits_t const*)vector, n);
|
|
133
|
-
case scalar_kind_t::b1x8_k: return index->search((b1x8_t const*)vector, n);
|
|
134
|
-
default: return search_result_t{}.failed("Unknown scalar kind!");
|
|
135
|
-
}
|
|
162
|
+
return reinterpret_cast<index_t*>(index)->contains(label);
|
|
136
163
|
}
|
|
137
164
|
|
|
138
165
|
size_t usearch_search( //
|
|
@@ -147,21 +174,14 @@ size_t usearch_search(
|
|
|
147
174
|
return result.dump_to(found_labels, found_distances);
|
|
148
175
|
}
|
|
149
176
|
|
|
150
|
-
bool get_(index_t* index, void const* vector, scalar_kind_t kind) {
|
|
151
|
-
switch (kind) {
|
|
152
|
-
case scalar_kind_t::f32_k: return index->get((f32_t const*)vector, n);
|
|
153
|
-
case scalar_kind_t::f64_k: return index->get((f64_t const*)vector, n);
|
|
154
|
-
case scalar_kind_t::f16_k: return index->get((f16_t const*)vector, n);
|
|
155
|
-
case scalar_kind_t::f8_k: return index->get((f8_bits_t const*)vector, n);
|
|
156
|
-
case scalar_kind_t::b1x8_k: return index->get((b1x8_t const*)vector, n);
|
|
157
|
-
default: return search_result_t{}.failed("Unknown scalar kind!");
|
|
158
|
-
}
|
|
159
|
-
}
|
|
160
|
-
|
|
161
177
|
bool usearch_get( //
|
|
162
178
|
usearch_index_t index, usearch_label_t label, //
|
|
163
179
|
void* vector, usearch_scalar_kind_t kind, usearch_error_t*) {
|
|
164
|
-
return
|
|
180
|
+
return get_(reinterpret_cast<index_t*>(index), label, vector, to_native_scalar(kind));
|
|
165
181
|
}
|
|
166
182
|
|
|
167
|
-
void usearch_remove(usearch_index_t, usearch_label_t, usearch_error_t*) {
|
|
183
|
+
void usearch_remove(usearch_index_t, usearch_label_t, usearch_error_t* error) {
|
|
184
|
+
if (error != nullptr)
|
|
185
|
+
*error = "USearch does not support removal of elements yet.";
|
|
186
|
+
}
|
|
187
|
+
}
|
package/c/test.c
ADDED
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
#include <assert.h>
|
|
2
|
+
#include <errno.h>
|
|
3
|
+
#include <stdio.h>
|
|
4
|
+
#include <stdlib.h>
|
|
5
|
+
#include <string.h>
|
|
6
|
+
#include <sys/stat.h>
|
|
7
|
+
#include <time.h>
|
|
8
|
+
|
|
9
|
+
#include "usearch.h"
|
|
10
|
+
|
|
11
|
+
float* read_fvecs_file(const char* filename, size_t* num_vectors, size_t* vector_dim) {
|
|
12
|
+
struct stat st;
|
|
13
|
+
FILE* file;
|
|
14
|
+
size_t cnt;
|
|
15
|
+
float skip;
|
|
16
|
+
|
|
17
|
+
;
|
|
18
|
+
if (stat(filename, &st) == -1) {
|
|
19
|
+
printf("Failed to read file: %s err: %d: %s\n", filename, errno, strerror(errno));
|
|
20
|
+
return NULL;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
file = fopen(filename, "rb");
|
|
24
|
+
if (file == NULL) {
|
|
25
|
+
printf("Failed to open file: %s\n", filename);
|
|
26
|
+
return NULL;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
// Read the number of vectors and vector dimension from the file
|
|
30
|
+
cnt = fread(vector_dim, sizeof(int), 1, file);
|
|
31
|
+
assert(cnt == 1);
|
|
32
|
+
if (st.st_size % ((*vector_dim + 1) * sizeof(float)) != 0) {
|
|
33
|
+
printf("File does not contain a whole number of vectors.\n Vector dim: %ld, file size: %ld\n", *vector_dim,
|
|
34
|
+
st.st_size);
|
|
35
|
+
fclose(file);
|
|
36
|
+
return NULL;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
*num_vectors = st.st_size / ((*vector_dim + 1) * sizeof(float));
|
|
40
|
+
|
|
41
|
+
// Allocate memory for the array
|
|
42
|
+
float* data = (float*)malloc(*num_vectors * *vector_dim * sizeof(float));
|
|
43
|
+
if (data == NULL) {
|
|
44
|
+
printf("Memory allocation failed.\n");
|
|
45
|
+
fclose(file);
|
|
46
|
+
return NULL;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
// Read the data from the file into the array
|
|
50
|
+
for (size_t i = 0; i < *num_vectors; i++) {
|
|
51
|
+
cnt = fread(data + i * *vector_dim, sizeof(float), *vector_dim, file);
|
|
52
|
+
assert(cnt == *vector_dim);
|
|
53
|
+
if (i == *num_vectors - 1) {
|
|
54
|
+
break;
|
|
55
|
+
}
|
|
56
|
+
cnt = fread(&skip, sizeof(float), 1, file);
|
|
57
|
+
assert(cnt == 1);
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
fclose(file);
|
|
61
|
+
return data;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
// log every this many iterations in test loops
|
|
65
|
+
const int LOG_GRANULARITY = 10000;
|
|
66
|
+
|
|
67
|
+
void test_index(float* data, size_t num_vectors, size_t vector_dim) {
|
|
68
|
+
usearch_init_options_t opts;
|
|
69
|
+
usearch_index_t idx;
|
|
70
|
+
usearch_error_t error = NULL;
|
|
71
|
+
clock_t start;
|
|
72
|
+
opts.connectivity = 2; // 32 in faiss
|
|
73
|
+
opts.dimensions = vector_dim;
|
|
74
|
+
opts.expansion_add = 40; // 40 in faiss
|
|
75
|
+
opts.expansion_search = 16; // 10 in faiss
|
|
76
|
+
opts.metric_kind = usearch_metric_ip_k;
|
|
77
|
+
opts.metric = NULL;
|
|
78
|
+
opts.quantization = usearch_scalar_f32_k;
|
|
79
|
+
|
|
80
|
+
// results arrays
|
|
81
|
+
int k = 10;
|
|
82
|
+
num_vectors /= 10;
|
|
83
|
+
usearch_label_t* labels = (usearch_label_t*)calloc(k, sizeof(usearch_label_t));
|
|
84
|
+
float* distances = (float*)calloc(k, sizeof(float));
|
|
85
|
+
|
|
86
|
+
fprintf(stderr, "read %ld vectors of dimension %ld\n", num_vectors, vector_dim);
|
|
87
|
+
fprintf(stderr, "initializing usearch...\n");
|
|
88
|
+
idx = usearch_init(&opts, &error);
|
|
89
|
+
if (error != NULL) {
|
|
90
|
+
fprintf(stderr, "usearch error: %s\n", error);
|
|
91
|
+
assert(false);
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
fprintf(stderr, "reserving capacity...\n");
|
|
95
|
+
usearch_reserve(idx, num_vectors, &error);
|
|
96
|
+
assert(!error);
|
|
97
|
+
|
|
98
|
+
fprintf(stderr, "adding vectors...\n");
|
|
99
|
+
for (size_t i = 0; i < num_vectors; i++) {
|
|
100
|
+
if (i % LOG_GRANULARITY == 0) {
|
|
101
|
+
fprintf(stderr, "added %ld vectors...\n", i);
|
|
102
|
+
}
|
|
103
|
+
usearch_label_t label = i;
|
|
104
|
+
usearch_add(idx, label, data + i * vector_dim, usearch_scalar_f32_k, &error);
|
|
105
|
+
assert(!error);
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
fprintf(stderr, "checking containment of vectors...\n");
|
|
109
|
+
for (size_t i = 0; i < num_vectors; i++) {
|
|
110
|
+
if (i % LOG_GRANULARITY == 0) {
|
|
111
|
+
fprintf(stderr, "checked %ld vectors...\n", i);
|
|
112
|
+
}
|
|
113
|
+
usearch_label_t label = i;
|
|
114
|
+
assert(usearch_contains(idx, label, &error));
|
|
115
|
+
assert(!error);
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
assert(!usearch_contains(idx, -1, &error));
|
|
119
|
+
assert(!error);
|
|
120
|
+
|
|
121
|
+
fprintf(stderr, "searching vectors...\n");
|
|
122
|
+
for (size_t i = 0; i < num_vectors; i++) {
|
|
123
|
+
if (i % LOG_GRANULARITY == 0) {
|
|
124
|
+
fprintf(stderr, "searched %ld vectors...\n", i);
|
|
125
|
+
}
|
|
126
|
+
const void *query_vector = data + i * vector_dim;
|
|
127
|
+
usearch_search(idx, query_vector, usearch_scalar_f32_k, k, labels, distances, &error);
|
|
128
|
+
assert(!error);
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
fprintf(stderr, "saving the index...\n");
|
|
132
|
+
start = clock();
|
|
133
|
+
usearch_save(idx, "usearch_index.bin", &error);
|
|
134
|
+
assert(!error);
|
|
135
|
+
fprintf(stderr, "saving took %.2f ms\n", ((double)(clock() - start)) / CLOCKS_PER_SEC * 1000);
|
|
136
|
+
|
|
137
|
+
fprintf(stderr, "destroying the index...\n");
|
|
138
|
+
start = clock();
|
|
139
|
+
usearch_free(idx, &error);
|
|
140
|
+
assert(!error);
|
|
141
|
+
fprintf(stderr, "destroying the index took %.2f ms\n", ((double)(clock() - start)) / CLOCKS_PER_SEC * 1000);
|
|
142
|
+
|
|
143
|
+
fprintf(stderr, "loading the index...\n");
|
|
144
|
+
start = clock();
|
|
145
|
+
idx = usearch_init(&opts, &error);
|
|
146
|
+
assert(!error);
|
|
147
|
+
assert(usearch_size(idx, &error) == 0);
|
|
148
|
+
assert(!error);
|
|
149
|
+
usearch_load(idx, "usearch_index.bin", &error);
|
|
150
|
+
assert(!error);
|
|
151
|
+
fprintf(stderr, "loading the index took %.2f ms\n", ((double)(clock() - start)) / CLOCKS_PER_SEC * 1000);
|
|
152
|
+
assert(usearch_size(idx, &error) == num_vectors);
|
|
153
|
+
assert(!error);
|
|
154
|
+
usearch_free(idx, &error);
|
|
155
|
+
|
|
156
|
+
fprintf(stderr, "viewing the index...\n");
|
|
157
|
+
start = clock();
|
|
158
|
+
idx = usearch_init(&opts, &error);
|
|
159
|
+
assert(!error);
|
|
160
|
+
assert(usearch_size(idx, &error) == 0);
|
|
161
|
+
assert(!error);
|
|
162
|
+
usearch_view(idx, "usearch_index.bin", &error);
|
|
163
|
+
assert(!error);
|
|
164
|
+
fprintf(stderr, "viewing the index took %.2f ms\n", ((double)(clock() - start)) / CLOCKS_PER_SEC * 1000);
|
|
165
|
+
assert(usearch_size(idx, &error) == num_vectors);
|
|
166
|
+
assert(!error);
|
|
167
|
+
usearch_free(idx, &error);
|
|
168
|
+
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
int main() {
|
|
172
|
+
const char* filename = "sift/sift_base.fvecs";
|
|
173
|
+
size_t num_vectors = 0, vector_dim = 0;
|
|
174
|
+
float* data = read_fvecs_file(filename, &num_vectors, &vector_dim);
|
|
175
|
+
if (data != NULL) {
|
|
176
|
+
|
|
177
|
+
test_index(data, num_vectors, vector_dim);
|
|
178
|
+
|
|
179
|
+
free(data); // Remember to free the dynamically allocated memory
|
|
180
|
+
}
|
|
181
|
+
return 0;
|
|
182
|
+
}
|
package/c/usearch.h
CHANGED
|
@@ -1,14 +1,18 @@
|
|
|
1
1
|
#ifndef UNUM_USEARCH_H
|
|
2
2
|
#define UNUM_USEARCH_H
|
|
3
|
-
|
|
4
|
-
#
|
|
3
|
+
// in case the header is included from cpp code
|
|
4
|
+
#ifdef __cplusplus
|
|
5
|
+
extern "C" {
|
|
6
|
+
#endif
|
|
7
|
+
#include <stdbool.h> // `bool`
|
|
8
|
+
#include <stdint.h> // `size_t`
|
|
5
9
|
|
|
6
10
|
typedef void* usearch_index_t;
|
|
7
11
|
typedef uint32_t usearch_label_t;
|
|
8
12
|
typedef float usearch_distance_t;
|
|
9
13
|
typedef char const* usearch_error_t;
|
|
10
14
|
|
|
11
|
-
typedef usearch_distance_t (*)(void const*, void const*)
|
|
15
|
+
typedef usearch_distance_t (*usearch_metric_t)(void const*, void const*);
|
|
12
16
|
|
|
13
17
|
typedef enum usearch_metric_kind_t {
|
|
14
18
|
usearch_metric_ip_k = 0,
|
|
@@ -39,13 +43,12 @@ typedef struct usearch_init_options_t {
|
|
|
39
43
|
|
|
40
44
|
usearch_scalar_kind_t quantization;
|
|
41
45
|
size_t dimensions;
|
|
42
|
-
size_t capacity;
|
|
43
46
|
size_t connectivity;
|
|
44
47
|
size_t expansion_add;
|
|
45
48
|
size_t expansion_search;
|
|
46
49
|
} usearch_init_options_t;
|
|
47
50
|
|
|
48
|
-
|
|
51
|
+
usearch_index_t usearch_init(usearch_init_options_t*, usearch_error_t*);
|
|
49
52
|
void usearch_free(usearch_index_t, usearch_error_t*);
|
|
50
53
|
|
|
51
54
|
void usearch_save(usearch_index_t, char const* path, usearch_error_t*);
|
|
@@ -79,4 +82,7 @@ bool usearch_get( //
|
|
|
79
82
|
|
|
80
83
|
void usearch_remove(usearch_index_t, usearch_label_t, usearch_error_t*);
|
|
81
84
|
|
|
85
|
+
#ifdef __cplusplus
|
|
86
|
+
}
|
|
87
|
+
#endif
|
|
82
88
|
#endif
|
package/conanfile.py
CHANGED
|
@@ -4,7 +4,7 @@ from conans import ConanFile
|
|
|
4
4
|
class USearchConan(ConanFile):
|
|
5
5
|
|
|
6
6
|
name = 'USearch'
|
|
7
|
-
version = '0.19.
|
|
7
|
+
version = '0.19.1'
|
|
8
8
|
license = 'Apache 2.0'
|
|
9
9
|
url = 'https://github.com/unum-cloud/usearch'
|
|
10
10
|
description = 'Smaller & Faster Single-File Vector Search Engine from Unum'
|
package/cpp/bench.cpp
CHANGED
|
@@ -71,7 +71,7 @@ struct alignas(32) persisted_matrix_gt {
|
|
|
71
71
|
persisted_matrix_gt(char const* path) noexcept(false) {
|
|
72
72
|
if (!path || !std::strlen(path))
|
|
73
73
|
throw std::invalid_argument("The file path is empty");
|
|
74
|
-
#if defined(
|
|
74
|
+
#if defined(USEARCH_DEFINED_WINDOWS)
|
|
75
75
|
|
|
76
76
|
HANDLE file_handle =
|
|
77
77
|
CreateFileA(path, GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr);
|
|
@@ -117,7 +117,7 @@ struct alignas(32) persisted_matrix_gt {
|
|
|
117
117
|
|
|
118
118
|
~persisted_matrix_gt() {
|
|
119
119
|
if (raw_handle != nullptr)
|
|
120
|
-
#if defined(
|
|
120
|
+
#if defined(USEARCH_DEFINED_WINDOWS)
|
|
121
121
|
UnmapViewOfFile(raw_handle);
|
|
122
122
|
#else
|
|
123
123
|
munmap((void*)raw_handle, raw_length);
|
|
@@ -377,7 +377,7 @@ void handler(int sig) {
|
|
|
377
377
|
size_t size;
|
|
378
378
|
|
|
379
379
|
// get void*'s for all entries on the stack
|
|
380
|
-
#if defined(
|
|
380
|
+
#if defined(USEARCH_DEFINED_WINDOWS)
|
|
381
381
|
size = CaptureStackBackTrace(0, 10, array, NULL);
|
|
382
382
|
#else
|
|
383
383
|
size = backtrace(array, 10);
|
|
@@ -386,7 +386,7 @@ void handler(int sig) {
|
|
|
386
386
|
// print out all the frames to stderr
|
|
387
387
|
fprintf(stderr, "Error: signal %d:\n", sig);
|
|
388
388
|
|
|
389
|
-
#if defined(
|
|
389
|
+
#if defined(USEARCH_DEFINED_WINDOWS)
|
|
390
390
|
SYMBOL_INFO* symbol = (SYMBOL_INFO*)calloc(sizeof(SYMBOL_INFO) + 256 * sizeof(char), 1);
|
|
391
391
|
symbol->MaxNameLen = 255;
|
|
392
392
|
symbol->SizeOfStruct = sizeof(SYMBOL_INFO);
|
package/docs/compilation.md
CHANGED
|
@@ -112,7 +112,7 @@ namespace usearch {
|
|
|
112
112
|
using f64_t = double;
|
|
113
113
|
using f32_t = float;
|
|
114
114
|
using byte_t = char;
|
|
115
|
-
enum
|
|
115
|
+
enum b1x8_t : unsigned char {};
|
|
116
116
|
|
|
117
117
|
enum class metric_kind_t : std::uint8_t {
|
|
118
118
|
unknown_k = 0,
|
|
@@ -311,7 +311,7 @@ template <typename scalar_at = float, typename result_at = scalar_at> struct cos
|
|
|
311
311
|
ab += result_t(a[i]) * result_t(b[i]), //
|
|
312
312
|
a2 += square<result_t>(a[i]), //
|
|
313
313
|
b2 += square<result_t>(b[i]);
|
|
314
|
-
return (ab != 0) ? (1 - ab / (std::sqrt(a2) * std::sqrt(b2))) :
|
|
314
|
+
return (ab != 0) ? (1 - ab / (std::sqrt(a2) * std::sqrt(b2))) : 1;
|
|
315
315
|
}
|
|
316
316
|
};
|
|
317
317
|
|
|
@@ -354,7 +354,10 @@ template <typename scalar_at = std::uint64_t, typename result_at = std::size_t>
|
|
|
354
354
|
using view_t = span_gt<scalar_t const>;
|
|
355
355
|
using result_t = result_at;
|
|
356
356
|
using result_type = result_t;
|
|
357
|
-
static_assert(
|
|
357
|
+
static_assert( //
|
|
358
|
+
std::is_unsigned<scalar_t>::value ||
|
|
359
|
+
(std::is_enum<scalar_t>::value && std::is_unsigned<typename std::underlying_type<scalar_t>::type>::value),
|
|
360
|
+
"Hamming distance requires unsigned integral words");
|
|
358
361
|
|
|
359
362
|
inline metric_kind_t kind() const noexcept { return metric_kind_t::hamming_k; }
|
|
360
363
|
inline scalar_kind_t scalar_kind() const noexcept { return common_scalar_kind<scalar_t>(); }
|
|
@@ -385,7 +388,10 @@ template <typename scalar_at = std::uint64_t, typename result_at = float> struct
|
|
|
385
388
|
using view_t = span_gt<scalar_t const>;
|
|
386
389
|
using result_t = result_at;
|
|
387
390
|
using result_type = result_t;
|
|
388
|
-
static_assert(
|
|
391
|
+
static_assert( //
|
|
392
|
+
std::is_unsigned<scalar_t>::value ||
|
|
393
|
+
(std::is_enum<scalar_t>::value && std::is_unsigned<typename std::underlying_type<scalar_t>::type>::value),
|
|
394
|
+
"Tanimoto distance requires unsigned integral words");
|
|
389
395
|
static_assert(std::is_floating_point<result_t>::value, "Tanimoto distance will be a fraction");
|
|
390
396
|
|
|
391
397
|
inline metric_kind_t kind() const noexcept { return metric_kind_t::tanimoto_k; }
|
|
@@ -419,7 +425,10 @@ template <typename scalar_at = std::uint64_t, typename result_at = float> struct
|
|
|
419
425
|
using view_t = span_gt<scalar_t const>;
|
|
420
426
|
using result_t = result_at;
|
|
421
427
|
using result_type = result_t;
|
|
422
|
-
static_assert(
|
|
428
|
+
static_assert( //
|
|
429
|
+
std::is_unsigned<scalar_t>::value ||
|
|
430
|
+
(std::is_enum<scalar_t>::value && std::is_unsigned<typename std::underlying_type<scalar_t>::type>::value),
|
|
431
|
+
"Sorensen-Dice distance requires unsigned integral words");
|
|
423
432
|
static_assert(std::is_floating_point<result_t>::value, "Sorensen-Dice distance will be a fraction");
|
|
424
433
|
|
|
425
434
|
inline metric_kind_t kind() const noexcept { return metric_kind_t::sorensen_k; }
|
|
@@ -1571,6 +1580,7 @@ class index_gt {
|
|
|
1571
1580
|
/**
|
|
1572
1581
|
* @brief Erases all the vectors from the index.
|
|
1573
1582
|
* Will change `size()` to zero, but will keep the same `capacity()`.
|
|
1583
|
+
* Will keep the number of available threads/contexts the same as it was.
|
|
1574
1584
|
*/
|
|
1575
1585
|
void clear() noexcept {
|
|
1576
1586
|
std::size_t n = size_;
|
|
@@ -1584,6 +1594,7 @@ class index_gt {
|
|
|
1584
1594
|
/**
|
|
1585
1595
|
* @brief Erases all the vectors from the index, also deallocating the registry.
|
|
1586
1596
|
* Will change both `size()` and `capacity()` to zero.
|
|
1597
|
+
* Will deallocate all threads/contexts.
|
|
1587
1598
|
*/
|
|
1588
1599
|
void reset() noexcept {
|
|
1589
1600
|
clear();
|
|
@@ -1732,6 +1743,11 @@ class index_gt {
|
|
|
1732
1743
|
std::size_t measurements{};
|
|
1733
1744
|
error_t error{};
|
|
1734
1745
|
|
|
1746
|
+
// todo: this is a hack to get a handle to a failed search_result_t without changing its member types
|
|
1747
|
+
// given that search_result_t can represent a failed result (which implies no valid top_candidates_t &top_),
|
|
1748
|
+
// the top_ probably needs to be a pointer.
|
|
1749
|
+
inline search_result_t(index_gt const& index) noexcept
|
|
1750
|
+
: index_(index), top_(index.contexts_[0].top_candidates) {}
|
|
1735
1751
|
inline search_result_t(search_result_t&&) = default;
|
|
1736
1752
|
inline search_result_t& operator=(search_result_t&&) = default;
|
|
1737
1753
|
|
|
@@ -2574,6 +2590,7 @@ class index_gt {
|
|
|
2574
2590
|
void search_exact_(vector_view_t query, std::size_t count, context_t& context) const noexcept {
|
|
2575
2591
|
top_candidates_t& top = context.top_candidates;
|
|
2576
2592
|
top.clear();
|
|
2593
|
+
top.reserve(count);
|
|
2577
2594
|
for (std::size_t i = 0; i != size(); ++i)
|
|
2578
2595
|
top.insert({context.measure(query, node_with_id_(i)), static_cast<id_t>(i)}, count);
|
|
2579
2596
|
}
|
|
@@ -288,6 +288,8 @@ class index_punned_dense_gt {
|
|
|
288
288
|
search_result_t search_around(label_t hint, f32_t const* vector, std::size_t wanted, search_config_t config) const { return search_around_(hint, vector, wanted, config, casts_.from_f32); }
|
|
289
289
|
search_result_t search_around(label_t hint, f64_t const* vector, std::size_t wanted, search_config_t config) const { return search_around_(hint, vector, wanted, config, casts_.from_f64); }
|
|
290
290
|
|
|
291
|
+
search_result_t empty_search_result() const { return search_result_t{*typed_}; }
|
|
292
|
+
|
|
291
293
|
bool get(label_t label, b1x8_t* vector) const { return get_(label, vector, casts_.to_b1x8); }
|
|
292
294
|
bool get(label_t label, f8_bits_t* vector) const { return get_(label, vector, casts_.to_f8); }
|
|
293
295
|
bool get(label_t label, f16_t* vector) const { return get_(label, vector, casts_.to_f16); }
|
|
@@ -641,10 +643,10 @@ class index_punned_dense_gt {
|
|
|
641
643
|
case metric_kind_t::l2sq_k: return l2sq_metric_(dimensions, accuracy);
|
|
642
644
|
case metric_kind_t::pearson_k: return pearson_metric_(accuracy);
|
|
643
645
|
case metric_kind_t::haversine_k: return haversine_metric_(accuracy);
|
|
644
|
-
case metric_kind_t::hamming_k: return hamming_gt<
|
|
646
|
+
case metric_kind_t::hamming_k: return hamming_gt<b1x8_t>{};
|
|
645
647
|
case metric_kind_t::jaccard_k: // Equivalent to Tanimoto
|
|
646
|
-
case metric_kind_t::tanimoto_k: return tanimoto_gt<
|
|
647
|
-
case metric_kind_t::sorensen_k: return sorensen_gt<
|
|
648
|
+
case metric_kind_t::tanimoto_k: return tanimoto_gt<b1x8_t>{};
|
|
649
|
+
case metric_kind_t::sorensen_k: return sorensen_gt<b1x8_t>{};
|
|
648
650
|
default: return {};
|
|
649
651
|
}
|
|
650
652
|
}
|