usearch 1.1.1 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/prerelease.yml +3 -3
- package/.github/workflows/release.yml +3 -3
- package/.github/workflows/update_version.sh +1 -1
- package/.gitmodules +1 -4
- package/Cargo.toml +1 -1
- package/Package.swift +2 -1
- package/README.md +86 -42
- package/binding.gyp +16 -12
- package/build.gradle +1 -1
- package/build.rs +0 -1
- package/c/CMakeLists.txt +11 -7
- package/c/lib.cpp +36 -24
- package/c/usearch.h +42 -12
- package/cluster.ipynb +167 -0
- package/conanfile.py +1 -1
- package/cpp/CMakeLists.txt +8 -3
- package/cpp/bench.cpp +2 -2
- package/cpp/test.cpp +42 -25
- package/docs/benchmarks.md +8 -0
- package/docs/compilation.md +7 -7
- package/docs/conf.py +27 -24
- package/docs/javascript/reference.rst +3 -0
- package/golang/README.md +31 -7
- package/golang/lib.go +1 -1
- package/include/usearch/index.hpp +348 -69
- package/include/usearch/index_dense.hpp +587 -179
- package/include/usearch/index_plugins.hpp +194 -16
- package/java/cloud/unum/usearch/Index.java +13 -0
- package/java/cloud/unum/usearch/cloud_unum_usearch_Index.cpp +40 -9
- package/java/cloud/unum/usearch/cloud_unum_usearch_Index.h +16 -0
- package/javascript/docs.js +111 -0
- package/javascript/lib.cpp +29 -9
- package/javascript/usearch.d.ts +9 -1
- package/objc/USearchObjective.mm +11 -11
- package/package.json +2 -2
- package/pyproject.toml +1 -1
- package/python/lib.cpp +488 -158
- package/python/scripts/index_faiss.py +3 -3
- package/python/scripts/join.py +6 -6
- package/python/scripts/test_index.py +219 -0
- package/python/scripts/test_sparse.py +21 -21
- package/python/scripts/test_tooling.py +111 -0
- package/python/usearch/eval.py +9 -5
- package/python/usearch/index.py +491 -216
- package/rust/lib.cpp +6 -24
- package/rust/lib.hpp +1 -3
- package/rust/lib.rs +89 -0
- package/setup.py +5 -6
- package/wasm/CMakeLists.txt +0 -1
- package/wasmer.toml +1 -1
- package/python/scripts/test.py +0 -364
- package/robin-map/.codecov.yml +0 -5
- package/robin-map/.github/workflows/ci.yml +0 -136
- package/robin-map/CMakeLists.txt +0 -86
- package/robin-map/LICENSE +0 -21
- package/robin-map/README.md +0 -483
- package/robin-map/cmake/tsl-robin-mapConfig.cmake.in +0 -9
- package/robin-map/doxygen.conf +0 -2483
- package/robin-map/include/tsl/robin_growth_policy.h +0 -406
- package/robin-map/include/tsl/robin_hash.h +0 -1639
- package/robin-map/include/tsl/robin_map.h +0 -807
- package/robin-map/include/tsl/robin_set.h +0 -660
- package/robin-map/tests/CMakeLists.txt +0 -26
- package/robin-map/tests/custom_allocator_tests.cpp +0 -138
- package/robin-map/tests/main.cpp +0 -26
- package/robin-map/tests/policy_tests.cpp +0 -97
- package/robin-map/tests/robin_map_tests.cpp +0 -1451
- package/robin-map/tests/robin_set_tests.cpp +0 -164
- package/robin-map/tests/utils.h +0 -443
- package/robin-map/tsl-robin-map.natvis +0 -78
package/c/usearch.h
CHANGED
|
@@ -4,9 +4,15 @@
|
|
|
4
4
|
#ifdef __cplusplus
|
|
5
5
|
extern "C" {
|
|
6
6
|
#endif
|
|
7
|
+
|
|
7
8
|
#ifndef USEARCH_EXPORT
|
|
9
|
+
#if defined(_WIN32) && !defined(__MINGW32__)
|
|
10
|
+
#define USEARCH_EXPORT __declspec(dllexport)
|
|
11
|
+
#else
|
|
8
12
|
#define USEARCH_EXPORT
|
|
9
13
|
#endif
|
|
14
|
+
#endif
|
|
15
|
+
|
|
10
16
|
#include <stdbool.h> // `bool`
|
|
11
17
|
#include <stdint.h> // `size_t`
|
|
12
18
|
|
|
@@ -83,6 +89,10 @@ USEARCH_EXPORT typedef struct usearch_init_options_t {
|
|
|
83
89
|
* @brief The @b optional expansion factor used for index construction during search operations.
|
|
84
90
|
*/
|
|
85
91
|
size_t expansion_search;
|
|
92
|
+
/**
|
|
93
|
+
* @brief When set allows multiple vectors to map to the same key.
|
|
94
|
+
*/
|
|
95
|
+
bool multi;
|
|
86
96
|
} usearch_init_options_t;
|
|
87
97
|
|
|
88
98
|
/**
|
|
@@ -151,41 +161,61 @@ USEARCH_EXPORT void usearch_add( //
|
|
|
151
161
|
*/
|
|
152
162
|
USEARCH_EXPORT bool usearch_contains(usearch_index_t, usearch_key_t, usearch_error_t* error);
|
|
153
163
|
|
|
164
|
+
/**
|
|
165
|
+
* @brief Counts the number of entries in the index under a specific key.
|
|
166
|
+
* @param[in] key The key to be checked.
|
|
167
|
+
* @param[out] error Pointer to a string where the error message will be stored, if an error occurs.
|
|
168
|
+
* @return Number of vectors found under that key.
|
|
169
|
+
*/
|
|
170
|
+
USEARCH_EXPORT size_t usearch_count(usearch_index_t, usearch_key_t, usearch_error_t* error);
|
|
171
|
+
|
|
154
172
|
/**
|
|
155
173
|
* @brief Performs k-Approximate Nearest Neighbors (kANN) Search for closest vectors to query.
|
|
156
174
|
* @param[in] query_vector Pointer to the query vector data.
|
|
157
175
|
* @param[in] query_kind The scalar type used in the query vector data.
|
|
158
|
-
* @param[in]
|
|
159
|
-
* @param[out]
|
|
160
|
-
* @param[out]
|
|
176
|
+
* @param[in] count Upper bound on the number of neighbors to search, the "k" in "kANN".
|
|
177
|
+
* @param[out] keys Output buffer for up to `count` nearest neighbors keys.
|
|
178
|
+
* @param[out] distances Output buffer for up to `count` distances to nearest neighbors.
|
|
161
179
|
* @param[out] error Pointer to a string where the error message will be stored, if an error occurs.
|
|
162
180
|
* @return Number of found matches.
|
|
163
181
|
*/
|
|
164
|
-
USEARCH_EXPORT size_t usearch_search(
|
|
165
|
-
usearch_index_t,
|
|
166
|
-
|
|
182
|
+
USEARCH_EXPORT size_t usearch_search( //
|
|
183
|
+
usearch_index_t, //
|
|
184
|
+
void const* query_vector, usearch_scalar_kind_t query_kind, //
|
|
185
|
+
size_t count, usearch_key_t* keys, usearch_distance_t* distances, usearch_error_t* error);
|
|
167
186
|
|
|
168
187
|
/**
|
|
169
188
|
* @brief Retrieves the vector associated with the given key from the index.
|
|
170
189
|
* @param[in] key The key of the vector to retrieve.
|
|
171
190
|
* @param[out] vector Pointer to the memory where the vector data will be copied.
|
|
191
|
+
* @param[in] count Number of vectors that can be fitted into `vector` for multi-vector entries.
|
|
172
192
|
* @param[in] vector_kind The scalar type used in the vector data.
|
|
173
193
|
* @param[out] error Pointer to a string where the error message will be stored, if an error occurs.
|
|
174
|
-
* @return
|
|
194
|
+
* @return Number of vectors found under that name and exported to `vector`.
|
|
175
195
|
*/
|
|
176
|
-
USEARCH_EXPORT
|
|
177
|
-
usearch_index_t, usearch_key_t key, //
|
|
196
|
+
USEARCH_EXPORT size_t usearch_get( //
|
|
197
|
+
usearch_index_t, usearch_key_t key, size_t count, //
|
|
178
198
|
void* vector, usearch_scalar_kind_t vector_kind, usearch_error_t* error);
|
|
179
199
|
|
|
180
200
|
/**
|
|
181
201
|
* @brief Removes the vector associated with the given key from the index.
|
|
182
202
|
* @param[in] key The key of the vector to be removed.
|
|
183
203
|
* @param[out] error Pointer to a string where the error message will be stored, if an error occurs.
|
|
184
|
-
* @return
|
|
204
|
+
* @return Number of vectors found under that name and dropped from the index.
|
|
205
|
+
*/
|
|
206
|
+
USEARCH_EXPORT size_t usearch_remove(usearch_index_t, usearch_key_t key, usearch_error_t* error);
|
|
207
|
+
|
|
208
|
+
/**
|
|
209
|
+
* @brief Renames the vector to map to a different key.
|
|
210
|
+
* @param[in] from The key of the vector to be renamed.
|
|
211
|
+
* @param[in] to New key for found entry.
|
|
212
|
+
* @param[out] error Pointer to a string where the error message will be stored, if an error occurs.
|
|
213
|
+
* @return Number of vectors found under that name and renamed.
|
|
185
214
|
*/
|
|
186
|
-
USEARCH_EXPORT
|
|
215
|
+
USEARCH_EXPORT size_t usearch_rename(usearch_index_t, usearch_key_t from, usearch_key_t to, usearch_error_t* error);
|
|
187
216
|
|
|
188
217
|
#ifdef __cplusplus
|
|
189
218
|
}
|
|
190
219
|
#endif
|
|
191
|
-
|
|
220
|
+
|
|
221
|
+
#endif // UNUM_USEARCH_H
|
package/cluster.ipynb
ADDED
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
{
|
|
2
|
+
"cells": [
|
|
3
|
+
{
|
|
4
|
+
"cell_type": "code",
|
|
5
|
+
"execution_count": null,
|
|
6
|
+
"metadata": {},
|
|
7
|
+
"outputs": [],
|
|
8
|
+
"source": [
|
|
9
|
+
"from usearch.io import load_matrix\n",
|
|
10
|
+
"import numpy as np"
|
|
11
|
+
]
|
|
12
|
+
},
|
|
13
|
+
{
|
|
14
|
+
"cell_type": "code",
|
|
15
|
+
"execution_count": null,
|
|
16
|
+
"metadata": {},
|
|
17
|
+
"outputs": [],
|
|
18
|
+
"source": [
|
|
19
|
+
"vectors = load_matrix(\"datasets/arxiv_2M/abstract.e5-base-v2.fbin\", view=True)"
|
|
20
|
+
]
|
|
21
|
+
},
|
|
22
|
+
{
|
|
23
|
+
"cell_type": "code",
|
|
24
|
+
"execution_count": null,
|
|
25
|
+
"metadata": {},
|
|
26
|
+
"outputs": [],
|
|
27
|
+
"source": [
|
|
28
|
+
"vectors.shape"
|
|
29
|
+
]
|
|
30
|
+
},
|
|
31
|
+
{
|
|
32
|
+
"cell_type": "code",
|
|
33
|
+
"execution_count": null,
|
|
34
|
+
"metadata": {},
|
|
35
|
+
"outputs": [],
|
|
36
|
+
"source": [
|
|
37
|
+
"from usearch.index import Index, USES_SIMSIMD, USES_NATIVE_F16\n",
|
|
38
|
+
"\n",
|
|
39
|
+
"index = Index(ndim=vectors.shape[1], metric=\"cos\", dtype=\"i8\")\n",
|
|
40
|
+
"index.hardware_acceleration, USES_SIMSIMD, USES_NATIVE_F16"
|
|
41
|
+
]
|
|
42
|
+
},
|
|
43
|
+
{
|
|
44
|
+
"cell_type": "code",
|
|
45
|
+
"execution_count": null,
|
|
46
|
+
"metadata": {},
|
|
47
|
+
"outputs": [],
|
|
48
|
+
"source": [
|
|
49
|
+
"\n",
|
|
50
|
+
"import os\n",
|
|
51
|
+
"\n",
|
|
52
|
+
"if os.path.exists(\"cluster.i8.usearch\"):\n",
|
|
53
|
+
" index.load(\"cluster.i8.usearch\")\n",
|
|
54
|
+
"\n",
|
|
55
|
+
"if len(index) == 0:\n",
|
|
56
|
+
" index.add(None, vectors, log=True)\n",
|
|
57
|
+
" index.save(\"cluster.i8.usearch\")"
|
|
58
|
+
]
|
|
59
|
+
},
|
|
60
|
+
{
|
|
61
|
+
"cell_type": "code",
|
|
62
|
+
"execution_count": null,
|
|
63
|
+
"metadata": {},
|
|
64
|
+
"outputs": [],
|
|
65
|
+
"source": [
|
|
66
|
+
"index"
|
|
67
|
+
]
|
|
68
|
+
},
|
|
69
|
+
{
|
|
70
|
+
"cell_type": "code",
|
|
71
|
+
"execution_count": null,
|
|
72
|
+
"metadata": {},
|
|
73
|
+
"outputs": [],
|
|
74
|
+
"source": [
|
|
75
|
+
"clustering = index.cluster(min_count=10, max_count=15, threads=60)\n",
|
|
76
|
+
"clustering"
|
|
77
|
+
]
|
|
78
|
+
},
|
|
79
|
+
{
|
|
80
|
+
"cell_type": "code",
|
|
81
|
+
"execution_count": null,
|
|
82
|
+
"metadata": {},
|
|
83
|
+
"outputs": [],
|
|
84
|
+
"source": [
|
|
85
|
+
"print(f\"{len(clustering.centroids_popularity[0])} unique clusters for {len(index)} members\")"
|
|
86
|
+
]
|
|
87
|
+
},
|
|
88
|
+
{
|
|
89
|
+
"cell_type": "code",
|
|
90
|
+
"execution_count": null,
|
|
91
|
+
"metadata": {},
|
|
92
|
+
"outputs": [],
|
|
93
|
+
"source": [
|
|
94
|
+
"clustering.plot_centroids_popularity()"
|
|
95
|
+
]
|
|
96
|
+
},
|
|
97
|
+
{
|
|
98
|
+
"cell_type": "code",
|
|
99
|
+
"execution_count": null,
|
|
100
|
+
"metadata": {},
|
|
101
|
+
"outputs": [],
|
|
102
|
+
"source": [
|
|
103
|
+
"nx.draw_networkx(g)"
|
|
104
|
+
]
|
|
105
|
+
},
|
|
106
|
+
{
|
|
107
|
+
"cell_type": "code",
|
|
108
|
+
"execution_count": null,
|
|
109
|
+
"metadata": {},
|
|
110
|
+
"outputs": [],
|
|
111
|
+
"source": [
|
|
112
|
+
"import matplotlib.pyplot as plt\n",
|
|
113
|
+
"import networkx as nx\n",
|
|
114
|
+
"\n",
|
|
115
|
+
"G = g\n",
|
|
116
|
+
"\n",
|
|
117
|
+
"node_sizes = nx.get_node_attributes(G, \"size\").values()\n",
|
|
118
|
+
"max_node_size = max(node_sizes)\n",
|
|
119
|
+
"node_sizes = [size * 10 / max_node_size for size in node_sizes]\n",
|
|
120
|
+
"\n",
|
|
121
|
+
"edge_labels = nx.get_edge_attributes(G, \"distance\")\n",
|
|
122
|
+
"edge_labels = {edge: f\"{label:.2}\" for edge, label in edge_labels.items()}\n",
|
|
123
|
+
"\n",
|
|
124
|
+
"pos = nx.spring_layout(G, seed=7)\n",
|
|
125
|
+
"nx.draw_networkx_nodes(G, pos, node_size=node_sizes)\n",
|
|
126
|
+
"nx.draw_networkx_edges(G, pos, edgelist=G.edges(data=False))\n",
|
|
127
|
+
"nx.draw_networkx_labels(G, pos, font_size=10, font_family=\"sans-serif\")\n",
|
|
128
|
+
"nx.draw_networkx_edge_labels(G, pos, edge_labels, font_size=5)\n",
|
|
129
|
+
"\n",
|
|
130
|
+
"ax = plt.gca()\n",
|
|
131
|
+
"ax.margins(0.08)\n",
|
|
132
|
+
"plt.axis(\"off\")\n",
|
|
133
|
+
"plt.tight_layout()\n",
|
|
134
|
+
"plt.show()"
|
|
135
|
+
]
|
|
136
|
+
},
|
|
137
|
+
{
|
|
138
|
+
"cell_type": "code",
|
|
139
|
+
"execution_count": null,
|
|
140
|
+
"metadata": {},
|
|
141
|
+
"outputs": [],
|
|
142
|
+
"source": []
|
|
143
|
+
}
|
|
144
|
+
],
|
|
145
|
+
"metadata": {
|
|
146
|
+
"kernelspec": {
|
|
147
|
+
"display_name": "base",
|
|
148
|
+
"language": "python",
|
|
149
|
+
"name": "python3"
|
|
150
|
+
},
|
|
151
|
+
"language_info": {
|
|
152
|
+
"codemirror_mode": {
|
|
153
|
+
"name": "ipython",
|
|
154
|
+
"version": 3
|
|
155
|
+
},
|
|
156
|
+
"file_extension": ".py",
|
|
157
|
+
"mimetype": "text/x-python",
|
|
158
|
+
"name": "python",
|
|
159
|
+
"nbconvert_exporter": "python",
|
|
160
|
+
"pygments_lexer": "ipython3",
|
|
161
|
+
"version": "3.10.11"
|
|
162
|
+
},
|
|
163
|
+
"orig_nbformat": 4
|
|
164
|
+
},
|
|
165
|
+
"nbformat": 4,
|
|
166
|
+
"nbformat_minor": 2
|
|
167
|
+
}
|
package/conanfile.py
CHANGED
|
@@ -4,7 +4,7 @@ from conans import ConanFile
|
|
|
4
4
|
class USearchConan(ConanFile):
|
|
5
5
|
|
|
6
6
|
name = 'USearch'
|
|
7
|
-
version = '1.
|
|
7
|
+
version = '1.2.0'
|
|
8
8
|
license = 'Apache 2.0'
|
|
9
9
|
url = 'https://github.com/unum-cloud/usearch'
|
|
10
10
|
description = 'Smaller & Faster Single-File Vector Search Engine from Unum'
|
package/cpp/CMakeLists.txt
CHANGED
|
@@ -19,18 +19,24 @@ if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
|
|
|
19
19
|
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Wall -Wextra -Wno-conversion -Wno-unknown-pragmas")
|
|
20
20
|
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native")
|
|
21
21
|
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fmax-errors=1")
|
|
22
|
-
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fopenmp")
|
|
23
22
|
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pedantic")
|
|
24
23
|
|
|
24
|
+
if(${USEARCH_USE_OPENMP})
|
|
25
|
+
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fopenmp")
|
|
26
|
+
endif()
|
|
27
|
+
|
|
25
28
|
elseif(CMAKE_CXX_COMPILER_ID MATCHES "Clang")
|
|
26
29
|
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O3")
|
|
27
30
|
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -g")
|
|
28
31
|
set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -O3 -g")
|
|
29
32
|
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -fsanitize=address -fsanitize=leak -fsanitize=alignment -fsanitize=undefined")
|
|
30
33
|
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Wfatal-errors")
|
|
31
|
-
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fopenmp")
|
|
32
34
|
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pedantic")
|
|
33
35
|
|
|
36
|
+
if(${USEARCH_USE_OPENMP})
|
|
37
|
+
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fopenmp")
|
|
38
|
+
endif()
|
|
39
|
+
|
|
34
40
|
elseif(CMAKE_CXX_COMPILER_ID STREQUAL "NVIDIA" OR CMAKE_CXX_COMPILER_ID STREQUAL "NVHPC")
|
|
35
41
|
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O3")
|
|
36
42
|
set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} --expt-relaxed-constexpr --extended-lambda")
|
|
@@ -73,7 +79,6 @@ include(clipp)
|
|
|
73
79
|
set(USEARCH_PUNNED_INCLUDE_DIRS
|
|
74
80
|
"${CMAKE_CURRENT_SOURCE_DIR}/../include"
|
|
75
81
|
"${CMAKE_CURRENT_SOURCE_DIR}/../fp16/include"
|
|
76
|
-
"${CMAKE_CURRENT_SOURCE_DIR}/../robin-map/include"
|
|
77
82
|
"${CMAKE_CURRENT_SOURCE_DIR}/../simsimd/include"
|
|
78
83
|
"${CMAKE_CURRENT_SOURCE_DIR}/"
|
|
79
84
|
)
|
package/cpp/bench.cpp
CHANGED
|
@@ -291,7 +291,7 @@ void index_many(index_at& index, std::size_t n, vector_id_at const* ids, real_at
|
|
|
291
291
|
config.thread = omp_get_thread_num();
|
|
292
292
|
#endif
|
|
293
293
|
float_span_t vector{vectors + dims * i, dims};
|
|
294
|
-
index.add(ids[i], vector, config);
|
|
294
|
+
index.add(ids[i], vector, config.thread);
|
|
295
295
|
printer.progress++;
|
|
296
296
|
if (config.thread == 0)
|
|
297
297
|
printer.refresh();
|
|
@@ -315,7 +315,7 @@ void search_many( //
|
|
|
315
315
|
config.thread = omp_get_thread_num();
|
|
316
316
|
#endif
|
|
317
317
|
float_span_t vector{vectors + dims * i, dims};
|
|
318
|
-
index.search(vector, wanted, config).dump_to(ids + wanted * i, distances + wanted * i);
|
|
318
|
+
index.search(vector, wanted, config.thread).dump_to(ids + wanted * i, distances + wanted * i);
|
|
319
319
|
printer.progress++;
|
|
320
320
|
if (config.thread == 0)
|
|
321
321
|
printer.refresh();
|
package/cpp/test.cpp
CHANGED
|
@@ -40,23 +40,21 @@ void test_cosine(index_at& index, std::vector<std::vector<scalar_at>> const& vec
|
|
|
40
40
|
index.add(key_first, vector_first, args...);
|
|
41
41
|
|
|
42
42
|
// Default approximate search
|
|
43
|
-
key_t
|
|
43
|
+
key_t matched_keys[10] = {0};
|
|
44
44
|
distance_t matched_distances[10] = {0};
|
|
45
|
-
std::size_t matched_count = index.search(vector_first, 5, args...).dump_to(
|
|
45
|
+
std::size_t matched_count = index.search(vector_first, 5, args...).dump_to(matched_keys, matched_distances);
|
|
46
46
|
|
|
47
47
|
expect(matched_count == 1);
|
|
48
|
-
expect(
|
|
48
|
+
expect(matched_keys[0] == key_first);
|
|
49
49
|
expect(std::abs(matched_distances[0]) < 0.01);
|
|
50
50
|
|
|
51
51
|
// Add more entries
|
|
52
|
-
index_search_config_t search_config;
|
|
53
|
-
search_config.exact = true;
|
|
54
52
|
index.add(key_second, vector_second, args...);
|
|
55
53
|
index.add(key_third, vector_third, args...);
|
|
56
54
|
expect(index.size() == 3);
|
|
57
55
|
|
|
58
56
|
// Perform exact search
|
|
59
|
-
matched_count = index.search(vector_first, 5, args
|
|
57
|
+
matched_count = index.search(vector_first, 5, args...).dump_to(matched_keys, matched_distances);
|
|
60
58
|
|
|
61
59
|
// Validate scans
|
|
62
60
|
std::size_t count = 0;
|
|
@@ -68,6 +66,9 @@ void test_cosine(index_at& index, std::vector<std::vector<scalar_at>> const& vec
|
|
|
68
66
|
expect((count == 3));
|
|
69
67
|
expect((index.stats(0).nodes == 3));
|
|
70
68
|
|
|
69
|
+
// Check if clustering endpoint compiles
|
|
70
|
+
index.cluster(vector_first, 0, args...);
|
|
71
|
+
|
|
71
72
|
// Try removals and replacements
|
|
72
73
|
if constexpr (punned_ak) {
|
|
73
74
|
using labeling_result_t = typename index_t::labeling_result_t;
|
|
@@ -81,9 +82,9 @@ void test_cosine(index_at& index, std::vector<std::vector<scalar_at>> const& vec
|
|
|
81
82
|
// Search again over reconstructed index
|
|
82
83
|
index.save("tmp.usearch");
|
|
83
84
|
index.load("tmp.usearch");
|
|
84
|
-
matched_count = index.search(vector_first, 5, args...).dump_to(
|
|
85
|
+
matched_count = index.search(vector_first, 5, args...).dump_to(matched_keys, matched_distances);
|
|
85
86
|
expect(matched_count == 3);
|
|
86
|
-
expect(
|
|
87
|
+
expect(matched_keys[0] == key_first);
|
|
87
88
|
expect(std::abs(matched_distances[0]) < 0.01);
|
|
88
89
|
|
|
89
90
|
if constexpr (punned_ak) {
|
|
@@ -95,19 +96,34 @@ void test_cosine(index_at& index, std::vector<std::vector<scalar_at>> const& vec
|
|
|
95
96
|
// Try batch requests
|
|
96
97
|
executor_default_t executor;
|
|
97
98
|
index.reserve({vectors.size(), executor.size()});
|
|
98
|
-
executor.
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
99
|
+
executor.fixed(vectors.size() - 3, [&](std::size_t thread, std::size_t task) {
|
|
100
|
+
if constexpr (punned_ak) {
|
|
101
|
+
index.add(key_max - task - 3, vectors[task + 3].data(), args...);
|
|
102
|
+
} else {
|
|
103
|
+
index_update_config_t config;
|
|
104
|
+
config.thread = thread;
|
|
105
|
+
index.add(key_max - task - 3, vectors[task + 3].data(), args..., config);
|
|
106
|
+
}
|
|
102
107
|
});
|
|
103
108
|
|
|
109
|
+
// Check for duplicates
|
|
110
|
+
if constexpr (punned_ak) {
|
|
111
|
+
index.reserve({vectors.size() + 1u, executor.size()});
|
|
112
|
+
auto result = index.add(key_first, vector_first, args...);
|
|
113
|
+
expect(!!result == index.multi());
|
|
114
|
+
result.error.release();
|
|
115
|
+
|
|
116
|
+
std::size_t first_key_count = index.count(key_first);
|
|
117
|
+
expect(first_key_count == (1ul + index.multi()));
|
|
118
|
+
}
|
|
119
|
+
|
|
104
120
|
// Search again over mapped index
|
|
105
121
|
// file_head_result_t head = index_dense_metadata("tmp.usearch");
|
|
106
122
|
// expect(head.size == 3);
|
|
107
123
|
index.view("tmp.usearch");
|
|
108
|
-
matched_count = index.search(vector_first, 5, args...).dump_to(
|
|
124
|
+
matched_count = index.search(vector_first, 5, args...).dump_to(matched_keys, matched_distances);
|
|
109
125
|
expect(matched_count == 3);
|
|
110
|
-
expect(
|
|
126
|
+
expect(matched_keys[0] == key_first);
|
|
111
127
|
expect(std::abs(matched_distances[0]) < 0.01);
|
|
112
128
|
|
|
113
129
|
if constexpr (punned_ak) {
|
|
@@ -176,13 +192,16 @@ void test_cosine(std::size_t collection_size, std::size_t dimensions) {
|
|
|
176
192
|
}
|
|
177
193
|
|
|
178
194
|
// Type-punned:
|
|
179
|
-
for (
|
|
180
|
-
std::
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
195
|
+
for (bool multi : {false, true}) {
|
|
196
|
+
for (std::size_t connectivity : {3, 13, 50}) {
|
|
197
|
+
std::printf("- punned with connectivity %zu \n", connectivity);
|
|
198
|
+
using index_t = index_dense_gt<key_t, slot_t>;
|
|
199
|
+
metric_punned_t metric(dimensions, metric_kind_t::cos_k, scalar_kind<scalar_at>());
|
|
200
|
+
index_dense_config_t config(connectivity);
|
|
201
|
+
config.multi = multi;
|
|
202
|
+
index_t index = index_t::make(metric, config);
|
|
203
|
+
test_cosine<true>(index, matrix);
|
|
204
|
+
}
|
|
186
205
|
}
|
|
187
206
|
}
|
|
188
207
|
|
|
@@ -203,10 +222,8 @@ template <typename key_at, typename slot_at> void test_tanimoto(std::size_t dime
|
|
|
203
222
|
std::generate(scalars.begin(), scalars.end(), [] { return static_cast<b1x8_t>(std::rand()); });
|
|
204
223
|
|
|
205
224
|
index.reserve({batch_size + index.size(), executor.size()});
|
|
206
|
-
executor.
|
|
207
|
-
|
|
208
|
-
config.thread = thread;
|
|
209
|
-
index.add(task + 25000, scalars.data() + index.scalar_words() * task, config);
|
|
225
|
+
executor.fixed(batch_size, [&](std::size_t thread, std::size_t task) {
|
|
226
|
+
index.add(task + 25000, scalars.data() + index.scalar_words() * task, thread);
|
|
210
227
|
});
|
|
211
228
|
}
|
|
212
229
|
|
package/docs/benchmarks.md
CHANGED
|
@@ -191,6 +191,14 @@ mkdir -p datasets/deep_1B/ && \
|
|
|
191
191
|
wget -nc https://storage.yandexcloud.net/yandex-research/ann-datasets/DEEP/groundtruth.public.10K.ibin -P datasets/deep_1B/
|
|
192
192
|
```
|
|
193
193
|
|
|
194
|
+
### Arxiv with E5
|
|
195
|
+
|
|
196
|
+
```sh
|
|
197
|
+
mkdir -p datasets/arxiv_2M/ && \
|
|
198
|
+
wget -nc https://huggingface.co/datasets/unum-cloud/ann-arxiv-2m/resolve/main/abstract.e5-base-v2.fbin -P datasets/arxiv_2M/ &&
|
|
199
|
+
wget -nc https://huggingface.co/datasets/unum-cloud/ann-arxiv-2m/resolve/main/title.e5-base-v2.fbin -P datasets/arxiv_2M/
|
|
200
|
+
```
|
|
201
|
+
|
|
194
202
|
## Profiling
|
|
195
203
|
|
|
196
204
|
With `perf`:
|
package/docs/compilation.md
CHANGED
|
@@ -53,7 +53,7 @@ cppcheck --enable=all --force --suppress=cstyleCast --suppress=unusedFunction \
|
|
|
53
53
|
Testing:
|
|
54
54
|
|
|
55
55
|
```sh
|
|
56
|
-
cmake -
|
|
56
|
+
cmake -DCMAKE_BUILD_TYPE=Debug -B ./build_debug && make -C ./build_debug && ./build_debug/test
|
|
57
57
|
```
|
|
58
58
|
|
|
59
59
|
## Python 3
|
|
@@ -63,7 +63,7 @@ The `-s` option will disable capturing the logs.
|
|
|
63
63
|
The `-x` option will exit after first failure to simplify debugging.
|
|
64
64
|
|
|
65
65
|
```sh
|
|
66
|
-
pip install -e . && pytest python/scripts/
|
|
66
|
+
pip install -e . && pytest python/scripts/ -s -x
|
|
67
67
|
```
|
|
68
68
|
|
|
69
69
|
Linting:
|
|
@@ -134,7 +134,6 @@ g++ -shared -o USearchJNI.dll cloud_unum_usearch_Index.o -Wl,--add-stdcall-alias
|
|
|
134
134
|
g++ -std=c++11 -c -fPIC \
|
|
135
135
|
-I../../../../include \
|
|
136
136
|
-I../../../../fp16/include \
|
|
137
|
-
-I../../../../robin-map/include \
|
|
138
137
|
-I../../../../simsimd/include \
|
|
139
138
|
-I${JAVA_HOME}/include -I${JAVA_HOME}/include/darwin cloud_unum_usearch_Index.cpp -o cloud_unum_usearch_Index.o
|
|
140
139
|
g++ -dynamiclib -o libusearch.dylib cloud_unum_usearch_Index.o -lc
|
|
@@ -156,19 +155,19 @@ There are a few ways to compile the C 99 USearch SDK.
|
|
|
156
155
|
Using the Makefile:
|
|
157
156
|
|
|
158
157
|
```sh
|
|
159
|
-
make -C ./c
|
|
158
|
+
make -C ./c libusearch_c.so -C ./c libusearch_c.so
|
|
160
159
|
```
|
|
161
160
|
|
|
162
161
|
Using CMake:
|
|
163
162
|
|
|
164
163
|
```sh
|
|
165
|
-
cmake -B ./build_release -
|
|
164
|
+
cmake -B ./build_release -DUSEARCH_BUILD_CLIB=1 && make -C ./build_release -j
|
|
166
165
|
```
|
|
167
166
|
|
|
168
167
|
Linux:
|
|
169
168
|
|
|
170
169
|
```sh
|
|
171
|
-
g++ -std=c++11 -shared -fPIC c/lib.cpp -I ./include/ -I ./fp16/include/ -
|
|
170
|
+
g++ -std=c++11 -shared -fPIC c/lib.cpp -I ./include/ -I ./fp16/include/ -o libusearch_c.so
|
|
172
171
|
```
|
|
173
172
|
|
|
174
173
|
|
|
@@ -178,7 +177,8 @@ GoLang bindings are based on C.
|
|
|
178
177
|
So one should first compile the C library, link it with GoLang, and only then run tests.
|
|
179
178
|
|
|
180
179
|
```sh
|
|
181
|
-
|
|
180
|
+
make -C ./c libusearch_c.so && mv ./c/libusearch_c.so ./golang/libusearch_c.so
|
|
181
|
+
cd golang && go test -v ; cd ..
|
|
182
182
|
```
|
|
183
183
|
|
|
184
184
|
## Wolfram
|
package/docs/conf.py
CHANGED
|
@@ -6,45 +6,48 @@
|
|
|
6
6
|
# -- Project information -----------------------------------------------------
|
|
7
7
|
# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
|
|
8
8
|
|
|
9
|
-
project =
|
|
10
|
-
copyright =
|
|
11
|
-
author =
|
|
12
|
-
release = open(
|
|
13
|
-
with open(
|
|
9
|
+
project = "Unum · USearch"
|
|
10
|
+
copyright = "2023, Unum"
|
|
11
|
+
author = "Unum"
|
|
12
|
+
release = open("../VERSION", "r").read().strip()
|
|
13
|
+
with open("_static/custom.js", "r+") as js:
|
|
14
14
|
content = js.read()
|
|
15
15
|
js.seek(0)
|
|
16
16
|
js.truncate()
|
|
17
|
-
js.write(content.replace(
|
|
17
|
+
js.write(content.replace("$(VERSION)", release))
|
|
18
18
|
|
|
19
19
|
# -- General configuration ---------------------------------------------------
|
|
20
20
|
# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
|
|
21
21
|
|
|
22
22
|
extensions = [
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
23
|
+
"breathe",
|
|
24
|
+
"m2r2",
|
|
25
|
+
"sphinx.ext.autodoc",
|
|
26
|
+
"sphinx_js",
|
|
27
|
+
"sphinx.ext.autosummary",
|
|
28
|
+
"sphinx.ext.intersphinx",
|
|
29
|
+
"sphinx.ext.napoleon",
|
|
30
|
+
"sphinxcontrib.jquery",
|
|
31
|
+
"sphinxcontrib.googleanalytics",
|
|
32
|
+
]
|
|
30
33
|
|
|
31
|
-
exclude_patterns = [
|
|
34
|
+
exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", "*.md"]
|
|
32
35
|
|
|
33
36
|
|
|
34
|
-
googleanalytics_id =
|
|
37
|
+
googleanalytics_id = "341385789"
|
|
35
38
|
googleanalytics_enabled = True
|
|
36
39
|
|
|
37
40
|
# -- Options for HTML output -------------------------------------------------
|
|
38
41
|
# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
|
|
39
42
|
|
|
40
|
-
html_logo =
|
|
41
|
-
html_theme =
|
|
42
|
-
html_static_path = [
|
|
43
|
-
html_css_files = [
|
|
44
|
-
html_js_files = [
|
|
45
|
-
html_baseurl =
|
|
43
|
+
html_logo = "../assets/unum.png"
|
|
44
|
+
html_theme = "furo"
|
|
45
|
+
html_static_path = ["_static"]
|
|
46
|
+
html_css_files = ["custom.css"]
|
|
47
|
+
html_js_files = ["custom.js"]
|
|
48
|
+
html_baseurl = "/docs/usearch/"
|
|
46
49
|
|
|
47
|
-
breathe_projects = {
|
|
48
|
-
breathe_default_project =
|
|
50
|
+
breathe_projects = {"USearch": "../build/xml"}
|
|
51
|
+
breathe_default_project = "USearch"
|
|
49
52
|
|
|
50
|
-
js_source_path =
|
|
53
|
+
js_source_path = "../javascript/docs.js"
|
package/golang/README.md
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
```golang
|
|
6
6
|
import (
|
|
7
|
-
"github.com/unum-cloud/usearch/golang"
|
|
7
|
+
"github.com/unum-cloud/usearch/golang-go"
|
|
8
8
|
)
|
|
9
9
|
```
|
|
10
10
|
|
|
@@ -15,14 +15,38 @@ package main
|
|
|
15
15
|
|
|
16
16
|
import (
|
|
17
17
|
"fmt"
|
|
18
|
-
"github.com/unum-cloud/usearch/golang"
|
|
18
|
+
"github.com/unum-cloud/usearch/golang-go"
|
|
19
19
|
)
|
|
20
20
|
|
|
21
21
|
func main() {
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
22
|
+
dim := uint(128)
|
|
23
|
+
conf := DefaultConfig(dim)
|
|
24
|
+
ind, err := NewIndex(conf)
|
|
25
|
+
if err != nil {
|
|
26
|
+
panic("Failed to construct the index: %s", err)
|
|
27
|
+
}
|
|
28
|
+
defer ind.Destroy()
|
|
29
|
+
|
|
30
|
+
err = ind.Reserve(100)
|
|
31
|
+
if err != nil {
|
|
32
|
+
panic("Failed to reserve capacity: %s", err)
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
vec := make([]float32, dim)
|
|
36
|
+
vec[0] = 40.0
|
|
37
|
+
vec[1] = 2.0
|
|
38
|
+
|
|
39
|
+
err = ind.Add(42, vec)
|
|
40
|
+
if err != nil {
|
|
41
|
+
panic("Failed to insert: %s", err)
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
keys, distances, err := ind.Search(vec, 10)
|
|
45
|
+
if err != nil {
|
|
46
|
+
panic("Failed to search: %s", err)
|
|
47
|
+
}
|
|
48
|
+
if keys[0] != 42 || distances[0] != 0.0 {
|
|
49
|
+
panic("Expected result 42 with distance 0, got key %d with distance %f", keys[0], distances[0])
|
|
50
|
+
}
|
|
27
51
|
}
|
|
28
52
|
```
|