usearch 2.8.3 → 2.8.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/release.yml +4 -2
- package/.vscode/settings.json +179 -179
- package/CITATION.cff +1 -1
- package/CONTRIBUTING.md +268 -6
- package/Cargo.toml +1 -1
- package/README.md +64 -59
- package/build.gradle +7 -9
- package/conanfile.py +1 -1
- package/cpp/README.md +89 -21
- package/csharp/DEV_SETUP.md +1 -1
- package/csharp/nuget/nuget-package.props +1 -1
- package/docs/{compilation.rst → contributing.rst} +2 -2
- package/docs/index.rst +1 -1
- package/include/usearch/index.hpp +1 -1
- package/include/usearch/index_dense.hpp +11 -7
- package/java/cloud/unum/usearch/Index.java +119 -53
- package/package.json +1 -1
- package/python/README.md +1 -1
- package/simsimd/CMakeLists.txt +1 -1
- package/simsimd/README.md +7 -1
- package/simsimd/include/simsimd/probability.h +24 -24
- package/simsimd/include/simsimd/types.h +8 -0
- package/simsimd/javascript/{test.js → test/simsimd.test.js} +15 -15
- package/simsimd/package.json +4 -8
- package/simsimd/setup.py +3 -0
- package/wasmer.toml +1 -1
- package/docs/compilation.md +0 -215
- package/simsimd/.clang-format +0 -10
- package/simsimd/.github/workflows/prerelease.yml +0 -89
- package/simsimd/.github/workflows/release.yml +0 -141
- package/simsimd/.github/workflows/update_version.sh +0 -5
- package/simsimd/.vscode/launch.json +0 -53
- package/simsimd/.vscode/settings.json +0 -114
- package/simsimd/.vscode/tasks.json +0 -23
- package/simsimd/go.mod +0 -3
- package/simsimd/golang/bench_test.go +0 -40
- package/simsimd/golang/simsimd.go +0 -47
- package/simsimd/golang/unit_test.go +0 -40
- package/simsimd/pyproject.toml +0 -42
- package/simsimd/python/bench.ipynb +0 -187
- package/simsimd/python/bench.py +0 -268
- package/simsimd/python/lib.c +0 -492
- package/simsimd/python/test.py +0 -195
package/Cargo.toml
CHANGED
package/README.md
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
<h1 align="center">USearch</h1>
|
|
2
2
|
<h3 align="center">
|
|
3
|
-
|
|
4
|
-
Search Engine for Vectors & Texts
|
|
3
|
+
Smaller & <a href="https://www.unum.cloud/blog/2023-11-11-scaling-vector-search">Faster</a> Single-File<br/>
|
|
4
|
+
Similarity Search Engine for <a href="https://github.com/ashvardanian/simsimd">Vectors</a> & 🔜 <a href="https://github.com/ashvardanian/stringzilla">Texts</a>
|
|
5
5
|
</h3>
|
|
6
6
|
<br/>
|
|
7
7
|
|
|
@@ -18,7 +18,7 @@ Search Engine for Vectors & Texts<br/>
|
|
|
18
18
|
</p>
|
|
19
19
|
|
|
20
20
|
<p align="center">
|
|
21
|
-
|
|
21
|
+
Spatial • Binary • Probabilistic • User-Defined Metrics
|
|
22
22
|
<br/>
|
|
23
23
|
<a href="https://unum-cloud.github.io/usearch/cpp">C++ 11</a> •
|
|
24
24
|
<a href="https://unum-cloud.github.io/usearch/python">Python 3</a> •
|
|
@@ -32,54 +32,72 @@ Euclidean • Angular • Bitwise • Haversine • User-Defined Metrics
|
|
|
32
32
|
<a href="https://unum-cloud.github.io/usearch/golang">GoLang</a> •
|
|
33
33
|
<a href="https://unum-cloud.github.io/usearch/wolfram">Wolfram</a>
|
|
34
34
|
<br/>
|
|
35
|
-
Linux • MacOS • Windows • iOS •
|
|
35
|
+
Linux • MacOS • Windows • iOS • WebAssembly
|
|
36
36
|
</p>
|
|
37
37
|
|
|
38
38
|
<div align="center">
|
|
39
|
-
<a href="https://pypi.org/project/usearch/"> <img alt="PyPI
|
|
40
|
-
<a href="https://www.npmjs.com/package/usearch"> <img alt="
|
|
41
|
-
<a href="https://crates.io/crates/usearch"> <img alt="
|
|
42
|
-
<a href="https://www.nuget.org/packages/Cloud.Unum.USearch"> <img alt="
|
|
43
|
-
<a href="https://
|
|
44
|
-
<
|
|
39
|
+
<a href="https://pypi.org/project/usearch/"> <img alt="PyPI" src="https://img.shields.io/pypi/dm/usearch?label=PyPi%20pulls"> </a>
|
|
40
|
+
<a href="https://www.npmjs.com/package/usearch"> <img alt="NPM" src="https://img.shields.io/npm/dy/usearch?label=NPM%20pulls"> </a>
|
|
41
|
+
<a href="https://crates.io/crates/usearch"> <img alt="Crate" src="https://img.shields.io/crates/d/usearch?label=Crate%20pulls"> </a>
|
|
42
|
+
<a href="https://www.nuget.org/packages/Cloud.Unum.USearch"> <img alt="NuGet" src="https://img.shields.io/nuget/dt/Cloud.Unum.USearch?label=NuGet%20pulls"> </a>
|
|
43
|
+
<a href="https://central.sonatype.com/artifact/cloud.unum/usearch/overview"> <img alt="Maven" src="https://img.shields.io/nexus/r/cloud.unum/usearch?server=https%3A%2F%2Fs01.oss.sonatype.org%2F&label=Maven%20version"> </a>
|
|
44
|
+
<a href="https://hub.docker.com/r/unum/usearch"> <img alt="Docker" src="https://img.shields.io/docker/pulls/unum/usearch?label=Docker%20pulls"> </a>
|
|
45
|
+
<img alt="GitHub code size in bytes" src="https://img.shields.io/github/languages/code-size/unum-cloud/usearch?label=Repo%20size">
|
|
45
46
|
</div>
|
|
46
47
|
|
|
47
48
|
---
|
|
48
49
|
|
|
49
|
-
- ✅
|
|
50
|
+
- ✅ 10x faster than [FAISS][faiss] implementation of [HNSW algorithm][hnsw-algorithm].
|
|
50
51
|
- ✅ Simple and extensible [single C++11 header][usearch-header] implementation.
|
|
51
|
-
- ✅
|
|
52
|
-
- ✅
|
|
53
|
-
- ✅ Bitwise Tanimoto and Sorensen coefficients for [Genomics and Chemistry applications](#usearch--rdkit--molecular-search).
|
|
52
|
+
- ✅ Compatible with a dozen programming languages out of the box.
|
|
53
|
+
- ✅ [SIMD][simd]-optimized and [user-defined metrics](#user-defined-functions) with JIT compilation.
|
|
54
54
|
- ✅ Hardware-agnostic `f16` & `i8` - [half-precision & quarter-precision support](#memory-efficiency-downcasting-and-quantization).
|
|
55
55
|
- ✅ [View large indexes from disk](#serving-index-from-disk) without loading into RAM.
|
|
56
|
+
- ✅ Heterogeneous lookups, renaming/relabeling, and on-the-fly deletions.
|
|
57
|
+
- ✅ Variable dimensionality vectors for unique applications, including search over compressed data.
|
|
58
|
+
- ✅ Binary Tanimoto and Sorensen coefficients for [Genomics and Chemistry applications](#usearch--rdkit--molecular-search).
|
|
56
59
|
- ✅ Space-efficient point-clouds with `uint40_t`, accommodating 4B+ size.
|
|
57
60
|
- ✅ Compatible with OpenMP and custom "executors", for fine-grained control over CPU utilization.
|
|
58
|
-
- ✅
|
|
59
|
-
- ✅ Near-real-time [clustering and sub-clusterings](#clustering) for Tens or Millions of clusters.
|
|
61
|
+
- ✅ Near-real-time [clustering and sub-clustering](#clustering) for Tens or Millions of clusters.
|
|
60
62
|
- ✅ [Semantic Search](#usearch--ai--multi-modal-semantic-search) and [Joins](#joins).
|
|
61
63
|
|
|
64
|
+
[faiss]: https://github.com/facebookresearch/faiss
|
|
62
65
|
[usearch-header]: https://github.com/unum-cloud/usearch/blob/main/include/usearch/index.hpp
|
|
63
66
|
[obscure-use-cases]: https://ashvardanian.com/posts/abusing-vector-search
|
|
67
|
+
[hnsw-algorithm]: https://arxiv.org/abs/1603.09320
|
|
68
|
+
[simd]: https://en.wikipedia.org/wiki/Single_instruction,_multiple_data
|
|
64
69
|
|
|
65
70
|
## Comparison with FAISS
|
|
66
71
|
|
|
67
72
|
FAISS is a widely recognized standard for high-performance vector search engines.
|
|
68
73
|
USearch and FAISS both employ the same HNSW algorithm, but they differ significantly in their design principles.
|
|
69
|
-
USearch is compact and broadly compatible without sacrificing performance,
|
|
70
|
-
|
|
71
|
-
|
|
|
72
|
-
|
|
|
73
|
-
|
|
|
74
|
-
|
|
|
75
|
-
|
|
|
76
|
-
|
|
|
77
|
-
|
|
|
78
|
-
|
|
|
79
|
-
|
|
|
74
|
+
USearch is compact and broadly compatible without sacrificing performance, primarily focusing on user-defined metrics and fewer dependencies.
|
|
75
|
+
|
|
76
|
+
| | FAISS | USearch | Improvement |
|
|
77
|
+
| :------------------------------------------- | ----------------------------: | -----------------------------: | -----------------------: |
|
|
78
|
+
| Indexing time | | | |
|
|
79
|
+
| 100 Million 96d `f32`, `f16`, `i8` vectors | 2.6 h, 2.6 h, 2.6 h | 0.3 h, 0.2 h, 0.2 h | __9.6x, 10.4x, 10.7x__ ⁰ |
|
|
80
|
+
| 100 Million 1536d `f32`, `f16`, `i8` vectors | 5.0 h, 4.1 h, 3.8 h | 2.1 h, 1.1 h, 0.8 h | __2.3x 3.6x, 4.4x__ ⁰ |
|
|
81
|
+
| | | | |
|
|
82
|
+
| Codebase length | 84 K [SLOC][sloc] in `faiss/` | 3 K [SLOC][sloc] in `usearch/` | maintainable ¹ |
|
|
83
|
+
| Supported metrics | 9 fixed metrics | any user-defined metrics | extendible ² |
|
|
84
|
+
| Supported languages | C++, Python | 10 languages | portable ³ |
|
|
85
|
+
| Supported ID types | 32-bit, 64-bit | 32-bit, 40-bit, 64-bit | efficient ⁴ |
|
|
86
|
+
| Required dependencies | BLAS, OpenMP | - | light-weight ⁵ |
|
|
87
|
+
| Bindings | SWIG | Native | low-latency ⁶ |
|
|
80
88
|
|
|
81
89
|
[sloc]: https://en.wikipedia.org/wiki/Source_lines_of_code
|
|
82
90
|
|
|
91
|
+
> ⁰ [Tested][intel-benchmarks] on Intel Sapphire Rapids, with the simplest inner-product distance, equivalent recall, and memory consumption, while also providing far superior search speed.
|
|
92
|
+
> ¹ A shorter codebase makes the project easier to maintain and audit.
|
|
93
|
+
> ² User-defined metrics allow you to customize your search for various applications, from GIS to creating custom metrics for composite embeddings from multiple AI models or hybrid full-text and semantic search.
|
|
94
|
+
> ³ With USearch, you can reuse the same preconstructed index in various programming languages.
|
|
95
|
+
> ⁴ The 40-bit integer allows you to store 4B+ vectors without allocating 8 bytes for every neighbor reference in the proximity graph.
|
|
96
|
+
> ⁵ Lack of obligatory dependencies makes USearch much more portable.
|
|
97
|
+
> ⁶ Native bindings introduce lower call latencies than more straightforward approaches.
|
|
98
|
+
|
|
99
|
+
[intel-benchmarks]: https://www.unum.cloud/blog/2023-11-11-scaling-vector-search
|
|
100
|
+
|
|
83
101
|
Base functionality is identical to FAISS, and the interface must be familiar if you have ever investigated Approximate Nearest Neighbors search:
|
|
84
102
|
|
|
85
103
|
```py
|
|
@@ -88,44 +106,30 @@ $ pip install usearch
|
|
|
88
106
|
import numpy as np
|
|
89
107
|
from usearch.index import Index
|
|
90
108
|
|
|
91
|
-
index = Index(
|
|
92
|
-
ndim=3, # Define the number of dimensions in input vectors
|
|
93
|
-
metric='cos', # Choose 'l2sq', 'haversine' or other metric, default = 'ip'
|
|
94
|
-
dtype='f32', # Quantize to 'f16' or 'i8' if needed, default = 'f32'
|
|
95
|
-
connectivity=16, # Optional: How frequent should the connections in the graph be
|
|
96
|
-
expansion_add=128, # Optional: Control the recall of indexing
|
|
97
|
-
expansion_search=64, # Optional: Control the quality of search
|
|
98
|
-
)
|
|
109
|
+
index = Index(ndim=3)
|
|
99
110
|
|
|
100
111
|
vector = np.array([0.2, 0.6, 0.4])
|
|
101
112
|
index.add(42, vector)
|
|
113
|
+
|
|
102
114
|
matches = index.search(vector, 10)
|
|
103
115
|
|
|
104
|
-
assert len(index) == 1
|
|
105
|
-
assert len(matches) == 1
|
|
106
116
|
assert matches[0].key == 42
|
|
107
117
|
assert matches[0].distance <= 0.001
|
|
108
118
|
assert np.allclose(index[42], vector)
|
|
109
119
|
```
|
|
110
120
|
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
| | FAISS, `f32` | USearch, `f32` | USearch, `f16` | USearch, `i8` |
|
|
114
|
-
| :----------- | -----------: | -------------: | -------------: | ----------------: |
|
|
115
|
-
| Batch Insert | 16 K/s | 73 K/s | 100 K/s | 104 K/s __+550%__ |
|
|
116
|
-
| Batch Search | 82 K/s | 103 K/s | 113 K/s | 134 K/s __+63%__ |
|
|
117
|
-
| Bulk Insert | 76 K/s | 105 K/s | 115 K/s | 202 K/s __+165%__ |
|
|
118
|
-
| Bulk Search | 118 K/s | 174 K/s | 173 K/s | 304 K/s __+157%__ |
|
|
119
|
-
| Recall @ 10 | 99% | 99.2% | 99.1% | 99.2% |
|
|
121
|
+
More settings are always available, and the API is designed to be as flexible as possible.
|
|
120
122
|
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
123
|
+
```py
|
|
124
|
+
index = Index(
|
|
125
|
+
ndim=3, # Define the number of dimensions in input vectors
|
|
126
|
+
metric='cos', # Choose 'l2sq', 'haversine' or other metric, default = 'ip'
|
|
127
|
+
dtype='f32', # Quantize to 'f16' or 'i8' if needed, default = 'f32'
|
|
128
|
+
connectivity=16, # Optional: How frequent should the connections in the graph be
|
|
129
|
+
expansion_add=128, # Optional: Control the recall of indexing
|
|
130
|
+
expansion_search=64, # Optional: Control the quality of search
|
|
131
|
+
)
|
|
132
|
+
```
|
|
129
133
|
|
|
130
134
|
## User-Defined Functions
|
|
131
135
|
|
|
@@ -220,7 +224,7 @@ multi_index.search(...)
|
|
|
220
224
|
## Clustering
|
|
221
225
|
|
|
222
226
|
Once the index is constructed, it can be used to cluster entries much faster.
|
|
223
|
-
In
|
|
227
|
+
In essence, the `Index` itself can be seen as a clustering, and it allows iterative deepening.
|
|
224
228
|
|
|
225
229
|
```py
|
|
226
230
|
clustering = index.cluster(
|
|
@@ -241,7 +245,7 @@ g = clustering.network
|
|
|
241
245
|
# Get members of a specific cluster
|
|
242
246
|
first_members = clustering.members_of(centroid_keys[0])
|
|
243
247
|
|
|
244
|
-
# Deepen into that cluster
|
|
248
|
+
# Deepen into that cluster splitting it into more parts, all same arguments supported
|
|
245
249
|
sub_clustering = clustering.subcluster(min_count=..., max_count=...)
|
|
246
250
|
```
|
|
247
251
|
|
|
@@ -342,7 +346,7 @@ Comparing molecule graphs and searching for similar structures is expensive and
|
|
|
342
346
|
It can be seen as a special case of the NP-Complete Subgraph Isomorphism problem.
|
|
343
347
|
Luckily, domain-specific approximate methods exist.
|
|
344
348
|
The one commonly used in Chemistry, is to generate structures from [SMILES][smiles], and later hash them into binary fingerprints.
|
|
345
|
-
The latter are searchable with
|
|
349
|
+
The latter are searchable with binary similarity metrics, like the Tanimoto coefficient.
|
|
346
350
|
Below is an example using the RDKit package.
|
|
347
351
|
|
|
348
352
|
```python
|
|
@@ -380,9 +384,10 @@ The search is based on the Haversine distance metric, but can easily be extended
|
|
|
380
384
|
## Integrations
|
|
381
385
|
|
|
382
386
|
- [x] GPTCache: [Python](https://github.com/zilliztech/GPTCache/releases/tag/0.1.29).
|
|
383
|
-
- [x] LangChain: [Python](https://github.com/langchain-ai/langchain/releases/tag/v0.0.257) and [
|
|
387
|
+
- [x] LangChain: [Python](https://github.com/langchain-ai/langchain/releases/tag/v0.0.257) and [JavaScript](https://github.com/hwchase17/langchainjs/releases/tag/0.0.125).
|
|
384
388
|
- [x] ClickHouse: [C++](https://github.com/ClickHouse/ClickHouse/pull/53447).
|
|
385
389
|
- [x] Microsoft Semantic Kernel: [Python](https://github.com/microsoft/semantic-kernel/releases/tag/python-0.3.9.dev) and C#.
|
|
390
|
+
- [x] LanternDB: [C++](https://github.com/lanterndata/lantern) and [Rust](https://github.com/lanterndata/lantern_extras).
|
|
386
391
|
|
|
387
392
|
## Citations
|
|
388
393
|
|
|
@@ -392,7 +397,7 @@ doi = {10.5281/zenodo.7949416},
|
|
|
392
397
|
author = {Vardanian, Ash},
|
|
393
398
|
title = {{USearch by Unum Cloud}},
|
|
394
399
|
url = {https://github.com/unum-cloud/usearch},
|
|
395
|
-
version = {2.8.
|
|
400
|
+
version = {2.8.5},
|
|
396
401
|
year = {2023},
|
|
397
402
|
month = oct,
|
|
398
403
|
}
|
package/build.gradle
CHANGED
|
@@ -109,11 +109,11 @@ jar {
|
|
|
109
109
|
}
|
|
110
110
|
}
|
|
111
111
|
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
112
|
+
signing {
|
|
113
|
+
def signingKey = findProperty("signingKey")
|
|
114
|
+
def signingPassword = findProperty("signingPassword")
|
|
115
|
+
useInMemoryPgpKeys(signingKey, signingPassword)
|
|
116
|
+
sign publishing.publications
|
|
117
117
|
}
|
|
118
118
|
|
|
119
119
|
publishing {
|
|
@@ -124,10 +124,8 @@ publishing {
|
|
|
124
124
|
artifact sourcesJar
|
|
125
125
|
artifact javadocJar
|
|
126
126
|
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
sign publishing.publications.library
|
|
130
|
-
}
|
|
127
|
+
signing {
|
|
128
|
+
sign publishing.publications.library
|
|
131
129
|
}
|
|
132
130
|
|
|
133
131
|
pom {
|
package/conanfile.py
CHANGED
|
@@ -7,7 +7,7 @@ required_conan_version = ">=1.53.0"
|
|
|
7
7
|
class USearchConan(ConanFile):
|
|
8
8
|
|
|
9
9
|
name = "usearch"
|
|
10
|
-
version = '2.8.
|
|
10
|
+
version = '2.8.5'
|
|
11
11
|
license = "Apache-2.0"
|
|
12
12
|
description = "Smaller & Faster Single-File Vector Search Engine from Unum"
|
|
13
13
|
homepage = "https://github.com/unum-cloud/usearch"
|
package/cpp/README.md
CHANGED
|
@@ -25,14 +25,20 @@ metric_punned_t metric(256, metric_kind_t::l2sq_k, scalar_kind_t::f32_k);
|
|
|
25
25
|
index_dense_t index = index_dense_t::make(metric);
|
|
26
26
|
float vec[3] = {0.1, 0.3, 0.2};
|
|
27
27
|
|
|
28
|
-
index.reserve(10);
|
|
29
|
-
index.add(
|
|
30
|
-
auto results = index.search(
|
|
28
|
+
index.reserve(10); // Pre-allocate memory for 10 vectors
|
|
29
|
+
index.add(42, &vec[0]); // Pass a key and a vector
|
|
30
|
+
auto results = index.search(&vec[0], 5); // Pass a query and limit number of results
|
|
31
31
|
|
|
32
32
|
for (std::size_t i = 0; i != results.size(); ++i)
|
|
33
33
|
results[i].element.key, results[i].element.vector, results[i].distance;
|
|
34
34
|
```
|
|
35
35
|
|
|
36
|
+
Here we:
|
|
37
|
+
|
|
38
|
+
- define a metric of kind [`metric_kind_t::l2sq_k`](https://unum-cloud.github.io/usearch/cpp/reference.html#_CPPv413metric_kind_t),
|
|
39
|
+
- to be applied to [`scalar_kind_t::f32_k`](https://unum-cloud.github.io/usearch/cpp/reference.html#_CPPv413scalar_kind_t) floating-point vectors,
|
|
40
|
+
- instantiate an [`index_dense_t`](https://unum-cloud.github.io/usearch/cpp/reference.html#_CPPv4I00EN4unum7usearch14index_dense_gtE) index.
|
|
41
|
+
|
|
36
42
|
The `add` is thread-safe for concurrent index construction.
|
|
37
43
|
It also has an overload for different vector types, casting them under the hood.
|
|
38
44
|
The same applies to the `search`, `get`, `cluster`, and `distance_between` functions.
|
|
@@ -52,20 +58,78 @@ index.load("index.usearch"); // Copying from disk
|
|
|
52
58
|
index.view("index.usearch"); // Memory-mapping from disk
|
|
53
59
|
```
|
|
54
60
|
|
|
55
|
-
##
|
|
61
|
+
## Multi-Threading
|
|
56
62
|
|
|
57
|
-
|
|
63
|
+
Most AI, HPC, or Big Data packages use some form of a thread pool.
|
|
64
|
+
Instead of spawning additional threads within USearch, we focus on the thread safety of `add()` function, simplifying resource management.
|
|
58
65
|
|
|
59
66
|
```cpp
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
67
|
+
#pragma omp parallel for
|
|
68
|
+
for (std::size_t i = 0; i < n; ++i)
|
|
69
|
+
native.add(key, span_t{vector, dims});
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
During initialization, we allocate enough temporary memory for all the cores on the machine.
|
|
73
|
+
On the call, the user can supply the identifier of the current thread, making this library easy to integrate with OpenMP and similar tools.
|
|
74
|
+
|
|
75
|
+
Moreover, you can take advantage of one of the provided "executors" to parallelize the search:
|
|
76
|
+
|
|
77
|
+
- `executor_openmp_t`, that would use OpenMP under the hood.
|
|
78
|
+
- `executor_stl_t`, that will spawn `std::thread` instances.
|
|
79
|
+
- `dummy_executor_t`, that will run everything sequentially.
|
|
80
|
+
|
|
81
|
+
## Clustering
|
|
82
|
+
|
|
83
|
+
Aside from basic Create-Read-Update-Delete (CRUD) operations and search, USearch also supports clustering.
|
|
84
|
+
Once the index is constructed, you can either:
|
|
85
|
+
|
|
86
|
+
- Identify a cluster to which any external vector belongs, once mapped onto the index.
|
|
87
|
+
- Split the entire index into a set of clusters, each with its own centroid.
|
|
88
|
+
|
|
89
|
+
For the first, the interface accepts a vector and a "clustering level", which is essentially the index of the HNSW graph layer, in which to search.
|
|
90
|
+
If you pass zero, the traversal will happen in every level except the bottom one.
|
|
91
|
+
Otherwise, the search will be limited to the specified level.
|
|
92
|
+
|
|
93
|
+
```cpp
|
|
94
|
+
some_scalar_t vector[3] = {0.1, 0.3, 0.2};
|
|
95
|
+
cluster_result_t result = index.cluster(&vector, index.max_level() / 2);
|
|
96
|
+
match_t cluster = result.cluster;
|
|
97
|
+
member_cref_t member = cluster.member;
|
|
98
|
+
distance_t distance = cluster.distance;
|
|
66
99
|
```
|
|
67
100
|
|
|
68
|
-
|
|
101
|
+
If you wish to split the whole structure into clusters, you must provide an iterator over a range of vectors, that will be processed in parallel using the previously described function.
|
|
102
|
+
Unlike the previous function, you don't have to manually specify the level, as the algorithm will pick the best one for you, depending on the number of clusters you want to highlight.
|
|
103
|
+
Aside from that auto-tuning, this function will regroup some of the clusters, if they are too small, and return the final number of clusters.
|
|
104
|
+
|
|
105
|
+
```cpp
|
|
106
|
+
std::size_t queries_count = queries_end - queries_begin;
|
|
107
|
+
index_dense_clustering_config_t config;
|
|
108
|
+
config.min_clusters = 1000;
|
|
109
|
+
config.max_clusters = 2000;
|
|
110
|
+
config.mode = index_dense_clustering_config_t::merge_smallest_k;
|
|
111
|
+
|
|
112
|
+
// Outputs:
|
|
113
|
+
vector_key_t cluster_centroids_keys[queries_count];
|
|
114
|
+
distance_t distances_to_cluster_centroids[queries_count];
|
|
115
|
+
executor_default_t thread_pool;
|
|
116
|
+
dummy_progress_t progress_bar;
|
|
117
|
+
|
|
118
|
+
clustering_result_t result = cluster(
|
|
119
|
+
queries_begin, queries_end,
|
|
120
|
+
config,
|
|
121
|
+
&cluster_centroids_keys, &distances_to_cluster_centroids,
|
|
122
|
+
thread_pool, progress_bar);
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
This approach requires basic understanding of templates meta-programming to implement the `queries_begin` and `queries_end` smart-iterators.
|
|
126
|
+
On the bright side, it allows iteratively deepening into a specific cluster.
|
|
127
|
+
|
|
128
|
+
As in many other bulk-processing APIs, the `executor` and `progress` are optional.
|
|
129
|
+
|
|
130
|
+
## User-Defined Metrics
|
|
131
|
+
|
|
132
|
+
In its high-level interface, USearch supports a variety of metrics, including the most popular ones:
|
|
69
133
|
|
|
70
134
|
- `metric_cos_gt<scalar_t>` for "Cosine" or "Angular" distance.
|
|
71
135
|
- `metric_ip_gt<scalar_t>` for "Inner Product" or "Dot Product" distance.
|
|
@@ -78,16 +142,20 @@ The following distances are pre-packaged:
|
|
|
78
142
|
- `metric_haversine_gt<scalar_t>` for "Haversine" or "Great Circle" distance between coordinates used in GIS applications.
|
|
79
143
|
- `metric_divergence_gt<scalar_t>` for the "Jensen Shannon" similarity between probability distributions.
|
|
80
144
|
|
|
81
|
-
|
|
145
|
+
In reality, for most common types, one of the [SimSIMD](https://github.com/ashvardanian/SimSIMD) backends will be triggered, providing hardware-acceleration for most common CPUs.
|
|
82
146
|
|
|
83
|
-
|
|
84
|
-
|
|
147
|
+
If you need a different metric, you can implement it yourself and wrap it into a `metric_punned_t`, which is our alternative to the `std::function`.
|
|
148
|
+
Unlike the `std::function`, it is a trivial type, which is important for performance.
|
|
149
|
+
|
|
150
|
+
## Advanced Interface
|
|
151
|
+
|
|
152
|
+
If you are proficient in C++ and ready to get your hands dirty, you can use the low-level interface.
|
|
85
153
|
|
|
86
154
|
```cpp
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
155
|
+
template <typename distance_at = default_distance_t, // `float`
|
|
156
|
+
typename key_at = default_key_t, // `int64_t`, `uuid_t`
|
|
157
|
+
typename compressed_slot_at = default_slot_t, // `uint32_t`, `uint40_t`
|
|
158
|
+
typename dynamic_allocator_at = std::allocator<byte_t>, //
|
|
159
|
+
typename tape_allocator_at = dynamic_allocator_at> //
|
|
160
|
+
class index_gt;
|
|
90
161
|
```
|
|
91
|
-
|
|
92
|
-
During initialization, we allocate enough temporary memory for all the cores on the machine.
|
|
93
|
-
On the call, the user can supply the identifier of the current thread, making this library easy to integrate with OpenMP and similar tools.
|
package/csharp/DEV_SETUP.md
CHANGED
|
@@ -11,7 +11,7 @@ To obtain all libraries, download the artifacts generated by the GitHub workflow
|
|
|
11
11
|
|
|
12
12
|
To get a library for your current platform (Linux or Macos), execute: `./eng/build_lib.sh`.
|
|
13
13
|
|
|
14
|
-
For more details, refer to the [compilation guide](https://github.com/unum-cloud/usearch/blob/main/
|
|
14
|
+
For more details, refer to the [compilation guide](https://github.com/unum-cloud/usearch/blob/main/CONTRIBUTING.md).
|
|
15
15
|
|
|
16
16
|
# Build and Test
|
|
17
17
|
|
package/docs/index.rst
CHANGED
|
@@ -323,7 +323,7 @@ class index_dense_gt {
|
|
|
323
323
|
/// @brief Schema: input buffer, bytes in input buffer, output buffer.
|
|
324
324
|
using cast_t = std::function<bool(byte_t const*, std::size_t, byte_t*)>;
|
|
325
325
|
/// @brief Punned index.
|
|
326
|
-
using index_t = index_gt<
|
|
326
|
+
using index_t = index_gt< //
|
|
327
327
|
distance_t, vector_key_t, compressed_slot_t, //
|
|
328
328
|
dynamic_allocator_t, tape_allocator_t>;
|
|
329
329
|
using index_allocator_t = aligned_allocator_gt<index_t, 64>;
|
|
@@ -1512,10 +1512,14 @@ class index_dense_gt {
|
|
|
1512
1512
|
* @brief Implements clustering, classifying the given objects (vectors of member keys)
|
|
1513
1513
|
* into a given number of clusters.
|
|
1514
1514
|
*
|
|
1515
|
-
* @param[in] queries_begin Iterator
|
|
1516
|
-
* @param[in] queries_end
|
|
1515
|
+
* @param[in] queries_begin Iterator pointing to the first query.
|
|
1516
|
+
* @param[in] queries_end Iterator pointing to the last query.
|
|
1517
1517
|
* @param[in] executor Thread-pool to execute the job in parallel.
|
|
1518
1518
|
* @param[in] progress Callback to report the execution progress.
|
|
1519
|
+
* @param[in] config Configuration parameters for clustering.
|
|
1520
|
+
*
|
|
1521
|
+
* @param[out] cluster_keys Pointer to the array where the cluster keys will be exported.
|
|
1522
|
+
* @param[out] cluster_distances Pointer to the array where the distances to those centroids will be exported.
|
|
1519
1523
|
*/
|
|
1520
1524
|
template < //
|
|
1521
1525
|
typename queries_iterator_at, //
|
|
@@ -1526,7 +1530,7 @@ class index_dense_gt {
|
|
|
1526
1530
|
queries_iterator_at queries_begin, //
|
|
1527
1531
|
queries_iterator_at queries_end, //
|
|
1528
1532
|
index_dense_clustering_config_t config, //
|
|
1529
|
-
vector_key_t* cluster_keys,
|
|
1533
|
+
vector_key_t* cluster_keys, //
|
|
1530
1534
|
distance_t* cluster_distances, //
|
|
1531
1535
|
executor_at&& executor = executor_at{}, //
|
|
1532
1536
|
progress_at&& progress = progress_at{}) {
|
|
@@ -1715,7 +1719,7 @@ class index_dense_gt {
|
|
|
1715
1719
|
}
|
|
1716
1720
|
|
|
1717
1721
|
template <typename scalar_at>
|
|
1718
|
-
add_result_t add_(
|
|
1722
|
+
add_result_t add_( //
|
|
1719
1723
|
vector_key_t key, scalar_at const* vector, //
|
|
1720
1724
|
std::size_t thread, bool force_vector_copy, cast_t const& cast) {
|
|
1721
1725
|
|
|
@@ -1811,8 +1815,8 @@ class index_dense_gt {
|
|
|
1811
1815
|
}
|
|
1812
1816
|
|
|
1813
1817
|
template <typename scalar_at>
|
|
1814
|
-
aggregated_distances_t distance_between_(
|
|
1815
|
-
vector_key_t key, scalar_at const* vector,
|
|
1818
|
+
aggregated_distances_t distance_between_( //
|
|
1819
|
+
vector_key_t key, scalar_at const* vector, //
|
|
1816
1820
|
std::size_t thread, cast_t const& cast) const {
|
|
1817
1821
|
|
|
1818
1822
|
// Cast the vector, if needed for compatibility with `metric_`
|