vicinity 0.3.0__tar.gz → 0.3.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {vicinity-0.3.0 → vicinity-0.3.2}/Makefile +1 -1
- {vicinity-0.3.0 → vicinity-0.3.2}/PKG-INFO +28 -13
- {vicinity-0.3.0 → vicinity-0.3.2}/README.md +24 -12
- vicinity-0.3.2/assets/images/vicinity_logo.png +0 -0
- {vicinity-0.3.0 → vicinity-0.3.2}/pyproject.toml +4 -1
- {vicinity-0.3.0 → vicinity-0.3.2}/tests/conftest.py +51 -3
- {vicinity-0.3.0 → vicinity-0.3.2}/tests/test_vicinity.py +41 -1
- {vicinity-0.3.0 → vicinity-0.3.2}/uv.lock +19 -1
- {vicinity-0.3.0 → vicinity-0.3.2}/vicinity/backends/__init__.py +7 -2
- {vicinity-0.3.0 → vicinity-0.3.2}/vicinity/backends/annoy.py +13 -11
- {vicinity-0.3.0 → vicinity-0.3.2}/vicinity/backends/base.py +9 -3
- {vicinity-0.3.0 → vicinity-0.3.2}/vicinity/backends/basic.py +87 -61
- {vicinity-0.3.0 → vicinity-0.3.2}/vicinity/backends/faiss.py +2 -3
- {vicinity-0.3.0 → vicinity-0.3.2}/vicinity/backends/hnsw.py +5 -4
- {vicinity-0.3.0 → vicinity-0.3.2}/vicinity/backends/pynndescent.py +3 -6
- {vicinity-0.3.0 → vicinity-0.3.2}/vicinity/backends/usearch.py +5 -5
- vicinity-0.3.2/vicinity/backends/voyager.py +116 -0
- {vicinity-0.3.0 → vicinity-0.3.2}/vicinity/datatypes.py +1 -1
- {vicinity-0.3.0 → vicinity-0.3.2}/vicinity/version.py +1 -1
- {vicinity-0.3.0 → vicinity-0.3.2}/vicinity/vicinity.py +33 -5
- {vicinity-0.3.0 → vicinity-0.3.2}/vicinity.egg-info/PKG-INFO +28 -13
- {vicinity-0.3.0 → vicinity-0.3.2}/vicinity.egg-info/SOURCES.txt +3 -1
- {vicinity-0.3.0 → vicinity-0.3.2}/vicinity.egg-info/requires.txt +4 -0
- {vicinity-0.3.0 → vicinity-0.3.2}/.github/workflows/ci.yaml +0 -0
- {vicinity-0.3.0 → vicinity-0.3.2}/.gitignore +0 -0
- {vicinity-0.3.0 → vicinity-0.3.2}/.pre-commit-config.yaml +0 -0
- {vicinity-0.3.0 → vicinity-0.3.2}/LICENSE +0 -0
- {vicinity-0.3.0 → vicinity-0.3.2}/setup.cfg +0 -0
- {vicinity-0.3.0 → vicinity-0.3.2}/tests/test_utils.py +0 -0
- {vicinity-0.3.0 → vicinity-0.3.2}/vicinity/__init__.py +0 -0
- {vicinity-0.3.0 → vicinity-0.3.2}/vicinity/py.typed +0 -0
- {vicinity-0.3.0 → vicinity-0.3.2}/vicinity/utils.py +0 -0
- {vicinity-0.3.0 → vicinity-0.3.2}/vicinity.egg-info/dependency_links.txt +0 -0
- {vicinity-0.3.0 → vicinity-0.3.2}/vicinity.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: vicinity
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.2
|
|
4
4
|
Summary: Lightweight Nearest Neighbors with Flexible Backends
|
|
5
5
|
Author-email: Stéphan Tulkens <stephantul@gmail.com>, Thomas van Dongen <thomas123@live.nl>
|
|
6
6
|
License: MIT License
|
|
@@ -67,6 +67,8 @@ Provides-Extra: faiss
|
|
|
67
67
|
Requires-Dist: faiss-cpu; extra == "faiss"
|
|
68
68
|
Provides-Extra: usearch
|
|
69
69
|
Requires-Dist: usearch; extra == "usearch"
|
|
70
|
+
Provides-Extra: voyager
|
|
71
|
+
Requires-Dist: voyager; extra == "voyager"
|
|
70
72
|
Provides-Extra: all
|
|
71
73
|
Requires-Dist: hnswlib; extra == "all"
|
|
72
74
|
Requires-Dist: pynndescent>=0.5.10; extra == "all"
|
|
@@ -76,13 +78,19 @@ Requires-Dist: numpy>=1.24.0; extra == "all"
|
|
|
76
78
|
Requires-Dist: annoy; extra == "all"
|
|
77
79
|
Requires-Dist: faiss-cpu; extra == "all"
|
|
78
80
|
Requires-Dist: usearch; extra == "all"
|
|
81
|
+
Requires-Dist: voyager; extra == "all"
|
|
79
82
|
|
|
80
|
-
<div align="center">
|
|
81
|
-
|
|
82
|
-
# Vicinity: Lightweight Nearest Neighbors
|
|
83
83
|
|
|
84
|
+
<div align="center">
|
|
85
|
+
<picture>
|
|
86
|
+
<img width="40%" alt="Model2Vec logo" src="assets/images/vicinity_logo.png">
|
|
87
|
+
</picture>
|
|
88
|
+
</a>
|
|
84
89
|
</div>
|
|
85
90
|
|
|
91
|
+
<div align="center">
|
|
92
|
+
<h2>Lightweight Nearest Neighbors with Flexible Backends</h2>
|
|
93
|
+
</div>
|
|
86
94
|
|
|
87
95
|
<div align="center">
|
|
88
96
|
<h2>
|
|
@@ -118,7 +126,7 @@ Install the package with:
|
|
|
118
126
|
```bash
|
|
119
127
|
pip install vicinity
|
|
120
128
|
```
|
|
121
|
-
|
|
129
|
+
Optionally, [install any of the supported backends](#installation), or simply install all of them with:
|
|
122
130
|
```bash
|
|
123
131
|
pip install vicinity[all]
|
|
124
132
|
```
|
|
@@ -127,15 +135,19 @@ pip install vicinity[all]
|
|
|
127
135
|
The following code snippet demonstrates how to use Vicinity for nearest neighbor search:
|
|
128
136
|
```python
|
|
129
137
|
import numpy as np
|
|
130
|
-
from vicinity import Vicinity
|
|
131
|
-
from vicinity.datatypes import Backend, Metric
|
|
138
|
+
from vicinity import Vicinity, Backend, Metric
|
|
132
139
|
|
|
133
140
|
# Create some dummy data
|
|
134
141
|
items = ["triforce", "master sword", "hylian shield", "boomerang", "hookshot"]
|
|
135
142
|
vectors = np.random.rand(len(items), 128)
|
|
136
143
|
|
|
137
144
|
# Initialize the Vicinity instance (using the basic backend and cosine metric)
|
|
138
|
-
vicinity = Vicinity.from_vectors_and_items(
|
|
145
|
+
vicinity = Vicinity.from_vectors_and_items(
|
|
146
|
+
vectors=vectors,
|
|
147
|
+
items=items,
|
|
148
|
+
backend_type=Backend.BASIC,
|
|
149
|
+
metric=Metric.COSINE
|
|
150
|
+
)
|
|
139
151
|
|
|
140
152
|
# Create a query vector
|
|
141
153
|
query_vector = np.random.rand(128)
|
|
@@ -190,7 +202,7 @@ The following backends are supported:
|
|
|
190
202
|
- `ivf_scalar`: Inverted file search with scalar quantizer.
|
|
191
203
|
- `ivfpq`: Inverted file search with product quantizer.
|
|
192
204
|
- `ivfpqr`: Inverted file search with product quantizer and refinement.
|
|
193
|
-
|
|
205
|
+
- [VOYAGER](https://github.com/spotify/voyager): Voyager is a library for performing fast approximate nearest-neighbor searches on an in-memory collection of vectors.
|
|
194
206
|
|
|
195
207
|
|
|
196
208
|
|
|
@@ -201,7 +213,7 @@ NOTE: the ANN backends do not support dynamic deletion. To delete items, you nee
|
|
|
201
213
|
|
|
202
214
|
| Backend | Parameter | Description | Default Value |
|
|
203
215
|
|-----------------|---------------------|-----------------------------------------------------------------------------------------------|---------------------|
|
|
204
|
-
| **
|
|
216
|
+
| **ANNOY** | `metric` | Similarity metric to use (`dot`, `euclidean`, `cosine`). | `"cosine"` |
|
|
205
217
|
| | `trees` | Number of trees to use for indexing. | `100` |
|
|
206
218
|
| | `length` | Optional length of the dataset. | `None` |
|
|
207
219
|
| **FAISS** | `metric` | Similarity metric to use (`cosine`, `l2`). | `"cosine"` |
|
|
@@ -213,13 +225,15 @@ NOTE: the ANN backends do not support dynamic deletion. To delete items, you nee
|
|
|
213
225
|
| **HNSW** | `metric` | Similarity space to use (`cosine`, `l2`). | `"cosine"` |
|
|
214
226
|
| | `ef_construction` | Size of the dynamic list during index construction. | `200` |
|
|
215
227
|
| | `m` | Number of connections per layer. | `16` |
|
|
216
|
-
| **
|
|
228
|
+
| **PYNNDESCENT** | `metric` | Similarity metric to use (`cosine`, `euclidean`, `manhattan`). | `"cosine"` |
|
|
217
229
|
| | `n_neighbors` | Number of neighbors to use for search. | `15` |
|
|
218
|
-
| **
|
|
230
|
+
| **USEARCH** | `metric` | Similarity metric to use (`cos`, `ip`, `l2sq`, `hamming`, `tanimoto`). | `"cos"` |
|
|
219
231
|
| | `connectivity` | Number of connections per node in the graph. | `16` |
|
|
220
232
|
| | `expansion_add` | Number of candidates considered during graph construction. | `128` |
|
|
221
233
|
| | `expansion_search` | Number of candidates considered during search. | `64` |
|
|
222
|
-
|
|
234
|
+
| **VOYAGER** | `metric` | Similarity space to use (`cosine`, `l2`). | `"cosine"` |
|
|
235
|
+
| | `ef_construction` | The number of vectors that this index searches through when inserting a new vector into the index. | `200` |
|
|
236
|
+
| | `m` | The number of connections between nodes in the tree’s internal data structure. | `16` |
|
|
223
237
|
|
|
224
238
|
## Installation
|
|
225
239
|
The following installation options are available:
|
|
@@ -236,6 +250,7 @@ pip install vicinity[faiss]
|
|
|
236
250
|
pip install vicinity[hnsw]
|
|
237
251
|
pip install vicinity[pynndescent]
|
|
238
252
|
pip install vicinity[usearch]
|
|
253
|
+
pip install vicinity[voyager]
|
|
239
254
|
```
|
|
240
255
|
|
|
241
256
|
## License
|
|
@@ -1,9 +1,14 @@
|
|
|
1
|
-
<div align="center">
|
|
2
|
-
|
|
3
|
-
# Vicinity: Lightweight Nearest Neighbors
|
|
4
1
|
|
|
2
|
+
<div align="center">
|
|
3
|
+
<picture>
|
|
4
|
+
<img width="40%" alt="Model2Vec logo" src="assets/images/vicinity_logo.png">
|
|
5
|
+
</picture>
|
|
6
|
+
</a>
|
|
5
7
|
</div>
|
|
6
8
|
|
|
9
|
+
<div align="center">
|
|
10
|
+
<h2>Lightweight Nearest Neighbors with Flexible Backends</h2>
|
|
11
|
+
</div>
|
|
7
12
|
|
|
8
13
|
<div align="center">
|
|
9
14
|
<h2>
|
|
@@ -39,7 +44,7 @@ Install the package with:
|
|
|
39
44
|
```bash
|
|
40
45
|
pip install vicinity
|
|
41
46
|
```
|
|
42
|
-
|
|
47
|
+
Optionally, [install any of the supported backends](#installation), or simply install all of them with:
|
|
43
48
|
```bash
|
|
44
49
|
pip install vicinity[all]
|
|
45
50
|
```
|
|
@@ -48,15 +53,19 @@ pip install vicinity[all]
|
|
|
48
53
|
The following code snippet demonstrates how to use Vicinity for nearest neighbor search:
|
|
49
54
|
```python
|
|
50
55
|
import numpy as np
|
|
51
|
-
from vicinity import Vicinity
|
|
52
|
-
from vicinity.datatypes import Backend, Metric
|
|
56
|
+
from vicinity import Vicinity, Backend, Metric
|
|
53
57
|
|
|
54
58
|
# Create some dummy data
|
|
55
59
|
items = ["triforce", "master sword", "hylian shield", "boomerang", "hookshot"]
|
|
56
60
|
vectors = np.random.rand(len(items), 128)
|
|
57
61
|
|
|
58
62
|
# Initialize the Vicinity instance (using the basic backend and cosine metric)
|
|
59
|
-
vicinity = Vicinity.from_vectors_and_items(
|
|
63
|
+
vicinity = Vicinity.from_vectors_and_items(
|
|
64
|
+
vectors=vectors,
|
|
65
|
+
items=items,
|
|
66
|
+
backend_type=Backend.BASIC,
|
|
67
|
+
metric=Metric.COSINE
|
|
68
|
+
)
|
|
60
69
|
|
|
61
70
|
# Create a query vector
|
|
62
71
|
query_vector = np.random.rand(128)
|
|
@@ -111,7 +120,7 @@ The following backends are supported:
|
|
|
111
120
|
- `ivf_scalar`: Inverted file search with scalar quantizer.
|
|
112
121
|
- `ivfpq`: Inverted file search with product quantizer.
|
|
113
122
|
- `ivfpqr`: Inverted file search with product quantizer and refinement.
|
|
114
|
-
|
|
123
|
+
- [VOYAGER](https://github.com/spotify/voyager): Voyager is a library for performing fast approximate nearest-neighbor searches on an in-memory collection of vectors.
|
|
115
124
|
|
|
116
125
|
|
|
117
126
|
|
|
@@ -122,7 +131,7 @@ NOTE: the ANN backends do not support dynamic deletion. To delete items, you nee
|
|
|
122
131
|
|
|
123
132
|
| Backend | Parameter | Description | Default Value |
|
|
124
133
|
|-----------------|---------------------|-----------------------------------------------------------------------------------------------|---------------------|
|
|
125
|
-
| **
|
|
134
|
+
| **ANNOY** | `metric` | Similarity metric to use (`dot`, `euclidean`, `cosine`). | `"cosine"` |
|
|
126
135
|
| | `trees` | Number of trees to use for indexing. | `100` |
|
|
127
136
|
| | `length` | Optional length of the dataset. | `None` |
|
|
128
137
|
| **FAISS** | `metric` | Similarity metric to use (`cosine`, `l2`). | `"cosine"` |
|
|
@@ -134,13 +143,15 @@ NOTE: the ANN backends do not support dynamic deletion. To delete items, you nee
|
|
|
134
143
|
| **HNSW** | `metric` | Similarity space to use (`cosine`, `l2`). | `"cosine"` |
|
|
135
144
|
| | `ef_construction` | Size of the dynamic list during index construction. | `200` |
|
|
136
145
|
| | `m` | Number of connections per layer. | `16` |
|
|
137
|
-
| **
|
|
146
|
+
| **PYNNDESCENT** | `metric` | Similarity metric to use (`cosine`, `euclidean`, `manhattan`). | `"cosine"` |
|
|
138
147
|
| | `n_neighbors` | Number of neighbors to use for search. | `15` |
|
|
139
|
-
| **
|
|
148
|
+
| **USEARCH** | `metric` | Similarity metric to use (`cos`, `ip`, `l2sq`, `hamming`, `tanimoto`). | `"cos"` |
|
|
140
149
|
| | `connectivity` | Number of connections per node in the graph. | `16` |
|
|
141
150
|
| | `expansion_add` | Number of candidates considered during graph construction. | `128` |
|
|
142
151
|
| | `expansion_search` | Number of candidates considered during search. | `64` |
|
|
143
|
-
|
|
152
|
+
| **VOYAGER** | `metric` | Similarity space to use (`cosine`, `l2`). | `"cosine"` |
|
|
153
|
+
| | `ef_construction` | The number of vectors that this index searches through when inserting a new vector into the index. | `200` |
|
|
154
|
+
| | `m` | The number of connections between nodes in the tree’s internal data structure. | `16` |
|
|
144
155
|
|
|
145
156
|
## Installation
|
|
146
157
|
The following installation options are available:
|
|
@@ -157,6 +168,7 @@ pip install vicinity[faiss]
|
|
|
157
168
|
pip install vicinity[hnsw]
|
|
158
169
|
pip install vicinity[pynndescent]
|
|
159
170
|
pip install vicinity[usearch]
|
|
171
|
+
pip install vicinity[voyager]
|
|
160
172
|
```
|
|
161
173
|
|
|
162
174
|
## License
|
|
Binary file
|
|
@@ -52,6 +52,7 @@ pynndescent = [
|
|
|
52
52
|
annoy = ["annoy"]
|
|
53
53
|
faiss = ["faiss-cpu"]
|
|
54
54
|
usearch = ["usearch"]
|
|
55
|
+
voyager = ["voyager"]
|
|
55
56
|
all = [
|
|
56
57
|
"hnswlib",
|
|
57
58
|
"pynndescent>=0.5.10",
|
|
@@ -60,7 +61,8 @@ all = [
|
|
|
60
61
|
"numpy>=1.24.0",
|
|
61
62
|
"annoy",
|
|
62
63
|
"faiss-cpu",
|
|
63
|
-
"usearch"
|
|
64
|
+
"usearch",
|
|
65
|
+
"voyager"
|
|
64
66
|
]
|
|
65
67
|
|
|
66
68
|
[project.urls]
|
|
@@ -87,6 +89,7 @@ select = [
|
|
|
87
89
|
"NPY",
|
|
88
90
|
# Print: Forbid print statements
|
|
89
91
|
"T20",
|
|
92
|
+
"F",
|
|
90
93
|
]
|
|
91
94
|
ignore = [
|
|
92
95
|
# Allow self and cls to be untyped, and allow Any type
|
|
@@ -8,7 +8,17 @@ from vicinity.datatypes import Backend
|
|
|
8
8
|
|
|
9
9
|
random_gen = np.random.default_rng(42)
|
|
10
10
|
|
|
11
|
-
_faiss_index_types = [
|
|
11
|
+
_faiss_index_types = [
|
|
12
|
+
"flat",
|
|
13
|
+
"ivf",
|
|
14
|
+
"hnsw",
|
|
15
|
+
"lsh",
|
|
16
|
+
"scalar",
|
|
17
|
+
"pq",
|
|
18
|
+
"ivf_scalar",
|
|
19
|
+
"ivfpq",
|
|
20
|
+
"ivfpqr",
|
|
21
|
+
]
|
|
12
22
|
|
|
13
23
|
|
|
14
24
|
@pytest.fixture(scope="session")
|
|
@@ -35,6 +45,7 @@ BACKEND_PARAMS = [(Backend.FAISS, index_type) for index_type in _faiss_index_typ
|
|
|
35
45
|
(Backend.ANNOY, None),
|
|
36
46
|
(Backend.PYNNDESCENT, None),
|
|
37
47
|
(Backend.USEARCH, None),
|
|
48
|
+
(Backend.VOYAGER, None),
|
|
38
49
|
]
|
|
39
50
|
|
|
40
51
|
|
|
@@ -57,11 +68,48 @@ def vicinity_instance(request: pytest.FixtureRequest, items: list[str], vectors:
|
|
|
57
68
|
if index_type in ("pq", "ivfpq", "ivfpqr"):
|
|
58
69
|
# Use smaller values for pq indexes since the dataset is small
|
|
59
70
|
return Vicinity.from_vectors_and_items(
|
|
60
|
-
vectors,
|
|
71
|
+
vectors,
|
|
72
|
+
items,
|
|
73
|
+
backend_type=backend_type,
|
|
74
|
+
index_type=index_type,
|
|
75
|
+
m=2,
|
|
76
|
+
nbits=4,
|
|
61
77
|
)
|
|
62
78
|
else:
|
|
63
79
|
return Vicinity.from_vectors_and_items(
|
|
64
|
-
vectors,
|
|
80
|
+
vectors,
|
|
81
|
+
items,
|
|
82
|
+
backend_type=backend_type,
|
|
83
|
+
index_type=index_type,
|
|
84
|
+
nlist=2,
|
|
85
|
+
nbits=32,
|
|
65
86
|
)
|
|
66
87
|
|
|
67
88
|
return Vicinity.from_vectors_and_items(vectors, items, backend_type=backend_type)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
@pytest.fixture(params=BACKEND_PARAMS, ids=BACKEND_IDS)
|
|
92
|
+
def vicinity_instance_with_stored_vectors(
|
|
93
|
+
request: pytest.FixtureRequest, items: list[str], vectors: np.ndarray
|
|
94
|
+
) -> Vicinity:
|
|
95
|
+
"""Fixture providing a Vicinity instance for each backend type."""
|
|
96
|
+
backend_type, index_type = request.param
|
|
97
|
+
# Handle FAISS backend with specific FAISS index types
|
|
98
|
+
if backend_type == Backend.FAISS:
|
|
99
|
+
if index_type in ("pq", "ivfpq", "ivfpqr"):
|
|
100
|
+
# Use smaller values for pq indexes since the dataset is small
|
|
101
|
+
return Vicinity.from_vectors_and_items(
|
|
102
|
+
vectors, items, backend_type=backend_type, index_type=index_type, m=2, nbits=4, store_vectors=True
|
|
103
|
+
)
|
|
104
|
+
else:
|
|
105
|
+
return Vicinity.from_vectors_and_items(
|
|
106
|
+
vectors, items, backend_type=backend_type, index_type=index_type, nlist=2, nbits=32, store_vectors=True
|
|
107
|
+
)
|
|
108
|
+
|
|
109
|
+
return Vicinity.from_vectors_and_items(vectors, items, backend_type=backend_type, store_vectors=True)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
@pytest.fixture()
|
|
113
|
+
def vicinity_with_basic_backend(vectors: np.ndarray, items: list[str]) -> Vicinity:
|
|
114
|
+
"""Fixture providing a BasicBackend instance."""
|
|
115
|
+
return Vicinity.from_vectors_and_items(vectors, items, backend_type=Backend.BASIC, store_vectors=True)
|
|
@@ -131,8 +131,48 @@ def test_vicinity_save_and_load(tmp_path: Path, vicinity_instance: Vicinity) ->
|
|
|
131
131
|
"""
|
|
132
132
|
save_path = tmp_path / "vicinity_data"
|
|
133
133
|
vicinity_instance.save(save_path)
|
|
134
|
+
assert vicinity_instance.vector_store is None
|
|
134
135
|
|
|
135
|
-
Vicinity.load(save_path)
|
|
136
|
+
v = Vicinity.load(save_path)
|
|
137
|
+
assert v.vector_store is None
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def test_vicinity_save_and_load_vector_store(tmp_path: Path, vicinity_instance_with_stored_vectors: Vicinity) -> None:
|
|
141
|
+
"""
|
|
142
|
+
Test Vicinity.save and Vicinity.load.
|
|
143
|
+
|
|
144
|
+
:param tmp_path: Temporary directory provided by pytest.
|
|
145
|
+
:param vicinity_instance: A Vicinity instance.
|
|
146
|
+
"""
|
|
147
|
+
save_path = tmp_path / "vicinity_data"
|
|
148
|
+
vicinity_instance_with_stored_vectors.save(save_path)
|
|
149
|
+
|
|
150
|
+
assert (save_path / "store").exists()
|
|
151
|
+
assert (save_path / "store" / "vectors.npy").exists()
|
|
152
|
+
|
|
153
|
+
v = Vicinity.load(save_path)
|
|
154
|
+
assert v.vector_store is not None
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def test_index_vector_store(vicinity_with_basic_backend: Vicinity, vectors: np.ndarray) -> None:
|
|
158
|
+
"""
|
|
159
|
+
Index vectors in the Vicinity instance.
|
|
160
|
+
|
|
161
|
+
:param vicinity_instance: A Vicinity instance.
|
|
162
|
+
:param vectors: Array of vectors to index.
|
|
163
|
+
"""
|
|
164
|
+
v = vicinity_with_basic_backend.get_vector_by_index(0)
|
|
165
|
+
assert np.allclose(v, vectors[0])
|
|
166
|
+
|
|
167
|
+
idx = [0, 1, 2, 3, 4, 10]
|
|
168
|
+
v = vicinity_with_basic_backend.get_vector_by_index(idx)
|
|
169
|
+
assert np.allclose(v, vectors[idx])
|
|
170
|
+
|
|
171
|
+
with pytest.raises(ValueError):
|
|
172
|
+
vicinity_with_basic_backend.get_vector_by_index([10_000])
|
|
173
|
+
|
|
174
|
+
with pytest.raises(ValueError):
|
|
175
|
+
vicinity_with_basic_backend.get_vector_by_index([-1])
|
|
136
176
|
|
|
137
177
|
|
|
138
178
|
def test_vicinity_insert_duplicate(vicinity_instance: Vicinity, query_vector: np.ndarray) -> None:
|
|
@@ -1087,7 +1087,7 @@ wheels = [
|
|
|
1087
1087
|
|
|
1088
1088
|
[[package]]
|
|
1089
1089
|
name = "vicinity"
|
|
1090
|
-
version = "0.
|
|
1090
|
+
version = "0.3.0"
|
|
1091
1091
|
source = { editable = "." }
|
|
1092
1092
|
dependencies = [
|
|
1093
1093
|
{ name = "numpy" },
|
|
@@ -1096,6 +1096,16 @@ dependencies = [
|
|
|
1096
1096
|
]
|
|
1097
1097
|
|
|
1098
1098
|
[package.optional-dependencies]
|
|
1099
|
+
all = [
|
|
1100
|
+
{ name = "annoy" },
|
|
1101
|
+
{ name = "faiss-cpu" },
|
|
1102
|
+
{ name = "hnswlib" },
|
|
1103
|
+
{ name = "llvmlite" },
|
|
1104
|
+
{ name = "numba" },
|
|
1105
|
+
{ name = "numpy" },
|
|
1106
|
+
{ name = "pynndescent" },
|
|
1107
|
+
{ name = "usearch" },
|
|
1108
|
+
]
|
|
1099
1109
|
annoy = [
|
|
1100
1110
|
{ name = "annoy" },
|
|
1101
1111
|
]
|
|
@@ -1127,24 +1137,32 @@ usearch = [
|
|
|
1127
1137
|
|
|
1128
1138
|
[package.metadata]
|
|
1129
1139
|
requires-dist = [
|
|
1140
|
+
{ name = "annoy", marker = "extra == 'all'" },
|
|
1130
1141
|
{ name = "annoy", marker = "extra == 'annoy'" },
|
|
1131
1142
|
{ name = "black", marker = "extra == 'dev'" },
|
|
1143
|
+
{ name = "faiss-cpu", marker = "extra == 'all'" },
|
|
1132
1144
|
{ name = "faiss-cpu", marker = "extra == 'faiss'" },
|
|
1145
|
+
{ name = "hnswlib", marker = "extra == 'all'" },
|
|
1133
1146
|
{ name = "hnswlib", marker = "extra == 'hnsw'" },
|
|
1134
1147
|
{ name = "ipython", marker = "extra == 'dev'" },
|
|
1148
|
+
{ name = "llvmlite", marker = "extra == 'all'", specifier = ">=0.42.0" },
|
|
1135
1149
|
{ name = "llvmlite", marker = "extra == 'pynndescent'", specifier = ">=0.42.0" },
|
|
1136
1150
|
{ name = "mypy", marker = "extra == 'dev'" },
|
|
1151
|
+
{ name = "numba", marker = "extra == 'all'", specifier = ">=0.59.0" },
|
|
1137
1152
|
{ name = "numba", marker = "extra == 'pynndescent'", specifier = ">=0.59.0" },
|
|
1138
1153
|
{ name = "numpy" },
|
|
1154
|
+
{ name = "numpy", marker = "extra == 'all'", specifier = ">=1.24.0" },
|
|
1139
1155
|
{ name = "numpy", marker = "extra == 'pynndescent'", specifier = ">=1.24.0" },
|
|
1140
1156
|
{ name = "orjson" },
|
|
1141
1157
|
{ name = "pre-commit", marker = "extra == 'dev'" },
|
|
1158
|
+
{ name = "pynndescent", marker = "extra == 'all'", specifier = ">=0.5.10" },
|
|
1142
1159
|
{ name = "pynndescent", marker = "extra == 'pynndescent'", specifier = ">=0.5.10" },
|
|
1143
1160
|
{ name = "pytest", marker = "extra == 'dev'" },
|
|
1144
1161
|
{ name = "pytest-coverage", marker = "extra == 'dev'" },
|
|
1145
1162
|
{ name = "ruff", marker = "extra == 'dev'" },
|
|
1146
1163
|
{ name = "setuptools", marker = "extra == 'dev'" },
|
|
1147
1164
|
{ name = "tqdm" },
|
|
1165
|
+
{ name = "usearch", marker = "extra == 'all'" },
|
|
1148
1166
|
{ name = "usearch", marker = "extra == 'usearch'" },
|
|
1149
1167
|
]
|
|
1150
1168
|
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
from typing import Union
|
|
2
2
|
|
|
3
3
|
from vicinity.backends.base import AbstractBackend
|
|
4
|
-
from vicinity.backends.basic import BasicBackend
|
|
4
|
+
from vicinity.backends.basic import BasicBackend, BasicVectorStore
|
|
5
5
|
from vicinity.datatypes import Backend
|
|
6
6
|
|
|
7
7
|
|
|
@@ -33,5 +33,10 @@ def get_backend_class(backend: Union[Backend, str]) -> type[AbstractBackend]:
|
|
|
33
33
|
|
|
34
34
|
return UsearchBackend
|
|
35
35
|
|
|
36
|
+
elif backend == Backend.VOYAGER:
|
|
37
|
+
from vicinity.backends.voyager import VoyagerBackend
|
|
36
38
|
|
|
37
|
-
|
|
39
|
+
return VoyagerBackend
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
__all__ = ["get_backend_class", "AbstractBackend", "BasicVectorStore"]
|
|
@@ -16,18 +16,18 @@ from vicinity.utils import Metric, normalize
|
|
|
16
16
|
@dataclass
|
|
17
17
|
class AnnoyArgs(BaseArgs):
|
|
18
18
|
dim: int = 0
|
|
19
|
-
metric:
|
|
19
|
+
metric: Metric = Metric.COSINE
|
|
20
|
+
internal_metric: str = "dot"
|
|
20
21
|
trees: int = 100
|
|
21
22
|
length: int | None = None
|
|
22
23
|
|
|
23
24
|
|
|
24
25
|
class AnnoyBackend(AbstractBackend[AnnoyArgs]):
|
|
25
26
|
argument_class = AnnoyArgs
|
|
26
|
-
supported_metrics = {Metric.COSINE, Metric.EUCLIDEAN
|
|
27
|
-
inverse_metric_mapping = {
|
|
27
|
+
supported_metrics = {Metric.COSINE, Metric.EUCLIDEAN}
|
|
28
|
+
inverse_metric_mapping: dict[Metric, str] = {
|
|
28
29
|
Metric.COSINE: "dot",
|
|
29
30
|
Metric.EUCLIDEAN: "euclidean",
|
|
30
|
-
Metric.INNER_PRODUCT: "dot",
|
|
31
31
|
}
|
|
32
32
|
|
|
33
33
|
def __init__(
|
|
@@ -56,18 +56,18 @@ class AnnoyBackend(AbstractBackend[AnnoyArgs]):
|
|
|
56
56
|
if metric_enum not in cls.supported_metrics:
|
|
57
57
|
raise ValueError(f"Metric '{metric_enum.value}' is not supported by AnnoyBackend.")
|
|
58
58
|
|
|
59
|
-
|
|
59
|
+
internal_metric = cls._map_metric_to_string(metric_enum)
|
|
60
60
|
|
|
61
|
-
if
|
|
61
|
+
if metric_enum == Metric.COSINE:
|
|
62
62
|
vectors = normalize(vectors)
|
|
63
63
|
|
|
64
64
|
dim = vectors.shape[1]
|
|
65
|
-
index = AnnoyIndex(f=dim, metric=
|
|
65
|
+
index = AnnoyIndex(f=dim, metric=internal_metric) # type: ignore
|
|
66
66
|
for i, vector in enumerate(vectors):
|
|
67
67
|
index.add_item(i, vector)
|
|
68
68
|
index.build(trees)
|
|
69
69
|
|
|
70
|
-
arguments = AnnoyArgs(dim=dim, metric=metric, trees=trees, length=len(vectors)) # type: ignore
|
|
70
|
+
arguments = AnnoyArgs(dim=dim, metric=metric, trees=trees, length=len(vectors), internal_metric=internal_metric) # type: ignore
|
|
71
71
|
return AnnoyBackend(index, arguments=arguments)
|
|
72
72
|
|
|
73
73
|
@property
|
|
@@ -88,8 +88,10 @@ class AnnoyBackend(AbstractBackend[AnnoyArgs]):
|
|
|
88
88
|
def load(cls: type[AnnoyBackend], base_path: Path) -> AnnoyBackend:
|
|
89
89
|
"""Load the vectors from a path."""
|
|
90
90
|
path = Path(base_path) / "index.bin"
|
|
91
|
+
|
|
91
92
|
arguments = AnnoyArgs.load(base_path / "arguments.json")
|
|
92
|
-
|
|
93
|
+
metric = cls._map_metric_to_string(arguments.metric)
|
|
94
|
+
index = AnnoyIndex(arguments.dim, metric) # type: ignore
|
|
93
95
|
index.load(str(path))
|
|
94
96
|
|
|
95
97
|
return cls(index, arguments=arguments)
|
|
@@ -106,11 +108,11 @@ class AnnoyBackend(AbstractBackend[AnnoyArgs]):
|
|
|
106
108
|
"""Query the backend."""
|
|
107
109
|
out = []
|
|
108
110
|
for vec in vectors:
|
|
109
|
-
if self.arguments.metric ==
|
|
111
|
+
if self.arguments.metric == Metric.COSINE:
|
|
110
112
|
vec = normalize(vec)
|
|
111
113
|
indices, scores = self.index.get_nns_by_vector(vec, k, include_distances=True)
|
|
112
114
|
scores_array = np.asarray(scores)
|
|
113
|
-
if self.arguments.metric ==
|
|
115
|
+
if self.arguments.metric == Metric.COSINE:
|
|
114
116
|
# Convert cosine similarity to cosine distance
|
|
115
117
|
scores_array = 1 - scores_array
|
|
116
118
|
out.append((np.asarray(indices), scores_array))
|
|
@@ -14,19 +14,25 @@ from vicinity.datatypes import Backend, QueryResult
|
|
|
14
14
|
|
|
15
15
|
@dataclass
|
|
16
16
|
class BaseArgs:
|
|
17
|
+
metric: Metric
|
|
18
|
+
|
|
17
19
|
def dump(self, file: Path) -> None:
|
|
18
20
|
"""Dump the arguments to a file."""
|
|
19
21
|
with open(file, "w") as f:
|
|
20
|
-
|
|
22
|
+
d = self.dict()
|
|
23
|
+
d["metric"] = d["metric"].value
|
|
24
|
+
json.dump(d, f)
|
|
21
25
|
|
|
22
26
|
@classmethod
|
|
23
27
|
def load(cls: type[ArgType], file: Path) -> ArgType:
|
|
24
28
|
"""Load the arguments from a file."""
|
|
25
29
|
with open(file, "r") as f:
|
|
26
|
-
|
|
30
|
+
data = json.load(f)
|
|
31
|
+
data["metric"] = Metric.from_string(data["metric"])
|
|
32
|
+
return cls(**data)
|
|
27
33
|
|
|
28
34
|
def dict(self) -> dict[str, Any]:
|
|
29
|
-
"""Dump the arguments to a
|
|
35
|
+
"""Dump the arguments to a dict."""
|
|
30
36
|
return asdict(self)
|
|
31
37
|
|
|
32
38
|
|