vicinity 0.1.0__tar.gz → 0.2.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- vicinity-0.2.1/.github/workflows/ci.yaml +67 -0
- vicinity-0.2.1/Makefile +21 -0
- vicinity-0.2.1/PKG-INFO +265 -0
- vicinity-0.2.1/README.md +195 -0
- {vicinity-0.1.0 → vicinity-0.2.1}/pyproject.toml +17 -1
- vicinity-0.2.1/tests/conftest.py +67 -0
- vicinity-0.1.0/tests/test_nearest.py → vicinity-0.2.1/tests/test_vicinity.py +59 -18
- vicinity-0.2.1/uv.lock +1172 -0
- vicinity-0.2.1/vicinity/backends/__init__.py +37 -0
- vicinity-0.2.1/vicinity/backends/annoy.py +128 -0
- {vicinity-0.1.0 → vicinity-0.2.1}/vicinity/backends/base.py +4 -6
- {vicinity-0.1.0 → vicinity-0.2.1}/vicinity/backends/basic.py +4 -6
- vicinity-0.2.1/vicinity/backends/faiss.py +220 -0
- {vicinity-0.1.0 → vicinity-0.2.1}/vicinity/backends/hnsw.py +7 -9
- vicinity-0.2.1/vicinity/backends/pynndescent.py +109 -0
- vicinity-0.2.1/vicinity/backends/usearch.py +132 -0
- vicinity-0.2.1/vicinity/datatypes.py +24 -0
- {vicinity-0.1.0 → vicinity-0.2.1}/vicinity/utils.py +3 -1
- {vicinity-0.1.0 → vicinity-0.2.1}/vicinity/version.py +1 -1
- {vicinity-0.1.0 → vicinity-0.2.1}/vicinity/vicinity.py +5 -4
- vicinity-0.2.1/vicinity.egg-info/PKG-INFO +265 -0
- {vicinity-0.1.0 → vicinity-0.2.1}/vicinity.egg-info/SOURCES.txt +7 -2
- vicinity-0.2.1/vicinity.egg-info/requires.txt +31 -0
- vicinity-0.1.0/Makefile +0 -15
- vicinity-0.1.0/PKG-INFO +0 -200
- vicinity-0.1.0/README.md +0 -146
- vicinity-0.1.0/tests/conftest.py +0 -39
- vicinity-0.1.0/uv.lock +0 -778
- vicinity-0.1.0/vicinity/backends/__init__.py +0 -17
- vicinity-0.1.0/vicinity/datatypes.py +0 -20
- vicinity-0.1.0/vicinity.egg-info/PKG-INFO +0 -200
- vicinity-0.1.0/vicinity.egg-info/requires.txt +0 -15
- {vicinity-0.1.0 → vicinity-0.2.1}/.gitignore +0 -0
- {vicinity-0.1.0 → vicinity-0.2.1}/.pre-commit-config.yaml +0 -0
- {vicinity-0.1.0 → vicinity-0.2.1}/LICENSE +0 -0
- {vicinity-0.1.0 → vicinity-0.2.1}/setup.cfg +0 -0
- {vicinity-0.1.0 → vicinity-0.2.1}/tests/test_utils.py +0 -0
- {vicinity-0.1.0 → vicinity-0.2.1}/vicinity/__init__.py +0 -0
- {vicinity-0.1.0 → vicinity-0.2.1}/vicinity/py.typed +0 -0
- {vicinity-0.1.0 → vicinity-0.2.1}/vicinity.egg-info/dependency_links.txt +0 -0
- {vicinity-0.1.0 → vicinity-0.2.1}/vicinity.egg-info/top_level.txt +0 -0
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
name: Run tests and upload coverage
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
pull_request:
|
|
5
|
+
types: [opened, synchronize, reopened, ready_for_review]
|
|
6
|
+
|
|
7
|
+
jobs:
|
|
8
|
+
test:
|
|
9
|
+
name: Run tests with pytest
|
|
10
|
+
if: ${{ !github.event.pull_request.draft }} # Skip draft PRs
|
|
11
|
+
runs-on: ${{ matrix.os }}
|
|
12
|
+
strategy:
|
|
13
|
+
matrix:
|
|
14
|
+
os: ["ubuntu-latest", "windows-latest"]
|
|
15
|
+
python-version: ["3.9", "3.10", "3.11", "3.12"]
|
|
16
|
+
exclude:
|
|
17
|
+
- os: windows-latest
|
|
18
|
+
python-version: "3.9"
|
|
19
|
+
- os: windows-latest
|
|
20
|
+
python-version: "3.11"
|
|
21
|
+
- os: windows-latest
|
|
22
|
+
python-version: "3.12"
|
|
23
|
+
fail-fast: false
|
|
24
|
+
|
|
25
|
+
steps:
|
|
26
|
+
- uses: actions/checkout@v4
|
|
27
|
+
|
|
28
|
+
- name: Set up Python ${{ matrix.python-version }} on ${{ matrix.os }}
|
|
29
|
+
uses: actions/setup-python@v5
|
|
30
|
+
with:
|
|
31
|
+
python-version: ${{ matrix.python-version }}
|
|
32
|
+
allow-prereleases: true
|
|
33
|
+
|
|
34
|
+
# Step for Windows: Create and activate a virtual environment
|
|
35
|
+
- name: Create and activate a virtual environment (Windows)
|
|
36
|
+
if: ${{ runner.os == 'Windows' }}
|
|
37
|
+
run: |
|
|
38
|
+
irm https://astral.sh/uv/install.ps1 | iex
|
|
39
|
+
$env:Path = "C:\Users\runneradmin\.local\bin;$env:Path"
|
|
40
|
+
uv venv .venv
|
|
41
|
+
"VIRTUAL_ENV=.venv" | Out-File -FilePath $env:GITHUB_ENV -Append
|
|
42
|
+
"$PWD/.venv/Scripts" | Out-File -FilePath $env:GITHUB_PATH -Append
|
|
43
|
+
|
|
44
|
+
# Step for Unix: Create and activate a virtual environment
|
|
45
|
+
- name: Create and activate a virtual environment (Unix)
|
|
46
|
+
if: ${{ runner.os != 'Windows' }}
|
|
47
|
+
run: |
|
|
48
|
+
curl -LsSf https://astral.sh/uv/install.sh | sh
|
|
49
|
+
uv venv .venv
|
|
50
|
+
echo "VIRTUAL_ENV=.venv" >> $GITHUB_ENV
|
|
51
|
+
echo "$PWD/.venv/bin" >> $GITHUB_PATH
|
|
52
|
+
|
|
53
|
+
# Install dependencies using uv pip
|
|
54
|
+
- name: Install dependencies
|
|
55
|
+
run: make install-no-pre-commit
|
|
56
|
+
|
|
57
|
+
# Run tests with coverage
|
|
58
|
+
- name: Run tests under coverage
|
|
59
|
+
run: |
|
|
60
|
+
coverage run -m pytest
|
|
61
|
+
coverage report
|
|
62
|
+
|
|
63
|
+
# Upload results to Codecov
|
|
64
|
+
- name: Upload results to Codecov
|
|
65
|
+
uses: codecov/codecov-action@v4
|
|
66
|
+
with:
|
|
67
|
+
token: ${{ secrets.CODECOV_TOKEN }}
|
vicinity-0.2.1/Makefile
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
clean:
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
venv:
|
|
5
|
+
uv venv
|
|
6
|
+
|
|
7
|
+
install: venv
|
|
8
|
+
uv sync --all-extras
|
|
9
|
+
uv run pre-commit install
|
|
10
|
+
|
|
11
|
+
install-no-pre-commit:
|
|
12
|
+
uv pip install ".[dev,hnsw,pynndescent,annoy,faiss,usearch]"
|
|
13
|
+
|
|
14
|
+
install-base:
|
|
15
|
+
uv sync --extra dev
|
|
16
|
+
|
|
17
|
+
fix:
|
|
18
|
+
uv run pre-commit run --all-files
|
|
19
|
+
|
|
20
|
+
test:
|
|
21
|
+
uv run pytest --cov=vicinity --cov-report=term-missing
|
vicinity-0.2.1/PKG-INFO
ADDED
|
@@ -0,0 +1,265 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: vicinity
|
|
3
|
+
Version: 0.2.1
|
|
4
|
+
Summary: Lightweight Nearest Neighbors with Flexible Backends
|
|
5
|
+
Author-email: Stéphan Tulkens <stephantul@gmail.com>, Thomas van Dongen <thomas123@live.nl>
|
|
6
|
+
License: MIT License
|
|
7
|
+
|
|
8
|
+
Copyright (c) 2024 The Minish Lab
|
|
9
|
+
|
|
10
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
11
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
12
|
+
in the Software without restriction, including without limitation the rights
|
|
13
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
14
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
15
|
+
furnished to do so, subject to the following conditions:
|
|
16
|
+
|
|
17
|
+
The above copyright notice and this permission notice shall be included in all
|
|
18
|
+
copies or substantial portions of the Software.
|
|
19
|
+
|
|
20
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
21
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
22
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
23
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
24
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
25
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
26
|
+
SOFTWARE.
|
|
27
|
+
|
|
28
|
+
Project-URL: Homepage, https://github.com/MinishLab
|
|
29
|
+
Project-URL: Bug Reports, https://github.com/MinishLab/vicinity/issues
|
|
30
|
+
Project-URL: Source, https://github.com/MinishLab/vicinity
|
|
31
|
+
Classifier: Development Status :: 4 - Beta
|
|
32
|
+
Classifier: Intended Audience :: Developers
|
|
33
|
+
Classifier: Intended Audience :: Science/Research
|
|
34
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
35
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
36
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
37
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
38
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
39
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
40
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
41
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
42
|
+
Requires-Python: >=3.9
|
|
43
|
+
Description-Content-Type: text/markdown
|
|
44
|
+
License-File: LICENSE
|
|
45
|
+
Requires-Dist: numpy
|
|
46
|
+
Requires-Dist: orjson
|
|
47
|
+
Requires-Dist: tqdm
|
|
48
|
+
Provides-Extra: dev
|
|
49
|
+
Requires-Dist: black; extra == "dev"
|
|
50
|
+
Requires-Dist: ipython; extra == "dev"
|
|
51
|
+
Requires-Dist: mypy; extra == "dev"
|
|
52
|
+
Requires-Dist: pre-commit; extra == "dev"
|
|
53
|
+
Requires-Dist: pytest; extra == "dev"
|
|
54
|
+
Requires-Dist: pytest-coverage; extra == "dev"
|
|
55
|
+
Requires-Dist: ruff; extra == "dev"
|
|
56
|
+
Requires-Dist: setuptools; extra == "dev"
|
|
57
|
+
Provides-Extra: hnsw
|
|
58
|
+
Requires-Dist: hnswlib; extra == "hnsw"
|
|
59
|
+
Provides-Extra: pynndescent
|
|
60
|
+
Requires-Dist: pynndescent>=0.5.10; extra == "pynndescent"
|
|
61
|
+
Requires-Dist: numba>=0.59.0; extra == "pynndescent"
|
|
62
|
+
Requires-Dist: llvmlite>=0.42.0; extra == "pynndescent"
|
|
63
|
+
Requires-Dist: numpy>=1.24.0; extra == "pynndescent"
|
|
64
|
+
Provides-Extra: annoy
|
|
65
|
+
Requires-Dist: annoy; extra == "annoy"
|
|
66
|
+
Provides-Extra: faiss
|
|
67
|
+
Requires-Dist: faiss-cpu; extra == "faiss"
|
|
68
|
+
Provides-Extra: usearch
|
|
69
|
+
Requires-Dist: usearch; extra == "usearch"
|
|
70
|
+
|
|
71
|
+
<div align="center">
|
|
72
|
+
|
|
73
|
+
# Vicinity: The Lightweight Vector Store
|
|
74
|
+
|
|
75
|
+
</div>
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
<div align="center">
|
|
79
|
+
<h2>
|
|
80
|
+
<a href="https://pypi.org/project/vicinity/"><img src="https://img.shields.io/pypi/v/vicinity?color=%23007ec6&label=pypi%20package" alt="Package version"></a>
|
|
81
|
+
<a href="https://pypi.org/project/vicinity/"><img src="https://img.shields.io/pypi/pyversions/vicinity" alt="Supported Python versions"></a>
|
|
82
|
+
<a href="https://pepy.tech/project/vicinity">
|
|
83
|
+
<img src="https://static.pepy.tech/badge/vicinity" alt="Downloads">
|
|
84
|
+
</a>
|
|
85
|
+
<a href="https://app.codecov.io/gh/MinishLab/vicinity">
|
|
86
|
+
<img src="https://codecov.io/gh/MinishLab/vicinity/graph/badge.svg?token=0MQ2945OZL" alt="Codecov">
|
|
87
|
+
</a>
|
|
88
|
+
<a href="https://github.com/MinishLab/vicinity/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-MIT-green" alt="License - MIT"></a>
|
|
89
|
+
</h2>
|
|
90
|
+
</div>
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
## Table of contents
|
|
94
|
+
|
|
95
|
+
- [Quickstart](#quickstart)
|
|
96
|
+
- [Main Features](#main-features)
|
|
97
|
+
- [Supported Backends](#supported-backends)
|
|
98
|
+
- [Backend Parameters](#backend-parameters)
|
|
99
|
+
- [Usage](#usage)
|
|
100
|
+
|
|
101
|
+
Vicinity is the lightest-weight vector store. Just put in some vectors, calculate query vectors, and off you go. It provides a simple and intuitive API for nearest neighbor search, with support for different backends.
|
|
102
|
+
|
|
103
|
+
## Quickstart
|
|
104
|
+
|
|
105
|
+
Install the package with:
|
|
106
|
+
```bash
|
|
107
|
+
pip install vicinity
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
The following code snippet demonstrates how to use Vicinity for nearest neighbor search:
|
|
112
|
+
```python
|
|
113
|
+
import numpy as np
|
|
114
|
+
from vicinity import Vicinity
|
|
115
|
+
from vicinity.datatypes import Backend
|
|
116
|
+
|
|
117
|
+
# Create some dummy data
|
|
118
|
+
items = ["triforce", "master sword", "hylian shield", "boomerang", "hookshot"]
|
|
119
|
+
vectors = np.random.rand(len(items), 128)
|
|
120
|
+
|
|
121
|
+
# Initialize the Vicinity instance (using the basic backend)
|
|
122
|
+
vicinity = Vicinity.from_vectors_and_items(vectors=vectors, items=items, backend_type=Backend.BASIC)
|
|
123
|
+
|
|
124
|
+
# Query for nearest neighbors with a top-k search
|
|
125
|
+
query_vector = np.random.rand(128)
|
|
126
|
+
results = vicinity.query([query_vector], k=3)
|
|
127
|
+
|
|
128
|
+
# Query for nearest neighbors with a threshold search
|
|
129
|
+
results = vicinity.query_threshold([query_vector], threshold=0.9)
|
|
130
|
+
|
|
131
|
+
# Save the vector store
|
|
132
|
+
vicinity.save('my_vector_store')
|
|
133
|
+
|
|
134
|
+
# Load the vector store
|
|
135
|
+
vicinity = Vicinity.load('my_vector_store')
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
## Main Features
|
|
139
|
+
Vicinity provides the following features:
|
|
140
|
+
- Lightweight: Minimal dependencies and fast performance.
|
|
141
|
+
- Flexible Backend Support: Use different backends for vector storage and search.
|
|
142
|
+
- Serialization: Save and load vector stores for persistence.
|
|
143
|
+
- Easy to Use: Simple and intuitive API.
|
|
144
|
+
|
|
145
|
+
## Supported Backends
|
|
146
|
+
The following backends are supported:
|
|
147
|
+
- `BASIC`: A simple flat index for vector storage and search.
|
|
148
|
+
- [HNSW](https://github.com/nmslib/hnswlib): Hierarchical Navigable Small World Graph (HNSW) for ANN search using hnswlib.
|
|
149
|
+
- [FAISS](https://github.com/facebookresearch/faiss): ANN search using FAISS. All FAISS indexes are supported.
|
|
150
|
+
- [ANNOY](https://github.com/spotify/annoy): "Approximate Nearest Neighbors Oh Yeah" for approximate nearest neighbor search.
|
|
151
|
+
- [PYNNDescent](https://github.com/lmcinnes/pynndescent): ANN search using PyNNDescent.
|
|
152
|
+
- [USEARCH](https://github.com/unum-cloud/usearch): ANN search using Usearch. This uses a highly optimized version of the HNSW algorithm.
|
|
153
|
+
|
|
154
|
+
NOTE: the ANN backends do not support dynamic deletion. To delete items, you need to recreate the index. Insertion is supported in the following backends: `FAISS`, `HNSW`, and `Usearch`. The `BASIC` backend supports both insertion and deletion.
|
|
155
|
+
|
|
156
|
+
### Backend Parameters
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
| Backend | Parameter | Description | Default Value |
|
|
160
|
+
|-----------------|---------------------|-----------------------------------------------------------------------------------------------|---------------------|
|
|
161
|
+
| **Annoy** | `metric` | Similarity metric to use (`dot`, `euclidean`, `cosine`). | `"cosine"` |
|
|
162
|
+
| | `trees` | Number of trees to use for indexing. | `100` |
|
|
163
|
+
| | `length` | Optional length of the dataset. | `None` |
|
|
164
|
+
| **FAISS** | `metric` | Similarity metric to use (`cosine`, `l2`). | `"cosine"` |
|
|
165
|
+
| | `index_type` | Type of FAISS index (`flat`, `ivf`, `hnsw`, `lsh`, `scalar`, `pq`, `ivf_scalar`, `ivfpq`, `ivfpqr`). | `"hnsw"` |
|
|
166
|
+
| | `nlist` | Number of cells for IVF indexes. | `100` |
|
|
167
|
+
| | `m` | Number of subquantizers for PQ and HNSW indexes. | `8` |
|
|
168
|
+
| | `nbits` | Number of bits for LSH and PQ indexes. | `8` |
|
|
169
|
+
| | `refine_nbits` | Number of bits for the refinement stage in IVFPQR indexes. | `8` |
|
|
170
|
+
| **HNSW** | `metric` | Similarity space to use (`cosine`, `l2`). | `"cosine"` |
|
|
171
|
+
| | `ef_construction` | Size of the dynamic list during index construction. | `200` |
|
|
172
|
+
| | `m` | Number of connections per layer. | `16` |
|
|
173
|
+
| **PyNNDescent** | `metric` | Similarity metric to use (`cosine`, `euclidean`, `manhattan`). | `"cosine"` |
|
|
174
|
+
| | `n_neighbors` | Number of neighbors to use for search. | `15` |
|
|
175
|
+
| **Usearch** | `metric` | Similarity metric to use (`cos`, `ip`, `l2sq`, `hamming`, `tanimoto`). | `"cos"` |
|
|
176
|
+
| | `connectivity` | Number of connections per node in the graph. | `16` |
|
|
177
|
+
| | `expansion_add` | Number of candidates considered during graph construction. | `128` |
|
|
178
|
+
| | `expansion_search` | Number of candidates considered during search. | `64` |
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
## Usage
|
|
183
|
+
|
|
184
|
+
<details>
|
|
185
|
+
<summary> Creating a Vector Store
|
|
186
|
+
</summary>
|
|
187
|
+
<br>
|
|
188
|
+
|
|
189
|
+
You can create a Vicinity instance by providing items and their corresponding vectors:
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
```python
|
|
193
|
+
from vicinity import Vicinity
|
|
194
|
+
import numpy as np
|
|
195
|
+
|
|
196
|
+
items = ["triforce", "master sword", "hylian shield", "boomerang", "hookshot"]
|
|
197
|
+
vectors = np.random.rand(len(items), 128)
|
|
198
|
+
|
|
199
|
+
vicinity = Vicinity.from_vectors_and_items(vectors=vectors, items=items)
|
|
200
|
+
```
|
|
201
|
+
|
|
202
|
+
</details>
|
|
203
|
+
|
|
204
|
+
<details>
|
|
205
|
+
<summary> Querying
|
|
206
|
+
</summary>
|
|
207
|
+
<br>
|
|
208
|
+
|
|
209
|
+
Find the k nearest neighbors for a given vector:
|
|
210
|
+
|
|
211
|
+
```python
|
|
212
|
+
query_vector = np.random.rand(128)
|
|
213
|
+
results = vicinity.query([query_vector], k=3)
|
|
214
|
+
```
|
|
215
|
+
|
|
216
|
+
Find all neighbors within a given threshold:
|
|
217
|
+
|
|
218
|
+
```python
|
|
219
|
+
query_vector = np.random.rand(128)
|
|
220
|
+
results = vicinity.query_threshold([query_vector], threshold=0.9)
|
|
221
|
+
```
|
|
222
|
+
</details>
|
|
223
|
+
|
|
224
|
+
<details>
|
|
225
|
+
|
|
226
|
+
<summary> Inserting and Deleting Items
|
|
227
|
+
</summary>
|
|
228
|
+
<br>
|
|
229
|
+
|
|
230
|
+
Insert new items:
|
|
231
|
+
|
|
232
|
+
```python
|
|
233
|
+
new_items = ["ocarina", "bow"]
|
|
234
|
+
new_vectors = np.random.rand(2, 128)
|
|
235
|
+
vicinity.insert(new_items, new_vectors)
|
|
236
|
+
```
|
|
237
|
+
|
|
238
|
+
Delete items:
|
|
239
|
+
|
|
240
|
+
```python
|
|
241
|
+
vicinity.delete(["hookshot"])
|
|
242
|
+
```
|
|
243
|
+
</details>
|
|
244
|
+
|
|
245
|
+
<details>
|
|
246
|
+
<summary> Saving and Loading
|
|
247
|
+
</summary>
|
|
248
|
+
<br>
|
|
249
|
+
|
|
250
|
+
Save the vector store:
|
|
251
|
+
|
|
252
|
+
```python
|
|
253
|
+
vicinity.save('my_vector_store')
|
|
254
|
+
```
|
|
255
|
+
|
|
256
|
+
Load the vector store:
|
|
257
|
+
|
|
258
|
+
```python
|
|
259
|
+
vicinity = Vicinity.load('my_vector_store')
|
|
260
|
+
```
|
|
261
|
+
</details>
|
|
262
|
+
|
|
263
|
+
## License
|
|
264
|
+
|
|
265
|
+
MIT
|
vicinity-0.2.1/README.md
ADDED
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
<div align="center">
|
|
2
|
+
|
|
3
|
+
# Vicinity: The Lightweight Vector Store
|
|
4
|
+
|
|
5
|
+
</div>
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
<div align="center">
|
|
9
|
+
<h2>
|
|
10
|
+
<a href="https://pypi.org/project/vicinity/"><img src="https://img.shields.io/pypi/v/vicinity?color=%23007ec6&label=pypi%20package" alt="Package version"></a>
|
|
11
|
+
<a href="https://pypi.org/project/vicinity/"><img src="https://img.shields.io/pypi/pyversions/vicinity" alt="Supported Python versions"></a>
|
|
12
|
+
<a href="https://pepy.tech/project/vicinity">
|
|
13
|
+
<img src="https://static.pepy.tech/badge/vicinity" alt="Downloads">
|
|
14
|
+
</a>
|
|
15
|
+
<a href="https://app.codecov.io/gh/MinishLab/vicinity">
|
|
16
|
+
<img src="https://codecov.io/gh/MinishLab/vicinity/graph/badge.svg?token=0MQ2945OZL" alt="Codecov">
|
|
17
|
+
</a>
|
|
18
|
+
<a href="https://github.com/MinishLab/vicinity/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-MIT-green" alt="License - MIT"></a>
|
|
19
|
+
</h2>
|
|
20
|
+
</div>
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
## Table of contents
|
|
24
|
+
|
|
25
|
+
- [Quickstart](#quickstart)
|
|
26
|
+
- [Main Features](#main-features)
|
|
27
|
+
- [Supported Backends](#supported-backends)
|
|
28
|
+
- [Backend Parameters](#backend-parameters)
|
|
29
|
+
- [Usage](#usage)
|
|
30
|
+
|
|
31
|
+
Vicinity is the lightest-weight vector store. Just put in some vectors, calculate query vectors, and off you go. It provides a simple and intuitive API for nearest neighbor search, with support for different backends.
|
|
32
|
+
|
|
33
|
+
## Quickstart
|
|
34
|
+
|
|
35
|
+
Install the package with:
|
|
36
|
+
```bash
|
|
37
|
+
pip install vicinity
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
The following code snippet demonstrates how to use Vicinity for nearest neighbor search:
|
|
42
|
+
```python
|
|
43
|
+
import numpy as np
|
|
44
|
+
from vicinity import Vicinity
|
|
45
|
+
from vicinity.datatypes import Backend
|
|
46
|
+
|
|
47
|
+
# Create some dummy data
|
|
48
|
+
items = ["triforce", "master sword", "hylian shield", "boomerang", "hookshot"]
|
|
49
|
+
vectors = np.random.rand(len(items), 128)
|
|
50
|
+
|
|
51
|
+
# Initialize the Vicinity instance (using the basic backend)
|
|
52
|
+
vicinity = Vicinity.from_vectors_and_items(vectors=vectors, items=items, backend_type=Backend.BASIC)
|
|
53
|
+
|
|
54
|
+
# Query for nearest neighbors with a top-k search
|
|
55
|
+
query_vector = np.random.rand(128)
|
|
56
|
+
results = vicinity.query([query_vector], k=3)
|
|
57
|
+
|
|
58
|
+
# Query for nearest neighbors with a threshold search
|
|
59
|
+
results = vicinity.query_threshold([query_vector], threshold=0.9)
|
|
60
|
+
|
|
61
|
+
# Save the vector store
|
|
62
|
+
vicinity.save('my_vector_store')
|
|
63
|
+
|
|
64
|
+
# Load the vector store
|
|
65
|
+
vicinity = Vicinity.load('my_vector_store')
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
## Main Features
|
|
69
|
+
Vicinity provides the following features:
|
|
70
|
+
- Lightweight: Minimal dependencies and fast performance.
|
|
71
|
+
- Flexible Backend Support: Use different backends for vector storage and search.
|
|
72
|
+
- Serialization: Save and load vector stores for persistence.
|
|
73
|
+
- Easy to Use: Simple and intuitive API.
|
|
74
|
+
|
|
75
|
+
## Supported Backends
|
|
76
|
+
The following backends are supported:
|
|
77
|
+
- `BASIC`: A simple flat index for vector storage and search.
|
|
78
|
+
- [HNSW](https://github.com/nmslib/hnswlib): Hierarchical Navigable Small World Graph (HNSW) for ANN search using hnswlib.
|
|
79
|
+
- [FAISS](https://github.com/facebookresearch/faiss): ANN search using FAISS. All FAISS indexes are supported.
|
|
80
|
+
- [ANNOY](https://github.com/spotify/annoy): "Approximate Nearest Neighbors Oh Yeah" for approximate nearest neighbor search.
|
|
81
|
+
- [PYNNDescent](https://github.com/lmcinnes/pynndescent): ANN search using PyNNDescent.
|
|
82
|
+
- [USEARCH](https://github.com/unum-cloud/usearch): ANN search using Usearch. This uses a highly optimized version of the HNSW algorithm.
|
|
83
|
+
|
|
84
|
+
NOTE: the ANN backends do not support dynamic deletion. To delete items, you need to recreate the index. Insertion is supported in the following backends: `FAISS`, `HNSW`, and `Usearch`. The `BASIC` backend supports both insertion and deletion.
|
|
85
|
+
|
|
86
|
+
### Backend Parameters
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
| Backend | Parameter | Description | Default Value |
|
|
90
|
+
|-----------------|---------------------|-----------------------------------------------------------------------------------------------|---------------------|
|
|
91
|
+
| **Annoy** | `metric` | Similarity metric to use (`dot`, `euclidean`, `cosine`). | `"cosine"` |
|
|
92
|
+
| | `trees` | Number of trees to use for indexing. | `100` |
|
|
93
|
+
| | `length` | Optional length of the dataset. | `None` |
|
|
94
|
+
| **FAISS** | `metric` | Similarity metric to use (`cosine`, `l2`). | `"cosine"` |
|
|
95
|
+
| | `index_type` | Type of FAISS index (`flat`, `ivf`, `hnsw`, `lsh`, `scalar`, `pq`, `ivf_scalar`, `ivfpq`, `ivfpqr`). | `"hnsw"` |
|
|
96
|
+
| | `nlist` | Number of cells for IVF indexes. | `100` |
|
|
97
|
+
| | `m` | Number of subquantizers for PQ and HNSW indexes. | `8` |
|
|
98
|
+
| | `nbits` | Number of bits for LSH and PQ indexes. | `8` |
|
|
99
|
+
| | `refine_nbits` | Number of bits for the refinement stage in IVFPQR indexes. | `8` |
|
|
100
|
+
| **HNSW** | `metric` | Similarity space to use (`cosine`, `l2`). | `"cosine"` |
|
|
101
|
+
| | `ef_construction` | Size of the dynamic list during index construction. | `200` |
|
|
102
|
+
| | `m` | Number of connections per layer. | `16` |
|
|
103
|
+
| **PyNNDescent** | `metric` | Similarity metric to use (`cosine`, `euclidean`, `manhattan`). | `"cosine"` |
|
|
104
|
+
| | `n_neighbors` | Number of neighbors to use for search. | `15` |
|
|
105
|
+
| **Usearch** | `metric` | Similarity metric to use (`cos`, `ip`, `l2sq`, `hamming`, `tanimoto`). | `"cos"` |
|
|
106
|
+
| | `connectivity` | Number of connections per node in the graph. | `16` |
|
|
107
|
+
| | `expansion_add` | Number of candidates considered during graph construction. | `128` |
|
|
108
|
+
| | `expansion_search` | Number of candidates considered during search. | `64` |
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
## Usage
|
|
113
|
+
|
|
114
|
+
<details>
|
|
115
|
+
<summary> Creating a Vector Store
|
|
116
|
+
</summary>
|
|
117
|
+
<br>
|
|
118
|
+
|
|
119
|
+
You can create a Vicinity instance by providing items and their corresponding vectors:
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
```python
|
|
123
|
+
from vicinity import Vicinity
|
|
124
|
+
import numpy as np
|
|
125
|
+
|
|
126
|
+
items = ["triforce", "master sword", "hylian shield", "boomerang", "hookshot"]
|
|
127
|
+
vectors = np.random.rand(len(items), 128)
|
|
128
|
+
|
|
129
|
+
vicinity = Vicinity.from_vectors_and_items(vectors=vectors, items=items)
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
</details>
|
|
133
|
+
|
|
134
|
+
<details>
|
|
135
|
+
<summary> Querying
|
|
136
|
+
</summary>
|
|
137
|
+
<br>
|
|
138
|
+
|
|
139
|
+
Find the k nearest neighbors for a given vector:
|
|
140
|
+
|
|
141
|
+
```python
|
|
142
|
+
query_vector = np.random.rand(128)
|
|
143
|
+
results = vicinity.query([query_vector], k=3)
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
Find all neighbors within a given threshold:
|
|
147
|
+
|
|
148
|
+
```python
|
|
149
|
+
query_vector = np.random.rand(128)
|
|
150
|
+
results = vicinity.query_threshold([query_vector], threshold=0.9)
|
|
151
|
+
```
|
|
152
|
+
</details>
|
|
153
|
+
|
|
154
|
+
<details>
|
|
155
|
+
|
|
156
|
+
<summary> Inserting and Deleting Items
|
|
157
|
+
</summary>
|
|
158
|
+
<br>
|
|
159
|
+
|
|
160
|
+
Insert new items:
|
|
161
|
+
|
|
162
|
+
```python
|
|
163
|
+
new_items = ["ocarina", "bow"]
|
|
164
|
+
new_vectors = np.random.rand(2, 128)
|
|
165
|
+
vicinity.insert(new_items, new_vectors)
|
|
166
|
+
```
|
|
167
|
+
|
|
168
|
+
Delete items:
|
|
169
|
+
|
|
170
|
+
```python
|
|
171
|
+
vicinity.delete(["hookshot"])
|
|
172
|
+
```
|
|
173
|
+
</details>
|
|
174
|
+
|
|
175
|
+
<details>
|
|
176
|
+
<summary> Saving and Loading
|
|
177
|
+
</summary>
|
|
178
|
+
<br>
|
|
179
|
+
|
|
180
|
+
Save the vector store:
|
|
181
|
+
|
|
182
|
+
```python
|
|
183
|
+
vicinity.save('my_vector_store')
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
Load the vector store:
|
|
187
|
+
|
|
188
|
+
```python
|
|
189
|
+
vicinity = Vicinity.load('my_vector_store')
|
|
190
|
+
```
|
|
191
|
+
</details>
|
|
192
|
+
|
|
193
|
+
## License
|
|
194
|
+
|
|
195
|
+
MIT
|
|
@@ -3,7 +3,7 @@ name = "vicinity"
|
|
|
3
3
|
description = "Lightweight Nearest Neighbors with Flexible Backends"
|
|
4
4
|
readme = { file = "README.md", content-type = "text/markdown" }
|
|
5
5
|
dynamic = ["version"]
|
|
6
|
-
requires-python = ">=3.
|
|
6
|
+
requires-python = ">=3.9"
|
|
7
7
|
license = { file = "LICENSE" }
|
|
8
8
|
authors = [{ name = "Stéphan Tulkens", email = "stephantul@gmail.com"}, {name = "Thomas van Dongen", email = "thomas123@live.nl"}]
|
|
9
9
|
|
|
@@ -15,6 +15,7 @@ classifiers = [
|
|
|
15
15
|
"Topic :: Software Development :: Libraries",
|
|
16
16
|
"License :: OSI Approved :: MIT License",
|
|
17
17
|
"Programming Language :: Python :: 3 :: Only",
|
|
18
|
+
"Programming Language :: Python :: 3.9",
|
|
18
19
|
"Programming Language :: Python :: 3.10",
|
|
19
20
|
"Programming Language :: Python :: 3.11",
|
|
20
21
|
"Programming Language :: Python :: 3.12",
|
|
@@ -39,8 +40,23 @@ dev = [
|
|
|
39
40
|
"pytest",
|
|
40
41
|
"pytest-coverage",
|
|
41
42
|
"ruff",
|
|
43
|
+
"setuptools"
|
|
42
44
|
]
|
|
43
45
|
hnsw = ["hnswlib"]
|
|
46
|
+
pynndescent = [
|
|
47
|
+
"pynndescent>=0.5.10",
|
|
48
|
+
"numba>=0.59.0",
|
|
49
|
+
"llvmlite>=0.42.0",
|
|
50
|
+
"numpy>=1.24.0"
|
|
51
|
+
]
|
|
52
|
+
annoy = ["annoy"]
|
|
53
|
+
faiss = ["faiss-cpu"]
|
|
54
|
+
usearch = ["usearch"]
|
|
55
|
+
|
|
56
|
+
[project.urls]
|
|
57
|
+
"Homepage" = "https://github.com/MinishLab"
|
|
58
|
+
"Bug Reports" = "https://github.com/MinishLab/vicinity/issues"
|
|
59
|
+
"Source" = "https://github.com/MinishLab/vicinity"
|
|
44
60
|
|
|
45
61
|
[tool.ruff]
|
|
46
62
|
exclude = [".venv/"]
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import numpy as np
|
|
4
|
+
import pytest
|
|
5
|
+
|
|
6
|
+
from vicinity import Vicinity
|
|
7
|
+
from vicinity.datatypes import Backend
|
|
8
|
+
|
|
9
|
+
random_gen = np.random.default_rng(42)
|
|
10
|
+
|
|
11
|
+
_faiss_index_types = ["flat", "ivf", "hnsw", "lsh", "scalar", "pq", "ivf_scalar", "ivfpq", "ivfpqr"]
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@pytest.fixture(scope="session")
|
|
15
|
+
def items() -> list[str]:
|
|
16
|
+
"""Fixture providing a list of item names."""
|
|
17
|
+
return [f"item{i}" for i in range(1, 10001)]
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@pytest.fixture(scope="session")
|
|
21
|
+
def vectors() -> np.ndarray:
|
|
22
|
+
"""Fixture providing an array of vectors corresponding to items."""
|
|
23
|
+
return random_gen.random((10000, 8))
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@pytest.fixture(scope="session")
|
|
27
|
+
def query_vector() -> np.ndarray:
|
|
28
|
+
"""Fixture providing a query vector."""
|
|
29
|
+
return random_gen.random(8)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
BACKEND_PARAMS = [(Backend.FAISS, index_type) for index_type in _faiss_index_types] + [
|
|
33
|
+
(Backend.BASIC, None),
|
|
34
|
+
(Backend.HNSW, None),
|
|
35
|
+
(Backend.ANNOY, None),
|
|
36
|
+
(Backend.PYNNDESCENT, None),
|
|
37
|
+
(Backend.USEARCH, None),
|
|
38
|
+
]
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
# Create human-readable ids for each backend type
|
|
42
|
+
BACKEND_IDS = [f"{backend.name}-{index_type}" if index_type else backend.name for backend, index_type in BACKEND_PARAMS]
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
@pytest.fixture(params=BACKEND_PARAMS)
|
|
46
|
+
def backend_type(request: pytest.FixtureRequest) -> Backend:
|
|
47
|
+
"""Fixture parametrizing over all backend types defined in Backend."""
|
|
48
|
+
return request.param
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
@pytest.fixture(params=BACKEND_PARAMS, ids=BACKEND_IDS)
|
|
52
|
+
def vicinity_instance(request: pytest.FixtureRequest, items: list[str], vectors: np.ndarray) -> Vicinity:
|
|
53
|
+
"""Fixture providing a Vicinity instance for each backend type."""
|
|
54
|
+
backend_type, index_type = request.param
|
|
55
|
+
# Handle FAISS backend with specific FAISS index types
|
|
56
|
+
if backend_type == Backend.FAISS:
|
|
57
|
+
if index_type in ("pq", "ivfpq", "ivfpqr"):
|
|
58
|
+
# Use smaller values for pq indexes since the dataset is small
|
|
59
|
+
return Vicinity.from_vectors_and_items(
|
|
60
|
+
vectors, items, backend_type=backend_type, index_type=index_type, m=2, nbits=4
|
|
61
|
+
)
|
|
62
|
+
else:
|
|
63
|
+
return Vicinity.from_vectors_and_items(
|
|
64
|
+
vectors, items, backend_type=backend_type, index_type=index_type, nlist=2, nbits=32
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
return Vicinity.from_vectors_and_items(vectors, items, backend_type=backend_type)
|