fromscratch-pyvecdb 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fromscratch_pyvecdb-0.3.0.dist-info/METADATA +147 -0
- fromscratch_pyvecdb-0.3.0.dist-info/RECORD +16 -0
- fromscratch_pyvecdb-0.3.0.dist-info/WHEEL +4 -0
- fromscratch_pyvecdb-0.3.0.dist-info/licenses/LICENSE +21 -0
- pyvecdb/__init__.py +68 -0
- pyvecdb/_version.py +24 -0
- pyvecdb/atomic.py +66 -0
- pyvecdb/distance.py +138 -0
- pyvecdb/errors.py +61 -0
- pyvecdb/filtering.py +128 -0
- pyvecdb/persistence.py +149 -0
- pyvecdb/py.typed +0 -0
- pyvecdb/record.py +94 -0
- pyvecdb/search.py +165 -0
- pyvecdb/store.py +250 -0
- pyvecdb/vector.py +180 -0
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: fromscratch-pyvecdb
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Summary: An educational vector database library built from scratch in Python
|
|
5
|
+
Project-URL: Homepage, https://github.com/robcox2910/pyvecdb
|
|
6
|
+
Project-URL: Repository, https://github.com/robcox2910/pyvecdb
|
|
7
|
+
Project-URL: Issues, https://github.com/robcox2910/pyvecdb/issues
|
|
8
|
+
Project-URL: Documentation, https://robcox2910.github.io/pyvecdb/
|
|
9
|
+
Author: Rob Cox
|
|
10
|
+
License: MIT
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Keywords: database,education,search,similarity,vector
|
|
13
|
+
Classifier: Development Status :: 3 - Alpha
|
|
14
|
+
Classifier: Intended Audience :: Education
|
|
15
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
18
|
+
Classifier: Topic :: Education
|
|
19
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
20
|
+
Classifier: Typing :: Typed
|
|
21
|
+
Requires-Python: >=3.14
|
|
22
|
+
Provides-Extra: dev
|
|
23
|
+
Requires-Dist: commitizen>=4.4.0; extra == 'dev'
|
|
24
|
+
Requires-Dist: hypothesis>=6.170; extra == 'dev'
|
|
25
|
+
Requires-Dist: pip-audit>=2.10.0; extra == 'dev'
|
|
26
|
+
Requires-Dist: pre-commit>=4.0.1; extra == 'dev'
|
|
27
|
+
Requires-Dist: pyright>=1.1.400; extra == 'dev'
|
|
28
|
+
Requires-Dist: pytest-cov>=6.0.0; extra == 'dev'
|
|
29
|
+
Requires-Dist: pytest>=8.3.4; extra == 'dev'
|
|
30
|
+
Requires-Dist: ruff>=0.9.3; extra == 'dev'
|
|
31
|
+
Description-Content-Type: text/markdown
|
|
32
|
+
|
|
33
|
+
# pyvecdb
|
|
34
|
+
|
|
35
|
+
An educational vector database library built from scratch in Python. Part of the **PyLearn** platform for kids aged 12+.
|
|
36
|
+
|
|
37
|
+
## What is a Vector Database?
|
|
38
|
+
|
|
39
|
+
A vector database stores **vectors** (lists of numbers) and finds **similar** ones. Think of it like a music app that turns songs into lists of numbers and finds other songs with similar numbers!
|
|
40
|
+
|
|
41
|
+
## Features
|
|
42
|
+
|
|
43
|
+
- **Vector Operations** -- add, subtract, scale, normalize, dot product, magnitude
|
|
44
|
+
- **Vector Store** -- CRUD operations for vector records with dimensionality checks
|
|
45
|
+
- **Distance Metrics** -- cosine similarity, Euclidean, Manhattan, dot product
|
|
46
|
+
- **KNN Search** -- find the K most similar vectors to a query
|
|
47
|
+
- **Metadata Filtering** -- filter results by metadata (eq, ne, gt, gte, lt, lte, in)
|
|
48
|
+
- **Batch Operations** -- insert and search multiple vectors at once
|
|
49
|
+
- **Persistence** -- save/load to JSON files
|
|
50
|
+
- **Pure Python** -- no external dependencies
|
|
51
|
+
|
|
52
|
+
## Quick Start
|
|
53
|
+
|
|
54
|
+
```python
|
|
55
|
+
from pyvecdb import VectorStore, knn_search
|
|
56
|
+
|
|
57
|
+
# Create a store for 3-dimensional vectors
|
|
58
|
+
store = VectorStore(dimensions=3)
|
|
59
|
+
|
|
60
|
+
# Insert some songs as vectors (energy, happiness, danceability)
|
|
61
|
+
store.insert("thunder", [0.9, 0.6, 0.7], metadata={"genre": "rock", "year": 2017})
|
|
62
|
+
store.insert("lullaby", [0.1, 0.8, 0.2], metadata={"genre": "folk", "year": 2020})
|
|
63
|
+
store.insert("party-rock", [0.95, 0.9, 0.99], metadata={"genre": "pop", "year": 2011})
|
|
64
|
+
store.insert("ballad", [0.3, 0.4, 0.1], metadata={"genre": "rock", "year": 2019})
|
|
65
|
+
|
|
66
|
+
# Find the 2 most similar songs to a high-energy query
|
|
67
|
+
results = knn_search(store, query=[0.85, 0.7, 0.8], k=2)
|
|
68
|
+
for record, score in results:
|
|
69
|
+
print(f"{record.record_id}: {score:.3f}")
|
|
70
|
+
# party-rock: 0.998
|
|
71
|
+
# thunder: 0.995
|
|
72
|
+
|
|
73
|
+
# Filter by metadata -- only rock songs
|
|
74
|
+
results = knn_search(
|
|
75
|
+
store,
|
|
76
|
+
query=[0.85, 0.7, 0.8],
|
|
77
|
+
k=2,
|
|
78
|
+
metadata_filter={"genre": "rock"},
|
|
79
|
+
)
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
## Installation
|
|
83
|
+
|
|
84
|
+
```bash
|
|
85
|
+
git clone https://github.com/robcox2910/pyvecdb.git
|
|
86
|
+
cd pyvecdb
|
|
87
|
+
uv sync --all-extras
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
## Development
|
|
91
|
+
|
|
92
|
+
```bash
|
|
93
|
+
uv run ruff check src/ tests/ # Lint
|
|
94
|
+
uv run ruff format src/ tests/ # Format
|
|
95
|
+
uv run pyright src tests # Type check
|
|
96
|
+
uv run pytest # Test (90%+ coverage)
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
## Releasing
|
|
100
|
+
|
|
101
|
+
Other projects in the series ([PyStack](https://github.com/robcox2910/pystack)) use a pinned release of this one,
|
|
102
|
+
not whatever is on `main`. To publish a change for them:
|
|
103
|
+
|
|
104
|
+
1. Merge the change to `main`.
|
|
105
|
+
2. Tag it: `git tag -a vX.Y.Z -m vX.Y.Z && git push origin vX.Y.Z`.
|
|
106
|
+
A change to the public API bumps the minor version (`v0.1.0` → `v0.2.0`);
|
|
107
|
+
anything else bumps the patch (`v0.1.0` → `v0.1.1`).
|
|
108
|
+
3. In each project that uses it, edit the tag in `[tool.uv.sources]`, run
|
|
109
|
+
`uv lock --upgrade-package <name>`, and open a pull request so CI checks
|
|
110
|
+
the pair together.
|
|
111
|
+
|
|
112
|
+
<!-- series:start -->
|
|
113
|
+
## Related Projects
|
|
114
|
+
|
|
115
|
+
This project is one of a series that builds real systems "from scratch in Python" to teach how they work. See the [series home](https://robcox2910.github.io/) for the full story.
|
|
116
|
+
|
|
117
|
+
| Project | What it teaches | Repository | Docs |
|
|
118
|
+
| --- | --- | --- | --- |
|
|
119
|
+
| PyOS | Operating systems | [py-os](https://github.com/robcox2910/py-os) | [Docs](https://robcox2910.github.io/py-os/) |
|
|
120
|
+
| Pebble | Compilers and programming languages | [pebble-lang](https://github.com/robcox2910/pebble-lang) | [Docs](https://robcox2910.github.io/pebble-lang/) |
|
|
121
|
+
| PyDB | Relational databases | [pydb](https://github.com/robcox2910/pydb) | [Docs](https://robcox2910.github.io/pydb/) |
|
|
122
|
+
| PyKV | Key-value stores | [pykv](https://github.com/robcox2910/pykv) | [Docs](https://robcox2910.github.io/pykv/) |
|
|
123
|
+
| PyDocDB | Document databases | [pydoc-db](https://github.com/robcox2910/pydoc-db) | [Docs](https://robcox2910.github.io/pydoc-db/) |
|
|
124
|
+
| PyGraphDB | Graph databases | [pygraphdb](https://github.com/robcox2910/pygraphdb) | [Docs](https://robcox2910.github.io/pygraphdb/) |
|
|
125
|
+
| PyTSDB | Time-series databases | [pytsdb](https://github.com/robcox2910/pytsdb) | [Docs](https://robcox2910.github.io/pytsdb/) |
|
|
126
|
+
| **PyVecDB** | **Vector databases** | [pyvecdb](https://github.com/robcox2910/pyvecdb) | [Docs](https://robcox2910.github.io/pyvecdb/) |
|
|
127
|
+
| PyColDB | Column-family databases | [pycoldb](https://github.com/robcox2910/pycoldb) | [Docs](https://robcox2910.github.io/pycoldb/) |
|
|
128
|
+
| PyCrypt | Cryptography | [pycrypt](https://github.com/robcox2910/pycrypt) | [Docs](https://robcox2910.github.io/pycrypt/) |
|
|
129
|
+
| PyNet | Computer networking | [pynet](https://github.com/robcox2910/pynet) | [Docs](https://robcox2910.github.io/pynet/) |
|
|
130
|
+
| PyWeb | HTTP web servers | [pyweb](https://github.com/robcox2910/pyweb) | [Docs](https://robcox2910.github.io/pyweb/) |
|
|
131
|
+
| PyGit | Version control | [pygit](https://github.com/robcox2910/pygit) | [Docs](https://robcox2910.github.io/pygit/) |
|
|
132
|
+
| PySearch | Search engines | [pysearch](https://github.com/robcox2910/pysearch) | [Docs](https://robcox2910.github.io/pysearch/) |
|
|
133
|
+
| PyMQ | Message queues | [pymq](https://github.com/robcox2910/pymq) | [Docs](https://robcox2910.github.io/pymq/) |
|
|
134
|
+
| PyLLM | Language models (the brain) | [pyllm](https://github.com/robcox2910/pyllm) | [Docs](https://robcox2910.github.io/pyllm/) |
|
|
135
|
+
| PyScheduler | Job scheduling | [pyscheduler](https://github.com/robcox2910/pyscheduler) | [Docs](https://robcox2910.github.io/pyscheduler/) |
|
|
136
|
+
| PyStack | Full-stack integration | [pystack](https://github.com/robcox2910/pystack) | [Docs](https://robcox2910.github.io/pystack/) |
|
|
137
|
+
<!-- series:end -->
|
|
138
|
+
|
|
139
|
+
<!-- docs:start -->
|
|
140
|
+
## Documentation
|
|
141
|
+
|
|
142
|
+
Full docs (with kid-friendly explanations of every idea): https://robcox2910.github.io/pyvecdb/
|
|
143
|
+
<!-- docs:end -->
|
|
144
|
+
|
|
145
|
+
## License
|
|
146
|
+
|
|
147
|
+
MIT
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
pyvecdb/__init__.py,sha256=60yt6pheS_PQEOV63U2Iqqpgtmnd4akAvS-5NQZdrmA,2000
|
|
2
|
+
pyvecdb/_version.py,sha256=RCY04NY0kBCyn6ttH6lnVL6Gz2gceOrt45f0yDOOU8A,520
|
|
3
|
+
pyvecdb/atomic.py,sha256=QLLPGZqpA-TgVV5lDEKCLB7vVr05bxj97_Fwykz_gi4,2345
|
|
4
|
+
pyvecdb/distance.py,sha256=Tq_GGed8SdcZ8RQ4GgEwUGoyXcd10KqUyyBb5EUQQgQ,4341
|
|
5
|
+
pyvecdb/errors.py,sha256=7ajMrizSEMn_e4rd7mYvdtIQL9dWv38oU3p7OrCa5NQ,1711
|
|
6
|
+
pyvecdb/filtering.py,sha256=s1S6_aV82vLBiKHx_11Y-FtrNFyBkLoAkjUijoT6HT8,4041
|
|
7
|
+
pyvecdb/persistence.py,sha256=BCIWp0alRku6ATia-WvhZOkpEOLkV5IjfZJTW0nirVY,5076
|
|
8
|
+
pyvecdb/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
9
|
+
pyvecdb/record.py,sha256=sUUgaTG-_7wjp9NaZ5V9Iv0ozcZSTWrTTuEYMp3btwo,3174
|
|
10
|
+
pyvecdb/search.py,sha256=MKtLNSbsvjM2S4Mg2WNQoJd6quHn0h2N4R-tciFf_Cw,5395
|
|
11
|
+
pyvecdb/store.py,sha256=l_rsP5piqlEbo4Je2N-QdsoXu9za4JtCVtDFdo6-LUs,8212
|
|
12
|
+
pyvecdb/vector.py,sha256=uhC4q4aeF9QV1_ste_nV8PHpvzKO8Yp7DE5tnrIwGxs,5208
|
|
13
|
+
fromscratch_pyvecdb-0.3.0.dist-info/METADATA,sha256=nicgBasl2jhM8RwBMAnt6yZyMPOAvM9F8qwJH5oNzSQ,6957
|
|
14
|
+
fromscratch_pyvecdb-0.3.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
|
|
15
|
+
fromscratch_pyvecdb-0.3.0.dist-info/licenses/LICENSE,sha256=NtHM2ak--oE71VCcY50kcCoC1cKS0LRUsxq_xA89RNc,1064
|
|
16
|
+
fromscratch_pyvecdb-0.3.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Rob Cox
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
pyvecdb/__init__.py
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
"""pyvecdb -- an educational vector database built from scratch in Python.
|
|
2
|
+
|
|
3
|
+
A vector database stores vectors (lists of numbers) and finds similar ones.
|
|
4
|
+
Think of it like a music app that knows your taste -- it turns songs into
|
|
5
|
+
lists of numbers and finds other songs with similar numbers.
|
|
6
|
+
|
|
7
|
+
This is part of the PyLearn educational platform for kids aged 12+.
|
|
8
|
+
|
|
9
|
+
Quick Start:
|
|
10
|
+
>>> from pyvecdb import VectorStore, knn_search
|
|
11
|
+
>>> store = VectorStore(dimensions=3)
|
|
12
|
+
>>> store.insert("rock-song", [0.9, 0.1, 0.3], metadata={"genre": "rock"})
|
|
13
|
+
>>> store.insert("pop-song", [0.2, 0.8, 0.4], metadata={"genre": "pop"})
|
|
14
|
+
>>> store.insert("rock-ballad", [0.7, 0.3, 0.5], metadata={"genre": "rock"})
|
|
15
|
+
>>> results = knn_search(store, query=[0.85, 0.15, 0.35], k=2)
|
|
16
|
+
>>> [r.record_id for r, _ in results]
|
|
17
|
+
['rock-song', 'rock-ballad']
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from pyvecdb.distance import (
|
|
21
|
+
cosine_similarity,
|
|
22
|
+
dot_product,
|
|
23
|
+
euclidean_distance,
|
|
24
|
+
manhattan_distance,
|
|
25
|
+
)
|
|
26
|
+
from pyvecdb.errors import (
|
|
27
|
+
DimensionMismatchError,
|
|
28
|
+
DuplicateIDError,
|
|
29
|
+
EmptyStoreError,
|
|
30
|
+
EmptyVectorError,
|
|
31
|
+
PersistenceError,
|
|
32
|
+
RecordNotFoundError,
|
|
33
|
+
VecDBError,
|
|
34
|
+
)
|
|
35
|
+
from pyvecdb.filtering import filter_records, matches_filter
|
|
36
|
+
from pyvecdb.persistence import load_store, save_store
|
|
37
|
+
from pyvecdb.record import VectorRecord
|
|
38
|
+
from pyvecdb.search import batch_search, knn_search
|
|
39
|
+
from pyvecdb.store import VectorStore
|
|
40
|
+
from pyvecdb.vector import add, dot, magnitude, normalize, scale, subtract
|
|
41
|
+
|
|
42
|
+
__all__ = [
|
|
43
|
+
"DimensionMismatchError",
|
|
44
|
+
"DuplicateIDError",
|
|
45
|
+
"EmptyStoreError",
|
|
46
|
+
"EmptyVectorError",
|
|
47
|
+
"PersistenceError",
|
|
48
|
+
"RecordNotFoundError",
|
|
49
|
+
"VecDBError",
|
|
50
|
+
"VectorRecord",
|
|
51
|
+
"VectorStore",
|
|
52
|
+
"add",
|
|
53
|
+
"batch_search",
|
|
54
|
+
"cosine_similarity",
|
|
55
|
+
"dot",
|
|
56
|
+
"dot_product",
|
|
57
|
+
"euclidean_distance",
|
|
58
|
+
"filter_records",
|
|
59
|
+
"knn_search",
|
|
60
|
+
"load_store",
|
|
61
|
+
"magnitude",
|
|
62
|
+
"manhattan_distance",
|
|
63
|
+
"matches_filter",
|
|
64
|
+
"normalize",
|
|
65
|
+
"save_store",
|
|
66
|
+
"scale",
|
|
67
|
+
"subtract",
|
|
68
|
+
]
|
pyvecdb/_version.py
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# file generated by vcs-versioning
|
|
2
|
+
# don't change, don't track in version control
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
__all__ = [
|
|
6
|
+
"__version__",
|
|
7
|
+
"__version_tuple__",
|
|
8
|
+
"version",
|
|
9
|
+
"version_tuple",
|
|
10
|
+
"__commit_id__",
|
|
11
|
+
"commit_id",
|
|
12
|
+
]
|
|
13
|
+
|
|
14
|
+
version: str
|
|
15
|
+
__version__: str
|
|
16
|
+
__version_tuple__: tuple[int | str, ...]
|
|
17
|
+
version_tuple: tuple[int | str, ...]
|
|
18
|
+
commit_id: str | None
|
|
19
|
+
__commit_id__: str | None
|
|
20
|
+
|
|
21
|
+
__version__ = version = '0.3.0'
|
|
22
|
+
__version_tuple__ = version_tuple = (0, 3, 0)
|
|
23
|
+
|
|
24
|
+
__commit_id__ = commit_id = None
|
pyvecdb/atomic.py
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
"""Safe saving -- never leave a half-written file behind.
|
|
2
|
+
|
|
3
|
+
The usual way to save a file is: open it, wipe it, write the new
|
|
4
|
+
contents. If the computer crashes after the wipe but before the writing
|
|
5
|
+
finishes, your data is gone -- the old version was wiped and the new one
|
|
6
|
+
never arrived.
|
|
7
|
+
|
|
8
|
+
So we save like a careful student copying out their homework:
|
|
9
|
+
|
|
10
|
+
1. Write the new version on a **fresh sheet** (a temporary file sitting
|
|
11
|
+
right next to the real one).
|
|
12
|
+
2. Make sure the **ink is dry** -- ``fsync`` forces the bytes out of the
|
|
13
|
+
computer's memory and onto the disk.
|
|
14
|
+
3. **Swap** the fresh sheet in for the old one in a single instant step
|
|
15
|
+
(``Path.replace``). There is never a moment with no complete file: you
|
|
16
|
+
see either the old version or the new one, never half of each.
|
|
17
|
+
|
|
18
|
+
Real databases (SQLite, PostgreSQL, Redis) all use this same trick.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
import contextlib
|
|
22
|
+
import os
|
|
23
|
+
import stat
|
|
24
|
+
import tempfile
|
|
25
|
+
from pathlib import Path
|
|
26
|
+
|
|
27
|
+
# Permissions for a brand-new file: owner can read/write, others can read.
|
|
28
|
+
_NEW_FILE_MODE = 0o644
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def atomic_write_text(path: Path, text: str) -> None:
|
|
32
|
+
"""Replace the file at *path* with *text*, so a crash can never leave it half-written.
|
|
33
|
+
|
|
34
|
+
Args:
|
|
35
|
+
path: The file to create or replace (its folder must already exist).
|
|
36
|
+
text: The new contents, saved as UTF-8.
|
|
37
|
+
|
|
38
|
+
"""
|
|
39
|
+
mode = stat.S_IMODE(path.stat().st_mode) if path.exists() else _NEW_FILE_MODE
|
|
40
|
+
fd, temp_name = tempfile.mkstemp(dir=path.parent, prefix=f".{path.name}.", suffix=".tmp")
|
|
41
|
+
temp_path = Path(temp_name)
|
|
42
|
+
try:
|
|
43
|
+
with os.fdopen(fd, "w", encoding="utf-8") as temp_file:
|
|
44
|
+
temp_file.write(text)
|
|
45
|
+
temp_file.flush()
|
|
46
|
+
os.fsync(temp_file.fileno())
|
|
47
|
+
temp_path.chmod(mode)
|
|
48
|
+
temp_path.replace(path)
|
|
49
|
+
except BaseException:
|
|
50
|
+
temp_path.unlink(missing_ok=True) # don't leave the scrap sheet lying around
|
|
51
|
+
raise
|
|
52
|
+
_sync_folder(path.parent)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _sync_folder(folder: Path) -> None:
|
|
56
|
+
"""Make the swap itself durable by syncing the folder's list of files.
|
|
57
|
+
|
|
58
|
+
Some systems (like Windows) can't sync a folder; there the swap is
|
|
59
|
+
still safe, just not guaranteed to survive a power cut.
|
|
60
|
+
"""
|
|
61
|
+
with contextlib.suppress(OSError):
|
|
62
|
+
fd = os.open(folder, os.O_RDONLY)
|
|
63
|
+
try:
|
|
64
|
+
os.fsync(fd)
|
|
65
|
+
finally:
|
|
66
|
+
os.close(fd)
|
pyvecdb/distance.py
ADDED
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
"""Distance and similarity metrics for comparing vectors.
|
|
2
|
+
|
|
3
|
+
Imagine you and your friend each rate 5 movies from 1 to 10.
|
|
4
|
+
Your ratings are a vector like [8, 3, 9, 2, 7] and theirs might be [7, 4, 8, 3, 6].
|
|
5
|
+
Distance metrics tell you how similar your taste in movies is!
|
|
6
|
+
|
|
7
|
+
This module provides four ways to measure how "close" two vectors are:
|
|
8
|
+
|
|
9
|
+
- **Cosine similarity** -- Are you pointing the same direction? (most popular for AI)
|
|
10
|
+
- **Euclidean distance** -- How far apart are you in a straight line?
|
|
11
|
+
- **Dot product** -- How much do your ratings "agree"?
|
|
12
|
+
- **Manhattan distance** -- How far apart are you walking on a grid?
|
|
13
|
+
|
|
14
|
+
Example:
|
|
15
|
+
>>> from pyvecdb.distance import cosine_similarity
|
|
16
|
+
>>> cosine_similarity([1.0, 0.0], [1.0, 0.0])
|
|
17
|
+
1.0
|
|
18
|
+
>>> cosine_similarity([1.0, 0.0], [0.0, 1.0])
|
|
19
|
+
0.0
|
|
20
|
+
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
import math
|
|
24
|
+
|
|
25
|
+
from pyvecdb.vector import check_not_empty, check_same_dimensions
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def cosine_similarity(a: list[float], b: list[float]) -> float:
|
|
29
|
+
"""Compute cosine similarity between two vectors.
|
|
30
|
+
|
|
31
|
+
Cosine similarity measures the angle between two vectors, ignoring their length.
|
|
32
|
+
It returns a value between -1 (opposite) and 1 (identical direction).
|
|
33
|
+
Think of it like comparing which direction two arrows point, not how long they are.
|
|
34
|
+
|
|
35
|
+
Args:
|
|
36
|
+
a: The first vector.
|
|
37
|
+
b: The second vector.
|
|
38
|
+
|
|
39
|
+
Returns:
|
|
40
|
+
A float between -1.0 and 1.0 (1.0 means identical direction).
|
|
41
|
+
|
|
42
|
+
Raises:
|
|
43
|
+
DimensionMismatchError: If the vectors have different lengths.
|
|
44
|
+
EmptyVectorError: If either vector is empty.
|
|
45
|
+
ValueError: If either vector is a zero vector.
|
|
46
|
+
|
|
47
|
+
"""
|
|
48
|
+
check_not_empty(a)
|
|
49
|
+
check_not_empty(b)
|
|
50
|
+
check_same_dimensions(a, b)
|
|
51
|
+
|
|
52
|
+
dot_product = sum(x * y for x, y in zip(a, b, strict=False))
|
|
53
|
+
mag_a = math.sqrt(sum(x * x for x in a))
|
|
54
|
+
mag_b = math.sqrt(sum(x * x for x in b))
|
|
55
|
+
|
|
56
|
+
if mag_a == 0.0 or mag_b == 0.0:
|
|
57
|
+
msg = "Cannot compute cosine similarity with a zero vector"
|
|
58
|
+
raise ValueError(msg)
|
|
59
|
+
|
|
60
|
+
return dot_product / (mag_a * mag_b)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def euclidean_distance(a: list[float], b: list[float]) -> float:
|
|
64
|
+
"""Compute the Euclidean (straight-line) distance between two vectors.
|
|
65
|
+
|
|
66
|
+
This is the "as the crow flies" distance -- the shortest path between
|
|
67
|
+
two points. Like using a ruler to measure between two dots on paper.
|
|
68
|
+
|
|
69
|
+
Args:
|
|
70
|
+
a: The first vector.
|
|
71
|
+
b: The second vector.
|
|
72
|
+
|
|
73
|
+
Returns:
|
|
74
|
+
A non-negative float (0.0 means the vectors are identical).
|
|
75
|
+
|
|
76
|
+
Raises:
|
|
77
|
+
DimensionMismatchError: If the vectors have different lengths.
|
|
78
|
+
EmptyVectorError: If either vector is empty.
|
|
79
|
+
|
|
80
|
+
"""
|
|
81
|
+
check_not_empty(a)
|
|
82
|
+
check_not_empty(b)
|
|
83
|
+
check_same_dimensions(a, b)
|
|
84
|
+
|
|
85
|
+
# ``d * d`` rather than ``d ** 2``: for a number too big to square, ``**``
|
|
86
|
+
# raises OverflowError, while ``*`` gives infinity, just like magnitude().
|
|
87
|
+
return math.sqrt(sum((x - y) * (x - y) for x, y in zip(a, b, strict=False)))
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def dot_product(a: list[float], b: list[float]) -> float:
|
|
91
|
+
"""Compute the dot product of two vectors.
|
|
92
|
+
|
|
93
|
+
Multiply each pair of matching numbers and add them all up.
|
|
94
|
+
A bigger result means the vectors "agree" more.
|
|
95
|
+
|
|
96
|
+
Args:
|
|
97
|
+
a: The first vector.
|
|
98
|
+
b: The second vector.
|
|
99
|
+
|
|
100
|
+
Returns:
|
|
101
|
+
The sum of component-wise products.
|
|
102
|
+
|
|
103
|
+
Raises:
|
|
104
|
+
DimensionMismatchError: If the vectors have different lengths.
|
|
105
|
+
EmptyVectorError: If either vector is empty.
|
|
106
|
+
|
|
107
|
+
"""
|
|
108
|
+
check_not_empty(a)
|
|
109
|
+
check_not_empty(b)
|
|
110
|
+
check_same_dimensions(a, b)
|
|
111
|
+
|
|
112
|
+
return sum(x * y for x, y in zip(a, b, strict=False))
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def manhattan_distance(a: list[float], b: list[float]) -> float:
|
|
116
|
+
"""Compute the Manhattan (taxicab) distance between two vectors.
|
|
117
|
+
|
|
118
|
+
Imagine walking through a city grid -- you can only go along streets,
|
|
119
|
+
not cut diagonally through buildings. Manhattan distance is the total
|
|
120
|
+
number of blocks you walk (up/down + left/right).
|
|
121
|
+
|
|
122
|
+
Args:
|
|
123
|
+
a: The first vector.
|
|
124
|
+
b: The second vector.
|
|
125
|
+
|
|
126
|
+
Returns:
|
|
127
|
+
A non-negative float (0.0 means the vectors are identical).
|
|
128
|
+
|
|
129
|
+
Raises:
|
|
130
|
+
DimensionMismatchError: If the vectors have different lengths.
|
|
131
|
+
EmptyVectorError: If either vector is empty.
|
|
132
|
+
|
|
133
|
+
"""
|
|
134
|
+
check_not_empty(a)
|
|
135
|
+
check_not_empty(b)
|
|
136
|
+
check_same_dimensions(a, b)
|
|
137
|
+
|
|
138
|
+
return sum(abs(x - y) for x, y in zip(a, b, strict=False))
|
pyvecdb/errors.py
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
"""Custom exceptions for pyvecdb.
|
|
2
|
+
|
|
3
|
+
Think of these like warning signs at an amusement park -- they tell you
|
|
4
|
+
exactly what went wrong so you can fix it!
|
|
5
|
+
|
|
6
|
+
Example:
|
|
7
|
+
If you try to add a 3D vector to a store that expects 5D vectors,
|
|
8
|
+
you'll get a ``DimensionMismatchError`` telling you the sizes don't match.
|
|
9
|
+
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class VecDBError(Exception):
|
|
14
|
+
"""Base exception for all pyvecdb errors."""
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class DimensionMismatchError(VecDBError):
|
|
18
|
+
"""Raise when vector dimensions don't match.
|
|
19
|
+
|
|
20
|
+
Imagine trying to fit a square peg in a round hole -- if your vector
|
|
21
|
+
has 3 numbers but the store expects 5, that's a dimension mismatch!
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class DuplicateIDError(VecDBError):
|
|
26
|
+
"""Raise when a vector record with the same ID already exists.
|
|
27
|
+
|
|
28
|
+
Like how every student has a unique student ID -- you can't have
|
|
29
|
+
two students with the same number.
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class RecordNotFoundError(VecDBError):
|
|
34
|
+
"""Raise when a requested vector record is not found.
|
|
35
|
+
|
|
36
|
+
Like looking for a book in the library that doesn't exist -- the
|
|
37
|
+
ID you asked for isn't in the store.
|
|
38
|
+
"""
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class EmptyStoreError(VecDBError):
|
|
42
|
+
"""Raise when searching an empty store.
|
|
43
|
+
|
|
44
|
+
You can't find the closest pizza shop if there are no pizza shops!
|
|
45
|
+
"""
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class EmptyVectorError(VecDBError):
|
|
49
|
+
"""Raise when a vector has no components.
|
|
50
|
+
|
|
51
|
+
A vector needs at least one number -- an empty list isn't a direction!
|
|
52
|
+
"""
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
class PersistenceError(VecDBError):
|
|
56
|
+
"""Raise when a saved store can't be read back from disk.
|
|
57
|
+
|
|
58
|
+
Like opening your saved game and finding the file missing or
|
|
59
|
+
scrambled -- rather than starting an empty game and pretending all is
|
|
60
|
+
well, we stop and say which file is the problem.
|
|
61
|
+
"""
|
pyvecdb/filtering.py
ADDED
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
"""Metadata filtering for vector search results.
|
|
2
|
+
|
|
3
|
+
Imagine you're searching for the 5 most similar songs to your favorite,
|
|
4
|
+
but you only want songs released after 2020. That's metadata filtering!
|
|
5
|
+
|
|
6
|
+
The metadata is the "extra info" attached to each vector record -- things
|
|
7
|
+
like genre, year, artist, rating. Filters let you narrow down results
|
|
8
|
+
based on these fields.
|
|
9
|
+
|
|
10
|
+
Supported filter operators:
|
|
11
|
+
|
|
12
|
+
- **$eq** (equals): ``{"genre": "rock"}`` -- exact match (plain values are shorthand for $eq)
|
|
13
|
+
- **ne** (not equals): ``{"genre": {"$ne": "country"}}``
|
|
14
|
+
- **gt / gte / lt / lte**: ``{"year": {"$gt": 2020}}`` -- comparisons
|
|
15
|
+
- **in**: ``{"genre": {"$in": ["rock", "pop"]}}`` -- match any in list
|
|
16
|
+
|
|
17
|
+
Example:
|
|
18
|
+
>>> from pyvecdb.filtering import matches_filter
|
|
19
|
+
>>> metadata = {"genre": "rock", "year": 2023}
|
|
20
|
+
>>> matches_filter(metadata, {"genre": "rock"})
|
|
21
|
+
True
|
|
22
|
+
>>> matches_filter(metadata, {"year": {"$gt": 2020}})
|
|
23
|
+
True
|
|
24
|
+
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
from collections.abc import Mapping
|
|
28
|
+
|
|
29
|
+
from pyvecdb.record import VectorRecord
|
|
30
|
+
|
|
31
|
+
# Type alias for metadata values
|
|
32
|
+
MetadataValue = str | int | float | bool
|
|
33
|
+
|
|
34
|
+
# Type alias for a single filter condition
|
|
35
|
+
FilterCondition = MetadataValue | dict[str, MetadataValue | list[MetadataValue]]
|
|
36
|
+
|
|
37
|
+
# Type alias for a complete filter
|
|
38
|
+
MetadataFilter = Mapping[str, FilterCondition]
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _compare_values(
|
|
42
|
+
actual: MetadataValue,
|
|
43
|
+
operator: str,
|
|
44
|
+
expected: MetadataValue | list[MetadataValue],
|
|
45
|
+
) -> bool:
|
|
46
|
+
"""Compare a metadata value using the given operator.
|
|
47
|
+
|
|
48
|
+
Args:
|
|
49
|
+
actual: The value from the record's metadata.
|
|
50
|
+
operator: One of '$eq', '$ne', '$gt', '$gte', '$lt', '$lte', '$in'.
|
|
51
|
+
expected: The value (or list) to compare against.
|
|
52
|
+
|
|
53
|
+
Returns:
|
|
54
|
+
True if the comparison passes, False otherwise.
|
|
55
|
+
|
|
56
|
+
"""
|
|
57
|
+
match operator:
|
|
58
|
+
case "$eq":
|
|
59
|
+
return actual == expected
|
|
60
|
+
case "$ne":
|
|
61
|
+
return actual != expected
|
|
62
|
+
case "$in":
|
|
63
|
+
return actual in expected if isinstance(expected, list) else False
|
|
64
|
+
case "$gt" | "$gte" | "$lt" | "$lte":
|
|
65
|
+
if not isinstance(actual, (int, float)) or not isinstance(expected, (int, float)):
|
|
66
|
+
return False
|
|
67
|
+
ops = {
|
|
68
|
+
"$gt": actual > expected,
|
|
69
|
+
"$gte": actual >= expected,
|
|
70
|
+
"$lt": actual < expected,
|
|
71
|
+
"$lte": actual <= expected,
|
|
72
|
+
}
|
|
73
|
+
return ops[operator]
|
|
74
|
+
case _:
|
|
75
|
+
return False
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def matches_filter(
|
|
79
|
+
metadata: Mapping[str, MetadataValue],
|
|
80
|
+
filter_dict: MetadataFilter,
|
|
81
|
+
) -> bool:
|
|
82
|
+
"""Check whether a metadata dict matches all conditions in a filter.
|
|
83
|
+
|
|
84
|
+
A filter is a dictionary where each key is a metadata field name. The value
|
|
85
|
+
can be a plain value (shorthand for equals) or a dict of operator-value pairs.
|
|
86
|
+
|
|
87
|
+
All conditions must match (AND logic).
|
|
88
|
+
|
|
89
|
+
Args:
|
|
90
|
+
metadata: The record's metadata dictionary.
|
|
91
|
+
filter_dict: The filter conditions to check.
|
|
92
|
+
|
|
93
|
+
Returns:
|
|
94
|
+
True if all conditions match, False otherwise.
|
|
95
|
+
|
|
96
|
+
"""
|
|
97
|
+
for field_name, condition in filter_dict.items():
|
|
98
|
+
actual = metadata.get(field_name)
|
|
99
|
+
if actual is None:
|
|
100
|
+
return False
|
|
101
|
+
|
|
102
|
+
if isinstance(condition, dict):
|
|
103
|
+
for operator, expected in condition.items():
|
|
104
|
+
if not _compare_values(actual, operator, expected):
|
|
105
|
+
return False
|
|
106
|
+
elif actual != condition:
|
|
107
|
+
return False
|
|
108
|
+
|
|
109
|
+
return True
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def filter_records(
|
|
113
|
+
records: list[VectorRecord],
|
|
114
|
+
filter_dict: MetadataFilter,
|
|
115
|
+
) -> list[VectorRecord]:
|
|
116
|
+
"""Filter a list of vector records by metadata conditions.
|
|
117
|
+
|
|
118
|
+
Args:
|
|
119
|
+
records: The records to filter.
|
|
120
|
+
filter_dict: The filter conditions to apply.
|
|
121
|
+
|
|
122
|
+
Returns:
|
|
123
|
+
A new list containing only records whose metadata matches all conditions.
|
|
124
|
+
Note: the records themselves are the live objects from the store, so
|
|
125
|
+
editing one here also edits it in the store.
|
|
126
|
+
|
|
127
|
+
"""
|
|
128
|
+
return [r for r in records if matches_filter(r.metadata, filter_dict)]
|