xyberos-faiss 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- xyberos_faiss-0.1.0/PKG-INFO +69 -0
- xyberos_faiss-0.1.0/README.md +55 -0
- xyberos_faiss-0.1.0/pyproject.toml +27 -0
- xyberos_faiss-0.1.0/setup.cfg +4 -0
- xyberos_faiss-0.1.0/tests/test_fake_faiss.py +125 -0
- xyberos_faiss-0.1.0/tests/test_parity.py +54 -0
- xyberos_faiss-0.1.0/tests/test_plugin.py +31 -0
- xyberos_faiss-0.1.0/tests/test_vector.py +44 -0
- xyberos_faiss-0.1.0/xyberos_faiss/__init__.py +6 -0
- xyberos_faiss-0.1.0/xyberos_faiss/plugin.py +42 -0
- xyberos_faiss-0.1.0/xyberos_faiss/vector.py +142 -0
- xyberos_faiss-0.1.0/xyberos_faiss.egg-info/PKG-INFO +69 -0
- xyberos_faiss-0.1.0/xyberos_faiss.egg-info/SOURCES.txt +15 -0
- xyberos_faiss-0.1.0/xyberos_faiss.egg-info/dependency_links.txt +1 -0
- xyberos_faiss-0.1.0/xyberos_faiss.egg-info/entry_points.txt +2 -0
- xyberos_faiss-0.1.0/xyberos_faiss.egg-info/requires.txt +7 -0
- xyberos_faiss-0.1.0/xyberos_faiss.egg-info/top_level.txt +1 -0
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: xyberos-faiss
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: FAISS VectorStore plugin (RFC-0019, M4): local, dependency-free-of-server vector search
|
|
5
|
+
License: Apache-2.0
|
|
6
|
+
Keywords: xyberos,plugin,vector,faiss,rag
|
|
7
|
+
Requires-Python: >=3.10
|
|
8
|
+
Description-Content-Type: text/markdown
|
|
9
|
+
Requires-Dist: xyberos>=1.0
|
|
10
|
+
Provides-Extra: vectors
|
|
11
|
+
Requires-Dist: faiss-cpu; extra == "vectors"
|
|
12
|
+
Provides-Extra: test
|
|
13
|
+
Requires-Dist: pytest; extra == "test"
|
|
14
|
+
|
|
15
|
+
# xyberos-faiss
|
|
16
|
+
|
|
17
|
+
**FAISS `VectorStore` plugin — RFC-0019, M4.** A purely local, no-server vector
|
|
18
|
+
store backed by [FAISS](https://github.com/facebookresearch/faiss). Passes the
|
|
19
|
+
same parity scenarios as the stdlib `SqliteVectorStore` (the M4 Definition of
|
|
20
|
+
Done).
|
|
21
|
+
|
|
22
|
+
## Install
|
|
23
|
+
|
|
24
|
+
```bash
|
|
25
|
+
pip install -e ./faiss
|
|
26
|
+
pip install faiss-cpu # or: pip install xyberos[vectors]
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
> `faiss-cpu` wheels are available on PyPI for Linux/macOS; on Windows use a
|
|
30
|
+
> conda env or run the parity tests on a platform with wheels — the tests skip
|
|
31
|
+
> cleanly when the module is absent.
|
|
32
|
+
|
|
33
|
+
## Usage
|
|
34
|
+
|
|
35
|
+
```python
|
|
36
|
+
from xyberos import create_app
|
|
37
|
+
from xyberos_faiss import FaissPlugin
|
|
38
|
+
|
|
39
|
+
app = create_app()
|
|
40
|
+
app.load_plugin(FaissPlugin())
|
|
41
|
+
|
|
42
|
+
store = app.resolve("vector_store")
|
|
43
|
+
store.upsert("ns", "doc-1", [1.0, 0.0, 0.0, 0.0], {"text": "alpha"})
|
|
44
|
+
hits = store.query("ns", [1.0, 0.0, 0.0, 0.0], top_k=3)
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
## Design
|
|
48
|
+
|
|
49
|
+
- Each **namespace** is an in-memory `faiss.IndexFlatIP` over **L2-normalized**
|
|
50
|
+
vectors, so inner-product scores equal cosine similarity (higher = more
|
|
51
|
+
relevant, matching the `VectorStore` contract).
|
|
52
|
+
- `dim` is optional and auto-detected from the first upsert (`FAISS_DIM` env
|
|
53
|
+
honored).
|
|
54
|
+
- `faiss` is imported **lazily**; a clear `ProviderError` is raised when it is
|
|
55
|
+
missing.
|
|
56
|
+
|
|
57
|
+
## Tests
|
|
58
|
+
|
|
59
|
+
```bash
|
|
60
|
+
pip install pytest
|
|
61
|
+
pytest tests/ # unit + parity + plugin tests
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
Parity tests run the exact scenarios the `SqliteVectorStore` reference passes;
|
|
65
|
+
they skip cleanly when `faiss-cpu` is not importable on the current platform.
|
|
66
|
+
|
|
67
|
+
## Ship location
|
|
68
|
+
|
|
69
|
+
`[vectors]` extra (`faiss-cpu` already added to `pyproject.toml`).
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
# xyberos-faiss
|
|
2
|
+
|
|
3
|
+
**FAISS `VectorStore` plugin — RFC-0019, M4.** A purely local, no-server vector
|
|
4
|
+
store backed by [FAISS](https://github.com/facebookresearch/faiss). Passes the
|
|
5
|
+
same parity scenarios as the stdlib `SqliteVectorStore` (the M4 Definition of
|
|
6
|
+
Done).
|
|
7
|
+
|
|
8
|
+
## Install
|
|
9
|
+
|
|
10
|
+
```bash
|
|
11
|
+
pip install -e ./faiss
|
|
12
|
+
pip install faiss-cpu # or: pip install xyberos[vectors]
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
> `faiss-cpu` wheels are available on PyPI for Linux/macOS; on Windows use a
|
|
16
|
+
> conda env or run the parity tests on a platform with wheels — the tests skip
|
|
17
|
+
> cleanly when the module is absent.
|
|
18
|
+
|
|
19
|
+
## Usage
|
|
20
|
+
|
|
21
|
+
```python
|
|
22
|
+
from xyberos import create_app
|
|
23
|
+
from xyberos_faiss import FaissPlugin
|
|
24
|
+
|
|
25
|
+
app = create_app()
|
|
26
|
+
app.load_plugin(FaissPlugin())
|
|
27
|
+
|
|
28
|
+
store = app.resolve("vector_store")
|
|
29
|
+
store.upsert("ns", "doc-1", [1.0, 0.0, 0.0, 0.0], {"text": "alpha"})
|
|
30
|
+
hits = store.query("ns", [1.0, 0.0, 0.0, 0.0], top_k=3)
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
## Design
|
|
34
|
+
|
|
35
|
+
- Each **namespace** is an in-memory `faiss.IndexFlatIP` over **L2-normalized**
|
|
36
|
+
vectors, so inner-product scores equal cosine similarity (higher = more
|
|
37
|
+
relevant, matching the `VectorStore` contract).
|
|
38
|
+
- `dim` is optional and auto-detected from the first upsert (`FAISS_DIM` env
|
|
39
|
+
honored).
|
|
40
|
+
- `faiss` is imported **lazily**; a clear `ProviderError` is raised when it is
|
|
41
|
+
missing.
|
|
42
|
+
|
|
43
|
+
## Tests
|
|
44
|
+
|
|
45
|
+
```bash
|
|
46
|
+
pip install pytest
|
|
47
|
+
pytest tests/ # unit + parity + plugin tests
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
Parity tests run the exact scenarios the `SqliteVectorStore` reference passes;
|
|
51
|
+
they skip cleanly when `faiss-cpu` is not importable on the current platform.
|
|
52
|
+
|
|
53
|
+
## Ship location
|
|
54
|
+
|
|
55
|
+
`[vectors]` extra (`faiss-cpu` already added to `pyproject.toml`).
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=61.0"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "xyberos-faiss"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "FAISS VectorStore plugin (RFC-0019, M4): local, dependency-free-of-server vector search"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = {text = "Apache-2.0"}
|
|
12
|
+
dependencies = ["xyberos>=1.0"]
|
|
13
|
+
keywords = ["xyberos", "plugin", "vector", "faiss", "rag"]
|
|
14
|
+
|
|
15
|
+
[project.optional-dependencies]
|
|
16
|
+
vectors = ["faiss-cpu"]
|
|
17
|
+
test = ["pytest"]
|
|
18
|
+
|
|
19
|
+
[project.entry-points."xyberos.plugins"]
|
|
20
|
+
faiss = "xyberos_faiss.plugin:plugin"
|
|
21
|
+
|
|
22
|
+
[tool.setuptools]
|
|
23
|
+
packages = ["xyberos_faiss"]
|
|
24
|
+
|
|
25
|
+
[tool.pytest.ini_options]
|
|
26
|
+
testpaths = ["tests"]
|
|
27
|
+
pythonpath = ["."]
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
"""Exercise the FaissVectorStore logic against a numpy-backed fake faiss.
|
|
2
|
+
|
|
3
|
+
faiss-cpu has no wheels on every platform (e.g. Windows), so this module runs
|
|
4
|
+
the store's full logic — including the M4 parity scenarios — against a faithful
|
|
5
|
+
``IndexFlatIP`` stand-in. It gives real coverage wherever pytest runs; the
|
|
6
|
+
``test_parity.py`` module covers the real ``faiss-cpu`` when it is present.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import sys
|
|
12
|
+
import types
|
|
13
|
+
|
|
14
|
+
import numpy as np
|
|
15
|
+
import pytest
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class _IndexFlatIP:
|
|
19
|
+
"""An in-memory inner-product index matching the faiss API subset used."""
|
|
20
|
+
|
|
21
|
+
def __init__(self, dim: int) -> None:
|
|
22
|
+
self._dim = int(dim)
|
|
23
|
+
self._rows = np.empty((0, self._dim), dtype="float32")
|
|
24
|
+
|
|
25
|
+
def add(self, x: object) -> None:
|
|
26
|
+
self._rows = np.vstack([self._rows, np.asarray(x, dtype="float32")])
|
|
27
|
+
|
|
28
|
+
@property
|
|
29
|
+
def ntotal(self) -> int:
|
|
30
|
+
return self._rows.shape[0]
|
|
31
|
+
|
|
32
|
+
def search(self, x: object, k: int) -> tuple[np.ndarray, np.ndarray]:
|
|
33
|
+
matrix = np.asarray(x, dtype="float32")
|
|
34
|
+
if matrix.ndim == 1:
|
|
35
|
+
matrix = matrix[None, :]
|
|
36
|
+
scores = self._rows @ matrix.T
|
|
37
|
+
n_queries = matrix.shape[0]
|
|
38
|
+
k = min(int(k), self.ntotal)
|
|
39
|
+
distances = np.zeros((n_queries, k), dtype="float32")
|
|
40
|
+
indices = np.full((n_queries, k), -1, dtype="int64")
|
|
41
|
+
for query in range(n_queries):
|
|
42
|
+
order = np.argsort(scores[:, query])[::-1][:k]
|
|
43
|
+
distances[query] = scores[order, query]
|
|
44
|
+
indices[query] = order
|
|
45
|
+
return distances, indices
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@pytest.fixture(autouse=True)
|
|
49
|
+
def fake_faiss():
|
|
50
|
+
"""Inject a fake ``faiss`` module unless the real one is already present."""
|
|
51
|
+
if sys.modules.get("faiss") is None:
|
|
52
|
+
module = types.ModuleType("faiss")
|
|
53
|
+
module.IndexFlatIP = _IndexFlatIP
|
|
54
|
+
sys.modules["faiss"] = module
|
|
55
|
+
yield
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
QUERY = [1.0, 0.0, 0.0, 0.0]
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def run_parity_scenarios(store) -> None:
|
|
62
|
+
"""The full VectorStore contract; every adapter must pass these."""
|
|
63
|
+
store.upsert("ns", "a", [1.0, 0.0, 0.0, 0.0], {"text": "alpha"})
|
|
64
|
+
store.upsert("ns", "b", [0.8, 0.2, 0.0, 0.0], {"text": "beta"})
|
|
65
|
+
store.upsert("ns", "c", [0.5, 0.5, 0.0, 0.0], {"text": "gamma"})
|
|
66
|
+
store.upsert("other", "x", [1.0, 1.0, 1.0, 1.0], {"text": "other ns"})
|
|
67
|
+
|
|
68
|
+
hits = store.query("ns", QUERY, top_k=3)
|
|
69
|
+
assert [h.id for h in hits] == ["a", "b", "c"]
|
|
70
|
+
assert hits[0].payload["text"] == "alpha"
|
|
71
|
+
assert hits[0].score > hits[1].score > hits[2].score
|
|
72
|
+
assert len(store.query("ns", QUERY, top_k=1)) == 1
|
|
73
|
+
|
|
74
|
+
strict = store.query("ns", QUERY, top_k=3, threshold=0.99)
|
|
75
|
+
assert [h.id for h in strict] == ["a"]
|
|
76
|
+
|
|
77
|
+
store.delete("ns", "b")
|
|
78
|
+
assert [h.id for h in store.query("ns", QUERY, top_k=3)] == ["a", "c"]
|
|
79
|
+
assert [h.id for h in store.query("other", [1.0, 1.0, 1.0, 1.0], top_k=2)] == ["x"]
|
|
80
|
+
assert all(abs(h.score) < 1e-6 for h in store.query("ns", [0.0, 0.0, 0.0, 1.0], top_k=2))
|
|
81
|
+
|
|
82
|
+
store.upsert("ns", "a", [1.0, 0.0, 0.0, 0.0], {"text": "alpha-v2"})
|
|
83
|
+
assert store.query("ns", QUERY, top_k=1)[0].payload["text"] == "alpha-v2"
|
|
84
|
+
|
|
85
|
+
store.clear("ns")
|
|
86
|
+
assert store.query("ns", QUERY, top_k=3) == []
|
|
87
|
+
assert [h.id for h in store.query("other", [1.0, 1.0, 1.0, 1.0], top_k=1)] == ["x"]
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def test_parity_with_fake_faiss():
|
|
91
|
+
from xyberos_faiss import FaissVectorStore
|
|
92
|
+
|
|
93
|
+
run_parity_scenarios(FaissVectorStore())
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def test_upsert_and_query_order():
|
|
97
|
+
from xyberos_faiss import FaissVectorStore
|
|
98
|
+
|
|
99
|
+
store = FaissVectorStore()
|
|
100
|
+
store.upsert("ns", "a", [1.0, 0.0, 0.0], {"text": "alpha"})
|
|
101
|
+
store.upsert("ns", "b", [0.0, 1.0, 0.0], {"text": "beta"})
|
|
102
|
+
hits = store.query("ns", [1.0, 0.0, 0.0], top_k=2)
|
|
103
|
+
assert hits[0].id == "a"
|
|
104
|
+
assert hits[0].payload["text"] == "alpha"
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def test_delete_and_clear():
|
|
108
|
+
from xyberos_faiss import FaissVectorStore
|
|
109
|
+
|
|
110
|
+
store = FaissVectorStore()
|
|
111
|
+
store.upsert("ns", "a", [1.0, 0.0, 0.0, 0.0])
|
|
112
|
+
store.upsert("ns", "b", [0.0, 1.0, 0.0, 0.0])
|
|
113
|
+
store.delete("ns", "a")
|
|
114
|
+
assert [h.id for h in store.query("ns", [1.0, 0.0, 0.0, 0.0], top_k=2)] == ["b"]
|
|
115
|
+
store.clear("ns")
|
|
116
|
+
assert store.query("ns", [1.0, 0.0, 0.0, 0.0]) == []
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def test_dimension_mismatch_raises():
|
|
120
|
+
from xyberos_faiss import FaissVectorStore
|
|
121
|
+
|
|
122
|
+
store = FaissVectorStore()
|
|
123
|
+
store.upsert("ns", "a", [1.0, 0.0, 0.0])
|
|
124
|
+
with pytest.raises(ValueError, match="dimension"):
|
|
125
|
+
store.query("ns", [1.0, 0.0, 0.0, 0.0])
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
"""M4 parity: FaissVectorStore must behave like SqliteVectorStore."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import importlib.util
|
|
6
|
+
|
|
7
|
+
import pytest
|
|
8
|
+
from xyberos.vector import SqliteVectorStore
|
|
9
|
+
|
|
10
|
+
from xyberos_faiss import FaissVectorStore
|
|
11
|
+
|
|
12
|
+
QUERY = [1.0, 0.0, 0.0, 0.0]
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def run_parity_scenarios(store) -> None:
|
|
16
|
+
"""The full VectorStore contract; every adapter must pass these."""
|
|
17
|
+
store.upsert("ns", "a", [1.0, 0.0, 0.0, 0.0], {"text": "alpha"})
|
|
18
|
+
store.upsert("ns", "b", [0.8, 0.2, 0.0, 0.0], {"text": "beta"})
|
|
19
|
+
store.upsert("ns", "c", [0.5, 0.5, 0.0, 0.0], {"text": "gamma"})
|
|
20
|
+
store.upsert("other", "x", [1.0, 1.0, 1.0, 1.0], {"text": "other ns"})
|
|
21
|
+
|
|
22
|
+
hits = store.query("ns", QUERY, top_k=3)
|
|
23
|
+
assert [h.id for h in hits] == ["a", "b", "c"]
|
|
24
|
+
assert hits[0].payload["text"] == "alpha"
|
|
25
|
+
assert hits[0].score > hits[1].score > hits[2].score
|
|
26
|
+
assert len(store.query("ns", QUERY, top_k=1)) == 1
|
|
27
|
+
|
|
28
|
+
strict = store.query("ns", QUERY, top_k=3, threshold=0.99)
|
|
29
|
+
assert [h.id for h in strict] == ["a"]
|
|
30
|
+
|
|
31
|
+
store.delete("ns", "b")
|
|
32
|
+
assert [h.id for h in store.query("ns", QUERY, top_k=3)] == ["a", "c"]
|
|
33
|
+
assert [h.id for h in store.query("other", [1.0, 1.0, 1.0, 1.0], top_k=2)] == ["x"]
|
|
34
|
+
assert all(abs(h.score) < 1e-6 for h in store.query("ns", [0.0, 0.0, 0.0, 1.0], top_k=2))
|
|
35
|
+
|
|
36
|
+
store.upsert("ns", "a", [1.0, 0.0, 0.0, 0.0], {"text": "alpha-v2"})
|
|
37
|
+
assert store.query("ns", QUERY, top_k=1)[0].payload["text"] == "alpha-v2"
|
|
38
|
+
|
|
39
|
+
store.clear("ns")
|
|
40
|
+
assert store.query("ns", QUERY, top_k=3) == []
|
|
41
|
+
assert [h.id for h in store.query("other", [1.0, 1.0, 1.0, 1.0], top_k=1)] == ["x"]
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def test_sqlite_reference() -> None:
|
|
45
|
+
"""The stdlib reference passes the same scenarios the adapters must match."""
|
|
46
|
+
run_parity_scenarios(SqliteVectorStore(":memory:"))
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
@pytest.mark.skipif(
|
|
50
|
+
importlib.util.find_spec("faiss") is None,
|
|
51
|
+
reason="faiss-cpu is not installed on this platform",
|
|
52
|
+
)
|
|
53
|
+
def test_faiss_parity():
|
|
54
|
+
run_parity_scenarios(FaissVectorStore())
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"""Tests for loading the faiss plugin into a Xyberos app."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import importlib.util
|
|
6
|
+
|
|
7
|
+
import pytest
|
|
8
|
+
from xyberos import create_app
|
|
9
|
+
|
|
10
|
+
pytestmark = pytest.mark.skipif(
|
|
11
|
+
importlib.util.find_spec("faiss") is None,
|
|
12
|
+
reason="faiss-cpu is not installed on this platform",
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
from xyberos_faiss import FaissPlugin # noqa: E402
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def test_plugin_conforms_to_contract():
|
|
19
|
+
plugin = FaissPlugin()
|
|
20
|
+
assert plugin.name == "faiss"
|
|
21
|
+
assert callable(plugin.register) and callable(plugin.unregister)
|
|
22
|
+
assert plugin.vector_store() is plugin.vector_store()
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def test_plugin_registers_vector_store():
|
|
26
|
+
app = create_app()
|
|
27
|
+
app.load_plugin(FaissPlugin())
|
|
28
|
+
store = app.resolve("vector_store")
|
|
29
|
+
store.upsert("ns", "a", [1.0, 0.0, 0.0, 0.0], {"text": "alpha"})
|
|
30
|
+
assert store.query("ns", [1.0, 0.0, 0.0, 0.0], top_k=1)[0].payload["text"] == "alpha"
|
|
31
|
+
app.unload_plugin("faiss")
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"""Unit tests for the FaissVectorStore (skip cleanly when faiss is missing)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import importlib.util
|
|
6
|
+
|
|
7
|
+
import pytest
|
|
8
|
+
|
|
9
|
+
pytestmark = pytest.mark.skipif(
|
|
10
|
+
importlib.util.find_spec("faiss") is None,
|
|
11
|
+
reason="faiss-cpu is not installed on this platform",
|
|
12
|
+
)
|
|
13
|
+
|
|
14
|
+
from xyberos_faiss import FaissVectorStore # noqa: E402
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@pytest.fixture()
|
|
18
|
+
def store():
|
|
19
|
+
return FaissVectorStore()
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def test_upsert_and_query(store):
|
|
23
|
+
store.upsert("ns", "a", [1.0, 0.0, 0.0], {"text": "alpha"})
|
|
24
|
+
store.upsert("ns", "b", [0.0, 1.0, 0.0], {"text": "beta"})
|
|
25
|
+
hits = store.query("ns", [1.0, 0.0, 0.0], top_k=2)
|
|
26
|
+
assert hits[0].id == "a"
|
|
27
|
+
assert hits[0].payload["text"] == "alpha"
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def test_delete(store):
|
|
31
|
+
store.upsert("ns", "a", [1.0, 0.0, 0.0, 0.0])
|
|
32
|
+
store.upsert("ns", "b", [0.0, 1.0, 0.0, 0.0])
|
|
33
|
+
store.delete("ns", "a")
|
|
34
|
+
assert [h.id for h in store.query("ns", [1.0, 0.0, 0.0, 0.0], top_k=2)] == ["b"]
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def test_clear(store):
|
|
38
|
+
store.upsert("ns", "a", [1.0, 0.0, 0.0, 0.0])
|
|
39
|
+
store.clear("ns")
|
|
40
|
+
assert store.query("ns", [1.0, 0.0, 0.0, 0.0]) == []
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def test_missing_namespace_returns_empty(store):
|
|
44
|
+
assert store.query("missing", [1.0, 0.0, 0.0, 0.0]) == []
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
"""FAISS plugin entry point (RFC-0019, M4)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
from xyberos.contracts import Plugin, VectorStore
|
|
9
|
+
|
|
10
|
+
from .vector import FaissVectorStore
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class FaissPlugin(Plugin):
|
|
14
|
+
"""Registers a :class:`FaissVectorStore` as the app's ``vector_store``.
|
|
15
|
+
|
|
16
|
+
``dim`` is optional and auto-detected from the first upsert. ``FAISS_DIM``
|
|
17
|
+
env var is honored as a fallback.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
def __init__(self, *, dim: int | None = None) -> None:
|
|
21
|
+
raw_dim = dim if dim is not None else os.getenv("FAISS_DIM")
|
|
22
|
+
self._dim = int(raw_dim) if raw_dim else None
|
|
23
|
+
self._store: FaissVectorStore | None = None
|
|
24
|
+
|
|
25
|
+
@property
|
|
26
|
+
def name(self) -> str:
|
|
27
|
+
return "faiss"
|
|
28
|
+
|
|
29
|
+
def vector_store(self) -> FaissVectorStore:
|
|
30
|
+
if self._store is None:
|
|
31
|
+
self._store = FaissVectorStore(dim=self._dim)
|
|
32
|
+
return self._store
|
|
33
|
+
|
|
34
|
+
def register(self, kernel: object) -> None:
|
|
35
|
+
kernel.register("vector_store", self.vector_store(), replace=True)
|
|
36
|
+
|
|
37
|
+
def unregister(self, kernel: object) -> None:
|
|
38
|
+
pass
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
#: Auto-discovered by ``app.load_entry_points()``.
|
|
42
|
+
plugin = FaissPlugin()
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
"""FAISS-backed :class:`~xyberos.contracts.VectorStore` (lazy ``faiss-cpu``).
|
|
2
|
+
|
|
3
|
+
A purely local, no-server vector store. Each namespace is an in-memory
|
|
4
|
+
``faiss.IndexFlatIP`` over **L2-normalized** vectors, so inner-product scores
|
|
5
|
+
equal cosine similarity — matching the ``VectorStore`` contract where a higher
|
|
6
|
+
score means more relevant (and matching ``SqliteVectorStore`` / Qdrant
|
|
7
|
+
behavior).
|
|
8
|
+
|
|
9
|
+
The ``faiss`` module is imported lazily on first use and a clear
|
|
10
|
+
:class:`~xyberos.exceptions.provider.ProviderError` is raised when it is
|
|
11
|
+
missing (``pip install xyberos[vectors]``).
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import importlib
|
|
17
|
+
from collections.abc import Mapping, Sequence
|
|
18
|
+
from typing import Any
|
|
19
|
+
|
|
20
|
+
from xyberos.contracts.vector import ScoredHit, VectorStore
|
|
21
|
+
from xyberos.exceptions.provider import ProviderError
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _require_faiss() -> Any:
|
|
25
|
+
try:
|
|
26
|
+
faiss = importlib.import_module("faiss")
|
|
27
|
+
except ImportError as exc:
|
|
28
|
+
raise ProviderError(
|
|
29
|
+
"the 'faiss-cpu' package is required; install it with "
|
|
30
|
+
"'pip install xyberos[vectors]'"
|
|
31
|
+
) from exc
|
|
32
|
+
return faiss
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _normalize(vector: Sequence[float]) -> list[float]:
|
|
36
|
+
norm = sum(value * value for value in vector) ** 0.5
|
|
37
|
+
if norm == 0.0:
|
|
38
|
+
return [float(value) for value in vector]
|
|
39
|
+
return [float(value) / norm for value in vector]
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class FaissVectorStore(VectorStore):
|
|
43
|
+
"""A :class:`VectorStore` backed by an in-memory FAISS index per namespace."""
|
|
44
|
+
|
|
45
|
+
def __init__(self, dim: int | None = None, *, normalize: bool = True) -> None:
|
|
46
|
+
if dim is not None and dim <= 0:
|
|
47
|
+
raise ValueError("dim must be a positive integer")
|
|
48
|
+
self._dim = dim
|
|
49
|
+
self._normalize = normalize
|
|
50
|
+
# namespace -> ordered {id: (vector, payload)}
|
|
51
|
+
self._data: dict[str, dict[str, tuple[list[float], dict[str, Any] | None]]] = {}
|
|
52
|
+
# namespace -> faiss index
|
|
53
|
+
self._indexes: dict[str, Any] = {}
|
|
54
|
+
|
|
55
|
+
def upsert(
|
|
56
|
+
self,
|
|
57
|
+
namespace: str,
|
|
58
|
+
id: str,
|
|
59
|
+
vector: Sequence[float],
|
|
60
|
+
payload: Mapping[str, Any] | None = None,
|
|
61
|
+
) -> None:
|
|
62
|
+
faiss = _require_faiss()
|
|
63
|
+
vectors = [float(value) for value in vector]
|
|
64
|
+
if self._dim is None:
|
|
65
|
+
self._dim = len(vectors)
|
|
66
|
+
elif len(vectors) != self._dim:
|
|
67
|
+
raise ValueError(
|
|
68
|
+
f"vector dimension {len(vectors)} != store dimension {self._dim}"
|
|
69
|
+
)
|
|
70
|
+
bucket = self._data.setdefault(namespace, {})
|
|
71
|
+
bucket[str(id)] = (vectors, dict(payload or {}))
|
|
72
|
+
self._rebuild(namespace, faiss)
|
|
73
|
+
|
|
74
|
+
def query(
|
|
75
|
+
self,
|
|
76
|
+
namespace: str,
|
|
77
|
+
vector: Sequence[float],
|
|
78
|
+
*,
|
|
79
|
+
top_k: int = 5,
|
|
80
|
+
threshold: float | None = None,
|
|
81
|
+
) -> list[ScoredHit]:
|
|
82
|
+
faiss = _require_faiss()
|
|
83
|
+
bucket = self._data.get(namespace)
|
|
84
|
+
if not bucket:
|
|
85
|
+
return []
|
|
86
|
+
if len(vector) != self._dim:
|
|
87
|
+
raise ValueError(
|
|
88
|
+
f"query dimension {len(vector)} != store dimension {self._dim}"
|
|
89
|
+
)
|
|
90
|
+
index = self._indexes.get(namespace)
|
|
91
|
+
if index is None or index.ntotal != len(bucket):
|
|
92
|
+
self._rebuild(namespace, faiss)
|
|
93
|
+
index = self._indexes[namespace]
|
|
94
|
+
if index.ntotal == 0:
|
|
95
|
+
return []
|
|
96
|
+
|
|
97
|
+
numpy = importlib.import_module("numpy")
|
|
98
|
+
|
|
99
|
+
query_vector = _normalize(list(vector)) if self._normalize else list(vector)
|
|
100
|
+
k = min(max(top_k, 1), index.ntotal)
|
|
101
|
+
distances, indices = index.search(numpy.array([query_vector], dtype="float32"), k)
|
|
102
|
+
|
|
103
|
+
ids = list(bucket.keys())
|
|
104
|
+
hits: list[ScoredHit] = []
|
|
105
|
+
for distance, position in zip(distances[0], indices[0]):
|
|
106
|
+
if position < 0 or position >= len(ids):
|
|
107
|
+
continue
|
|
108
|
+
item_id = ids[int(position)]
|
|
109
|
+
score = float(distance)
|
|
110
|
+
if threshold is not None and score < threshold:
|
|
111
|
+
continue
|
|
112
|
+
_, payload = bucket[item_id]
|
|
113
|
+
hits.append(ScoredHit(id=item_id, score=score, payload=payload))
|
|
114
|
+
return hits
|
|
115
|
+
|
|
116
|
+
def delete(self, namespace: str, id: str) -> None:
|
|
117
|
+
faiss = _require_faiss()
|
|
118
|
+
bucket = self._data.get(namespace)
|
|
119
|
+
if bucket is None:
|
|
120
|
+
return
|
|
121
|
+
bucket.pop(str(id), None)
|
|
122
|
+
self._rebuild(namespace, faiss)
|
|
123
|
+
|
|
124
|
+
def clear(self, namespace: str) -> None:
|
|
125
|
+
self._data.pop(namespace, None)
|
|
126
|
+
self._indexes.pop(namespace, None)
|
|
127
|
+
|
|
128
|
+
# -- internals ----------------------------------------------------------
|
|
129
|
+
|
|
130
|
+
def _rebuild(self, namespace: str, faiss: Any) -> None:
|
|
131
|
+
bucket = self._data.get(namespace, {})
|
|
132
|
+
if not bucket:
|
|
133
|
+
self._indexes.pop(namespace, None)
|
|
134
|
+
return
|
|
135
|
+
numpy = importlib.import_module("numpy")
|
|
136
|
+
|
|
137
|
+
vectors = [v for v, _ in bucket.values()]
|
|
138
|
+
if self._normalize:
|
|
139
|
+
vectors = [_normalize(v) for v in vectors]
|
|
140
|
+
index = faiss.IndexFlatIP(self._dim or len(vectors))
|
|
141
|
+
index.add(numpy.array(vectors, dtype="float32"))
|
|
142
|
+
self._indexes[namespace] = index
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: xyberos-faiss
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: FAISS VectorStore plugin (RFC-0019, M4): local, dependency-free-of-server vector search
|
|
5
|
+
License: Apache-2.0
|
|
6
|
+
Keywords: xyberos,plugin,vector,faiss,rag
|
|
7
|
+
Requires-Python: >=3.10
|
|
8
|
+
Description-Content-Type: text/markdown
|
|
9
|
+
Requires-Dist: xyberos>=1.0
|
|
10
|
+
Provides-Extra: vectors
|
|
11
|
+
Requires-Dist: faiss-cpu; extra == "vectors"
|
|
12
|
+
Provides-Extra: test
|
|
13
|
+
Requires-Dist: pytest; extra == "test"
|
|
14
|
+
|
|
15
|
+
# xyberos-faiss
|
|
16
|
+
|
|
17
|
+
**FAISS `VectorStore` plugin — RFC-0019, M4.** A purely local, no-server vector
|
|
18
|
+
store backed by [FAISS](https://github.com/facebookresearch/faiss). Passes the
|
|
19
|
+
same parity scenarios as the stdlib `SqliteVectorStore` (the M4 Definition of
|
|
20
|
+
Done).
|
|
21
|
+
|
|
22
|
+
## Install
|
|
23
|
+
|
|
24
|
+
```bash
|
|
25
|
+
pip install -e ./faiss
|
|
26
|
+
pip install faiss-cpu # or: pip install xyberos[vectors]
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
> `faiss-cpu` wheels are available on PyPI for Linux/macOS; on Windows use a
|
|
30
|
+
> conda env or run the parity tests on a platform with wheels — the tests skip
|
|
31
|
+
> cleanly when the module is absent.
|
|
32
|
+
|
|
33
|
+
## Usage
|
|
34
|
+
|
|
35
|
+
```python
|
|
36
|
+
from xyberos import create_app
|
|
37
|
+
from xyberos_faiss import FaissPlugin
|
|
38
|
+
|
|
39
|
+
app = create_app()
|
|
40
|
+
app.load_plugin(FaissPlugin())
|
|
41
|
+
|
|
42
|
+
store = app.resolve("vector_store")
|
|
43
|
+
store.upsert("ns", "doc-1", [1.0, 0.0, 0.0, 0.0], {"text": "alpha"})
|
|
44
|
+
hits = store.query("ns", [1.0, 0.0, 0.0, 0.0], top_k=3)
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
## Design
|
|
48
|
+
|
|
49
|
+
- Each **namespace** is an in-memory `faiss.IndexFlatIP` over **L2-normalized**
|
|
50
|
+
vectors, so inner-product scores equal cosine similarity (higher = more
|
|
51
|
+
relevant, matching the `VectorStore` contract).
|
|
52
|
+
- `dim` is optional and auto-detected from the first upsert (`FAISS_DIM` env
|
|
53
|
+
honored).
|
|
54
|
+
- `faiss` is imported **lazily**; a clear `ProviderError` is raised when it is
|
|
55
|
+
missing.
|
|
56
|
+
|
|
57
|
+
## Tests
|
|
58
|
+
|
|
59
|
+
```bash
|
|
60
|
+
pip install pytest
|
|
61
|
+
pytest tests/ # unit + parity + plugin tests
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
Parity tests run the exact scenarios the `SqliteVectorStore` reference passes;
|
|
65
|
+
they skip cleanly when `faiss-cpu` is not importable on the current platform.
|
|
66
|
+
|
|
67
|
+
## Ship location
|
|
68
|
+
|
|
69
|
+
`[vectors]` extra (`faiss-cpu` already added to `pyproject.toml`).
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
README.md
|
|
2
|
+
pyproject.toml
|
|
3
|
+
tests/test_fake_faiss.py
|
|
4
|
+
tests/test_parity.py
|
|
5
|
+
tests/test_plugin.py
|
|
6
|
+
tests/test_vector.py
|
|
7
|
+
xyberos_faiss/__init__.py
|
|
8
|
+
xyberos_faiss/plugin.py
|
|
9
|
+
xyberos_faiss/vector.py
|
|
10
|
+
xyberos_faiss.egg-info/PKG-INFO
|
|
11
|
+
xyberos_faiss.egg-info/SOURCES.txt
|
|
12
|
+
xyberos_faiss.egg-info/dependency_links.txt
|
|
13
|
+
xyberos_faiss.egg-info/entry_points.txt
|
|
14
|
+
xyberos_faiss.egg-info/requires.txt
|
|
15
|
+
xyberos_faiss.egg-info/top_level.txt
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
xyberos_faiss
|