faiss-haystack 1.0.0__tar.gz → 2.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {faiss_haystack-1.0.0 → faiss_haystack-2.0.0}/PKG-INFO +12 -15
- faiss_haystack-2.0.0/README.md +11 -0
- {faiss_haystack-1.0.0 → faiss_haystack-2.0.0}/pyproject.toml +25 -7
- {faiss_haystack-1.0.0 → faiss_haystack-2.0.0}/src/haystack_integrations/components/retrievers/faiss/embedding_retriever.py +3 -1
- {faiss_haystack-1.0.0 → faiss_haystack-2.0.0}/src/haystack_integrations/document_stores/faiss/document_store.py +7 -6
- {faiss_haystack-1.0.0 → faiss_haystack-2.0.0}/tests/test_document_store.py +10 -47
- faiss_haystack-1.0.0/README.md +0 -17
- faiss_haystack-1.0.0/src/haystack_integrations/components/__init__.py +0 -3
- faiss_haystack-1.0.0/src/haystack_integrations/components/retrievers/__init__.py +0 -3
- faiss_haystack-1.0.0/src/haystack_integrations/document_stores/__init__.py +0 -0
- faiss_haystack-1.0.0/tests/__init__.py +0 -0
- {faiss_haystack-1.0.0 → faiss_haystack-2.0.0}/.gitignore +0 -0
- {faiss_haystack-1.0.0 → faiss_haystack-2.0.0}/pydoc/config_docusaurus.yml +0 -0
- {faiss_haystack-1.0.0 → faiss_haystack-2.0.0}/src/haystack_integrations/components/retrievers/faiss/__init__.py +0 -0
- {faiss_haystack-1.0.0 → faiss_haystack-2.0.0}/src/haystack_integrations/components/retrievers/faiss/py.typed +0 -0
- {faiss_haystack-1.0.0 → faiss_haystack-2.0.0}/src/haystack_integrations/document_stores/faiss/__init__.py +0 -0
- {faiss_haystack-1.0.0 → faiss_haystack-2.0.0}/src/haystack_integrations/document_stores/faiss/py.typed +0 -0
- {faiss_haystack-1.0.0/src/haystack_integrations → faiss_haystack-2.0.0/tests}/__init__.py +0 -0
- {faiss_haystack-1.0.0 → faiss_haystack-2.0.0}/tests/test_embedding_retriever.py +0 -0
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: faiss-haystack
|
|
3
|
-
Version:
|
|
3
|
+
Version: 2.0.0
|
|
4
|
+
Summary: An integration of FAISS (library for efficient similarity search of dense vectors) with Haystack
|
|
4
5
|
Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/faiss#readme
|
|
5
6
|
Project-URL: Issues, https://github.com/deepset-ai/haystack-core-integrations/issues
|
|
6
7
|
Project-URL: Source, https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/faiss
|
|
7
|
-
Author-email:
|
|
8
|
+
Author-email: deepset GmbH <info@deepset.ai>
|
|
8
9
|
License-Expression: Apache-2.0
|
|
9
10
|
Classifier: Development Status :: 4 - Beta
|
|
10
11
|
Classifier: License :: OSI Approved :: Apache Software License
|
|
@@ -12,28 +13,24 @@ Classifier: Programming Language :: Python
|
|
|
12
13
|
Classifier: Programming Language :: Python :: 3.10
|
|
13
14
|
Classifier: Programming Language :: Python :: 3.11
|
|
14
15
|
Classifier: Programming Language :: Python :: 3.12
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
15
18
|
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
16
19
|
Classifier: Programming Language :: Python :: Implementation :: PyPy
|
|
17
20
|
Requires-Python: >=3.10
|
|
18
21
|
Requires-Dist: faiss-cpu>=1.8.0
|
|
19
|
-
Requires-Dist: haystack-ai>=2.
|
|
22
|
+
Requires-Dist: haystack-ai>=2.26.1
|
|
20
23
|
Requires-Dist: numpy<2,>=1.22; python_version < '3.13'
|
|
21
24
|
Description-Content-Type: text/markdown
|
|
22
25
|
|
|
23
26
|
# faiss-haystack
|
|
24
27
|
|
|
25
|
-
|
|
28
|
+
[](https://pypi.org/project/faiss-haystack)
|
|
29
|
+
[](https://pypi.org/project/faiss-haystack)
|
|
26
30
|
|
|
27
|
-
|
|
31
|
+
- [Integration page](https://haystack.deepset.ai/integrations/faiss-documentstore)
|
|
32
|
+
- [Changelog](https://github.com/deepset-ai/haystack-core-integrations/blob/main/integrations/faiss/CHANGELOG.md)
|
|
28
33
|
|
|
29
|
-
|
|
30
|
-
pip install faiss-haystack
|
|
31
|
-
```
|
|
34
|
+
---
|
|
32
35
|
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
```python
|
|
36
|
-
from haystack_integrations.document_stores.faiss import FAISSDocumentStore
|
|
37
|
-
|
|
38
|
-
document_store = FAISSDocumentStore(index_path="my_index")
|
|
39
|
-
```
|
|
36
|
+
Refer to the general [Contribution Guidelines](https://github.com/deepset-ai/haystack-core-integrations/blob/main/CONTRIBUTING.md).
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
# faiss-haystack
|
|
2
|
+
|
|
3
|
+
[](https://pypi.org/project/faiss-haystack)
|
|
4
|
+
[](https://pypi.org/project/faiss-haystack)
|
|
5
|
+
|
|
6
|
+
- [Integration page](https://haystack.deepset.ai/integrations/faiss-documentstore)
|
|
7
|
+
- [Changelog](https://github.com/deepset-ai/haystack-core-integrations/blob/main/integrations/faiss/CHANGELOG.md)
|
|
8
|
+
|
|
9
|
+
---
|
|
10
|
+
|
|
11
|
+
Refer to the general [Contribution Guidelines](https://github.com/deepset-ai/haystack-core-integrations/blob/main/CONTRIBUTING.md).
|
|
@@ -5,12 +5,12 @@ build-backend = "hatchling.build"
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "faiss-haystack"
|
|
7
7
|
dynamic = ["version"]
|
|
8
|
-
description = ''
|
|
8
|
+
description = 'An integration of FAISS (library for efficient similarity search of dense vectors) with Haystack'
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
11
11
|
license = "Apache-2.0"
|
|
12
12
|
keywords = []
|
|
13
|
-
authors = [{ name = "
|
|
13
|
+
authors = [{ name = "deepset GmbH", email = "info@deepset.ai" }]
|
|
14
14
|
classifiers = [
|
|
15
15
|
"License :: OSI Approved :: Apache Software License",
|
|
16
16
|
"Development Status :: 4 - Beta",
|
|
@@ -18,11 +18,13 @@ classifiers = [
|
|
|
18
18
|
"Programming Language :: Python :: 3.10",
|
|
19
19
|
"Programming Language :: Python :: 3.11",
|
|
20
20
|
"Programming Language :: Python :: 3.12",
|
|
21
|
+
"Programming Language :: Python :: 3.13",
|
|
22
|
+
"Programming Language :: Python :: 3.14",
|
|
21
23
|
"Programming Language :: Python :: Implementation :: CPython",
|
|
22
24
|
"Programming Language :: Python :: Implementation :: PyPy",
|
|
23
25
|
]
|
|
24
26
|
dependencies = [
|
|
25
|
-
"haystack-ai>=2.
|
|
27
|
+
"haystack-ai>=2.26.1",
|
|
26
28
|
"faiss-cpu>=1.8.0",
|
|
27
29
|
"numpy>=1.22,<2; python_version < '3.13'",
|
|
28
30
|
]
|
|
@@ -48,7 +50,7 @@ installer = "uv"
|
|
|
48
50
|
dependencies = ["haystack-pydoc-tools", "ruff"]
|
|
49
51
|
|
|
50
52
|
[tool.hatch.envs.default.scripts]
|
|
51
|
-
docs = ["pydoc
|
|
53
|
+
docs = ["haystack-pydoc pydoc/config_docusaurus.yml"]
|
|
52
54
|
fmt = "ruff check --fix {args}; ruff format {args}"
|
|
53
55
|
fmt-check = "ruff check {args} && ruff format --check {args}"
|
|
54
56
|
|
|
@@ -59,6 +61,7 @@ dependencies = [
|
|
|
59
61
|
"pytest-rerunfailures",
|
|
60
62
|
"mypy",
|
|
61
63
|
"pandas",
|
|
64
|
+
"pip", # needed for mypy to install type stubs
|
|
62
65
|
]
|
|
63
66
|
|
|
64
67
|
[tool.hatch.envs.test.scripts]
|
|
@@ -75,8 +78,12 @@ non_interactive = true
|
|
|
75
78
|
check_untyped_defs = true
|
|
76
79
|
disallow_incomplete_defs = true
|
|
77
80
|
|
|
78
|
-
[tool.
|
|
79
|
-
|
|
81
|
+
[[tool.mypy.overrides]]
|
|
82
|
+
module = [
|
|
83
|
+
# faiss does not provide types
|
|
84
|
+
"faiss.*",
|
|
85
|
+
]
|
|
86
|
+
ignore_missing_imports = true
|
|
80
87
|
|
|
81
88
|
[tool.ruff]
|
|
82
89
|
line-length = 120
|
|
@@ -84,9 +91,17 @@ line-length = 120
|
|
|
84
91
|
[tool.ruff.lint]
|
|
85
92
|
select = [
|
|
86
93
|
"A",
|
|
94
|
+
"ANN",
|
|
87
95
|
"ARG",
|
|
88
96
|
"B",
|
|
89
97
|
"C",
|
|
98
|
+
"D102", # Missing docstring in public method
|
|
99
|
+
"D103", # Missing docstring in public function
|
|
100
|
+
"D205", # 1 blank line required between summary line and description
|
|
101
|
+
"D209", # Closing triple quotes go to new line
|
|
102
|
+
"D213", # summary lines must be positioned on the second physical line of the docstring
|
|
103
|
+
"D417", # Missing argument descriptions in the docstring
|
|
104
|
+
"D419", # Docstring is empty
|
|
90
105
|
"DTZ",
|
|
91
106
|
"E",
|
|
92
107
|
"EM",
|
|
@@ -128,6 +143,8 @@ ignore = [
|
|
|
128
143
|
"ARG002",
|
|
129
144
|
# Allow assertions
|
|
130
145
|
"S101",
|
|
146
|
+
# Allow `Any` - used legitimately for dynamic types and SDK boundaries
|
|
147
|
+
"ANN401",
|
|
131
148
|
]
|
|
132
149
|
exclude = ["example"]
|
|
133
150
|
|
|
@@ -139,7 +156,7 @@ ban-relative-imports = "parents"
|
|
|
139
156
|
|
|
140
157
|
[tool.ruff.lint.per-file-ignores]
|
|
141
158
|
# Tests can use magic values, assertions, and relative imports
|
|
142
|
-
"tests/**/*" = ["PLR2004", "S101", "TID252"]
|
|
159
|
+
"tests/**/*" = ["D", "PLR2004", "S101", "TID252", "ANN"]
|
|
143
160
|
"example/**/*" = ["T201"]
|
|
144
161
|
|
|
145
162
|
[tool.coverage.run]
|
|
@@ -156,4 +173,5 @@ exclude_lines = ["no cov", "if __name__ == .__main__.:", "if TYPE_CHECKING:"]
|
|
|
156
173
|
|
|
157
174
|
[tool.pytest.ini_options]
|
|
158
175
|
minversion = "6.0"
|
|
176
|
+
addopts = "--strict-markers"
|
|
159
177
|
markers = ["integration: integration tests"]
|
|
@@ -59,8 +59,10 @@ class FAISSEmbeddingRetriever:
|
|
|
59
59
|
filters: dict[str, Any] | None = None,
|
|
60
60
|
top_k: int = 10,
|
|
61
61
|
filter_policy: str | FilterPolicy = FilterPolicy.REPLACE,
|
|
62
|
-
):
|
|
62
|
+
) -> None:
|
|
63
63
|
"""
|
|
64
|
+
Initialize FAISSEmbeddingRetriever.
|
|
65
|
+
|
|
64
66
|
:param document_store: An instance of `FAISSDocumentStore`.
|
|
65
67
|
:param filters: Filters applied to the retrieved Documents at initialisation time. At runtime, these are merged
|
|
66
68
|
with any runtime filters according to the `filter_policy`.
|
|
@@ -9,7 +9,6 @@ from dataclasses import replace
|
|
|
9
9
|
from pathlib import Path
|
|
10
10
|
from typing import Any
|
|
11
11
|
|
|
12
|
-
import faiss # type: ignore[import-untyped]
|
|
13
12
|
import numpy as np
|
|
14
13
|
from haystack import default_from_dict, default_to_dict
|
|
15
14
|
from haystack.dataclasses import Document
|
|
@@ -17,6 +16,8 @@ from haystack.document_stores.errors import DocumentStoreError, DuplicateDocumen
|
|
|
17
16
|
from haystack.document_stores.types import DuplicatePolicy
|
|
18
17
|
from haystack.errors import FilterError
|
|
19
18
|
|
|
19
|
+
import faiss
|
|
20
|
+
|
|
20
21
|
logger = logging.getLogger(__name__)
|
|
21
22
|
|
|
22
23
|
|
|
@@ -33,7 +34,7 @@ class FAISSDocumentStore:
|
|
|
33
34
|
index_path: str | None = None,
|
|
34
35
|
index_string: str = "Flat",
|
|
35
36
|
embedding_dim: int = 768,
|
|
36
|
-
):
|
|
37
|
+
) -> None:
|
|
37
38
|
"""
|
|
38
39
|
Initializes the FAISSDocumentStore.
|
|
39
40
|
|
|
@@ -60,7 +61,7 @@ class FAISSDocumentStore:
|
|
|
60
61
|
else:
|
|
61
62
|
self._create_new_index()
|
|
62
63
|
|
|
63
|
-
def _create_new_index(self):
|
|
64
|
+
def _create_new_index(self) -> None:
|
|
64
65
|
"""Creates a new FAISS index."""
|
|
65
66
|
try:
|
|
66
67
|
# We use IndexIDMap to support add_with_ids
|
|
@@ -488,18 +489,18 @@ class FAISSDocumentStore:
|
|
|
488
489
|
values.add(val)
|
|
489
490
|
return list(values)
|
|
490
491
|
|
|
491
|
-
def count_unique_metadata_by_filter(self, filters: dict[str, Any],
|
|
492
|
+
def count_unique_metadata_by_filter(self, filters: dict[str, Any], metadata_fields: list[str]) -> dict[str, int]:
|
|
492
493
|
"""
|
|
493
494
|
Returns a count of unique values for multiple metadata fields, optionally scoped by a filter.
|
|
494
495
|
|
|
495
496
|
:param filters: A dictionary of filters to apply.
|
|
496
|
-
:param
|
|
497
|
+
:param metadata_fields: A list of metadata field names to count unique values for.
|
|
497
498
|
:returns: A dictionary mapping each field name to the count of its unique values.
|
|
498
499
|
"""
|
|
499
500
|
filtered_docs = self.filter_documents(filters)
|
|
500
501
|
counts = {}
|
|
501
502
|
|
|
502
|
-
for field in
|
|
503
|
+
for field in metadata_fields:
|
|
503
504
|
unique_vals = set()
|
|
504
505
|
for doc in filtered_docs:
|
|
505
506
|
val = FAISSDocumentStore._get_doc_value(doc, field)
|
|
@@ -6,11 +6,16 @@ import pytest
|
|
|
6
6
|
from haystack.dataclasses import Document
|
|
7
7
|
from haystack.errors import FilterError
|
|
8
8
|
from haystack.testing.document_store import (
|
|
9
|
+
CountDocumentsByFilterTest,
|
|
9
10
|
CountDocumentsTest,
|
|
11
|
+
CountUniqueMetadataByFilterTest,
|
|
10
12
|
DeleteAllTest,
|
|
11
13
|
DeleteByFilterTest,
|
|
12
14
|
DeleteDocumentsTest,
|
|
13
15
|
FilterDocumentsTest,
|
|
16
|
+
GetMetadataFieldMinMaxTest,
|
|
17
|
+
GetMetadataFieldsInfoTest,
|
|
18
|
+
GetMetadataFieldUniqueValuesTest,
|
|
14
19
|
UpdateByFilterTest,
|
|
15
20
|
)
|
|
16
21
|
|
|
@@ -24,20 +29,23 @@ class TestFAISSDocumentStore(
|
|
|
24
29
|
UpdateByFilterTest,
|
|
25
30
|
DeleteAllTest,
|
|
26
31
|
DeleteByFilterTest,
|
|
32
|
+
CountDocumentsByFilterTest,
|
|
33
|
+
CountUniqueMetadataByFilterTest,
|
|
34
|
+
GetMetadataFieldsInfoTest,
|
|
35
|
+
GetMetadataFieldMinMaxTest,
|
|
36
|
+
GetMetadataFieldUniqueValuesTest,
|
|
27
37
|
):
|
|
28
38
|
@pytest.fixture
|
|
29
39
|
def document_store(self, tmp_path):
|
|
30
40
|
return FAISSDocumentStore(index_path=str(tmp_path / "test_index"))
|
|
31
41
|
|
|
32
42
|
def test_write_documents(self, document_store):
|
|
33
|
-
|
|
34
43
|
doc = Document(content="test")
|
|
35
44
|
document_store.write_documents([doc])
|
|
36
45
|
assert document_store.count_documents() == 1
|
|
37
46
|
assert document_store.filter_documents()[0].id == doc.id
|
|
38
47
|
|
|
39
48
|
def test_persistence(self, tmp_path):
|
|
40
|
-
|
|
41
49
|
path = tmp_path / "persistent_index"
|
|
42
50
|
ds = FAISSDocumentStore(index_path=str(path), embedding_dim=3)
|
|
43
51
|
|
|
@@ -73,7 +81,6 @@ class TestFAISSDocumentStore(
|
|
|
73
81
|
ds.load(path)
|
|
74
82
|
|
|
75
83
|
def test_search_with_and_without_filters(self, document_store):
|
|
76
|
-
|
|
77
84
|
# Setup documents with missing/varied embeddings to test edge cases
|
|
78
85
|
doc1 = Document(content="test1", embedding=[0.1, 0.2, 0.3], meta={"category": "A"})
|
|
79
86
|
doc2 = Document(content="test2", embedding=[0.4, 0.5, 0.6], meta={"category": "B"})
|
|
@@ -97,7 +104,6 @@ class TestFAISSDocumentStore(
|
|
|
97
104
|
|
|
98
105
|
def test_to_dict_from_dict(self):
|
|
99
106
|
ds = FAISSDocumentStore(index_path="test_index", index_string="Flat", embedding_dim=128)
|
|
100
|
-
|
|
101
107
|
data = ds.to_dict()
|
|
102
108
|
assert data["type"] == "haystack_integrations.document_stores.faiss.document_store.FAISSDocumentStore"
|
|
103
109
|
assert data["init_parameters"]["index_path"] == "test_index"
|
|
@@ -109,50 +115,7 @@ class TestFAISSDocumentStore(
|
|
|
109
115
|
assert ds_loaded.index_string == "Flat"
|
|
110
116
|
assert ds_loaded.embedding_dim == 128
|
|
111
117
|
|
|
112
|
-
def test_count_documents_by_filter(self, document_store):
|
|
113
|
-
|
|
114
|
-
docs = [
|
|
115
|
-
Document(content="test1", meta={"category": "A"}),
|
|
116
|
-
Document(content="test2", meta={"category": "B"}),
|
|
117
|
-
Document(content="test3", meta={"category": "A"}),
|
|
118
|
-
]
|
|
119
|
-
document_store.write_documents(docs)
|
|
120
|
-
|
|
121
|
-
count = document_store.count_documents_by_filter(
|
|
122
|
-
filters={"field": "meta.category", "operator": "==", "value": "A"}
|
|
123
|
-
)
|
|
124
|
-
assert count == 2
|
|
125
|
-
|
|
126
|
-
def test_get_metadata_fields_info(self, document_store):
|
|
127
|
-
|
|
128
|
-
docs = [Document(content="test1", meta={"category": "A", "count": 1, "is_active": True})]
|
|
129
|
-
document_store.write_documents(docs)
|
|
130
|
-
|
|
131
|
-
info = document_store.get_metadata_fields_info()
|
|
132
|
-
assert "category" in info
|
|
133
|
-
assert info["category"]["type"] == "keyword"
|
|
134
|
-
assert "count" in info
|
|
135
|
-
assert info["count"]["type"] == "long"
|
|
136
|
-
assert "is_active" in info
|
|
137
|
-
assert info["is_active"]["type"] == "boolean"
|
|
138
|
-
|
|
139
|
-
def test_count_unique_metadata_by_filter(self, document_store):
|
|
140
|
-
|
|
141
|
-
docs = [
|
|
142
|
-
Document(content="test1", meta={"category": "A", "status": "active"}),
|
|
143
|
-
Document(content="test2", meta={"category": "B", "status": "inactive"}),
|
|
144
|
-
Document(content="test3", meta={"category": "A", "status": "active"}),
|
|
145
|
-
]
|
|
146
|
-
document_store.write_documents(docs)
|
|
147
|
-
|
|
148
|
-
counts = document_store.count_unique_metadata_by_filter(
|
|
149
|
-
filters={"field": "meta.category", "operator": "==", "value": "A"}, fields=["meta.status"]
|
|
150
|
-
)
|
|
151
|
-
assert "meta.status" in counts
|
|
152
|
-
assert counts["meta.status"] == 1 # Only "active" status for category A
|
|
153
|
-
|
|
154
118
|
def test_not_filter_with_empty_conditions_raises_filter_error(self, document_store):
|
|
155
119
|
document_store.write_documents([Document(content="test", meta={"category": "A"})])
|
|
156
|
-
|
|
157
120
|
with pytest.raises(FilterError, match="NOT operator expects at least one condition"):
|
|
158
121
|
document_store.filter_documents(filters={"operator": "NOT", "conditions": []})
|
faiss_haystack-1.0.0/README.md
DELETED
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
# faiss-haystack
|
|
2
|
-
|
|
3
|
-
This package provides a [FAISS](https://github.com/facebookresearch/faiss) document store for [Haystack](https://github.com/deepset-ai/haystack).
|
|
4
|
-
|
|
5
|
-
## Installation
|
|
6
|
-
|
|
7
|
-
```bash
|
|
8
|
-
pip install faiss-haystack
|
|
9
|
-
```
|
|
10
|
-
|
|
11
|
-
## Usage
|
|
12
|
-
|
|
13
|
-
```python
|
|
14
|
-
from haystack_integrations.document_stores.faiss import FAISSDocumentStore
|
|
15
|
-
|
|
16
|
-
document_store = FAISSDocumentStore(index_path="my_index")
|
|
17
|
-
```
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|