pgvector-haystack 6.3.0__tar.gz → 6.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pgvector_haystack-6.3.0 → pgvector_haystack-6.4.0}/CHANGELOG.md +31 -0
- {pgvector_haystack-6.3.0 → pgvector_haystack-6.4.0}/PKG-INFO +3 -2
- {pgvector_haystack-6.3.0 → pgvector_haystack-6.4.0}/examples/embedding_retrieval.py +4 -1
- {pgvector_haystack-6.3.0 → pgvector_haystack-6.4.0}/examples/hybrid_retrieval.py +4 -1
- {pgvector_haystack-6.3.0 → pgvector_haystack-6.4.0}/pyproject.toml +17 -4
- {pgvector_haystack-6.3.0 → pgvector_haystack-6.4.0}/src/haystack_integrations/components/retrievers/pgvector/embedding_retriever.py +18 -3
- {pgvector_haystack-6.3.0 → pgvector_haystack-6.4.0}/src/haystack_integrations/components/retrievers/pgvector/keyword_retriever.py +15 -1
- {pgvector_haystack-6.3.0 → pgvector_haystack-6.4.0}/src/haystack_integrations/document_stores/pgvector/converters.py +3 -2
- {pgvector_haystack-6.3.0 → pgvector_haystack-6.4.0}/src/haystack_integrations/document_stores/pgvector/document_store.py +70 -43
- {pgvector_haystack-6.3.0 → pgvector_haystack-6.4.0}/src/haystack_integrations/document_stores/pgvector/filters.py +3 -4
- {pgvector_haystack-6.3.0 → pgvector_haystack-6.4.0}/tests/conftest.py +14 -1
- {pgvector_haystack-6.3.0 → pgvector_haystack-6.4.0}/tests/test_document_store.py +254 -140
- pgvector_haystack-6.4.0/tests/test_document_store_async.py +346 -0
- {pgvector_haystack-6.3.0 → pgvector_haystack-6.4.0}/tests/test_filters.py +17 -2
- {pgvector_haystack-6.3.0 → pgvector_haystack-6.4.0}/tests/test_retrievers.py +41 -1
- pgvector_haystack-6.3.0/tests/test_document_store_async.py +0 -799
- {pgvector_haystack-6.3.0 → pgvector_haystack-6.4.0}/.gitignore +0 -0
- {pgvector_haystack-6.3.0 → pgvector_haystack-6.4.0}/LICENSE.txt +0 -0
- {pgvector_haystack-6.3.0 → pgvector_haystack-6.4.0}/README.md +0 -0
- {pgvector_haystack-6.3.0 → pgvector_haystack-6.4.0}/pydoc/config_docusaurus.yml +0 -0
- {pgvector_haystack-6.3.0 → pgvector_haystack-6.4.0}/src/haystack_integrations/components/retrievers/pgvector/__init__.py +0 -0
- {pgvector_haystack-6.3.0 → pgvector_haystack-6.4.0}/src/haystack_integrations/components/retrievers/py.typed +0 -0
- {pgvector_haystack-6.3.0 → pgvector_haystack-6.4.0}/src/haystack_integrations/document_stores/pgvector/__init__.py +0 -0
- {pgvector_haystack-6.3.0 → pgvector_haystack-6.4.0}/src/haystack_integrations/document_stores/py.typed +0 -0
- {pgvector_haystack-6.3.0 → pgvector_haystack-6.4.0}/tests/__init__.py +0 -0
- {pgvector_haystack-6.3.0 → pgvector_haystack-6.4.0}/tests/test_converters.py +0 -0
- {pgvector_haystack-6.3.0 → pgvector_haystack-6.4.0}/tests/test_retrieval.py +0 -0
- {pgvector_haystack-6.3.0 → pgvector_haystack-6.4.0}/tests/test_retrieval_async.py +0 -0
|
@@ -1,5 +1,36 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [integrations/pgvector-v6.3.1] - 2026-06-01
|
|
4
|
+
|
|
5
|
+
### 🐛 Bug Fixes
|
|
6
|
+
|
|
7
|
+
- Replace in place dataclass mutations for pgvector #11087 (#3142)
|
|
8
|
+
- *(pgvector)* Order retrieval by distance operator (#3370)
|
|
9
|
+
|
|
10
|
+
### 🚜 Refactor
|
|
11
|
+
|
|
12
|
+
- *(pgvector)* Use async DocumentStore mixin tests (#3094)
|
|
13
|
+
|
|
14
|
+
### 🧪 Testing
|
|
15
|
+
|
|
16
|
+
- `PGVectorDocumentStore`e use Mixin tests (#3003)
|
|
17
|
+
- Test compatible integrations with python 3.14; update pyproject (#3001)
|
|
18
|
+
- Track test coverage for all integrations (#3065)
|
|
19
|
+
- Add tests for pgvector document store (#3156)
|
|
20
|
+
|
|
21
|
+
### 🧹 Chores
|
|
22
|
+
|
|
23
|
+
- Add ANN ruff ruleset to optimum, paddleocr, pgvector, pinecone, pyversity, qdrant, ragas, snowflake (#2992)
|
|
24
|
+
- Enforce ruff docstring rules in integrations 31-40 (openrouter, opensearch, optimum, paddleocr, pgvector, pinecone, pyversity, qdrant, ragas, snowflake) (#3011)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
## [integrations/pgvector-v6.3.0] - 2026-03-17
|
|
28
|
+
|
|
29
|
+
### 🐛 Bug Fixes
|
|
30
|
+
|
|
31
|
+
- PgVectorDocumentStore `_treat_meta_field` and comparison functions now return Composed - string escaping done by `psycopg` (#2964)
|
|
32
|
+
|
|
33
|
+
|
|
3
34
|
## [integrations/pgvector-v6.2.0] - 2026-03-02
|
|
4
35
|
|
|
5
36
|
### 🐛 Bug Fixes
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: pgvector-haystack
|
|
3
|
-
Version: 6.
|
|
3
|
+
Version: 6.4.0
|
|
4
4
|
Summary: An integration of pgvector (vector search extension for Postgres) with Haystack
|
|
5
5
|
Project-URL: Source, https://github.com/deepset-ai/haystack-core-integrations
|
|
6
6
|
Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/blob/main/integrations/pgvector/README.md
|
|
@@ -15,10 +15,11 @@ Classifier: Programming Language :: Python :: 3.10
|
|
|
15
15
|
Classifier: Programming Language :: Python :: 3.11
|
|
16
16
|
Classifier: Programming Language :: Python :: 3.12
|
|
17
17
|
Classifier: Programming Language :: Python :: 3.13
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
18
19
|
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
19
20
|
Classifier: Programming Language :: Python :: Implementation :: PyPy
|
|
20
21
|
Requires-Python: >=3.10
|
|
21
|
-
Requires-Dist: haystack-ai>=2.
|
|
22
|
+
Requires-Dist: haystack-ai>=2.28.0
|
|
22
23
|
Requires-Dist: pgvector>=0.3.0
|
|
23
24
|
Requires-Dist: psycopg[binary]
|
|
24
25
|
Description-Content-Type: text/markdown
|
|
@@ -14,10 +14,13 @@ import glob
|
|
|
14
14
|
|
|
15
15
|
from haystack import Pipeline
|
|
16
16
|
from haystack.components.converters import MarkdownToDocument
|
|
17
|
-
from haystack.components.embedders import SentenceTransformersDocumentEmbedder, SentenceTransformersTextEmbedder
|
|
18
17
|
from haystack.components.preprocessors import DocumentSplitter
|
|
19
18
|
from haystack.components.writers import DocumentWriter
|
|
20
19
|
|
|
20
|
+
from haystack_integrations.components.embedders.sentence_transformers import (
|
|
21
|
+
SentenceTransformersDocumentEmbedder,
|
|
22
|
+
SentenceTransformersTextEmbedder,
|
|
23
|
+
)
|
|
21
24
|
from haystack_integrations.components.retrievers.pgvector import PgvectorEmbeddingRetriever
|
|
22
25
|
from haystack_integrations.document_stores.pgvector import PgvectorDocumentStore
|
|
23
26
|
|
|
@@ -14,11 +14,14 @@ import glob
|
|
|
14
14
|
|
|
15
15
|
from haystack import Pipeline
|
|
16
16
|
from haystack.components.converters import MarkdownToDocument
|
|
17
|
-
from haystack.components.embedders import SentenceTransformersDocumentEmbedder, SentenceTransformersTextEmbedder
|
|
18
17
|
from haystack.components.joiners import DocumentJoiner
|
|
19
18
|
from haystack.components.preprocessors import DocumentSplitter
|
|
20
19
|
from haystack.components.writers import DocumentWriter
|
|
21
20
|
|
|
21
|
+
from haystack_integrations.components.embedders.sentence_transformers import (
|
|
22
|
+
SentenceTransformersDocumentEmbedder,
|
|
23
|
+
SentenceTransformersTextEmbedder,
|
|
24
|
+
)
|
|
22
25
|
from haystack_integrations.components.retrievers.pgvector import PgvectorEmbeddingRetriever, PgvectorKeywordRetriever
|
|
23
26
|
from haystack_integrations.document_stores.pgvector import PgvectorDocumentStore
|
|
24
27
|
|
|
@@ -19,11 +19,12 @@ classifiers = [
|
|
|
19
19
|
"Programming Language :: Python :: 3.11",
|
|
20
20
|
"Programming Language :: Python :: 3.12",
|
|
21
21
|
"Programming Language :: Python :: 3.13",
|
|
22
|
+
"Programming Language :: Python :: 3.14",
|
|
22
23
|
"Programming Language :: Python :: Implementation :: CPython",
|
|
23
24
|
"Programming Language :: Python :: Implementation :: PyPy",
|
|
24
25
|
]
|
|
25
26
|
dependencies = [
|
|
26
|
-
"haystack-ai>=2.
|
|
27
|
+
"haystack-ai>=2.28.0",
|
|
27
28
|
"pgvector>=0.3.0",
|
|
28
29
|
"psycopg[binary]"
|
|
29
30
|
]
|
|
@@ -67,7 +68,8 @@ dependencies = [
|
|
|
67
68
|
unit = 'pytest -m "not integration" {args:tests}'
|
|
68
69
|
integration = 'pytest -m "integration" {args:tests}'
|
|
69
70
|
all = 'pytest {args:tests}'
|
|
70
|
-
cov-retry = 'pytest --cov=haystack_integrations --reruns 3 --reruns-delay 30 -x {args:tests}'
|
|
71
|
+
unit-cov-retry = 'pytest --cov=haystack_integrations --reruns 3 --reruns-delay 30 -x -m "not integration" {args:tests}'
|
|
72
|
+
integration-cov-append-retry = 'pytest --cov=haystack_integrations --cov-append --reruns 3 --reruns-delay 30 -x -m "integration" {args:tests}'
|
|
71
73
|
|
|
72
74
|
types = "mypy -p haystack_integrations.document_stores.pgvector -p haystack_integrations.components.retrievers.pgvector {args}"
|
|
73
75
|
|
|
@@ -91,9 +93,17 @@ line-length = 120
|
|
|
91
93
|
[tool.ruff.lint]
|
|
92
94
|
select = [
|
|
93
95
|
"A",
|
|
96
|
+
"ANN",
|
|
94
97
|
"ARG",
|
|
95
98
|
"B",
|
|
96
99
|
"C",
|
|
100
|
+
"D102", # Missing docstring in public method
|
|
101
|
+
"D103", # Missing docstring in public function
|
|
102
|
+
"D205", # 1 blank line required between summary line and description
|
|
103
|
+
"D209", # Closing triple quotes go to new line
|
|
104
|
+
"D213", # summary lines must be positioned on the second physical line of the docstring
|
|
105
|
+
"D417", # Missing argument descriptions in the docstring
|
|
106
|
+
"D419", # Docstring is empty
|
|
97
107
|
"DTZ",
|
|
98
108
|
"E",
|
|
99
109
|
"EM",
|
|
@@ -119,6 +129,8 @@ select = [
|
|
|
119
129
|
ignore = [
|
|
120
130
|
# Allow non-abstract empty methods in abstract base classes
|
|
121
131
|
"B027",
|
|
132
|
+
# Allow Any in type annotations at dynamic boundaries
|
|
133
|
+
"ANN401",
|
|
122
134
|
# Allow boolean positional values in function calls, like `dict.get(... True)`
|
|
123
135
|
"FBT003",
|
|
124
136
|
# Ignore checks for possible passwords
|
|
@@ -145,14 +157,15 @@ ban-relative-imports = "parents"
|
|
|
145
157
|
|
|
146
158
|
[tool.ruff.lint.per-file-ignores]
|
|
147
159
|
# Tests can use magic values, assertions, and relative imports
|
|
148
|
-
"tests/**/*" = ["PLR2004", "S101", "TID252"]
|
|
160
|
+
"tests/**/*" = ["D", "PLR2004", "S101", "TID252", "ANN"]
|
|
149
161
|
# examples can contain "print" commands
|
|
150
|
-
"examples/**/*" = ["T201"]
|
|
162
|
+
"examples/**/*" = ["D", "T201"]
|
|
151
163
|
|
|
152
164
|
|
|
153
165
|
[tool.coverage.run]
|
|
154
166
|
source = ["haystack_integrations"]
|
|
155
167
|
branch = true
|
|
168
|
+
relative_files = true
|
|
156
169
|
parallel = false
|
|
157
170
|
|
|
158
171
|
[tool.coverage.report]
|
|
@@ -21,7 +21,9 @@ class PgvectorEmbeddingRetriever:
|
|
|
21
21
|
```python
|
|
22
22
|
from haystack.document_stores import DuplicatePolicy
|
|
23
23
|
from haystack import Document, Pipeline
|
|
24
|
-
|
|
24
|
+
# Requires: pip install sentence-transformers-haystack
|
|
25
|
+
from haystack_integrations.components.embedders.sentence_transformers import SentenceTransformersTextEmbedder
|
|
26
|
+
from haystack_integrations.components.embedders.sentence_transformers import SentenceTransformersDocumentEmbedder
|
|
25
27
|
|
|
26
28
|
from haystack_integrations.document_stores.pgvector import PgvectorDocumentStore
|
|
27
29
|
from haystack_integrations.components.retrievers.pgvector import PgvectorEmbeddingRetriever
|
|
@@ -40,7 +42,6 @@ class PgvectorEmbeddingRetriever:
|
|
|
40
42
|
Document(content="In certain places, you can witness the phenomenon of bioluminescent waves.")]
|
|
41
43
|
|
|
42
44
|
document_embedder = SentenceTransformersDocumentEmbedder()
|
|
43
|
-
document_embedder.warm_up()
|
|
44
45
|
documents_with_embeddings = document_embedder.run(documents)
|
|
45
46
|
|
|
46
47
|
document_store.write_documents(documents_with_embeddings.get("documents"), policy=DuplicatePolicy.OVERWRITE)
|
|
@@ -66,8 +67,10 @@ class PgvectorEmbeddingRetriever:
|
|
|
66
67
|
top_k: int = 10,
|
|
67
68
|
vector_function: Literal["cosine_similarity", "inner_product", "l2_distance"] | None = None,
|
|
68
69
|
filter_policy: str | FilterPolicy = FilterPolicy.REPLACE,
|
|
69
|
-
):
|
|
70
|
+
) -> None:
|
|
70
71
|
"""
|
|
72
|
+
Initialize the PgvectorEmbeddingRetriever.
|
|
73
|
+
|
|
71
74
|
:param document_store: An instance of `PgvectorDocumentStore`.
|
|
72
75
|
:param filters: Filters applied to the retrieved Documents.
|
|
73
76
|
:param top_k: Maximum number of Documents to return.
|
|
@@ -133,6 +136,18 @@ class PgvectorEmbeddingRetriever:
|
|
|
133
136
|
data["init_parameters"]["filter_policy"] = FilterPolicy.from_str(filter_policy)
|
|
134
137
|
return default_from_dict(cls, data)
|
|
135
138
|
|
|
139
|
+
def close(self) -> None:
|
|
140
|
+
"""
|
|
141
|
+
Release the synchronous resources of the underlying Document Store.
|
|
142
|
+
"""
|
|
143
|
+
self.document_store.close()
|
|
144
|
+
|
|
145
|
+
async def close_async(self) -> None:
|
|
146
|
+
"""
|
|
147
|
+
Release the asynchronous resources of the underlying Document Store.
|
|
148
|
+
"""
|
|
149
|
+
await self.document_store.close_async()
|
|
150
|
+
|
|
136
151
|
@component.output_types(documents=list[Document])
|
|
137
152
|
def run(
|
|
138
153
|
self,
|
|
@@ -55,8 +55,10 @@ class PgvectorKeywordRetriever:
|
|
|
55
55
|
filters: dict[str, Any] | None = None,
|
|
56
56
|
top_k: int = 10,
|
|
57
57
|
filter_policy: str | FilterPolicy = FilterPolicy.REPLACE,
|
|
58
|
-
):
|
|
58
|
+
) -> None:
|
|
59
59
|
"""
|
|
60
|
+
Initialize the PgvectorKeywordRetriever.
|
|
61
|
+
|
|
60
62
|
:param document_store: An instance of `PgvectorDocumentStore`.
|
|
61
63
|
:param filters: Filters applied to the retrieved Documents.
|
|
62
64
|
:param top_k: Maximum number of Documents to return.
|
|
@@ -107,6 +109,18 @@ class PgvectorKeywordRetriever:
|
|
|
107
109
|
data["init_parameters"]["filter_policy"] = FilterPolicy.from_str(filter_policy)
|
|
108
110
|
return default_from_dict(cls, data)
|
|
109
111
|
|
|
112
|
+
def close(self) -> None:
|
|
113
|
+
"""
|
|
114
|
+
Release the synchronous resources of the underlying Document Store.
|
|
115
|
+
"""
|
|
116
|
+
self.document_store.close()
|
|
117
|
+
|
|
118
|
+
async def close_async(self) -> None:
|
|
119
|
+
"""
|
|
120
|
+
Release the asynchronous resources of the underlying Document Store.
|
|
121
|
+
"""
|
|
122
|
+
await self.document_store.close_async()
|
|
123
|
+
|
|
110
124
|
@component.output_types(documents=list[Document])
|
|
111
125
|
def run(
|
|
112
126
|
self,
|
|
@@ -10,8 +10,9 @@ logger = logging.getLogger(__name__)
|
|
|
10
10
|
|
|
11
11
|
def _from_haystack_to_pg_documents(documents: list[Document]) -> list[dict[str, Any]]:
|
|
12
12
|
"""
|
|
13
|
-
Internal method to convert a list of Haystack Documents to a list of dictionaries
|
|
14
|
-
|
|
13
|
+
Internal method to convert a list of Haystack Documents to a list of dictionaries.
|
|
14
|
+
|
|
15
|
+
The resulting dictionaries can be used to insert documents into the PgvectorDocumentStore.
|
|
15
16
|
"""
|
|
16
17
|
|
|
17
18
|
db_documents = []
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
#
|
|
3
3
|
# SPDX-License-Identifier: Apache-2.0
|
|
4
4
|
|
|
5
|
+
from contextlib import suppress
|
|
5
6
|
from typing import Any, Literal, overload
|
|
6
7
|
|
|
7
8
|
from haystack import default_from_dict, default_to_dict, logging
|
|
@@ -96,9 +97,10 @@ class PgvectorDocumentStore:
|
|
|
96
97
|
hnsw_index_name: str = "haystack_hnsw_index",
|
|
97
98
|
hnsw_ef_search: int | None = None,
|
|
98
99
|
keyword_index_name: str = "haystack_keyword_index",
|
|
99
|
-
):
|
|
100
|
+
) -> None:
|
|
100
101
|
"""
|
|
101
102
|
Creates a new PgvectorDocumentStore instance.
|
|
103
|
+
|
|
102
104
|
It is meant to be connected to a PostgreSQL database with the pgvector extension installed.
|
|
103
105
|
A specific table to store Haystack documents will be created if it doesn't exist yet.
|
|
104
106
|
|
|
@@ -222,8 +224,30 @@ class PgvectorDocumentStore:
|
|
|
222
224
|
deserialize_secrets_inplace(data["init_parameters"], ["connection_string"])
|
|
223
225
|
return default_from_dict(cls, data)
|
|
224
226
|
|
|
227
|
+
def close(self) -> None:
|
|
228
|
+
"""
|
|
229
|
+
Release the associated synchronous resources.
|
|
230
|
+
"""
|
|
231
|
+
if self._connection is not None:
|
|
232
|
+
with suppress(Exception):
|
|
233
|
+
self._connection.close()
|
|
234
|
+
self._connection = None
|
|
235
|
+
self._cursor = None
|
|
236
|
+
self._dict_cursor = None
|
|
237
|
+
|
|
238
|
+
async def close_async(self) -> None:
|
|
239
|
+
"""
|
|
240
|
+
Release the associated asynchronous resources.
|
|
241
|
+
"""
|
|
242
|
+
if self._async_connection is not None:
|
|
243
|
+
with suppress(Exception):
|
|
244
|
+
await self._async_connection.close()
|
|
245
|
+
self._async_connection = None
|
|
246
|
+
self._async_cursor = None
|
|
247
|
+
self._async_dict_cursor = None
|
|
248
|
+
|
|
225
249
|
@staticmethod
|
|
226
|
-
def _connection_is_valid(connection):
|
|
250
|
+
def _connection_is_valid(connection: Connection) -> bool:
|
|
227
251
|
"""
|
|
228
252
|
Internal method to check if the connection is still valid.
|
|
229
253
|
"""
|
|
@@ -238,7 +262,7 @@ class PgvectorDocumentStore:
|
|
|
238
262
|
return True
|
|
239
263
|
|
|
240
264
|
@staticmethod
|
|
241
|
-
async def _connection_is_valid_async(connection):
|
|
265
|
+
async def _connection_is_valid_async(connection: AsyncConnection) -> bool:
|
|
242
266
|
"""
|
|
243
267
|
Internal method to check if the async connection is still valid.
|
|
244
268
|
"""
|
|
@@ -250,18 +274,18 @@ class PgvectorDocumentStore:
|
|
|
250
274
|
|
|
251
275
|
@overload
|
|
252
276
|
def _execute_sql(
|
|
253
|
-
self, cursor: Cursor, sql_query: Composed, params: tuple | None = None, error_msg: str = ""
|
|
277
|
+
self, cursor: Cursor, sql_query: SQL | Composed, params: tuple | None = None, error_msg: str = ""
|
|
254
278
|
) -> Cursor: ...
|
|
255
279
|
|
|
256
280
|
@overload
|
|
257
281
|
def _execute_sql(
|
|
258
|
-
self, cursor: Cursor[DictRow], sql_query: Composed, params: tuple | None = None, error_msg: str = ""
|
|
282
|
+
self, cursor: Cursor[DictRow], sql_query: SQL | Composed, params: tuple | None = None, error_msg: str = ""
|
|
259
283
|
) -> Cursor[DictRow]: ...
|
|
260
284
|
|
|
261
285
|
def _execute_sql(
|
|
262
286
|
self,
|
|
263
287
|
cursor: Cursor | Cursor[DictRow],
|
|
264
|
-
sql_query: Composed,
|
|
288
|
+
sql_query: SQL | Composed,
|
|
265
289
|
params: tuple | None = None,
|
|
266
290
|
error_msg: str = "",
|
|
267
291
|
) -> Cursor | Cursor[DictRow]:
|
|
@@ -299,18 +323,18 @@ class PgvectorDocumentStore:
|
|
|
299
323
|
|
|
300
324
|
@overload
|
|
301
325
|
async def _execute_sql_async(
|
|
302
|
-
self, cursor: AsyncCursor, sql_query: Composed, params: tuple | None = None, error_msg: str = ""
|
|
326
|
+
self, cursor: AsyncCursor, sql_query: SQL | Composed, params: tuple | None = None, error_msg: str = ""
|
|
303
327
|
) -> AsyncCursor: ...
|
|
304
328
|
|
|
305
329
|
@overload
|
|
306
330
|
async def _execute_sql_async(
|
|
307
|
-
self, cursor: AsyncCursor[DictRow], sql_query: Composed, params: tuple | None = None, error_msg: str = ""
|
|
331
|
+
self, cursor: AsyncCursor[DictRow], sql_query: SQL | Composed, params: tuple | None = None, error_msg: str = ""
|
|
308
332
|
) -> AsyncCursor[DictRow]: ...
|
|
309
333
|
|
|
310
334
|
async def _execute_sql_async(
|
|
311
335
|
self,
|
|
312
336
|
cursor: AsyncCursor | AsyncCursor[DictRow],
|
|
313
|
-
sql_query: Composed,
|
|
337
|
+
sql_query: SQL | Composed,
|
|
314
338
|
params: tuple | None = None,
|
|
315
339
|
error_msg: str = "",
|
|
316
340
|
) -> AsyncCursor | AsyncCursor[DictRow]:
|
|
@@ -346,9 +370,10 @@ class PgvectorDocumentStore:
|
|
|
346
370
|
|
|
347
371
|
return result
|
|
348
372
|
|
|
349
|
-
def _ensure_db_setup(self):
|
|
373
|
+
def _ensure_db_setup(self) -> None:
|
|
350
374
|
"""
|
|
351
375
|
Ensures that the connection to the PostgreSQL database exists and is valid.
|
|
376
|
+
|
|
352
377
|
If not, connection and cursors are created.
|
|
353
378
|
If the table is not initialized, it will be set up.
|
|
354
379
|
"""
|
|
@@ -384,9 +409,10 @@ class PgvectorDocumentStore:
|
|
|
384
409
|
if not self._table_initialized:
|
|
385
410
|
self._initialize_table()
|
|
386
411
|
|
|
387
|
-
async def _ensure_db_setup_async(self):
|
|
412
|
+
async def _ensure_db_setup_async(self) -> None:
|
|
388
413
|
"""
|
|
389
414
|
Async internal method.
|
|
415
|
+
|
|
390
416
|
Ensures that the connection to the PostgreSQL database exists and is valid.
|
|
391
417
|
If not, connection and cursors are created.
|
|
392
418
|
If the table is not initialized, it will be set up.
|
|
@@ -426,7 +452,7 @@ class PgvectorDocumentStore:
|
|
|
426
452
|
if not self._table_initialized:
|
|
427
453
|
await self._initialize_table_async()
|
|
428
454
|
|
|
429
|
-
def _build_table_creation_queries(self):
|
|
455
|
+
def _build_table_creation_queries(self) -> tuple[SQL, Composed, SQL, Composed]:
|
|
430
456
|
"""
|
|
431
457
|
Internal method to build the SQL queries for table creation.
|
|
432
458
|
"""
|
|
@@ -453,7 +479,7 @@ class PgvectorDocumentStore:
|
|
|
453
479
|
|
|
454
480
|
return sql_table_exists, sql_create_table, sql_keyword_index_exists, sql_create_keyword_index
|
|
455
481
|
|
|
456
|
-
def _initialize_table(self):
|
|
482
|
+
def _initialize_table(self) -> None:
|
|
457
483
|
"""
|
|
458
484
|
Internal method to initialize the table.
|
|
459
485
|
"""
|
|
@@ -497,7 +523,7 @@ class PgvectorDocumentStore:
|
|
|
497
523
|
|
|
498
524
|
self._table_initialized = True
|
|
499
525
|
|
|
500
|
-
async def _initialize_table_async(self):
|
|
526
|
+
async def _initialize_table_async(self) -> None:
|
|
501
527
|
"""
|
|
502
528
|
Internal async method to initialize the table.
|
|
503
529
|
"""
|
|
@@ -547,9 +573,10 @@ class PgvectorDocumentStore:
|
|
|
547
573
|
|
|
548
574
|
self._table_initialized = True
|
|
549
575
|
|
|
550
|
-
def delete_table(self):
|
|
576
|
+
def delete_table(self) -> None:
|
|
551
577
|
"""
|
|
552
578
|
Deletes the table used to store Haystack documents.
|
|
579
|
+
|
|
553
580
|
The name of the schema (`schema_name`) and the name of the table (`table_name`)
|
|
554
581
|
are defined when initializing the `PgvectorDocumentStore`.
|
|
555
582
|
"""
|
|
@@ -567,7 +594,7 @@ class PgvectorDocumentStore:
|
|
|
567
594
|
error_msg=f"Could not delete table {self.schema_name}.{self.table_name} in PgvectorDocumentStore",
|
|
568
595
|
)
|
|
569
596
|
|
|
570
|
-
async def delete_table_async(self):
|
|
597
|
+
async def delete_table_async(self) -> None:
|
|
571
598
|
"""
|
|
572
599
|
Async method to delete the table used to store Haystack documents.
|
|
573
600
|
"""
|
|
@@ -585,7 +612,7 @@ class PgvectorDocumentStore:
|
|
|
585
612
|
error_msg=f"Could not delete table {self.schema_name}.{self.table_name} in PgvectorDocumentStore",
|
|
586
613
|
)
|
|
587
614
|
|
|
588
|
-
def _build_hnsw_queries(self):
|
|
615
|
+
def _build_hnsw_queries(self) -> tuple[Composed | None, SQL, Composed, Composed]:
|
|
589
616
|
"""Common method to build all HNSW-related SQL queries"""
|
|
590
617
|
|
|
591
618
|
sql_set_hnsw_ef_search = (
|
|
@@ -633,9 +660,10 @@ class PgvectorDocumentStore:
|
|
|
633
660
|
|
|
634
661
|
return sql_set_hnsw_ef_search, sql_hnsw_index_exists, sql_drop_hnsw_index, sql_create_hnsw_index
|
|
635
662
|
|
|
636
|
-
def _handle_hnsw(self):
|
|
663
|
+
def _handle_hnsw(self) -> None:
|
|
637
664
|
"""
|
|
638
665
|
Internal method to handle the HNSW index creation.
|
|
666
|
+
|
|
639
667
|
It also sets the `hnsw.ef_search` parameter for queries if it is specified.
|
|
640
668
|
"""
|
|
641
669
|
|
|
@@ -645,7 +673,7 @@ class PgvectorDocumentStore:
|
|
|
645
673
|
|
|
646
674
|
assert self._cursor is not None
|
|
647
675
|
|
|
648
|
-
if
|
|
676
|
+
if sql_set_hnsw_ef_search is not None:
|
|
649
677
|
self._execute_sql(
|
|
650
678
|
cursor=self._cursor, sql_query=sql_set_hnsw_ef_search, error_msg="Could not set hnsw.ef_search"
|
|
651
679
|
)
|
|
@@ -671,7 +699,7 @@ class PgvectorDocumentStore:
|
|
|
671
699
|
|
|
672
700
|
self._execute_sql(cursor=self._cursor, sql_query=sql_create_hnsw_index, error_msg="Could not create HNSW index")
|
|
673
701
|
|
|
674
|
-
async def _handle_hnsw_async(self):
|
|
702
|
+
async def _handle_hnsw_async(self) -> None:
|
|
675
703
|
"""
|
|
676
704
|
Internal async method to handle the HNSW index creation.
|
|
677
705
|
"""
|
|
@@ -682,7 +710,7 @@ class PgvectorDocumentStore:
|
|
|
682
710
|
|
|
683
711
|
assert self._async_cursor is not None
|
|
684
712
|
|
|
685
|
-
if
|
|
713
|
+
if sql_set_hnsw_ef_search is not None:
|
|
686
714
|
await self._execute_sql_async(
|
|
687
715
|
cursor=self._async_cursor, sql_query=sql_set_hnsw_ef_search, error_msg="Could not set hnsw.ef_search"
|
|
688
716
|
)
|
|
@@ -1346,10 +1374,13 @@ class PgvectorDocumentStore:
|
|
|
1346
1374
|
# cosine_similarity and inner_product are modified from the result of the operator
|
|
1347
1375
|
if vector_function == "cosine_similarity":
|
|
1348
1376
|
score_definition = f"1 - (embedding <=> {query_embedding_for_postgres}) AS score"
|
|
1377
|
+
order_by_definition = f"embedding <=> {query_embedding_for_postgres}"
|
|
1349
1378
|
elif vector_function == "inner_product":
|
|
1350
1379
|
score_definition = f"(embedding <#> {query_embedding_for_postgres}) * -1 AS score"
|
|
1380
|
+
order_by_definition = f"embedding <#> {query_embedding_for_postgres}"
|
|
1351
1381
|
elif vector_function == "l2_distance":
|
|
1352
1382
|
score_definition = f"embedding <-> {query_embedding_for_postgres} AS score"
|
|
1383
|
+
order_by_definition = f"embedding <-> {query_embedding_for_postgres}"
|
|
1353
1384
|
|
|
1354
1385
|
sql_select = SQL("SELECT *, {score} FROM {schema_name}.{table_name}").format(
|
|
1355
1386
|
schema_name=Identifier(self.schema_name),
|
|
@@ -1362,13 +1393,9 @@ class PgvectorDocumentStore:
|
|
|
1362
1393
|
if filters:
|
|
1363
1394
|
sql_where_clause, params = _convert_filters_to_where_clause_and_params(filters)
|
|
1364
1395
|
|
|
1365
|
-
|
|
1366
|
-
# so when using l2_distance, the sort order must be ASC
|
|
1367
|
-
sort_order = "ASC" if vector_function == "l2_distance" else "DESC"
|
|
1368
|
-
|
|
1369
|
-
sql_sort = SQL(" ORDER BY score {sort_order} LIMIT {top_k}").format(
|
|
1396
|
+
sql_sort = SQL(" ORDER BY {order_by} ASC LIMIT {top_k}").format(
|
|
1370
1397
|
top_k=SQLLiteral(top_k),
|
|
1371
|
-
|
|
1398
|
+
order_by=SQL(order_by_definition),
|
|
1372
1399
|
)
|
|
1373
1400
|
|
|
1374
1401
|
sql_query = sql_select + sql_where_clause + sql_sort
|
|
@@ -1418,8 +1445,9 @@ class PgvectorDocumentStore:
|
|
|
1418
1445
|
vector_function: Literal["cosine_similarity", "inner_product", "l2_distance"] | None = None,
|
|
1419
1446
|
) -> list[Document]:
|
|
1420
1447
|
"""
|
|
1421
|
-
Asynchronously retrieves documents that are most similar to the query embedding
|
|
1422
|
-
|
|
1448
|
+
Asynchronously retrieves documents that are most similar to the query embedding.
|
|
1449
|
+
|
|
1450
|
+
Uses a vector similarity metric for comparison.
|
|
1423
1451
|
"""
|
|
1424
1452
|
|
|
1425
1453
|
sql_query, params = self._check_and_build_embedding_retrieval_query(
|
|
@@ -1439,7 +1467,7 @@ class PgvectorDocumentStore:
|
|
|
1439
1467
|
docs = _from_pg_to_haystack_documents(records)
|
|
1440
1468
|
return docs
|
|
1441
1469
|
|
|
1442
|
-
def _prepare_filters_count_documents(self, filters):
|
|
1470
|
+
def _prepare_filters_count_documents(self, filters: dict[str, Any] | None) -> tuple[tuple[Any, ...], Composed]:
|
|
1443
1471
|
_validate_filters(filters)
|
|
1444
1472
|
sql_count = SQL("SELECT COUNT(*) FROM {schema_name}.{table_name}").format(
|
|
1445
1473
|
schema_name=Identifier(self.schema_name), table_name=Identifier(self.table_name)
|
|
@@ -1571,8 +1599,9 @@ class PgvectorDocumentStore:
|
|
|
1571
1599
|
|
|
1572
1600
|
def count_unique_metadata_by_filter(self, filters: dict[str, Any], metadata_fields: list[str]) -> dict[str, int]:
|
|
1573
1601
|
"""
|
|
1574
|
-
Returns the count of unique values for each specified metadata field
|
|
1575
|
-
|
|
1602
|
+
Returns the count of unique values for each specified metadata field.
|
|
1603
|
+
|
|
1604
|
+
Considers only documents that match the provided filters.
|
|
1576
1605
|
|
|
1577
1606
|
:param filters: The filters to apply to select documents.
|
|
1578
1607
|
For filter syntax, see [Haystack metadata filtering](https://docs.haystack.deepset.ai/docs/metadata-filtering)
|
|
@@ -1604,8 +1633,9 @@ class PgvectorDocumentStore:
|
|
|
1604
1633
|
self, filters: dict[str, Any], metadata_fields: list[str]
|
|
1605
1634
|
) -> dict[str, int]:
|
|
1606
1635
|
"""
|
|
1607
|
-
Asynchronously returns the count of unique values for each specified metadata field
|
|
1608
|
-
|
|
1636
|
+
Asynchronously returns the count of unique values for each specified metadata field.
|
|
1637
|
+
|
|
1638
|
+
Considers only documents that match the provided filters.
|
|
1609
1639
|
|
|
1610
1640
|
:param filters: The filters to apply to select documents.
|
|
1611
1641
|
For filter syntax, see [Haystack metadata filtering](https://docs.haystack.deepset.ai/docs/metadata-filtering)
|
|
@@ -1661,7 +1691,7 @@ class PgvectorDocumentStore:
|
|
|
1661
1691
|
:param records: List of database records containing 'meta' field.
|
|
1662
1692
|
:returns: A dictionary mapping field names to their type information.
|
|
1663
1693
|
"""
|
|
1664
|
-
fields_info: dict[str, dict[str, str]] = {
|
|
1694
|
+
fields_info: dict[str, dict[str, str]] = {}
|
|
1665
1695
|
|
|
1666
1696
|
# Analyze metadata from all documents
|
|
1667
1697
|
for record in records:
|
|
@@ -1681,7 +1711,7 @@ class PgvectorDocumentStore:
|
|
|
1681
1711
|
|
|
1682
1712
|
return fields_info
|
|
1683
1713
|
|
|
1684
|
-
def _analyze_metadata_from_docs_query(self):
|
|
1714
|
+
def _analyze_metadata_from_docs_query(self) -> Composed:
|
|
1685
1715
|
# query all documents to analyze metadata structure
|
|
1686
1716
|
sql_query = SQL("SELECT meta FROM {schema_name}.{table_name} WHERE meta IS NOT NULL").format(
|
|
1687
1717
|
schema_name=Identifier(self.schema_name), table_name=Identifier(self.table_name)
|
|
@@ -1698,7 +1728,6 @@ class PgvectorDocumentStore:
|
|
|
1698
1728
|
Example return:
|
|
1699
1729
|
```python
|
|
1700
1730
|
{
|
|
1701
|
-
'content': {'type': 'text'},
|
|
1702
1731
|
'category': {'type': 'text'},
|
|
1703
1732
|
'status': {'type': 'text'},
|
|
1704
1733
|
'priority': {'type': 'integer'},
|
|
@@ -1838,15 +1867,14 @@ class PgvectorDocumentStore:
|
|
|
1838
1867
|
:returns: A dictionary with 'min' and 'max' keys containing the minimum and maximum values.
|
|
1839
1868
|
For numeric fields (integer, real), returns numeric min/max.
|
|
1840
1869
|
For text fields, returns lexicographic min/max based on database collation.
|
|
1841
|
-
|
|
1870
|
+
Returns `{"min": None, "max": None}` when the field has no values or the store is empty.
|
|
1842
1871
|
"""
|
|
1843
1872
|
normalized_field = PgvectorDocumentStore._normalize_metadata_field_name(metadata_field)
|
|
1844
1873
|
|
|
1845
1874
|
# Get field type information from metadata fields info
|
|
1846
1875
|
fields_info = self.get_metadata_fields_info()
|
|
1847
1876
|
if normalized_field not in fields_info:
|
|
1848
|
-
|
|
1849
|
-
raise ValueError(msg)
|
|
1877
|
+
return {"min": None, "max": None}
|
|
1850
1878
|
|
|
1851
1879
|
field_type = fields_info[normalized_field]["type"]
|
|
1852
1880
|
sql_query = self._build_min_max_query(normalized_field, field_type)
|
|
@@ -1871,15 +1899,14 @@ class PgvectorDocumentStore:
|
|
|
1871
1899
|
:returns: A dictionary with 'min' and 'max' keys containing the minimum and maximum values.
|
|
1872
1900
|
For numeric fields (integer, real), returns numeric min/max.
|
|
1873
1901
|
For text fields, returns lexicographic min/max based on database collation.
|
|
1874
|
-
|
|
1902
|
+
Returns ``{"min": None, "max": None}`` when the field has no values or the store is empty.
|
|
1875
1903
|
"""
|
|
1876
1904
|
normalized_field = PgvectorDocumentStore._normalize_metadata_field_name(metadata_field)
|
|
1877
1905
|
|
|
1878
1906
|
# Get field type information from metadata fields info
|
|
1879
1907
|
fields_info = await self.get_metadata_fields_info_async()
|
|
1880
1908
|
if normalized_field not in fields_info:
|
|
1881
|
-
|
|
1882
|
-
raise ValueError(msg)
|
|
1909
|
+
return {"min": None, "max": None}
|
|
1883
1910
|
|
|
1884
1911
|
field_type = fields_info[normalized_field]["type"]
|
|
1885
1912
|
sql_query = self._build_min_max_query(normalized_field, field_type)
|
|
@@ -20,7 +20,7 @@ PYTHON_TYPES_TO_PG_TYPES = {
|
|
|
20
20
|
bool: "boolean",
|
|
21
21
|
}
|
|
22
22
|
|
|
23
|
-
NO_VALUE =
|
|
23
|
+
NO_VALUE = object() # unique sentinel; can never equal a user-supplied filter value
|
|
24
24
|
|
|
25
25
|
|
|
26
26
|
def _validate_filters(filters: dict[str, Any] | None = None) -> None:
|
|
@@ -48,7 +48,7 @@ def _convert_filters_to_where_clause_and_params(
|
|
|
48
48
|
query, values = _parse_logical_condition(filters)
|
|
49
49
|
|
|
50
50
|
where_clause = SQL(f" {operator} ") + query
|
|
51
|
-
params = tuple(value for value in values if value
|
|
51
|
+
params = tuple(value for value in values if value is not NO_VALUE)
|
|
52
52
|
|
|
53
53
|
return where_clause, params
|
|
54
54
|
|
|
@@ -116,8 +116,7 @@ def _parse_comparison_condition(condition: dict[str, Any]) -> tuple[Composed, li
|
|
|
116
116
|
|
|
117
117
|
def _treat_meta_field(field: str, value: Any) -> Composed:
|
|
118
118
|
"""
|
|
119
|
-
Internal method that returns a psycopg Composed object
|
|
120
|
-
to make the meta JSONB field queryable safely.
|
|
119
|
+
Internal method that returns a psycopg Composed object to make the meta JSONB field queryable safely.
|
|
121
120
|
|
|
122
121
|
Uses psycopg.sql.Literal to embed the field name, preventing SQL injection
|
|
123
122
|
via metadata field names without requiring regex validation.
|
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
from unittest.mock import patch
|
|
1
|
+
from unittest.mock import Mock, patch
|
|
2
2
|
|
|
3
3
|
import pytest
|
|
4
|
+
from psycopg import AsyncConnection, Connection
|
|
4
5
|
|
|
5
6
|
from haystack_integrations.document_stores.pgvector import PgvectorDocumentStore
|
|
6
7
|
|
|
@@ -110,3 +111,15 @@ def mock_store(patches_for_unit_tests, monkeypatch): # noqa: ARG001 patches ar
|
|
|
110
111
|
)
|
|
111
112
|
|
|
112
113
|
yield store
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
@pytest.fixture
|
|
117
|
+
def mock_store_with_mock_connection(mock_store):
|
|
118
|
+
mock_store._connection = Mock(spec=Connection)
|
|
119
|
+
return mock_store
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
@pytest.fixture
|
|
123
|
+
def mock_store_with_mock_async_connection(mock_store):
|
|
124
|
+
mock_store._async_connection = Mock(spec=AsyncConnection)
|
|
125
|
+
return mock_store
|