pgvector-haystack 5.3.0__tar.gz → 5.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pgvector_haystack-5.3.0 → pgvector_haystack-5.4.0}/CHANGELOG.md +11 -0
- {pgvector_haystack-5.3.0 → pgvector_haystack-5.4.0}/PKG-INFO +1 -1
- pgvector_haystack-5.4.0/pydoc/config_docusaurus.yml +30 -0
- {pgvector_haystack-5.3.0 → pgvector_haystack-5.4.0}/pyproject.toml +3 -3
- {pgvector_haystack-5.3.0 → pgvector_haystack-5.4.0}/src/haystack_integrations/components/retrievers/pgvector/embedding_retriever.py +12 -12
- {pgvector_haystack-5.3.0 → pgvector_haystack-5.4.0}/src/haystack_integrations/components/retrievers/pgvector/keyword_retriever.py +10 -10
- {pgvector_haystack-5.3.0 → pgvector_haystack-5.4.0}/src/haystack_integrations/document_stores/pgvector/converters.py +3 -3
- {pgvector_haystack-5.3.0 → pgvector_haystack-5.4.0}/src/haystack_integrations/document_stores/pgvector/document_store.py +47 -30
- {pgvector_haystack-5.3.0 → pgvector_haystack-5.4.0}/src/haystack_integrations/document_stores/pgvector/filters.py +16 -16
- {pgvector_haystack-5.3.0 → pgvector_haystack-5.4.0}/tests/conftest.py +9 -5
- {pgvector_haystack-5.3.0 → pgvector_haystack-5.4.0}/tests/test_document_store.py +8 -1
- {pgvector_haystack-5.3.0 → pgvector_haystack-5.4.0}/tests/test_document_store_async.py +8 -1
- {pgvector_haystack-5.3.0 → pgvector_haystack-5.4.0}/tests/test_filters.py +1 -3
- {pgvector_haystack-5.3.0 → pgvector_haystack-5.4.0}/tests/test_retrieval.py +1 -3
- {pgvector_haystack-5.3.0 → pgvector_haystack-5.4.0}/tests/test_retrieval_async.py +1 -2
- pgvector_haystack-5.3.0/pydoc/config.yml +0 -31
- {pgvector_haystack-5.3.0 → pgvector_haystack-5.4.0}/.gitignore +0 -0
- {pgvector_haystack-5.3.0 → pgvector_haystack-5.4.0}/LICENSE.txt +0 -0
- {pgvector_haystack-5.3.0 → pgvector_haystack-5.4.0}/README.md +0 -0
- {pgvector_haystack-5.3.0 → pgvector_haystack-5.4.0}/examples/embedding_retrieval.py +0 -0
- {pgvector_haystack-5.3.0 → pgvector_haystack-5.4.0}/examples/hybrid_retrieval.py +0 -0
- {pgvector_haystack-5.3.0 → pgvector_haystack-5.4.0}/src/haystack_integrations/components/retrievers/pgvector/__init__.py +0 -0
- {pgvector_haystack-5.3.0 → pgvector_haystack-5.4.0}/src/haystack_integrations/components/retrievers/py.typed +0 -0
- {pgvector_haystack-5.3.0 → pgvector_haystack-5.4.0}/src/haystack_integrations/document_stores/pgvector/__init__.py +0 -0
- {pgvector_haystack-5.3.0 → pgvector_haystack-5.4.0}/src/haystack_integrations/document_stores/py.typed +0 -0
- {pgvector_haystack-5.3.0 → pgvector_haystack-5.4.0}/tests/__init__.py +0 -0
- {pgvector_haystack-5.3.0 → pgvector_haystack-5.4.0}/tests/test_converters.py +0 -0
- {pgvector_haystack-5.3.0 → pgvector_haystack-5.4.0}/tests/test_retrievers.py +0 -0
|
@@ -1,5 +1,16 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [integrations/pgvector-v5.3.0] - 2025-10-17
|
|
4
|
+
|
|
5
|
+
### 🚀 Features
|
|
6
|
+
|
|
7
|
+
- `PgvectorDocumentStore` now supports `delete_all_documents` and `delete_all_documents_async` (#2394)
|
|
8
|
+
|
|
9
|
+
### 🧹 Chores
|
|
10
|
+
|
|
11
|
+
- Standardize readmes - part 2 (#2205)
|
|
12
|
+
|
|
13
|
+
|
|
3
14
|
## [integrations/pgvector-v5.2.1] - 2025-07-03
|
|
4
15
|
|
|
5
16
|
### 🐛 Bug Fixes
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: pgvector-haystack
|
|
3
|
-
Version: 5.
|
|
3
|
+
Version: 5.4.0
|
|
4
4
|
Summary: An integration of pgvector (vector search extension for Postgres) with Haystack
|
|
5
5
|
Project-URL: Source, https://github.com/deepset-ai/haystack-core-integrations
|
|
6
6
|
Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/blob/main/integrations/pgvector/README.md
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
loaders:
|
|
2
|
+
- ignore_when_discovered:
|
|
3
|
+
- __init__
|
|
4
|
+
modules:
|
|
5
|
+
- haystack_integrations.components.retrievers.pgvector.embedding_retriever
|
|
6
|
+
- haystack_integrations.components.retrievers.pgvector.keyword_retriever
|
|
7
|
+
- haystack_integrations.document_stores.pgvector.document_store
|
|
8
|
+
search_path:
|
|
9
|
+
- ../src
|
|
10
|
+
type: haystack_pydoc_tools.loaders.CustomPythonLoader
|
|
11
|
+
processors:
|
|
12
|
+
- do_not_filter_modules: false
|
|
13
|
+
documented_only: true
|
|
14
|
+
expression: null
|
|
15
|
+
skip_empty_modules: true
|
|
16
|
+
type: filter
|
|
17
|
+
- type: smart
|
|
18
|
+
- type: crossref
|
|
19
|
+
renderer:
|
|
20
|
+
description: Pgvector integration for Haystack
|
|
21
|
+
id: integrations-pgvector
|
|
22
|
+
markdown:
|
|
23
|
+
add_member_class_prefix: false
|
|
24
|
+
add_method_class_prefix: true
|
|
25
|
+
classdef_code_block: false
|
|
26
|
+
descriptive_class_title: false
|
|
27
|
+
descriptive_module_title: true
|
|
28
|
+
filename: pgvector.md
|
|
29
|
+
title: Pgvector
|
|
30
|
+
type: haystack_pydoc_tools.renderers.DocusaurusRenderer
|
|
@@ -46,7 +46,7 @@ installer = "uv"
|
|
|
46
46
|
dependencies = ["haystack-pydoc-tools", "ruff"]
|
|
47
47
|
|
|
48
48
|
[tool.hatch.envs.default.scripts]
|
|
49
|
-
docs = ["pydoc-markdown pydoc/
|
|
49
|
+
docs = ["pydoc-markdown pydoc/config_docusaurus.yml"]
|
|
50
50
|
fmt = "ruff check --fix {args} && ruff format {args}"
|
|
51
51
|
fmt-check = "ruff check {args} && ruff format --check {args}"
|
|
52
52
|
|
|
@@ -64,7 +64,7 @@ dependencies = [
|
|
|
64
64
|
unit = 'pytest -m "not integration" {args:tests}'
|
|
65
65
|
integration = 'pytest -m "integration" {args:tests}'
|
|
66
66
|
all = 'pytest {args:tests}'
|
|
67
|
-
cov-retry = '
|
|
67
|
+
cov-retry = 'pytest --cov=haystack_integrations --reruns 3 --reruns-delay 30 -x {args:tests}'
|
|
68
68
|
|
|
69
69
|
types = "mypy -p haystack_integrations.document_stores.pgvector -p haystack_integrations.components.retrievers.pgvector {args}"
|
|
70
70
|
|
|
@@ -83,7 +83,7 @@ ignore_missing_imports = true
|
|
|
83
83
|
|
|
84
84
|
|
|
85
85
|
[tool.ruff]
|
|
86
|
-
target-version = "
|
|
86
|
+
target-version = "py39"
|
|
87
87
|
line-length = 120
|
|
88
88
|
|
|
89
89
|
[tool.ruff.lint]
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# SPDX-FileCopyrightText: 2023-present deepset GmbH <info@deepset.ai>
|
|
2
2
|
#
|
|
3
3
|
# SPDX-License-Identifier: Apache-2.0
|
|
4
|
-
from typing import Any,
|
|
4
|
+
from typing import Any, Literal, Optional, Union
|
|
5
5
|
|
|
6
6
|
from haystack import component, default_from_dict, default_to_dict
|
|
7
7
|
from haystack.dataclasses import Document
|
|
@@ -62,7 +62,7 @@ class PgvectorEmbeddingRetriever:
|
|
|
62
62
|
self,
|
|
63
63
|
*,
|
|
64
64
|
document_store: PgvectorDocumentStore,
|
|
65
|
-
filters: Optional[
|
|
65
|
+
filters: Optional[dict[str, Any]] = None,
|
|
66
66
|
top_k: int = 10,
|
|
67
67
|
vector_function: Optional[Literal["cosine_similarity", "inner_product", "l2_distance"]] = None,
|
|
68
68
|
filter_policy: Union[str, FilterPolicy] = FilterPolicy.REPLACE,
|
|
@@ -99,7 +99,7 @@ class PgvectorEmbeddingRetriever:
|
|
|
99
99
|
filter_policy if isinstance(filter_policy, FilterPolicy) else FilterPolicy.from_str(filter_policy)
|
|
100
100
|
)
|
|
101
101
|
|
|
102
|
-
def to_dict(self) ->
|
|
102
|
+
def to_dict(self) -> dict[str, Any]:
|
|
103
103
|
"""
|
|
104
104
|
Serializes the component to a dictionary.
|
|
105
105
|
|
|
@@ -116,7 +116,7 @@ class PgvectorEmbeddingRetriever:
|
|
|
116
116
|
)
|
|
117
117
|
|
|
118
118
|
@classmethod
|
|
119
|
-
def from_dict(cls, data:
|
|
119
|
+
def from_dict(cls, data: dict[str, Any]) -> "PgvectorEmbeddingRetriever":
|
|
120
120
|
"""
|
|
121
121
|
Deserializes the component from a dictionary.
|
|
122
122
|
|
|
@@ -133,14 +133,14 @@ class PgvectorEmbeddingRetriever:
|
|
|
133
133
|
data["init_parameters"]["filter_policy"] = FilterPolicy.from_str(filter_policy)
|
|
134
134
|
return default_from_dict(cls, data)
|
|
135
135
|
|
|
136
|
-
@component.output_types(documents=
|
|
136
|
+
@component.output_types(documents=list[Document])
|
|
137
137
|
def run(
|
|
138
138
|
self,
|
|
139
|
-
query_embedding:
|
|
140
|
-
filters: Optional[
|
|
139
|
+
query_embedding: list[float],
|
|
140
|
+
filters: Optional[dict[str, Any]] = None,
|
|
141
141
|
top_k: Optional[int] = None,
|
|
142
142
|
vector_function: Optional[Literal["cosine_similarity", "inner_product", "l2_distance"]] = None,
|
|
143
|
-
) ->
|
|
143
|
+
) -> dict[str, list[Document]]:
|
|
144
144
|
"""
|
|
145
145
|
Retrieve documents from the `PgvectorDocumentStore`, based on their embeddings.
|
|
146
146
|
|
|
@@ -166,14 +166,14 @@ class PgvectorEmbeddingRetriever:
|
|
|
166
166
|
)
|
|
167
167
|
return {"documents": docs}
|
|
168
168
|
|
|
169
|
-
@component.output_types(documents=
|
|
169
|
+
@component.output_types(documents=list[Document])
|
|
170
170
|
async def run_async(
|
|
171
171
|
self,
|
|
172
|
-
query_embedding:
|
|
173
|
-
filters: Optional[
|
|
172
|
+
query_embedding: list[float],
|
|
173
|
+
filters: Optional[dict[str, Any]] = None,
|
|
174
174
|
top_k: Optional[int] = None,
|
|
175
175
|
vector_function: Optional[Literal["cosine_similarity", "inner_product", "l2_distance"]] = None,
|
|
176
|
-
) ->
|
|
176
|
+
) -> dict[str, list[Document]]:
|
|
177
177
|
"""
|
|
178
178
|
Asynchronously retrieve documents from the `PgvectorDocumentStore`, based on their embeddings.
|
|
179
179
|
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# SPDX-FileCopyrightText: 2023-present deepset GmbH <info@deepset.ai>
|
|
2
2
|
#
|
|
3
3
|
# SPDX-License-Identifier: Apache-2.0
|
|
4
|
-
from typing import Any,
|
|
4
|
+
from typing import Any, Optional, Union
|
|
5
5
|
|
|
6
6
|
from haystack import component, default_from_dict, default_to_dict
|
|
7
7
|
from haystack.dataclasses import Document
|
|
@@ -52,7 +52,7 @@ class PgvectorKeywordRetriever:
|
|
|
52
52
|
self,
|
|
53
53
|
*,
|
|
54
54
|
document_store: PgvectorDocumentStore,
|
|
55
|
-
filters: Optional[
|
|
55
|
+
filters: Optional[dict[str, Any]] = None,
|
|
56
56
|
top_k: int = 10,
|
|
57
57
|
filter_policy: Union[str, FilterPolicy] = FilterPolicy.REPLACE,
|
|
58
58
|
):
|
|
@@ -74,7 +74,7 @@ class PgvectorKeywordRetriever:
|
|
|
74
74
|
filter_policy if isinstance(filter_policy, FilterPolicy) else FilterPolicy.from_str(filter_policy)
|
|
75
75
|
)
|
|
76
76
|
|
|
77
|
-
def to_dict(self) ->
|
|
77
|
+
def to_dict(self) -> dict[str, Any]:
|
|
78
78
|
"""
|
|
79
79
|
Serializes the component to a dictionary.
|
|
80
80
|
|
|
@@ -90,7 +90,7 @@ class PgvectorKeywordRetriever:
|
|
|
90
90
|
)
|
|
91
91
|
|
|
92
92
|
@classmethod
|
|
93
|
-
def from_dict(cls, data:
|
|
93
|
+
def from_dict(cls, data: dict[str, Any]) -> "PgvectorKeywordRetriever":
|
|
94
94
|
"""
|
|
95
95
|
Deserializes the component from a dictionary.
|
|
96
96
|
|
|
@@ -107,13 +107,13 @@ class PgvectorKeywordRetriever:
|
|
|
107
107
|
data["init_parameters"]["filter_policy"] = FilterPolicy.from_str(filter_policy)
|
|
108
108
|
return default_from_dict(cls, data)
|
|
109
109
|
|
|
110
|
-
@component.output_types(documents=
|
|
110
|
+
@component.output_types(documents=list[Document])
|
|
111
111
|
def run(
|
|
112
112
|
self,
|
|
113
113
|
query: str,
|
|
114
|
-
filters: Optional[
|
|
114
|
+
filters: Optional[dict[str, Any]] = None,
|
|
115
115
|
top_k: Optional[int] = None,
|
|
116
|
-
) ->
|
|
116
|
+
) -> dict[str, list[Document]]:
|
|
117
117
|
"""
|
|
118
118
|
Retrieve documents from the `PgvectorDocumentStore`, based on keywords.
|
|
119
119
|
|
|
@@ -137,13 +137,13 @@ class PgvectorKeywordRetriever:
|
|
|
137
137
|
)
|
|
138
138
|
return {"documents": docs}
|
|
139
139
|
|
|
140
|
-
@component.output_types(documents=
|
|
140
|
+
@component.output_types(documents=list[Document])
|
|
141
141
|
async def run_async(
|
|
142
142
|
self,
|
|
143
143
|
query: str,
|
|
144
|
-
filters: Optional[
|
|
144
|
+
filters: Optional[dict[str, Any]] = None,
|
|
145
145
|
top_k: Optional[int] = None,
|
|
146
|
-
) ->
|
|
146
|
+
) -> dict[str, list[Document]]:
|
|
147
147
|
"""
|
|
148
148
|
Asynchronously retrieve documents from the `PgvectorDocumentStore`, based on keywords.
|
|
149
149
|
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
from typing import Any
|
|
1
|
+
from typing import Any
|
|
2
2
|
|
|
3
3
|
from haystack import logging
|
|
4
4
|
from haystack.dataclasses import ByteStream, Document
|
|
@@ -7,7 +7,7 @@ from psycopg.types.json import Jsonb
|
|
|
7
7
|
logger = logging.getLogger(__name__)
|
|
8
8
|
|
|
9
9
|
|
|
10
|
-
def _from_haystack_to_pg_documents(documents:
|
|
10
|
+
def _from_haystack_to_pg_documents(documents: list[Document]) -> list[dict[str, Any]]:
|
|
11
11
|
"""
|
|
12
12
|
Internal method to convert a list of Haystack Documents to a list of dictionaries that can be used to insert
|
|
13
13
|
documents into the PgvectorDocumentStore.
|
|
@@ -38,7 +38,7 @@ def _from_haystack_to_pg_documents(documents: List[Document]) -> List[Dict[str,
|
|
|
38
38
|
return db_documents
|
|
39
39
|
|
|
40
40
|
|
|
41
|
-
def _from_pg_to_haystack_documents(documents:
|
|
41
|
+
def _from_pg_to_haystack_documents(documents: list[dict[str, Any]]) -> list[Document]:
|
|
42
42
|
"""
|
|
43
43
|
Internal method to convert a list of dictionaries from pgvector to a list of Haystack Documents.
|
|
44
44
|
"""
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# SPDX-FileCopyrightText: 2023-present deepset GmbH <info@deepset.ai>
|
|
2
2
|
#
|
|
3
3
|
# SPDX-License-Identifier: Apache-2.0
|
|
4
|
-
from typing import Any,
|
|
4
|
+
from typing import Any, Literal, Optional, Union, overload
|
|
5
5
|
|
|
6
6
|
from haystack import default_from_dict, default_to_dict, logging
|
|
7
7
|
from haystack.dataclasses.document import Document
|
|
@@ -90,7 +90,7 @@ class PgvectorDocumentStore:
|
|
|
90
90
|
recreate_table: bool = False,
|
|
91
91
|
search_strategy: Literal["exact_nearest_neighbor", "hnsw"] = "exact_nearest_neighbor",
|
|
92
92
|
hnsw_recreate_index_if_exists: bool = False,
|
|
93
|
-
hnsw_index_creation_kwargs: Optional[
|
|
93
|
+
hnsw_index_creation_kwargs: Optional[dict[str, int]] = None,
|
|
94
94
|
hnsw_index_name: str = "haystack_hnsw_index",
|
|
95
95
|
hnsw_ef_search: Optional[int] = None,
|
|
96
96
|
keyword_index_name: str = "haystack_keyword_index",
|
|
@@ -101,9 +101,10 @@ class PgvectorDocumentStore:
|
|
|
101
101
|
A specific table to store Haystack documents will be created if it doesn't exist yet.
|
|
102
102
|
|
|
103
103
|
:param connection_string: The connection string to use to connect to the PostgreSQL database, defined as an
|
|
104
|
-
environment variable.
|
|
105
|
-
e.g
|
|
106
|
-
|
|
104
|
+
environment variable. Supported formats:
|
|
105
|
+
- URI, e.g. `PG_CONN_STR="postgresql://USER:PASSWORD@HOST:PORT/DB_NAME"` (use percent-encoding for special
|
|
106
|
+
characters)
|
|
107
|
+
- keyword/value format, e.g. `PG_CONN_STR="host=HOST port=PORT dbname=DBNAME user=USER password=PASSWORD"`
|
|
107
108
|
See [PostgreSQL Documentation](https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING)
|
|
108
109
|
for more details.
|
|
109
110
|
:param create_extension: Whether to create the pgvector extension if it doesn't exist.
|
|
@@ -180,7 +181,7 @@ class PgvectorDocumentStore:
|
|
|
180
181
|
self._async_dict_cursor: Optional[AsyncCursor[DictRow]] = None
|
|
181
182
|
self._table_initialized = False
|
|
182
183
|
|
|
183
|
-
def to_dict(self) ->
|
|
184
|
+
def to_dict(self) -> dict[str, Any]:
|
|
184
185
|
"""
|
|
185
186
|
Serializes the component to a dictionary.
|
|
186
187
|
|
|
@@ -207,7 +208,7 @@ class PgvectorDocumentStore:
|
|
|
207
208
|
)
|
|
208
209
|
|
|
209
210
|
@classmethod
|
|
210
|
-
def from_dict(cls, data:
|
|
211
|
+
def from_dict(cls, data: dict[str, Any]) -> "PgvectorDocumentStore":
|
|
211
212
|
"""
|
|
212
213
|
Deserializes the component from a dictionary.
|
|
213
214
|
|
|
@@ -360,7 +361,15 @@ class PgvectorDocumentStore:
|
|
|
360
361
|
logger.debug("Failed to close connection: {e}", e=str(e))
|
|
361
362
|
|
|
362
363
|
conn_str = self.connection_string.resolve_value() or ""
|
|
363
|
-
|
|
364
|
+
try:
|
|
365
|
+
connection = Connection.connect(conn_str)
|
|
366
|
+
except Error as e:
|
|
367
|
+
msg = (
|
|
368
|
+
"Failed to connect to PostgreSQL database. Ensure the connection string follows the "
|
|
369
|
+
"PostgreSQL connection specification: "
|
|
370
|
+
"https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING."
|
|
371
|
+
)
|
|
372
|
+
raise DocumentStoreError(msg) from e
|
|
364
373
|
connection.autocommit = True
|
|
365
374
|
if self.create_extension:
|
|
366
375
|
connection.execute("CREATE EXTENSION IF NOT EXISTS vector")
|
|
@@ -394,7 +403,15 @@ class PgvectorDocumentStore:
|
|
|
394
403
|
await self._async_connection.close()
|
|
395
404
|
|
|
396
405
|
conn_str = self.connection_string.resolve_value() or ""
|
|
397
|
-
|
|
406
|
+
try:
|
|
407
|
+
async_connection = await AsyncConnection.connect(conn_str)
|
|
408
|
+
except Error as e:
|
|
409
|
+
msg = (
|
|
410
|
+
"Failed to connect to PostgreSQL database. Ensure the connection string follows the "
|
|
411
|
+
"PostgreSQL connection specification: "
|
|
412
|
+
"https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING."
|
|
413
|
+
)
|
|
414
|
+
raise DocumentStoreError(msg) from e
|
|
398
415
|
await async_connection.set_autocommit(True)
|
|
399
416
|
if self.create_extension:
|
|
400
417
|
await async_connection.execute("CREATE EXTENSION IF NOT EXISTS vector")
|
|
@@ -740,7 +757,7 @@ class PgvectorDocumentStore:
|
|
|
740
757
|
return result[0]
|
|
741
758
|
return 0
|
|
742
759
|
|
|
743
|
-
def filter_documents(self, filters: Optional[
|
|
760
|
+
def filter_documents(self, filters: Optional[dict[str, Any]] = None) -> list[Document]:
|
|
744
761
|
"""
|
|
745
762
|
Returns the documents that match the filters provided.
|
|
746
763
|
|
|
@@ -777,7 +794,7 @@ class PgvectorDocumentStore:
|
|
|
777
794
|
docs = _from_pg_to_haystack_documents(records)
|
|
778
795
|
return docs
|
|
779
796
|
|
|
780
|
-
async def filter_documents_async(self, filters: Optional[
|
|
797
|
+
async def filter_documents_async(self, filters: Optional[dict[str, Any]] = None) -> list[Document]:
|
|
781
798
|
"""
|
|
782
799
|
Asynchronously returns the documents that match the filters provided.
|
|
783
800
|
|
|
@@ -832,7 +849,7 @@ class PgvectorDocumentStore:
|
|
|
832
849
|
|
|
833
850
|
return sql_insert
|
|
834
851
|
|
|
835
|
-
def write_documents(self, documents:
|
|
852
|
+
def write_documents(self, documents: list[Document], policy: DuplicatePolicy = DuplicatePolicy.NONE) -> int:
|
|
836
853
|
"""
|
|
837
854
|
Writes documents to the document store.
|
|
838
855
|
|
|
@@ -888,7 +905,7 @@ class PgvectorDocumentStore:
|
|
|
888
905
|
return written_docs
|
|
889
906
|
|
|
890
907
|
async def write_documents_async(
|
|
891
|
-
self, documents:
|
|
908
|
+
self, documents: list[Document], policy: DuplicatePolicy = DuplicatePolicy.NONE
|
|
892
909
|
) -> int:
|
|
893
910
|
"""
|
|
894
911
|
Asynchronously writes documents to the document store.
|
|
@@ -939,7 +956,7 @@ class PgvectorDocumentStore:
|
|
|
939
956
|
|
|
940
957
|
return written_docs
|
|
941
958
|
|
|
942
|
-
def delete_documents(self, document_ids:
|
|
959
|
+
def delete_documents(self, document_ids: list[str]) -> None:
|
|
943
960
|
"""
|
|
944
961
|
Deletes documents that match the provided `document_ids` from the document store.
|
|
945
962
|
|
|
@@ -962,7 +979,7 @@ class PgvectorDocumentStore:
|
|
|
962
979
|
cursor=self._cursor, sql_query=delete_sql, error_msg="Could not delete documents from PgvectorDocumentStore"
|
|
963
980
|
)
|
|
964
981
|
|
|
965
|
-
async def delete_documents_async(self, document_ids:
|
|
982
|
+
async def delete_documents_async(self, document_ids: list[str]) -> None:
|
|
966
983
|
"""
|
|
967
984
|
Asynchronously deletes documents that match the provided `document_ids` from the document store.
|
|
968
985
|
|
|
@@ -1020,8 +1037,8 @@ class PgvectorDocumentStore:
|
|
|
1020
1037
|
)
|
|
1021
1038
|
|
|
1022
1039
|
def _build_keyword_retrieval_query(
|
|
1023
|
-
self, query: str, top_k: int, filters: Optional[
|
|
1024
|
-
) ->
|
|
1040
|
+
self, query: str, top_k: int, filters: Optional[dict[str, Any]] = None
|
|
1041
|
+
) -> tuple[Composed, tuple]:
|
|
1025
1042
|
"""
|
|
1026
1043
|
Builds the SQL query and the where parameters for keyword retrieval.
|
|
1027
1044
|
"""
|
|
@@ -1049,9 +1066,9 @@ class PgvectorDocumentStore:
|
|
|
1049
1066
|
self,
|
|
1050
1067
|
query: str,
|
|
1051
1068
|
*,
|
|
1052
|
-
filters: Optional[
|
|
1069
|
+
filters: Optional[dict[str, Any]] = None,
|
|
1053
1070
|
top_k: int = 10,
|
|
1054
|
-
) ->
|
|
1071
|
+
) -> list[Document]:
|
|
1055
1072
|
"""
|
|
1056
1073
|
Retrieves documents that are most similar to the query using a full-text search.
|
|
1057
1074
|
|
|
@@ -1084,9 +1101,9 @@ class PgvectorDocumentStore:
|
|
|
1084
1101
|
self,
|
|
1085
1102
|
query: str,
|
|
1086
1103
|
*,
|
|
1087
|
-
filters: Optional[
|
|
1104
|
+
filters: Optional[dict[str, Any]] = None,
|
|
1088
1105
|
top_k: int = 10,
|
|
1089
|
-
) ->
|
|
1106
|
+
) -> list[Document]:
|
|
1090
1107
|
"""
|
|
1091
1108
|
Retrieves documents that are most similar to the query using a full-text search asynchronously.
|
|
1092
1109
|
"""
|
|
@@ -1111,11 +1128,11 @@ class PgvectorDocumentStore:
|
|
|
1111
1128
|
|
|
1112
1129
|
def _check_and_build_embedding_retrieval_query(
|
|
1113
1130
|
self,
|
|
1114
|
-
query_embedding:
|
|
1131
|
+
query_embedding: list[float],
|
|
1115
1132
|
vector_function: Optional[Literal["cosine_similarity", "inner_product", "l2_distance"]],
|
|
1116
1133
|
top_k: int,
|
|
1117
|
-
filters: Optional[
|
|
1118
|
-
) ->
|
|
1134
|
+
filters: Optional[dict[str, Any]] = None,
|
|
1135
|
+
) -> tuple[Composed, tuple]:
|
|
1119
1136
|
"""
|
|
1120
1137
|
Performs checks and builds the SQL query and the where parameters for embedding retrieval.
|
|
1121
1138
|
"""
|
|
@@ -1174,12 +1191,12 @@ class PgvectorDocumentStore:
|
|
|
1174
1191
|
|
|
1175
1192
|
def _embedding_retrieval(
|
|
1176
1193
|
self,
|
|
1177
|
-
query_embedding:
|
|
1194
|
+
query_embedding: list[float],
|
|
1178
1195
|
*,
|
|
1179
|
-
filters: Optional[
|
|
1196
|
+
filters: Optional[dict[str, Any]] = None,
|
|
1180
1197
|
top_k: int = 10,
|
|
1181
1198
|
vector_function: Optional[Literal["cosine_similarity", "inner_product", "l2_distance"]] = None,
|
|
1182
|
-
) ->
|
|
1199
|
+
) -> list[Document]:
|
|
1183
1200
|
"""
|
|
1184
1201
|
Retrieves documents that are most similar to the query embedding using a vector similarity metric.
|
|
1185
1202
|
|
|
@@ -1208,12 +1225,12 @@ class PgvectorDocumentStore:
|
|
|
1208
1225
|
|
|
1209
1226
|
async def _embedding_retrieval_async(
|
|
1210
1227
|
self,
|
|
1211
|
-
query_embedding:
|
|
1228
|
+
query_embedding: list[float],
|
|
1212
1229
|
*,
|
|
1213
|
-
filters: Optional[
|
|
1230
|
+
filters: Optional[dict[str, Any]] = None,
|
|
1214
1231
|
top_k: int = 10,
|
|
1215
1232
|
vector_function: Optional[Literal["cosine_similarity", "inner_product", "l2_distance"]] = None,
|
|
1216
|
-
) ->
|
|
1233
|
+
) -> list[Document]:
|
|
1217
1234
|
"""
|
|
1218
1235
|
Asynchronously retrieves documents that are most similar to the query embedding using a
|
|
1219
1236
|
vector similarity metric.
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
# SPDX-License-Identifier: Apache-2.0
|
|
4
4
|
from datetime import datetime
|
|
5
5
|
from itertools import chain
|
|
6
|
-
from typing import Any,
|
|
6
|
+
from typing import Any, Literal, Optional
|
|
7
7
|
|
|
8
8
|
from haystack.errors import FilterError
|
|
9
9
|
from psycopg.sql import SQL, Composed
|
|
@@ -21,7 +21,7 @@ PYTHON_TYPES_TO_PG_TYPES = {
|
|
|
21
21
|
NO_VALUE = "no_value"
|
|
22
22
|
|
|
23
23
|
|
|
24
|
-
def _validate_filters(filters: Optional[
|
|
24
|
+
def _validate_filters(filters: Optional[dict[str, Any]] = None) -> None:
|
|
25
25
|
"""
|
|
26
26
|
Validates the filters provided.
|
|
27
27
|
"""
|
|
@@ -35,8 +35,8 @@ def _validate_filters(filters: Optional[Dict[str, Any]] = None) -> None:
|
|
|
35
35
|
|
|
36
36
|
|
|
37
37
|
def _convert_filters_to_where_clause_and_params(
|
|
38
|
-
filters:
|
|
39
|
-
) ->
|
|
38
|
+
filters: dict[str, Any], operator: Literal["WHERE", "AND"] = "WHERE"
|
|
39
|
+
) -> tuple[Composed, tuple]:
|
|
40
40
|
"""
|
|
41
41
|
Convert Haystack filters to a WHERE clause and a tuple of params to query PostgreSQL.
|
|
42
42
|
"""
|
|
@@ -51,7 +51,7 @@ def _convert_filters_to_where_clause_and_params(
|
|
|
51
51
|
return where_clause, params
|
|
52
52
|
|
|
53
53
|
|
|
54
|
-
def _parse_logical_condition(condition:
|
|
54
|
+
def _parse_logical_condition(condition: dict[str, Any]) -> tuple[str, list[Any]]:
|
|
55
55
|
if "operator" not in condition:
|
|
56
56
|
msg = f"'operator' key missing in {condition}"
|
|
57
57
|
raise FilterError(msg)
|
|
@@ -91,7 +91,7 @@ def _parse_logical_condition(condition: Dict[str, Any]) -> Tuple[str, List[Any]]
|
|
|
91
91
|
return sql_query, values
|
|
92
92
|
|
|
93
93
|
|
|
94
|
-
def _parse_comparison_condition(condition:
|
|
94
|
+
def _parse_comparison_condition(condition: dict[str, Any]) -> tuple[str, list[Any]]:
|
|
95
95
|
field: str = condition["field"]
|
|
96
96
|
if "operator" not in condition:
|
|
97
97
|
msg = f"'operator' key missing in {condition}"
|
|
@@ -142,20 +142,20 @@ def _treat_meta_field(field: str, value: Any) -> str:
|
|
|
142
142
|
return field
|
|
143
143
|
|
|
144
144
|
|
|
145
|
-
def _equal(field: str, value: Any) ->
|
|
145
|
+
def _equal(field: str, value: Any) -> tuple[str, Any]:
|
|
146
146
|
if value is None:
|
|
147
147
|
# NO_VALUE is a placeholder that will be removed in _convert_filters_to_where_clause_and_params
|
|
148
148
|
return f"{field} IS NULL", NO_VALUE
|
|
149
149
|
return f"{field} = %s", value
|
|
150
150
|
|
|
151
151
|
|
|
152
|
-
def _not_equal(field: str, value: Any) ->
|
|
152
|
+
def _not_equal(field: str, value: Any) -> tuple[str, Any]:
|
|
153
153
|
# we use IS DISTINCT FROM to correctly handle NULL values
|
|
154
154
|
# (not handled by !=)
|
|
155
155
|
return f"{field} IS DISTINCT FROM %s", value
|
|
156
156
|
|
|
157
157
|
|
|
158
|
-
def _greater_than(field: str, value: Any) ->
|
|
158
|
+
def _greater_than(field: str, value: Any) -> tuple[str, Any]:
|
|
159
159
|
if isinstance(value, str):
|
|
160
160
|
try:
|
|
161
161
|
datetime.fromisoformat(value)
|
|
@@ -172,7 +172,7 @@ def _greater_than(field: str, value: Any) -> Tuple[str, Any]:
|
|
|
172
172
|
return f"{field} > %s", value
|
|
173
173
|
|
|
174
174
|
|
|
175
|
-
def _greater_than_equal(field: str, value: Any) ->
|
|
175
|
+
def _greater_than_equal(field: str, value: Any) -> tuple[str, Any]:
|
|
176
176
|
if isinstance(value, str):
|
|
177
177
|
try:
|
|
178
178
|
datetime.fromisoformat(value)
|
|
@@ -189,7 +189,7 @@ def _greater_than_equal(field: str, value: Any) -> Tuple[str, Any]:
|
|
|
189
189
|
return f"{field} >= %s", value
|
|
190
190
|
|
|
191
191
|
|
|
192
|
-
def _less_than(field: str, value: Any) ->
|
|
192
|
+
def _less_than(field: str, value: Any) -> tuple[str, Any]:
|
|
193
193
|
if isinstance(value, str):
|
|
194
194
|
try:
|
|
195
195
|
datetime.fromisoformat(value)
|
|
@@ -206,7 +206,7 @@ def _less_than(field: str, value: Any) -> Tuple[str, Any]:
|
|
|
206
206
|
return f"{field} < %s", value
|
|
207
207
|
|
|
208
208
|
|
|
209
|
-
def _less_than_equal(field: str, value: Any) ->
|
|
209
|
+
def _less_than_equal(field: str, value: Any) -> tuple[str, Any]:
|
|
210
210
|
if isinstance(value, str):
|
|
211
211
|
try:
|
|
212
212
|
datetime.fromisoformat(value)
|
|
@@ -223,7 +223,7 @@ def _less_than_equal(field: str, value: Any) -> Tuple[str, Any]:
|
|
|
223
223
|
return f"{field} <= %s", value
|
|
224
224
|
|
|
225
225
|
|
|
226
|
-
def _not_in(field: str, value: Any) ->
|
|
226
|
+
def _not_in(field: str, value: Any) -> tuple[str, list]:
|
|
227
227
|
if not isinstance(value, list):
|
|
228
228
|
msg = f"{field}'s value must be a list when using 'not in' comparator in Pinecone"
|
|
229
229
|
raise FilterError(msg)
|
|
@@ -231,7 +231,7 @@ def _not_in(field: str, value: Any) -> Tuple[str, List]:
|
|
|
231
231
|
return f"{field} IS NULL OR {field} != ALL(%s)", [value]
|
|
232
232
|
|
|
233
233
|
|
|
234
|
-
def _in(field: str, value: Any) ->
|
|
234
|
+
def _in(field: str, value: Any) -> tuple[str, list]:
|
|
235
235
|
if not isinstance(value, list):
|
|
236
236
|
msg = f"{field}'s value must be a list when using 'in' comparator in Pinecone"
|
|
237
237
|
raise FilterError(msg)
|
|
@@ -240,14 +240,14 @@ def _in(field: str, value: Any) -> Tuple[str, List]:
|
|
|
240
240
|
return f"{field} = ANY(%s)", [value]
|
|
241
241
|
|
|
242
242
|
|
|
243
|
-
def _like(field: str, value: Any) ->
|
|
243
|
+
def _like(field: str, value: Any) -> tuple[str, Any]:
|
|
244
244
|
if not isinstance(value, str):
|
|
245
245
|
msg = f"{field}'s value must be a str when using 'LIKE' "
|
|
246
246
|
raise FilterError(msg)
|
|
247
247
|
return f"{field} LIKE %s", value
|
|
248
248
|
|
|
249
249
|
|
|
250
|
-
def _not_like(field: str, value: Any) ->
|
|
250
|
+
def _not_like(field: str, value: Any) -> tuple[str, Any]:
|
|
251
251
|
if not isinstance(value, str):
|
|
252
252
|
msg = f"{field}'s value must be a str when using 'LIKE' "
|
|
253
253
|
raise FilterError(msg)
|
|
@@ -80,11 +80,15 @@ def document_store_w_halfvec_hnsw_index(request, monkeypatch):
|
|
|
80
80
|
|
|
81
81
|
@pytest.fixture
|
|
82
82
|
def patches_for_unit_tests():
|
|
83
|
-
with
|
|
84
|
-
"haystack_integrations.document_stores.pgvector.document_store.
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
83
|
+
with (
|
|
84
|
+
patch("haystack_integrations.document_stores.pgvector.document_store.register_vector") as mock_register,
|
|
85
|
+
patch(
|
|
86
|
+
"haystack_integrations.document_stores.pgvector.document_store.PgvectorDocumentStore.delete_table"
|
|
87
|
+
) as mock_delete,
|
|
88
|
+
patch(
|
|
89
|
+
"haystack_integrations.document_stores.pgvector.document_store.PgvectorDocumentStore._handle_hnsw"
|
|
90
|
+
) as mock_hnsw,
|
|
91
|
+
):
|
|
88
92
|
yield mock_register, mock_delete, mock_hnsw
|
|
89
93
|
|
|
90
94
|
|
|
@@ -7,7 +7,7 @@ from unittest.mock import patch
|
|
|
7
7
|
import psycopg
|
|
8
8
|
import pytest
|
|
9
9
|
from haystack.dataclasses.document import ByteStream, Document
|
|
10
|
-
from haystack.document_stores.errors import DuplicateDocumentError
|
|
10
|
+
from haystack.document_stores.errors import DocumentStoreError, DuplicateDocumentError
|
|
11
11
|
from haystack.document_stores.types import DuplicatePolicy
|
|
12
12
|
from haystack.testing.document_store import CountDocumentsTest, DeleteDocumentsTest, WriteDocumentsTest
|
|
13
13
|
from haystack.utils import Secret
|
|
@@ -60,6 +60,13 @@ class TestDocumentStore(CountDocumentsTest, WriteDocumentsTest, DeleteDocumentsT
|
|
|
60
60
|
document_store.write_documents([Document(id="1")])
|
|
61
61
|
assert document_store.count_documents() == 1
|
|
62
62
|
|
|
63
|
+
def test_invalid_connection_string(self, monkeypatch):
|
|
64
|
+
monkeypatch.setenv("PG_CONN_STR", "invalid_connection_string")
|
|
65
|
+
document_store = PgvectorDocumentStore()
|
|
66
|
+
with pytest.raises(DocumentStoreError) as e:
|
|
67
|
+
document_store._ensure_db_setup()
|
|
68
|
+
assert "Failed to connect to PostgreSQL database" in str(e)
|
|
69
|
+
|
|
63
70
|
|
|
64
71
|
@pytest.mark.usefixtures("patches_for_unit_tests")
|
|
65
72
|
def test_init(monkeypatch):
|
|
@@ -7,7 +7,7 @@ from unittest.mock import patch
|
|
|
7
7
|
import psycopg
|
|
8
8
|
import pytest
|
|
9
9
|
from haystack.dataclasses.document import ByteStream, Document
|
|
10
|
-
from haystack.document_stores.errors import DuplicateDocumentError
|
|
10
|
+
from haystack.document_stores.errors import DocumentStoreError, DuplicateDocumentError
|
|
11
11
|
from haystack.document_stores.types import DuplicatePolicy
|
|
12
12
|
from haystack.utils import Secret
|
|
13
13
|
from psycopg.sql import SQL
|
|
@@ -101,6 +101,13 @@ class TestDocumentStoreAsync:
|
|
|
101
101
|
document_store.write_documents([Document(id="1")])
|
|
102
102
|
assert document_store.count_documents() == 1
|
|
103
103
|
|
|
104
|
+
async def test_invalid_connection_string(self, monkeypatch):
|
|
105
|
+
monkeypatch.setenv("PG_CONN_STR", "invalid_connection_string")
|
|
106
|
+
document_store = PgvectorDocumentStore()
|
|
107
|
+
with pytest.raises(DocumentStoreError) as e:
|
|
108
|
+
await document_store._ensure_db_setup_async()
|
|
109
|
+
assert "Failed to connect to PostgreSQL database" in str(e)
|
|
110
|
+
|
|
104
111
|
|
|
105
112
|
@pytest.mark.integration
|
|
106
113
|
@pytest.mark.asyncio
|
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
from typing import List
|
|
2
|
-
|
|
3
1
|
import pytest
|
|
4
2
|
from haystack.dataclasses.document import Document
|
|
5
3
|
from haystack.testing.document_store import FilterDocumentsTest
|
|
@@ -17,7 +15,7 @@ from haystack_integrations.document_stores.pgvector.filters import (
|
|
|
17
15
|
|
|
18
16
|
@pytest.mark.integration
|
|
19
17
|
class TestFilters(FilterDocumentsTest):
|
|
20
|
-
def assert_documents_are_equal(self, received:
|
|
18
|
+
def assert_documents_are_equal(self, received: list[Document], expected: list[Document]):
|
|
21
19
|
"""
|
|
22
20
|
This overrides the default assert_documents_are_equal from FilterDocumentsTest.
|
|
23
21
|
It is needed because the embeddings are not exactly the same when they are retrieved from Postgres.
|
|
@@ -2,8 +2,6 @@
|
|
|
2
2
|
#
|
|
3
3
|
# SPDX-License-Identifier: Apache-2.0
|
|
4
4
|
|
|
5
|
-
from typing import List
|
|
6
|
-
|
|
7
5
|
import pytest
|
|
8
6
|
from haystack.dataclasses.document import Document
|
|
9
7
|
from numpy.random import rand
|
|
@@ -101,7 +99,7 @@ class TestEmbeddingRetrieval:
|
|
|
101
99
|
assert results[0].score > results[1].score > results[2].score
|
|
102
100
|
|
|
103
101
|
def test_empty_query_embedding(self, document_store: PgvectorDocumentStore):
|
|
104
|
-
query_embedding:
|
|
102
|
+
query_embedding: list[float] = []
|
|
105
103
|
with pytest.raises(ValueError):
|
|
106
104
|
document_store._embedding_retrieval(query_embedding=query_embedding)
|
|
107
105
|
|
|
@@ -2,7 +2,6 @@
|
|
|
2
2
|
#
|
|
3
3
|
# SPDX-License-Identifier: Apache-2.0
|
|
4
4
|
|
|
5
|
-
from typing import List
|
|
6
5
|
|
|
7
6
|
import pytest
|
|
8
7
|
from haystack.dataclasses.document import Document
|
|
@@ -104,7 +103,7 @@ class TestEmbeddingRetrievalAsync:
|
|
|
104
103
|
assert results[0].score > results[1].score > results[2].score
|
|
105
104
|
|
|
106
105
|
async def test_empty_query_embedding_async(self, document_store: PgvectorDocumentStore):
|
|
107
|
-
query_embedding:
|
|
106
|
+
query_embedding: list[float] = []
|
|
108
107
|
with pytest.raises(ValueError):
|
|
109
108
|
await document_store._embedding_retrieval_async(query_embedding=query_embedding)
|
|
110
109
|
|
|
@@ -1,31 +0,0 @@
|
|
|
1
|
-
loaders:
|
|
2
|
-
- type: haystack_pydoc_tools.loaders.CustomPythonLoader
|
|
3
|
-
search_path: [../src]
|
|
4
|
-
modules: [
|
|
5
|
-
"haystack_integrations.components.retrievers.pgvector.embedding_retriever",
|
|
6
|
-
"haystack_integrations.components.retrievers.pgvector.keyword_retriever",
|
|
7
|
-
"haystack_integrations.document_stores.pgvector.document_store",
|
|
8
|
-
]
|
|
9
|
-
ignore_when_discovered: ["__init__"]
|
|
10
|
-
processors:
|
|
11
|
-
- type: filter
|
|
12
|
-
expression:
|
|
13
|
-
documented_only: true
|
|
14
|
-
do_not_filter_modules: false
|
|
15
|
-
skip_empty_modules: true
|
|
16
|
-
- type: smart
|
|
17
|
-
- type: crossref
|
|
18
|
-
renderer:
|
|
19
|
-
type: haystack_pydoc_tools.renderers.ReadmeIntegrationRenderer
|
|
20
|
-
excerpt: Pgvector integration for Haystack
|
|
21
|
-
category_slug: integrations-api
|
|
22
|
-
title: Pgvector
|
|
23
|
-
slug: integrations-pgvector
|
|
24
|
-
order: 190
|
|
25
|
-
markdown:
|
|
26
|
-
descriptive_class_title: false
|
|
27
|
-
classdef_code_block: false
|
|
28
|
-
descriptive_module_title: true
|
|
29
|
-
add_method_class_prefix: true
|
|
30
|
-
add_member_class_prefix: false
|
|
31
|
-
filename: _readme_pgvector.md
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|