pgvector-haystack 5.3.0__tar.gz → 5.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pgvector_haystack-5.3.0 → pgvector_haystack-5.5.0}/CHANGELOG.md +30 -0
- {pgvector_haystack-5.3.0 → pgvector_haystack-5.5.0}/PKG-INFO +1 -1
- pgvector_haystack-5.5.0/pydoc/config_docusaurus.yml +30 -0
- {pgvector_haystack-5.3.0 → pgvector_haystack-5.5.0}/pyproject.toml +3 -3
- {pgvector_haystack-5.3.0 → pgvector_haystack-5.5.0}/src/haystack_integrations/components/retrievers/pgvector/embedding_retriever.py +12 -12
- {pgvector_haystack-5.3.0 → pgvector_haystack-5.5.0}/src/haystack_integrations/components/retrievers/pgvector/keyword_retriever.py +10 -10
- {pgvector_haystack-5.3.0 → pgvector_haystack-5.5.0}/src/haystack_integrations/document_stores/pgvector/converters.py +3 -3
- {pgvector_haystack-5.3.0 → pgvector_haystack-5.5.0}/src/haystack_integrations/document_stores/pgvector/document_store.py +233 -30
- {pgvector_haystack-5.3.0 → pgvector_haystack-5.5.0}/src/haystack_integrations/document_stores/pgvector/filters.py +16 -16
- {pgvector_haystack-5.3.0 → pgvector_haystack-5.5.0}/tests/conftest.py +9 -5
- {pgvector_haystack-5.3.0 → pgvector_haystack-5.5.0}/tests/test_document_store.py +226 -1
- pgvector_haystack-5.5.0/tests/test_document_store_async.py +450 -0
- {pgvector_haystack-5.3.0 → pgvector_haystack-5.5.0}/tests/test_filters.py +1 -3
- {pgvector_haystack-5.3.0 → pgvector_haystack-5.5.0}/tests/test_retrieval.py +1 -3
- {pgvector_haystack-5.3.0 → pgvector_haystack-5.5.0}/tests/test_retrieval_async.py +1 -2
- pgvector_haystack-5.3.0/pydoc/config.yml +0 -31
- pgvector_haystack-5.3.0/tests/test_document_store_async.py +0 -225
- {pgvector_haystack-5.3.0 → pgvector_haystack-5.5.0}/.gitignore +0 -0
- {pgvector_haystack-5.3.0 → pgvector_haystack-5.5.0}/LICENSE.txt +0 -0
- {pgvector_haystack-5.3.0 → pgvector_haystack-5.5.0}/README.md +0 -0
- {pgvector_haystack-5.3.0 → pgvector_haystack-5.5.0}/examples/embedding_retrieval.py +0 -0
- {pgvector_haystack-5.3.0 → pgvector_haystack-5.5.0}/examples/hybrid_retrieval.py +0 -0
- {pgvector_haystack-5.3.0 → pgvector_haystack-5.5.0}/src/haystack_integrations/components/retrievers/pgvector/__init__.py +0 -0
- {pgvector_haystack-5.3.0 → pgvector_haystack-5.5.0}/src/haystack_integrations/components/retrievers/py.typed +0 -0
- {pgvector_haystack-5.3.0 → pgvector_haystack-5.5.0}/src/haystack_integrations/document_stores/pgvector/__init__.py +0 -0
- {pgvector_haystack-5.3.0 → pgvector_haystack-5.5.0}/src/haystack_integrations/document_stores/py.typed +0 -0
- {pgvector_haystack-5.3.0 → pgvector_haystack-5.5.0}/tests/__init__.py +0 -0
- {pgvector_haystack-5.3.0 → pgvector_haystack-5.5.0}/tests/test_converters.py +0 -0
- {pgvector_haystack-5.3.0 → pgvector_haystack-5.5.0}/tests/test_retrievers.py +0 -0
|
@@ -1,5 +1,35 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [integrations/pgvector-v5.4.0] - 2025-12-05
|
|
4
|
+
|
|
5
|
+
### 📚 Documentation
|
|
6
|
+
|
|
7
|
+
- Add pydoc configurations for Docusaurus (#2411)
|
|
8
|
+
|
|
9
|
+
### ⚙️ CI
|
|
10
|
+
|
|
11
|
+
- Change pytest command (#2475)
|
|
12
|
+
|
|
13
|
+
### 🧹 Chores
|
|
14
|
+
|
|
15
|
+
- Remove Readme API CI workflow and configs (#2573)
|
|
16
|
+
- Pgvector - document expected connection string and raise informative error (#2583)
|
|
17
|
+
|
|
18
|
+
### 🌀 Miscellaneous
|
|
19
|
+
|
|
20
|
+
- Enhancement: Adopt PEP 585 type hinting (part 5) (#2528)
|
|
21
|
+
|
|
22
|
+
## [integrations/pgvector-v5.3.0] - 2025-10-17
|
|
23
|
+
|
|
24
|
+
### 🚀 Features
|
|
25
|
+
|
|
26
|
+
- `PgvectorDocumentStore` now supports `delete_all_documents` and `delete_all_documents_async` (#2394)
|
|
27
|
+
|
|
28
|
+
### 🧹 Chores
|
|
29
|
+
|
|
30
|
+
- Standardize readmes - part 2 (#2205)
|
|
31
|
+
|
|
32
|
+
|
|
3
33
|
## [integrations/pgvector-v5.2.1] - 2025-07-03
|
|
4
34
|
|
|
5
35
|
### 🐛 Bug Fixes
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: pgvector-haystack
|
|
3
|
-
Version: 5.
|
|
3
|
+
Version: 5.5.0
|
|
4
4
|
Summary: An integration of pgvector (vector search extension for Postgres) with Haystack
|
|
5
5
|
Project-URL: Source, https://github.com/deepset-ai/haystack-core-integrations
|
|
6
6
|
Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/blob/main/integrations/pgvector/README.md
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
loaders:
|
|
2
|
+
- ignore_when_discovered:
|
|
3
|
+
- __init__
|
|
4
|
+
modules:
|
|
5
|
+
- haystack_integrations.components.retrievers.pgvector.embedding_retriever
|
|
6
|
+
- haystack_integrations.components.retrievers.pgvector.keyword_retriever
|
|
7
|
+
- haystack_integrations.document_stores.pgvector.document_store
|
|
8
|
+
search_path:
|
|
9
|
+
- ../src
|
|
10
|
+
type: haystack_pydoc_tools.loaders.CustomPythonLoader
|
|
11
|
+
processors:
|
|
12
|
+
- do_not_filter_modules: false
|
|
13
|
+
documented_only: true
|
|
14
|
+
expression: null
|
|
15
|
+
skip_empty_modules: true
|
|
16
|
+
type: filter
|
|
17
|
+
- type: smart
|
|
18
|
+
- type: crossref
|
|
19
|
+
renderer:
|
|
20
|
+
description: Pgvector integration for Haystack
|
|
21
|
+
id: integrations-pgvector
|
|
22
|
+
markdown:
|
|
23
|
+
add_member_class_prefix: false
|
|
24
|
+
add_method_class_prefix: true
|
|
25
|
+
classdef_code_block: false
|
|
26
|
+
descriptive_class_title: false
|
|
27
|
+
descriptive_module_title: true
|
|
28
|
+
filename: pgvector.md
|
|
29
|
+
title: Pgvector
|
|
30
|
+
type: haystack_pydoc_tools.renderers.DocusaurusRenderer
|
|
@@ -46,7 +46,7 @@ installer = "uv"
|
|
|
46
46
|
dependencies = ["haystack-pydoc-tools", "ruff"]
|
|
47
47
|
|
|
48
48
|
[tool.hatch.envs.default.scripts]
|
|
49
|
-
docs = ["pydoc-markdown pydoc/
|
|
49
|
+
docs = ["pydoc-markdown pydoc/config_docusaurus.yml"]
|
|
50
50
|
fmt = "ruff check --fix {args} && ruff format {args}"
|
|
51
51
|
fmt-check = "ruff check {args} && ruff format --check {args}"
|
|
52
52
|
|
|
@@ -64,7 +64,7 @@ dependencies = [
|
|
|
64
64
|
unit = 'pytest -m "not integration" {args:tests}'
|
|
65
65
|
integration = 'pytest -m "integration" {args:tests}'
|
|
66
66
|
all = 'pytest {args:tests}'
|
|
67
|
-
cov-retry = '
|
|
67
|
+
cov-retry = 'pytest --cov=haystack_integrations --reruns 3 --reruns-delay 30 -x {args:tests}'
|
|
68
68
|
|
|
69
69
|
types = "mypy -p haystack_integrations.document_stores.pgvector -p haystack_integrations.components.retrievers.pgvector {args}"
|
|
70
70
|
|
|
@@ -83,7 +83,7 @@ ignore_missing_imports = true
|
|
|
83
83
|
|
|
84
84
|
|
|
85
85
|
[tool.ruff]
|
|
86
|
-
target-version = "
|
|
86
|
+
target-version = "py39"
|
|
87
87
|
line-length = 120
|
|
88
88
|
|
|
89
89
|
[tool.ruff.lint]
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# SPDX-FileCopyrightText: 2023-present deepset GmbH <info@deepset.ai>
|
|
2
2
|
#
|
|
3
3
|
# SPDX-License-Identifier: Apache-2.0
|
|
4
|
-
from typing import Any,
|
|
4
|
+
from typing import Any, Literal, Optional, Union
|
|
5
5
|
|
|
6
6
|
from haystack import component, default_from_dict, default_to_dict
|
|
7
7
|
from haystack.dataclasses import Document
|
|
@@ -62,7 +62,7 @@ class PgvectorEmbeddingRetriever:
|
|
|
62
62
|
self,
|
|
63
63
|
*,
|
|
64
64
|
document_store: PgvectorDocumentStore,
|
|
65
|
-
filters: Optional[
|
|
65
|
+
filters: Optional[dict[str, Any]] = None,
|
|
66
66
|
top_k: int = 10,
|
|
67
67
|
vector_function: Optional[Literal["cosine_similarity", "inner_product", "l2_distance"]] = None,
|
|
68
68
|
filter_policy: Union[str, FilterPolicy] = FilterPolicy.REPLACE,
|
|
@@ -99,7 +99,7 @@ class PgvectorEmbeddingRetriever:
|
|
|
99
99
|
filter_policy if isinstance(filter_policy, FilterPolicy) else FilterPolicy.from_str(filter_policy)
|
|
100
100
|
)
|
|
101
101
|
|
|
102
|
-
def to_dict(self) ->
|
|
102
|
+
def to_dict(self) -> dict[str, Any]:
|
|
103
103
|
"""
|
|
104
104
|
Serializes the component to a dictionary.
|
|
105
105
|
|
|
@@ -116,7 +116,7 @@ class PgvectorEmbeddingRetriever:
|
|
|
116
116
|
)
|
|
117
117
|
|
|
118
118
|
@classmethod
|
|
119
|
-
def from_dict(cls, data:
|
|
119
|
+
def from_dict(cls, data: dict[str, Any]) -> "PgvectorEmbeddingRetriever":
|
|
120
120
|
"""
|
|
121
121
|
Deserializes the component from a dictionary.
|
|
122
122
|
|
|
@@ -133,14 +133,14 @@ class PgvectorEmbeddingRetriever:
|
|
|
133
133
|
data["init_parameters"]["filter_policy"] = FilterPolicy.from_str(filter_policy)
|
|
134
134
|
return default_from_dict(cls, data)
|
|
135
135
|
|
|
136
|
-
@component.output_types(documents=
|
|
136
|
+
@component.output_types(documents=list[Document])
|
|
137
137
|
def run(
|
|
138
138
|
self,
|
|
139
|
-
query_embedding:
|
|
140
|
-
filters: Optional[
|
|
139
|
+
query_embedding: list[float],
|
|
140
|
+
filters: Optional[dict[str, Any]] = None,
|
|
141
141
|
top_k: Optional[int] = None,
|
|
142
142
|
vector_function: Optional[Literal["cosine_similarity", "inner_product", "l2_distance"]] = None,
|
|
143
|
-
) ->
|
|
143
|
+
) -> dict[str, list[Document]]:
|
|
144
144
|
"""
|
|
145
145
|
Retrieve documents from the `PgvectorDocumentStore`, based on their embeddings.
|
|
146
146
|
|
|
@@ -166,14 +166,14 @@ class PgvectorEmbeddingRetriever:
|
|
|
166
166
|
)
|
|
167
167
|
return {"documents": docs}
|
|
168
168
|
|
|
169
|
-
@component.output_types(documents=
|
|
169
|
+
@component.output_types(documents=list[Document])
|
|
170
170
|
async def run_async(
|
|
171
171
|
self,
|
|
172
|
-
query_embedding:
|
|
173
|
-
filters: Optional[
|
|
172
|
+
query_embedding: list[float],
|
|
173
|
+
filters: Optional[dict[str, Any]] = None,
|
|
174
174
|
top_k: Optional[int] = None,
|
|
175
175
|
vector_function: Optional[Literal["cosine_similarity", "inner_product", "l2_distance"]] = None,
|
|
176
|
-
) ->
|
|
176
|
+
) -> dict[str, list[Document]]:
|
|
177
177
|
"""
|
|
178
178
|
Asynchronously retrieve documents from the `PgvectorDocumentStore`, based on their embeddings.
|
|
179
179
|
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# SPDX-FileCopyrightText: 2023-present deepset GmbH <info@deepset.ai>
|
|
2
2
|
#
|
|
3
3
|
# SPDX-License-Identifier: Apache-2.0
|
|
4
|
-
from typing import Any,
|
|
4
|
+
from typing import Any, Optional, Union
|
|
5
5
|
|
|
6
6
|
from haystack import component, default_from_dict, default_to_dict
|
|
7
7
|
from haystack.dataclasses import Document
|
|
@@ -52,7 +52,7 @@ class PgvectorKeywordRetriever:
|
|
|
52
52
|
self,
|
|
53
53
|
*,
|
|
54
54
|
document_store: PgvectorDocumentStore,
|
|
55
|
-
filters: Optional[
|
|
55
|
+
filters: Optional[dict[str, Any]] = None,
|
|
56
56
|
top_k: int = 10,
|
|
57
57
|
filter_policy: Union[str, FilterPolicy] = FilterPolicy.REPLACE,
|
|
58
58
|
):
|
|
@@ -74,7 +74,7 @@ class PgvectorKeywordRetriever:
|
|
|
74
74
|
filter_policy if isinstance(filter_policy, FilterPolicy) else FilterPolicy.from_str(filter_policy)
|
|
75
75
|
)
|
|
76
76
|
|
|
77
|
-
def to_dict(self) ->
|
|
77
|
+
def to_dict(self) -> dict[str, Any]:
|
|
78
78
|
"""
|
|
79
79
|
Serializes the component to a dictionary.
|
|
80
80
|
|
|
@@ -90,7 +90,7 @@ class PgvectorKeywordRetriever:
|
|
|
90
90
|
)
|
|
91
91
|
|
|
92
92
|
@classmethod
|
|
93
|
-
def from_dict(cls, data:
|
|
93
|
+
def from_dict(cls, data: dict[str, Any]) -> "PgvectorKeywordRetriever":
|
|
94
94
|
"""
|
|
95
95
|
Deserializes the component from a dictionary.
|
|
96
96
|
|
|
@@ -107,13 +107,13 @@ class PgvectorKeywordRetriever:
|
|
|
107
107
|
data["init_parameters"]["filter_policy"] = FilterPolicy.from_str(filter_policy)
|
|
108
108
|
return default_from_dict(cls, data)
|
|
109
109
|
|
|
110
|
-
@component.output_types(documents=
|
|
110
|
+
@component.output_types(documents=list[Document])
|
|
111
111
|
def run(
|
|
112
112
|
self,
|
|
113
113
|
query: str,
|
|
114
|
-
filters: Optional[
|
|
114
|
+
filters: Optional[dict[str, Any]] = None,
|
|
115
115
|
top_k: Optional[int] = None,
|
|
116
|
-
) ->
|
|
116
|
+
) -> dict[str, list[Document]]:
|
|
117
117
|
"""
|
|
118
118
|
Retrieve documents from the `PgvectorDocumentStore`, based on keywords.
|
|
119
119
|
|
|
@@ -137,13 +137,13 @@ class PgvectorKeywordRetriever:
|
|
|
137
137
|
)
|
|
138
138
|
return {"documents": docs}
|
|
139
139
|
|
|
140
|
-
@component.output_types(documents=
|
|
140
|
+
@component.output_types(documents=list[Document])
|
|
141
141
|
async def run_async(
|
|
142
142
|
self,
|
|
143
143
|
query: str,
|
|
144
|
-
filters: Optional[
|
|
144
|
+
filters: Optional[dict[str, Any]] = None,
|
|
145
145
|
top_k: Optional[int] = None,
|
|
146
|
-
) ->
|
|
146
|
+
) -> dict[str, list[Document]]:
|
|
147
147
|
"""
|
|
148
148
|
Asynchronously retrieve documents from the `PgvectorDocumentStore`, based on keywords.
|
|
149
149
|
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
from typing import Any
|
|
1
|
+
from typing import Any
|
|
2
2
|
|
|
3
3
|
from haystack import logging
|
|
4
4
|
from haystack.dataclasses import ByteStream, Document
|
|
@@ -7,7 +7,7 @@ from psycopg.types.json import Jsonb
|
|
|
7
7
|
logger = logging.getLogger(__name__)
|
|
8
8
|
|
|
9
9
|
|
|
10
|
-
def _from_haystack_to_pg_documents(documents:
|
|
10
|
+
def _from_haystack_to_pg_documents(documents: list[Document]) -> list[dict[str, Any]]:
|
|
11
11
|
"""
|
|
12
12
|
Internal method to convert a list of Haystack Documents to a list of dictionaries that can be used to insert
|
|
13
13
|
documents into the PgvectorDocumentStore.
|
|
@@ -38,7 +38,7 @@ def _from_haystack_to_pg_documents(documents: List[Document]) -> List[Dict[str,
|
|
|
38
38
|
return db_documents
|
|
39
39
|
|
|
40
40
|
|
|
41
|
-
def _from_pg_to_haystack_documents(documents:
|
|
41
|
+
def _from_pg_to_haystack_documents(documents: list[dict[str, Any]]) -> list[Document]:
|
|
42
42
|
"""
|
|
43
43
|
Internal method to convert a list of dictionaries from pgvector to a list of Haystack Documents.
|
|
44
44
|
"""
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
# SPDX-FileCopyrightText: 2023-present deepset GmbH <info@deepset.ai>
|
|
2
2
|
#
|
|
3
3
|
# SPDX-License-Identifier: Apache-2.0
|
|
4
|
-
|
|
4
|
+
|
|
5
|
+
from typing import Any, Literal, Optional, Union, overload
|
|
5
6
|
|
|
6
7
|
from haystack import default_from_dict, default_to_dict, logging
|
|
7
8
|
from haystack.dataclasses.document import Document
|
|
@@ -13,6 +14,7 @@ from psycopg.cursor_async import AsyncCursor
|
|
|
13
14
|
from psycopg.rows import DictRow, dict_row
|
|
14
15
|
from psycopg.sql import SQL, Composed, Identifier
|
|
15
16
|
from psycopg.sql import Literal as SQLLiteral
|
|
17
|
+
from psycopg.types.json import Jsonb
|
|
16
18
|
|
|
17
19
|
from pgvector.psycopg import register_vector, register_vector_async
|
|
18
20
|
|
|
@@ -90,7 +92,7 @@ class PgvectorDocumentStore:
|
|
|
90
92
|
recreate_table: bool = False,
|
|
91
93
|
search_strategy: Literal["exact_nearest_neighbor", "hnsw"] = "exact_nearest_neighbor",
|
|
92
94
|
hnsw_recreate_index_if_exists: bool = False,
|
|
93
|
-
hnsw_index_creation_kwargs: Optional[
|
|
95
|
+
hnsw_index_creation_kwargs: Optional[dict[str, int]] = None,
|
|
94
96
|
hnsw_index_name: str = "haystack_hnsw_index",
|
|
95
97
|
hnsw_ef_search: Optional[int] = None,
|
|
96
98
|
keyword_index_name: str = "haystack_keyword_index",
|
|
@@ -101,9 +103,10 @@ class PgvectorDocumentStore:
|
|
|
101
103
|
A specific table to store Haystack documents will be created if it doesn't exist yet.
|
|
102
104
|
|
|
103
105
|
:param connection_string: The connection string to use to connect to the PostgreSQL database, defined as an
|
|
104
|
-
environment variable.
|
|
105
|
-
e.g
|
|
106
|
-
|
|
106
|
+
environment variable. Supported formats:
|
|
107
|
+
- URI, e.g. `PG_CONN_STR="postgresql://USER:PASSWORD@HOST:PORT/DB_NAME"` (use percent-encoding for special
|
|
108
|
+
characters)
|
|
109
|
+
- keyword/value format, e.g. `PG_CONN_STR="host=HOST port=PORT dbname=DBNAME user=USER password=PASSWORD"`
|
|
107
110
|
See [PostgreSQL Documentation](https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING)
|
|
108
111
|
for more details.
|
|
109
112
|
:param create_extension: Whether to create the pgvector extension if it doesn't exist.
|
|
@@ -180,7 +183,7 @@ class PgvectorDocumentStore:
|
|
|
180
183
|
self._async_dict_cursor: Optional[AsyncCursor[DictRow]] = None
|
|
181
184
|
self._table_initialized = False
|
|
182
185
|
|
|
183
|
-
def to_dict(self) ->
|
|
186
|
+
def to_dict(self) -> dict[str, Any]:
|
|
184
187
|
"""
|
|
185
188
|
Serializes the component to a dictionary.
|
|
186
189
|
|
|
@@ -207,7 +210,7 @@ class PgvectorDocumentStore:
|
|
|
207
210
|
)
|
|
208
211
|
|
|
209
212
|
@classmethod
|
|
210
|
-
def from_dict(cls, data:
|
|
213
|
+
def from_dict(cls, data: dict[str, Any]) -> "PgvectorDocumentStore":
|
|
211
214
|
"""
|
|
212
215
|
Deserializes the component from a dictionary.
|
|
213
216
|
|
|
@@ -360,7 +363,15 @@ class PgvectorDocumentStore:
|
|
|
360
363
|
logger.debug("Failed to close connection: {e}", e=str(e))
|
|
361
364
|
|
|
362
365
|
conn_str = self.connection_string.resolve_value() or ""
|
|
363
|
-
|
|
366
|
+
try:
|
|
367
|
+
connection = Connection.connect(conn_str)
|
|
368
|
+
except Error as e:
|
|
369
|
+
msg = (
|
|
370
|
+
"Failed to connect to PostgreSQL database. Ensure the connection string follows the "
|
|
371
|
+
"PostgreSQL connection specification: "
|
|
372
|
+
"https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING."
|
|
373
|
+
)
|
|
374
|
+
raise DocumentStoreError(msg) from e
|
|
364
375
|
connection.autocommit = True
|
|
365
376
|
if self.create_extension:
|
|
366
377
|
connection.execute("CREATE EXTENSION IF NOT EXISTS vector")
|
|
@@ -394,7 +405,15 @@ class PgvectorDocumentStore:
|
|
|
394
405
|
await self._async_connection.close()
|
|
395
406
|
|
|
396
407
|
conn_str = self.connection_string.resolve_value() or ""
|
|
397
|
-
|
|
408
|
+
try:
|
|
409
|
+
async_connection = await AsyncConnection.connect(conn_str)
|
|
410
|
+
except Error as e:
|
|
411
|
+
msg = (
|
|
412
|
+
"Failed to connect to PostgreSQL database. Ensure the connection string follows the "
|
|
413
|
+
"PostgreSQL connection specification: "
|
|
414
|
+
"https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING."
|
|
415
|
+
)
|
|
416
|
+
raise DocumentStoreError(msg) from e
|
|
398
417
|
await async_connection.set_autocommit(True)
|
|
399
418
|
if self.create_extension:
|
|
400
419
|
await async_connection.execute("CREATE EXTENSION IF NOT EXISTS vector")
|
|
@@ -740,7 +759,7 @@ class PgvectorDocumentStore:
|
|
|
740
759
|
return result[0]
|
|
741
760
|
return 0
|
|
742
761
|
|
|
743
|
-
def filter_documents(self, filters: Optional[
|
|
762
|
+
def filter_documents(self, filters: Optional[dict[str, Any]] = None) -> list[Document]:
|
|
744
763
|
"""
|
|
745
764
|
Returns the documents that match the filters provided.
|
|
746
765
|
|
|
@@ -777,7 +796,7 @@ class PgvectorDocumentStore:
|
|
|
777
796
|
docs = _from_pg_to_haystack_documents(records)
|
|
778
797
|
return docs
|
|
779
798
|
|
|
780
|
-
async def filter_documents_async(self, filters: Optional[
|
|
799
|
+
async def filter_documents_async(self, filters: Optional[dict[str, Any]] = None) -> list[Document]:
|
|
781
800
|
"""
|
|
782
801
|
Asynchronously returns the documents that match the filters provided.
|
|
783
802
|
|
|
@@ -832,7 +851,7 @@ class PgvectorDocumentStore:
|
|
|
832
851
|
|
|
833
852
|
return sql_insert
|
|
834
853
|
|
|
835
|
-
def write_documents(self, documents:
|
|
854
|
+
def write_documents(self, documents: list[Document], policy: DuplicatePolicy = DuplicatePolicy.NONE) -> int:
|
|
836
855
|
"""
|
|
837
856
|
Writes documents to the document store.
|
|
838
857
|
|
|
@@ -888,7 +907,7 @@ class PgvectorDocumentStore:
|
|
|
888
907
|
return written_docs
|
|
889
908
|
|
|
890
909
|
async def write_documents_async(
|
|
891
|
-
self, documents:
|
|
910
|
+
self, documents: list[Document], policy: DuplicatePolicy = DuplicatePolicy.NONE
|
|
892
911
|
) -> int:
|
|
893
912
|
"""
|
|
894
913
|
Asynchronously writes documents to the document store.
|
|
@@ -939,7 +958,7 @@ class PgvectorDocumentStore:
|
|
|
939
958
|
|
|
940
959
|
return written_docs
|
|
941
960
|
|
|
942
|
-
def delete_documents(self, document_ids:
|
|
961
|
+
def delete_documents(self, document_ids: list[str]) -> None:
|
|
943
962
|
"""
|
|
944
963
|
Deletes documents that match the provided `document_ids` from the document store.
|
|
945
964
|
|
|
@@ -962,7 +981,7 @@ class PgvectorDocumentStore:
|
|
|
962
981
|
cursor=self._cursor, sql_query=delete_sql, error_msg="Could not delete documents from PgvectorDocumentStore"
|
|
963
982
|
)
|
|
964
983
|
|
|
965
|
-
async def delete_documents_async(self, document_ids:
|
|
984
|
+
async def delete_documents_async(self, document_ids: list[str]) -> None:
|
|
966
985
|
"""
|
|
967
986
|
Asynchronously deletes documents that match the provided `document_ids` from the document store.
|
|
968
987
|
|
|
@@ -1019,9 +1038,193 @@ class PgvectorDocumentStore:
|
|
|
1019
1038
|
error_msg="Could not delete all documents from PgvectorDocumentStore",
|
|
1020
1039
|
)
|
|
1021
1040
|
|
|
1041
|
+
def delete_by_filter(self, filters: dict[str, Any]) -> int:
|
|
1042
|
+
"""
|
|
1043
|
+
Deletes all documents that match the provided filters.
|
|
1044
|
+
|
|
1045
|
+
:param filters: The filters to apply to select documents for deletion.
|
|
1046
|
+
For filter syntax, see [Haystack metadata filtering](https://docs.haystack.deepset.ai/docs/metadata-filtering)
|
|
1047
|
+
:returns: The number of documents deleted.
|
|
1048
|
+
"""
|
|
1049
|
+
_validate_filters(filters)
|
|
1050
|
+
|
|
1051
|
+
delete_sql = SQL("DELETE FROM {schema_name}.{table_name}").format(
|
|
1052
|
+
schema_name=Identifier(self.schema_name),
|
|
1053
|
+
table_name=Identifier(self.table_name),
|
|
1054
|
+
)
|
|
1055
|
+
|
|
1056
|
+
params = ()
|
|
1057
|
+
if filters:
|
|
1058
|
+
sql_where_clause, params = _convert_filters_to_where_clause_and_params(filters)
|
|
1059
|
+
delete_sql += sql_where_clause
|
|
1060
|
+
|
|
1061
|
+
self._ensure_db_setup()
|
|
1062
|
+
assert self._cursor is not None
|
|
1063
|
+
|
|
1064
|
+
try:
|
|
1065
|
+
self._execute_sql(
|
|
1066
|
+
cursor=self._cursor,
|
|
1067
|
+
sql_query=delete_sql,
|
|
1068
|
+
params=params,
|
|
1069
|
+
error_msg="Could not delete documents by filter from PgvectorDocumentStore",
|
|
1070
|
+
)
|
|
1071
|
+
deleted_count = self._cursor.rowcount
|
|
1072
|
+
logger.info(
|
|
1073
|
+
"Deleted {n_docs} documents from table '{schema}.{table}' using filters.",
|
|
1074
|
+
n_docs=deleted_count,
|
|
1075
|
+
schema=self.schema_name,
|
|
1076
|
+
table=self.table_name,
|
|
1077
|
+
)
|
|
1078
|
+
return deleted_count
|
|
1079
|
+
except Error as e:
|
|
1080
|
+
msg = f"Failed to delete documents by filter from PgvectorDocumentStore: {e!s}"
|
|
1081
|
+
raise DocumentStoreError(msg) from e
|
|
1082
|
+
|
|
1083
|
+
async def delete_by_filter_async(self, filters: dict[str, Any]) -> int:
|
|
1084
|
+
"""
|
|
1085
|
+
Asynchronously deletes all documents that match the provided filters.
|
|
1086
|
+
|
|
1087
|
+
:param filters: The filters to apply to select documents for deletion.
|
|
1088
|
+
For filter syntax, see [Haystack metadata filtering](https://docs.haystack.deepset.ai/docs/metadata-filtering)
|
|
1089
|
+
:returns: The number of documents deleted.
|
|
1090
|
+
"""
|
|
1091
|
+
_validate_filters(filters)
|
|
1092
|
+
|
|
1093
|
+
delete_sql = SQL("DELETE FROM {schema_name}.{table_name}").format(
|
|
1094
|
+
schema_name=Identifier(self.schema_name),
|
|
1095
|
+
table_name=Identifier(self.table_name),
|
|
1096
|
+
)
|
|
1097
|
+
|
|
1098
|
+
params = ()
|
|
1099
|
+
if filters:
|
|
1100
|
+
sql_where_clause, params = _convert_filters_to_where_clause_and_params(filters)
|
|
1101
|
+
delete_sql += sql_where_clause
|
|
1102
|
+
|
|
1103
|
+
await self._ensure_db_setup_async()
|
|
1104
|
+
assert self._async_cursor is not None
|
|
1105
|
+
|
|
1106
|
+
try:
|
|
1107
|
+
await self._execute_sql_async(
|
|
1108
|
+
cursor=self._async_cursor,
|
|
1109
|
+
sql_query=delete_sql,
|
|
1110
|
+
params=params,
|
|
1111
|
+
error_msg="Could not delete documents by filter from PgvectorDocumentStore",
|
|
1112
|
+
)
|
|
1113
|
+
deleted_count = self._async_cursor.rowcount
|
|
1114
|
+
logger.info(
|
|
1115
|
+
"Deleted {n_docs} documents from table '{schema}.{table}' using filters.",
|
|
1116
|
+
n_docs=deleted_count,
|
|
1117
|
+
schema=self.schema_name,
|
|
1118
|
+
table=self.table_name,
|
|
1119
|
+
)
|
|
1120
|
+
return deleted_count
|
|
1121
|
+
except Error as e:
|
|
1122
|
+
msg = f"Failed to delete documents by filter from PgvectorDocumentStore: {e!s}"
|
|
1123
|
+
raise DocumentStoreError(msg) from e
|
|
1124
|
+
|
|
1125
|
+
def update_by_filter(self, filters: dict[str, Any], meta: dict[str, Any]) -> int:
|
|
1126
|
+
"""
|
|
1127
|
+
Updates the metadata of all documents that match the provided filters.
|
|
1128
|
+
|
|
1129
|
+
:param filters: The filters to apply to select documents for updating.
|
|
1130
|
+
For filter syntax, see [Haystack metadata filtering](https://docs.haystack.deepset.ai/docs/metadata-filtering)
|
|
1131
|
+
:param meta: The metadata fields to update.
|
|
1132
|
+
:returns: The number of documents updated.
|
|
1133
|
+
"""
|
|
1134
|
+
_validate_filters(filters)
|
|
1135
|
+
|
|
1136
|
+
if not meta:
|
|
1137
|
+
msg = "meta must be a non-empty dictionary"
|
|
1138
|
+
raise ValueError(msg)
|
|
1139
|
+
|
|
1140
|
+
update_sql = SQL(
|
|
1141
|
+
"UPDATE {schema_name}.{table_name} SET meta = COALESCE(meta, '{{}}'::jsonb) || %s::jsonb"
|
|
1142
|
+
).format(
|
|
1143
|
+
schema_name=Identifier(self.schema_name),
|
|
1144
|
+
table_name=Identifier(self.table_name),
|
|
1145
|
+
)
|
|
1146
|
+
|
|
1147
|
+
params: tuple[Any, ...] = (Jsonb(meta),)
|
|
1148
|
+
if filters:
|
|
1149
|
+
sql_where_clause, where_params = _convert_filters_to_where_clause_and_params(filters)
|
|
1150
|
+
update_sql += sql_where_clause
|
|
1151
|
+
params = params + where_params
|
|
1152
|
+
|
|
1153
|
+
self._ensure_db_setup()
|
|
1154
|
+
assert self._cursor is not None
|
|
1155
|
+
|
|
1156
|
+
try:
|
|
1157
|
+
self._execute_sql(
|
|
1158
|
+
cursor=self._cursor,
|
|
1159
|
+
sql_query=update_sql,
|
|
1160
|
+
params=params,
|
|
1161
|
+
error_msg="Could not update documents by filter from PgvectorDocumentStore",
|
|
1162
|
+
)
|
|
1163
|
+
updated_count = self._cursor.rowcount
|
|
1164
|
+
logger.info(
|
|
1165
|
+
"Updated {n_docs} documents in table '{schema}.{table}' using filters.",
|
|
1166
|
+
n_docs=updated_count,
|
|
1167
|
+
schema=self.schema_name,
|
|
1168
|
+
table=self.table_name,
|
|
1169
|
+
)
|
|
1170
|
+
return updated_count
|
|
1171
|
+
except Error as e:
|
|
1172
|
+
msg = f"Failed to update documents by filter in PgvectorDocumentStore: {e!s}"
|
|
1173
|
+
raise DocumentStoreError(msg) from e
|
|
1174
|
+
|
|
1175
|
+
async def update_by_filter_async(self, filters: dict[str, Any], meta: dict[str, Any]) -> int:
|
|
1176
|
+
"""
|
|
1177
|
+
Asynchronously updates the metadata of all documents that match the provided filters.
|
|
1178
|
+
|
|
1179
|
+
:param filters: The filters to apply to select documents for updating.
|
|
1180
|
+
For filter syntax, see [Haystack metadata filtering](https://docs.haystack.deepset.ai/docs/metadata-filtering)
|
|
1181
|
+
:param meta: The metadata fields to update.
|
|
1182
|
+
:returns: The number of documents updated.
|
|
1183
|
+
"""
|
|
1184
|
+
_validate_filters(filters)
|
|
1185
|
+
|
|
1186
|
+
if not meta:
|
|
1187
|
+
msg = "meta must be a non-empty dictionary"
|
|
1188
|
+
raise ValueError(msg)
|
|
1189
|
+
|
|
1190
|
+
update_sql = SQL(
|
|
1191
|
+
"UPDATE {schema_name}.{table_name} SET meta = COALESCE(meta, '{{}}'::jsonb) || %s::jsonb"
|
|
1192
|
+
).format(
|
|
1193
|
+
schema_name=Identifier(self.schema_name),
|
|
1194
|
+
table_name=Identifier(self.table_name),
|
|
1195
|
+
)
|
|
1196
|
+
|
|
1197
|
+
params: tuple[Any, ...] = (Jsonb(meta),)
|
|
1198
|
+
if filters:
|
|
1199
|
+
sql_where_clause, where_params = _convert_filters_to_where_clause_and_params(filters)
|
|
1200
|
+
update_sql += sql_where_clause
|
|
1201
|
+
params = params + where_params
|
|
1202
|
+
|
|
1203
|
+
await self._ensure_db_setup_async()
|
|
1204
|
+
assert self._async_cursor is not None
|
|
1205
|
+
|
|
1206
|
+
try:
|
|
1207
|
+
await self._execute_sql_async(
|
|
1208
|
+
cursor=self._async_cursor,
|
|
1209
|
+
sql_query=update_sql,
|
|
1210
|
+
params=params,
|
|
1211
|
+
error_msg="Could not update documents by filter from PgvectorDocumentStore",
|
|
1212
|
+
)
|
|
1213
|
+
updated_count = self._async_cursor.rowcount
|
|
1214
|
+
logger.info(
|
|
1215
|
+
"Updated {n_docs} documents in table '{schema}.{table}' using filters.",
|
|
1216
|
+
n_docs=updated_count,
|
|
1217
|
+
schema=self.schema_name,
|
|
1218
|
+
table=self.table_name,
|
|
1219
|
+
)
|
|
1220
|
+
return updated_count
|
|
1221
|
+
except Error as e:
|
|
1222
|
+
msg = f"Failed to update documents by filter in PgvectorDocumentStore: {e!s}"
|
|
1223
|
+
raise DocumentStoreError(msg) from e
|
|
1224
|
+
|
|
1022
1225
|
def _build_keyword_retrieval_query(
|
|
1023
|
-
self, query: str, top_k: int, filters: Optional[
|
|
1024
|
-
) ->
|
|
1226
|
+
self, query: str, top_k: int, filters: Optional[dict[str, Any]] = None
|
|
1227
|
+
) -> tuple[Composed, tuple]:
|
|
1025
1228
|
"""
|
|
1026
1229
|
Builds the SQL query and the where parameters for keyword retrieval.
|
|
1027
1230
|
"""
|
|
@@ -1049,9 +1252,9 @@ class PgvectorDocumentStore:
|
|
|
1049
1252
|
self,
|
|
1050
1253
|
query: str,
|
|
1051
1254
|
*,
|
|
1052
|
-
filters: Optional[
|
|
1255
|
+
filters: Optional[dict[str, Any]] = None,
|
|
1053
1256
|
top_k: int = 10,
|
|
1054
|
-
) ->
|
|
1257
|
+
) -> list[Document]:
|
|
1055
1258
|
"""
|
|
1056
1259
|
Retrieves documents that are most similar to the query using a full-text search.
|
|
1057
1260
|
|
|
@@ -1084,9 +1287,9 @@ class PgvectorDocumentStore:
|
|
|
1084
1287
|
self,
|
|
1085
1288
|
query: str,
|
|
1086
1289
|
*,
|
|
1087
|
-
filters: Optional[
|
|
1290
|
+
filters: Optional[dict[str, Any]] = None,
|
|
1088
1291
|
top_k: int = 10,
|
|
1089
|
-
) ->
|
|
1292
|
+
) -> list[Document]:
|
|
1090
1293
|
"""
|
|
1091
1294
|
Retrieves documents that are most similar to the query using a full-text search asynchronously.
|
|
1092
1295
|
"""
|
|
@@ -1111,11 +1314,11 @@ class PgvectorDocumentStore:
|
|
|
1111
1314
|
|
|
1112
1315
|
def _check_and_build_embedding_retrieval_query(
|
|
1113
1316
|
self,
|
|
1114
|
-
query_embedding:
|
|
1317
|
+
query_embedding: list[float],
|
|
1115
1318
|
vector_function: Optional[Literal["cosine_similarity", "inner_product", "l2_distance"]],
|
|
1116
1319
|
top_k: int,
|
|
1117
|
-
filters: Optional[
|
|
1118
|
-
) ->
|
|
1320
|
+
filters: Optional[dict[str, Any]] = None,
|
|
1321
|
+
) -> tuple[Composed, tuple]:
|
|
1119
1322
|
"""
|
|
1120
1323
|
Performs checks and builds the SQL query and the where parameters for embedding retrieval.
|
|
1121
1324
|
"""
|
|
@@ -1174,12 +1377,12 @@ class PgvectorDocumentStore:
|
|
|
1174
1377
|
|
|
1175
1378
|
def _embedding_retrieval(
|
|
1176
1379
|
self,
|
|
1177
|
-
query_embedding:
|
|
1380
|
+
query_embedding: list[float],
|
|
1178
1381
|
*,
|
|
1179
|
-
filters: Optional[
|
|
1382
|
+
filters: Optional[dict[str, Any]] = None,
|
|
1180
1383
|
top_k: int = 10,
|
|
1181
1384
|
vector_function: Optional[Literal["cosine_similarity", "inner_product", "l2_distance"]] = None,
|
|
1182
|
-
) ->
|
|
1385
|
+
) -> list[Document]:
|
|
1183
1386
|
"""
|
|
1184
1387
|
Retrieves documents that are most similar to the query embedding using a vector similarity metric.
|
|
1185
1388
|
|
|
@@ -1208,12 +1411,12 @@ class PgvectorDocumentStore:
|
|
|
1208
1411
|
|
|
1209
1412
|
async def _embedding_retrieval_async(
|
|
1210
1413
|
self,
|
|
1211
|
-
query_embedding:
|
|
1414
|
+
query_embedding: list[float],
|
|
1212
1415
|
*,
|
|
1213
|
-
filters: Optional[
|
|
1416
|
+
filters: Optional[dict[str, Any]] = None,
|
|
1214
1417
|
top_k: int = 10,
|
|
1215
1418
|
vector_function: Optional[Literal["cosine_similarity", "inner_product", "l2_distance"]] = None,
|
|
1216
|
-
) ->
|
|
1419
|
+
) -> list[Document]:
|
|
1217
1420
|
"""
|
|
1218
1421
|
Asynchronously retrieves documents that are most similar to the query embedding using a
|
|
1219
1422
|
vector similarity metric.
|