pgvector-haystack 5.4.0__tar.gz → 6.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pgvector_haystack-5.4.0 → pgvector_haystack-6.0.0}/CHANGELOG.md +26 -0
- {pgvector_haystack-5.4.0 → pgvector_haystack-6.0.0}/PKG-INFO +3 -4
- {pgvector_haystack-5.4.0 → pgvector_haystack-6.0.0}/pyproject.toml +3 -9
- {pgvector_haystack-5.4.0 → pgvector_haystack-6.0.0}/src/haystack_integrations/components/retrievers/pgvector/embedding_retriever.py +10 -10
- {pgvector_haystack-5.4.0 → pgvector_haystack-6.0.0}/src/haystack_integrations/components/retrievers/pgvector/keyword_retriever.py +7 -7
- {pgvector_haystack-5.4.0 → pgvector_haystack-6.0.0}/src/haystack_integrations/document_stores/pgvector/document_store.py +220 -34
- {pgvector_haystack-5.4.0 → pgvector_haystack-6.0.0}/src/haystack_integrations/document_stores/pgvector/filters.py +2 -2
- {pgvector_haystack-5.4.0 → pgvector_haystack-6.0.0}/tests/test_document_store.py +219 -1
- pgvector_haystack-6.0.0/tests/test_document_store_async.py +450 -0
- {pgvector_haystack-5.4.0 → pgvector_haystack-6.0.0}/tests/test_filters.py +1 -1
- pgvector_haystack-5.4.0/tests/test_document_store_async.py +0 -232
- {pgvector_haystack-5.4.0 → pgvector_haystack-6.0.0}/.gitignore +0 -0
- {pgvector_haystack-5.4.0 → pgvector_haystack-6.0.0}/LICENSE.txt +0 -0
- {pgvector_haystack-5.4.0 → pgvector_haystack-6.0.0}/README.md +0 -0
- {pgvector_haystack-5.4.0 → pgvector_haystack-6.0.0}/examples/embedding_retrieval.py +0 -0
- {pgvector_haystack-5.4.0 → pgvector_haystack-6.0.0}/examples/hybrid_retrieval.py +0 -0
- {pgvector_haystack-5.4.0 → pgvector_haystack-6.0.0}/pydoc/config_docusaurus.yml +0 -0
- {pgvector_haystack-5.4.0 → pgvector_haystack-6.0.0}/src/haystack_integrations/components/retrievers/pgvector/__init__.py +0 -0
- {pgvector_haystack-5.4.0 → pgvector_haystack-6.0.0}/src/haystack_integrations/components/retrievers/py.typed +0 -0
- {pgvector_haystack-5.4.0 → pgvector_haystack-6.0.0}/src/haystack_integrations/document_stores/pgvector/__init__.py +0 -0
- {pgvector_haystack-5.4.0 → pgvector_haystack-6.0.0}/src/haystack_integrations/document_stores/pgvector/converters.py +0 -0
- {pgvector_haystack-5.4.0 → pgvector_haystack-6.0.0}/src/haystack_integrations/document_stores/py.typed +0 -0
- {pgvector_haystack-5.4.0 → pgvector_haystack-6.0.0}/tests/__init__.py +0 -0
- {pgvector_haystack-5.4.0 → pgvector_haystack-6.0.0}/tests/conftest.py +0 -0
- {pgvector_haystack-5.4.0 → pgvector_haystack-6.0.0}/tests/test_converters.py +0 -0
- {pgvector_haystack-5.4.0 → pgvector_haystack-6.0.0}/tests/test_retrieval.py +0 -0
- {pgvector_haystack-5.4.0 → pgvector_haystack-6.0.0}/tests/test_retrieval_async.py +0 -0
- {pgvector_haystack-5.4.0 → pgvector_haystack-6.0.0}/tests/test_retrievers.py +0 -0
|
@@ -1,5 +1,31 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [integrations/pgvector-v5.5.0] - 2026-01-02
|
|
4
|
+
|
|
5
|
+
### 🚀 Features
|
|
6
|
+
|
|
7
|
+
- Adding `update_by_filter` and `delete_by_filter` to PgVector document store (#2647)
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
## [integrations/pgvector-v5.4.0] - 2025-12-05
|
|
11
|
+
|
|
12
|
+
### 📚 Documentation
|
|
13
|
+
|
|
14
|
+
- Add pydoc configurations for Docusaurus (#2411)
|
|
15
|
+
|
|
16
|
+
### ⚙️ CI
|
|
17
|
+
|
|
18
|
+
- Change pytest command (#2475)
|
|
19
|
+
|
|
20
|
+
### 🧹 Chores
|
|
21
|
+
|
|
22
|
+
- Remove Readme API CI workflow and configs (#2573)
|
|
23
|
+
- Pgvector - document expected connection string and raise informative error (#2583)
|
|
24
|
+
|
|
25
|
+
### 🌀 Miscellaneous
|
|
26
|
+
|
|
27
|
+
- Enhancement: Adopt PEP 585 type hinting (part 5) (#2528)
|
|
28
|
+
|
|
3
29
|
## [integrations/pgvector-v5.3.0] - 2025-10-17
|
|
4
30
|
|
|
5
31
|
### 🚀 Features
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: pgvector-haystack
|
|
3
|
-
Version:
|
|
3
|
+
Version: 6.0.0
|
|
4
4
|
Summary: An integration of pgvector (vector search extension for Postgres) with Haystack
|
|
5
5
|
Project-URL: Source, https://github.com/deepset-ai/haystack-core-integrations
|
|
6
6
|
Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/blob/main/integrations/pgvector/README.md
|
|
@@ -11,15 +11,14 @@ License-File: LICENSE.txt
|
|
|
11
11
|
Classifier: Development Status :: 4 - Beta
|
|
12
12
|
Classifier: License :: OSI Approved :: Apache Software License
|
|
13
13
|
Classifier: Programming Language :: Python
|
|
14
|
-
Classifier: Programming Language :: Python :: 3.9
|
|
15
14
|
Classifier: Programming Language :: Python :: 3.10
|
|
16
15
|
Classifier: Programming Language :: Python :: 3.11
|
|
17
16
|
Classifier: Programming Language :: Python :: 3.12
|
|
18
17
|
Classifier: Programming Language :: Python :: 3.13
|
|
19
18
|
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
20
19
|
Classifier: Programming Language :: Python :: Implementation :: PyPy
|
|
21
|
-
Requires-Python: >=3.
|
|
22
|
-
Requires-Dist: haystack-ai>=2.
|
|
20
|
+
Requires-Python: >=3.10
|
|
21
|
+
Requires-Dist: haystack-ai>=2.22.0
|
|
23
22
|
Requires-Dist: pgvector>=0.3.0
|
|
24
23
|
Requires-Dist: psycopg[binary]
|
|
25
24
|
Description-Content-Type: text/markdown
|
|
@@ -7,7 +7,7 @@ name = "pgvector-haystack"
|
|
|
7
7
|
dynamic = ["version"]
|
|
8
8
|
description = "An integration of pgvector (vector search extension for Postgres) with Haystack"
|
|
9
9
|
readme = "README.md"
|
|
10
|
-
requires-python = ">=3.
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
11
|
license = "Apache-2.0"
|
|
12
12
|
keywords = []
|
|
13
13
|
authors = [{ name = "deepset GmbH", email = "info@deepset.ai" }]
|
|
@@ -15,7 +15,6 @@ classifiers = [
|
|
|
15
15
|
"License :: OSI Approved :: Apache Software License",
|
|
16
16
|
"Development Status :: 4 - Beta",
|
|
17
17
|
"Programming Language :: Python",
|
|
18
|
-
"Programming Language :: Python :: 3.9",
|
|
19
18
|
"Programming Language :: Python :: 3.10",
|
|
20
19
|
"Programming Language :: Python :: 3.11",
|
|
21
20
|
"Programming Language :: Python :: 3.12",
|
|
@@ -23,7 +22,7 @@ classifiers = [
|
|
|
23
22
|
"Programming Language :: Python :: Implementation :: CPython",
|
|
24
23
|
"Programming Language :: Python :: Implementation :: PyPy",
|
|
25
24
|
]
|
|
26
|
-
dependencies = ["haystack-ai>=2.
|
|
25
|
+
dependencies = ["haystack-ai>=2.22.0", "pgvector>=0.3.0", "psycopg[binary]"]
|
|
27
26
|
|
|
28
27
|
[project.urls]
|
|
29
28
|
Source = "https://github.com/deepset-ai/haystack-core-integrations"
|
|
@@ -47,7 +46,7 @@ dependencies = ["haystack-pydoc-tools", "ruff"]
|
|
|
47
46
|
|
|
48
47
|
[tool.hatch.envs.default.scripts]
|
|
49
48
|
docs = ["pydoc-markdown pydoc/config_docusaurus.yml"]
|
|
50
|
-
fmt = "ruff check --fix {args}
|
|
49
|
+
fmt = "ruff check --fix {args}; ruff format {args}"
|
|
51
50
|
fmt-check = "ruff check {args} && ruff format --check {args}"
|
|
52
51
|
|
|
53
52
|
[tool.hatch.envs.test]
|
|
@@ -83,7 +82,6 @@ ignore_missing_imports = true
|
|
|
83
82
|
|
|
84
83
|
|
|
85
84
|
[tool.ruff]
|
|
86
|
-
target-version = "py39"
|
|
87
85
|
line-length = 120
|
|
88
86
|
|
|
89
87
|
[tool.ruff.lint]
|
|
@@ -134,10 +132,6 @@ ignore = [
|
|
|
134
132
|
# ignore assertions
|
|
135
133
|
"S101",
|
|
136
134
|
]
|
|
137
|
-
unfixable = [
|
|
138
|
-
# Don't touch unused imports
|
|
139
|
-
"F401",
|
|
140
|
-
]
|
|
141
135
|
|
|
142
136
|
[tool.ruff.lint.isort]
|
|
143
137
|
known-first-party = ["haystack_integrations"]
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# SPDX-FileCopyrightText: 2023-present deepset GmbH <info@deepset.ai>
|
|
2
2
|
#
|
|
3
3
|
# SPDX-License-Identifier: Apache-2.0
|
|
4
|
-
from typing import Any, Literal
|
|
4
|
+
from typing import Any, Literal
|
|
5
5
|
|
|
6
6
|
from haystack import component, default_from_dict, default_to_dict
|
|
7
7
|
from haystack.dataclasses import Document
|
|
@@ -62,10 +62,10 @@ class PgvectorEmbeddingRetriever:
|
|
|
62
62
|
self,
|
|
63
63
|
*,
|
|
64
64
|
document_store: PgvectorDocumentStore,
|
|
65
|
-
filters:
|
|
65
|
+
filters: dict[str, Any] | None = None,
|
|
66
66
|
top_k: int = 10,
|
|
67
|
-
vector_function:
|
|
68
|
-
filter_policy:
|
|
67
|
+
vector_function: Literal["cosine_similarity", "inner_product", "l2_distance"] | None = None,
|
|
68
|
+
filter_policy: str | FilterPolicy = FilterPolicy.REPLACE,
|
|
69
69
|
):
|
|
70
70
|
"""
|
|
71
71
|
:param document_store: An instance of `PgvectorDocumentStore`.
|
|
@@ -137,9 +137,9 @@ class PgvectorEmbeddingRetriever:
|
|
|
137
137
|
def run(
|
|
138
138
|
self,
|
|
139
139
|
query_embedding: list[float],
|
|
140
|
-
filters:
|
|
141
|
-
top_k:
|
|
142
|
-
vector_function:
|
|
140
|
+
filters: dict[str, Any] | None = None,
|
|
141
|
+
top_k: int | None = None,
|
|
142
|
+
vector_function: Literal["cosine_similarity", "inner_product", "l2_distance"] | None = None,
|
|
143
143
|
) -> dict[str, list[Document]]:
|
|
144
144
|
"""
|
|
145
145
|
Retrieve documents from the `PgvectorDocumentStore`, based on their embeddings.
|
|
@@ -170,9 +170,9 @@ class PgvectorEmbeddingRetriever:
|
|
|
170
170
|
async def run_async(
|
|
171
171
|
self,
|
|
172
172
|
query_embedding: list[float],
|
|
173
|
-
filters:
|
|
174
|
-
top_k:
|
|
175
|
-
vector_function:
|
|
173
|
+
filters: dict[str, Any] | None = None,
|
|
174
|
+
top_k: int | None = None,
|
|
175
|
+
vector_function: Literal["cosine_similarity", "inner_product", "l2_distance"] | None = None,
|
|
176
176
|
) -> dict[str, list[Document]]:
|
|
177
177
|
"""
|
|
178
178
|
Asynchronously retrieve documents from the `PgvectorDocumentStore`, based on their embeddings.
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# SPDX-FileCopyrightText: 2023-present deepset GmbH <info@deepset.ai>
|
|
2
2
|
#
|
|
3
3
|
# SPDX-License-Identifier: Apache-2.0
|
|
4
|
-
from typing import Any
|
|
4
|
+
from typing import Any
|
|
5
5
|
|
|
6
6
|
from haystack import component, default_from_dict, default_to_dict
|
|
7
7
|
from haystack.dataclasses import Document
|
|
@@ -52,9 +52,9 @@ class PgvectorKeywordRetriever:
|
|
|
52
52
|
self,
|
|
53
53
|
*,
|
|
54
54
|
document_store: PgvectorDocumentStore,
|
|
55
|
-
filters:
|
|
55
|
+
filters: dict[str, Any] | None = None,
|
|
56
56
|
top_k: int = 10,
|
|
57
|
-
filter_policy:
|
|
57
|
+
filter_policy: str | FilterPolicy = FilterPolicy.REPLACE,
|
|
58
58
|
):
|
|
59
59
|
"""
|
|
60
60
|
:param document_store: An instance of `PgvectorDocumentStore`.
|
|
@@ -111,8 +111,8 @@ class PgvectorKeywordRetriever:
|
|
|
111
111
|
def run(
|
|
112
112
|
self,
|
|
113
113
|
query: str,
|
|
114
|
-
filters:
|
|
115
|
-
top_k:
|
|
114
|
+
filters: dict[str, Any] | None = None,
|
|
115
|
+
top_k: int | None = None,
|
|
116
116
|
) -> dict[str, list[Document]]:
|
|
117
117
|
"""
|
|
118
118
|
Retrieve documents from the `PgvectorDocumentStore`, based on keywords.
|
|
@@ -141,8 +141,8 @@ class PgvectorKeywordRetriever:
|
|
|
141
141
|
async def run_async(
|
|
142
142
|
self,
|
|
143
143
|
query: str,
|
|
144
|
-
filters:
|
|
145
|
-
top_k:
|
|
144
|
+
filters: dict[str, Any] | None = None,
|
|
145
|
+
top_k: int | None = None,
|
|
146
146
|
) -> dict[str, list[Document]]:
|
|
147
147
|
"""
|
|
148
148
|
Asynchronously retrieve documents from the `PgvectorDocumentStore`, based on keywords.
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
# SPDX-FileCopyrightText: 2023-present deepset GmbH <info@deepset.ai>
|
|
2
2
|
#
|
|
3
3
|
# SPDX-License-Identifier: Apache-2.0
|
|
4
|
-
|
|
4
|
+
|
|
5
|
+
from typing import Any, Literal, overload
|
|
5
6
|
|
|
6
7
|
from haystack import default_from_dict, default_to_dict, logging
|
|
7
8
|
from haystack.dataclasses.document import Document
|
|
@@ -13,6 +14,7 @@ from psycopg.cursor_async import AsyncCursor
|
|
|
13
14
|
from psycopg.rows import DictRow, dict_row
|
|
14
15
|
from psycopg.sql import SQL, Composed, Identifier
|
|
15
16
|
from psycopg.sql import Literal as SQLLiteral
|
|
17
|
+
from psycopg.types.json import Jsonb
|
|
16
18
|
|
|
17
19
|
from pgvector.psycopg import register_vector, register_vector_async
|
|
18
20
|
|
|
@@ -90,9 +92,9 @@ class PgvectorDocumentStore:
|
|
|
90
92
|
recreate_table: bool = False,
|
|
91
93
|
search_strategy: Literal["exact_nearest_neighbor", "hnsw"] = "exact_nearest_neighbor",
|
|
92
94
|
hnsw_recreate_index_if_exists: bool = False,
|
|
93
|
-
hnsw_index_creation_kwargs:
|
|
95
|
+
hnsw_index_creation_kwargs: dict[str, int] | None = None,
|
|
94
96
|
hnsw_index_name: str = "haystack_hnsw_index",
|
|
95
|
-
hnsw_ef_search:
|
|
97
|
+
hnsw_ef_search: int | None = None,
|
|
96
98
|
keyword_index_name: str = "haystack_keyword_index",
|
|
97
99
|
):
|
|
98
100
|
"""
|
|
@@ -173,12 +175,12 @@ class PgvectorDocumentStore:
|
|
|
173
175
|
self.keyword_index_name = keyword_index_name
|
|
174
176
|
self.language = language
|
|
175
177
|
|
|
176
|
-
self._connection:
|
|
177
|
-
self._async_connection:
|
|
178
|
-
self._cursor:
|
|
179
|
-
self._async_cursor:
|
|
180
|
-
self._dict_cursor:
|
|
181
|
-
self._async_dict_cursor:
|
|
178
|
+
self._connection: Connection | None = None
|
|
179
|
+
self._async_connection: AsyncConnection | None = None
|
|
180
|
+
self._cursor: Cursor | None = None
|
|
181
|
+
self._async_cursor: AsyncCursor | None = None
|
|
182
|
+
self._dict_cursor: Cursor[DictRow] | None = None
|
|
183
|
+
self._async_dict_cursor: AsyncCursor[DictRow] | None = None
|
|
182
184
|
self._table_initialized = False
|
|
183
185
|
|
|
184
186
|
def to_dict(self) -> dict[str, Any]:
|
|
@@ -248,21 +250,21 @@ class PgvectorDocumentStore:
|
|
|
248
250
|
|
|
249
251
|
@overload
|
|
250
252
|
def _execute_sql(
|
|
251
|
-
self, cursor: Cursor, sql_query: Composed, params:
|
|
253
|
+
self, cursor: Cursor, sql_query: Composed, params: tuple | None = None, error_msg: str = ""
|
|
252
254
|
) -> Cursor: ...
|
|
253
255
|
|
|
254
256
|
@overload
|
|
255
257
|
def _execute_sql(
|
|
256
|
-
self, cursor: Cursor[DictRow], sql_query: Composed, params:
|
|
258
|
+
self, cursor: Cursor[DictRow], sql_query: Composed, params: tuple | None = None, error_msg: str = ""
|
|
257
259
|
) -> Cursor[DictRow]: ...
|
|
258
260
|
|
|
259
261
|
def _execute_sql(
|
|
260
262
|
self,
|
|
261
|
-
cursor:
|
|
263
|
+
cursor: Cursor | Cursor[DictRow],
|
|
262
264
|
sql_query: Composed,
|
|
263
|
-
params:
|
|
265
|
+
params: tuple | None = None,
|
|
264
266
|
error_msg: str = "",
|
|
265
|
-
) ->
|
|
267
|
+
) -> Cursor | Cursor[DictRow]:
|
|
266
268
|
"""
|
|
267
269
|
Internal method to execute SQL statements and handle exceptions.
|
|
268
270
|
|
|
@@ -297,21 +299,21 @@ class PgvectorDocumentStore:
|
|
|
297
299
|
|
|
298
300
|
@overload
|
|
299
301
|
async def _execute_sql_async(
|
|
300
|
-
self, cursor: AsyncCursor, sql_query: Composed, params:
|
|
302
|
+
self, cursor: AsyncCursor, sql_query: Composed, params: tuple | None = None, error_msg: str = ""
|
|
301
303
|
) -> AsyncCursor: ...
|
|
302
304
|
|
|
303
305
|
@overload
|
|
304
306
|
async def _execute_sql_async(
|
|
305
|
-
self, cursor: AsyncCursor[DictRow], sql_query: Composed, params:
|
|
307
|
+
self, cursor: AsyncCursor[DictRow], sql_query: Composed, params: tuple | None = None, error_msg: str = ""
|
|
306
308
|
) -> AsyncCursor[DictRow]: ...
|
|
307
309
|
|
|
308
310
|
async def _execute_sql_async(
|
|
309
311
|
self,
|
|
310
|
-
cursor:
|
|
312
|
+
cursor: AsyncCursor | AsyncCursor[DictRow],
|
|
311
313
|
sql_query: Composed,
|
|
312
|
-
params:
|
|
314
|
+
params: tuple | None = None,
|
|
313
315
|
error_msg: str = "",
|
|
314
|
-
) ->
|
|
316
|
+
) -> AsyncCursor | AsyncCursor[DictRow]:
|
|
315
317
|
"""
|
|
316
318
|
Internal method to asynchronously execute SQL statements and handle exceptions.
|
|
317
319
|
|
|
@@ -757,12 +759,12 @@ class PgvectorDocumentStore:
|
|
|
757
759
|
return result[0]
|
|
758
760
|
return 0
|
|
759
761
|
|
|
760
|
-
def filter_documents(self, filters:
|
|
762
|
+
def filter_documents(self, filters: dict[str, Any] | None = None) -> list[Document]:
|
|
761
763
|
"""
|
|
762
764
|
Returns the documents that match the filters provided.
|
|
763
765
|
|
|
764
766
|
For a detailed specification of the filters,
|
|
765
|
-
refer to the [documentation](https://docs.haystack.deepset.ai/
|
|
767
|
+
refer to the [documentation](https://docs.haystack.deepset.ai/docs/metadata-filtering)
|
|
766
768
|
|
|
767
769
|
:param filters: The filters to apply to the document list.
|
|
768
770
|
:raises TypeError: If `filters` is not a dictionary.
|
|
@@ -794,12 +796,12 @@ class PgvectorDocumentStore:
|
|
|
794
796
|
docs = _from_pg_to_haystack_documents(records)
|
|
795
797
|
return docs
|
|
796
798
|
|
|
797
|
-
async def filter_documents_async(self, filters:
|
|
799
|
+
async def filter_documents_async(self, filters: dict[str, Any] | None = None) -> list[Document]:
|
|
798
800
|
"""
|
|
799
801
|
Asynchronously returns the documents that match the filters provided.
|
|
800
802
|
|
|
801
803
|
For a detailed specification of the filters,
|
|
802
|
-
refer to the [documentation](https://docs.haystack.deepset.ai/
|
|
804
|
+
refer to the [documentation](https://docs.haystack.deepset.ai/docs/metadata-filtering)
|
|
803
805
|
|
|
804
806
|
:param filters: The filters to apply to the document list.
|
|
805
807
|
|
|
@@ -1036,8 +1038,192 @@ class PgvectorDocumentStore:
|
|
|
1036
1038
|
error_msg="Could not delete all documents from PgvectorDocumentStore",
|
|
1037
1039
|
)
|
|
1038
1040
|
|
|
1041
|
+
def delete_by_filter(self, filters: dict[str, Any]) -> int:
|
|
1042
|
+
"""
|
|
1043
|
+
Deletes all documents that match the provided filters.
|
|
1044
|
+
|
|
1045
|
+
:param filters: The filters to apply to select documents for deletion.
|
|
1046
|
+
For filter syntax, see [Haystack metadata filtering](https://docs.haystack.deepset.ai/docs/metadata-filtering)
|
|
1047
|
+
:returns: The number of documents deleted.
|
|
1048
|
+
"""
|
|
1049
|
+
_validate_filters(filters)
|
|
1050
|
+
|
|
1051
|
+
delete_sql = SQL("DELETE FROM {schema_name}.{table_name}").format(
|
|
1052
|
+
schema_name=Identifier(self.schema_name),
|
|
1053
|
+
table_name=Identifier(self.table_name),
|
|
1054
|
+
)
|
|
1055
|
+
|
|
1056
|
+
params = ()
|
|
1057
|
+
if filters:
|
|
1058
|
+
sql_where_clause, params = _convert_filters_to_where_clause_and_params(filters)
|
|
1059
|
+
delete_sql += sql_where_clause
|
|
1060
|
+
|
|
1061
|
+
self._ensure_db_setup()
|
|
1062
|
+
assert self._cursor is not None
|
|
1063
|
+
|
|
1064
|
+
try:
|
|
1065
|
+
self._execute_sql(
|
|
1066
|
+
cursor=self._cursor,
|
|
1067
|
+
sql_query=delete_sql,
|
|
1068
|
+
params=params,
|
|
1069
|
+
error_msg="Could not delete documents by filter from PgvectorDocumentStore",
|
|
1070
|
+
)
|
|
1071
|
+
deleted_count = self._cursor.rowcount
|
|
1072
|
+
logger.info(
|
|
1073
|
+
"Deleted {n_docs} documents from table '{schema}.{table}' using filters.",
|
|
1074
|
+
n_docs=deleted_count,
|
|
1075
|
+
schema=self.schema_name,
|
|
1076
|
+
table=self.table_name,
|
|
1077
|
+
)
|
|
1078
|
+
return deleted_count
|
|
1079
|
+
except Error as e:
|
|
1080
|
+
msg = f"Failed to delete documents by filter from PgvectorDocumentStore: {e!s}"
|
|
1081
|
+
raise DocumentStoreError(msg) from e
|
|
1082
|
+
|
|
1083
|
+
async def delete_by_filter_async(self, filters: dict[str, Any]) -> int:
|
|
1084
|
+
"""
|
|
1085
|
+
Asynchronously deletes all documents that match the provided filters.
|
|
1086
|
+
|
|
1087
|
+
:param filters: The filters to apply to select documents for deletion.
|
|
1088
|
+
For filter syntax, see [Haystack metadata filtering](https://docs.haystack.deepset.ai/docs/metadata-filtering)
|
|
1089
|
+
:returns: The number of documents deleted.
|
|
1090
|
+
"""
|
|
1091
|
+
_validate_filters(filters)
|
|
1092
|
+
|
|
1093
|
+
delete_sql = SQL("DELETE FROM {schema_name}.{table_name}").format(
|
|
1094
|
+
schema_name=Identifier(self.schema_name),
|
|
1095
|
+
table_name=Identifier(self.table_name),
|
|
1096
|
+
)
|
|
1097
|
+
|
|
1098
|
+
params = ()
|
|
1099
|
+
if filters:
|
|
1100
|
+
sql_where_clause, params = _convert_filters_to_where_clause_and_params(filters)
|
|
1101
|
+
delete_sql += sql_where_clause
|
|
1102
|
+
|
|
1103
|
+
await self._ensure_db_setup_async()
|
|
1104
|
+
assert self._async_cursor is not None
|
|
1105
|
+
|
|
1106
|
+
try:
|
|
1107
|
+
await self._execute_sql_async(
|
|
1108
|
+
cursor=self._async_cursor,
|
|
1109
|
+
sql_query=delete_sql,
|
|
1110
|
+
params=params,
|
|
1111
|
+
error_msg="Could not delete documents by filter from PgvectorDocumentStore",
|
|
1112
|
+
)
|
|
1113
|
+
deleted_count = self._async_cursor.rowcount
|
|
1114
|
+
logger.info(
|
|
1115
|
+
"Deleted {n_docs} documents from table '{schema}.{table}' using filters.",
|
|
1116
|
+
n_docs=deleted_count,
|
|
1117
|
+
schema=self.schema_name,
|
|
1118
|
+
table=self.table_name,
|
|
1119
|
+
)
|
|
1120
|
+
return deleted_count
|
|
1121
|
+
except Error as e:
|
|
1122
|
+
msg = f"Failed to delete documents by filter from PgvectorDocumentStore: {e!s}"
|
|
1123
|
+
raise DocumentStoreError(msg) from e
|
|
1124
|
+
|
|
1125
|
+
def update_by_filter(self, filters: dict[str, Any], meta: dict[str, Any]) -> int:
|
|
1126
|
+
"""
|
|
1127
|
+
Updates the metadata of all documents that match the provided filters.
|
|
1128
|
+
|
|
1129
|
+
:param filters: The filters to apply to select documents for updating.
|
|
1130
|
+
For filter syntax, see [Haystack metadata filtering](https://docs.haystack.deepset.ai/docs/metadata-filtering)
|
|
1131
|
+
:param meta: The metadata fields to update.
|
|
1132
|
+
:returns: The number of documents updated.
|
|
1133
|
+
"""
|
|
1134
|
+
_validate_filters(filters)
|
|
1135
|
+
|
|
1136
|
+
if not meta:
|
|
1137
|
+
msg = "meta must be a non-empty dictionary"
|
|
1138
|
+
raise ValueError(msg)
|
|
1139
|
+
|
|
1140
|
+
update_sql = SQL(
|
|
1141
|
+
"UPDATE {schema_name}.{table_name} SET meta = COALESCE(meta, '{{}}'::jsonb) || %s::jsonb"
|
|
1142
|
+
).format(
|
|
1143
|
+
schema_name=Identifier(self.schema_name),
|
|
1144
|
+
table_name=Identifier(self.table_name),
|
|
1145
|
+
)
|
|
1146
|
+
|
|
1147
|
+
params: tuple[Any, ...] = (Jsonb(meta),)
|
|
1148
|
+
if filters:
|
|
1149
|
+
sql_where_clause, where_params = _convert_filters_to_where_clause_and_params(filters)
|
|
1150
|
+
update_sql += sql_where_clause
|
|
1151
|
+
params = params + where_params
|
|
1152
|
+
|
|
1153
|
+
self._ensure_db_setup()
|
|
1154
|
+
assert self._cursor is not None
|
|
1155
|
+
|
|
1156
|
+
try:
|
|
1157
|
+
self._execute_sql(
|
|
1158
|
+
cursor=self._cursor,
|
|
1159
|
+
sql_query=update_sql,
|
|
1160
|
+
params=params,
|
|
1161
|
+
error_msg="Could not update documents by filter from PgvectorDocumentStore",
|
|
1162
|
+
)
|
|
1163
|
+
updated_count = self._cursor.rowcount
|
|
1164
|
+
logger.info(
|
|
1165
|
+
"Updated {n_docs} documents in table '{schema}.{table}' using filters.",
|
|
1166
|
+
n_docs=updated_count,
|
|
1167
|
+
schema=self.schema_name,
|
|
1168
|
+
table=self.table_name,
|
|
1169
|
+
)
|
|
1170
|
+
return updated_count
|
|
1171
|
+
except Error as e:
|
|
1172
|
+
msg = f"Failed to update documents by filter in PgvectorDocumentStore: {e!s}"
|
|
1173
|
+
raise DocumentStoreError(msg) from e
|
|
1174
|
+
|
|
1175
|
+
async def update_by_filter_async(self, filters: dict[str, Any], meta: dict[str, Any]) -> int:
|
|
1176
|
+
"""
|
|
1177
|
+
Asynchronously updates the metadata of all documents that match the provided filters.
|
|
1178
|
+
|
|
1179
|
+
:param filters: The filters to apply to select documents for updating.
|
|
1180
|
+
For filter syntax, see [Haystack metadata filtering](https://docs.haystack.deepset.ai/docs/metadata-filtering)
|
|
1181
|
+
:param meta: The metadata fields to update.
|
|
1182
|
+
:returns: The number of documents updated.
|
|
1183
|
+
"""
|
|
1184
|
+
_validate_filters(filters)
|
|
1185
|
+
|
|
1186
|
+
if not meta:
|
|
1187
|
+
msg = "meta must be a non-empty dictionary"
|
|
1188
|
+
raise ValueError(msg)
|
|
1189
|
+
|
|
1190
|
+
update_sql = SQL(
|
|
1191
|
+
"UPDATE {schema_name}.{table_name} SET meta = COALESCE(meta, '{{}}'::jsonb) || %s::jsonb"
|
|
1192
|
+
).format(
|
|
1193
|
+
schema_name=Identifier(self.schema_name),
|
|
1194
|
+
table_name=Identifier(self.table_name),
|
|
1195
|
+
)
|
|
1196
|
+
|
|
1197
|
+
params: tuple[Any, ...] = (Jsonb(meta),)
|
|
1198
|
+
if filters:
|
|
1199
|
+
sql_where_clause, where_params = _convert_filters_to_where_clause_and_params(filters)
|
|
1200
|
+
update_sql += sql_where_clause
|
|
1201
|
+
params = params + where_params
|
|
1202
|
+
|
|
1203
|
+
await self._ensure_db_setup_async()
|
|
1204
|
+
assert self._async_cursor is not None
|
|
1205
|
+
|
|
1206
|
+
try:
|
|
1207
|
+
await self._execute_sql_async(
|
|
1208
|
+
cursor=self._async_cursor,
|
|
1209
|
+
sql_query=update_sql,
|
|
1210
|
+
params=params,
|
|
1211
|
+
error_msg="Could not update documents by filter from PgvectorDocumentStore",
|
|
1212
|
+
)
|
|
1213
|
+
updated_count = self._async_cursor.rowcount
|
|
1214
|
+
logger.info(
|
|
1215
|
+
"Updated {n_docs} documents in table '{schema}.{table}' using filters.",
|
|
1216
|
+
n_docs=updated_count,
|
|
1217
|
+
schema=self.schema_name,
|
|
1218
|
+
table=self.table_name,
|
|
1219
|
+
)
|
|
1220
|
+
return updated_count
|
|
1221
|
+
except Error as e:
|
|
1222
|
+
msg = f"Failed to update documents by filter in PgvectorDocumentStore: {e!s}"
|
|
1223
|
+
raise DocumentStoreError(msg) from e
|
|
1224
|
+
|
|
1039
1225
|
def _build_keyword_retrieval_query(
|
|
1040
|
-
self, query: str, top_k: int, filters:
|
|
1226
|
+
self, query: str, top_k: int, filters: dict[str, Any] | None = None
|
|
1041
1227
|
) -> tuple[Composed, tuple]:
|
|
1042
1228
|
"""
|
|
1043
1229
|
Builds the SQL query and the where parameters for keyword retrieval.
|
|
@@ -1050,7 +1236,7 @@ class PgvectorDocumentStore:
|
|
|
1050
1236
|
)
|
|
1051
1237
|
|
|
1052
1238
|
where_params = ()
|
|
1053
|
-
sql_where_clause:
|
|
1239
|
+
sql_where_clause: Composed | SQL = SQL("")
|
|
1054
1240
|
if filters:
|
|
1055
1241
|
sql_where_clause, where_params = _convert_filters_to_where_clause_and_params(
|
|
1056
1242
|
filters=filters, operator="AND"
|
|
@@ -1066,7 +1252,7 @@ class PgvectorDocumentStore:
|
|
|
1066
1252
|
self,
|
|
1067
1253
|
query: str,
|
|
1068
1254
|
*,
|
|
1069
|
-
filters:
|
|
1255
|
+
filters: dict[str, Any] | None = None,
|
|
1070
1256
|
top_k: int = 10,
|
|
1071
1257
|
) -> list[Document]:
|
|
1072
1258
|
"""
|
|
@@ -1101,7 +1287,7 @@ class PgvectorDocumentStore:
|
|
|
1101
1287
|
self,
|
|
1102
1288
|
query: str,
|
|
1103
1289
|
*,
|
|
1104
|
-
filters:
|
|
1290
|
+
filters: dict[str, Any] | None = None,
|
|
1105
1291
|
top_k: int = 10,
|
|
1106
1292
|
) -> list[Document]:
|
|
1107
1293
|
"""
|
|
@@ -1129,9 +1315,9 @@ class PgvectorDocumentStore:
|
|
|
1129
1315
|
def _check_and_build_embedding_retrieval_query(
|
|
1130
1316
|
self,
|
|
1131
1317
|
query_embedding: list[float],
|
|
1132
|
-
vector_function:
|
|
1318
|
+
vector_function: Literal["cosine_similarity", "inner_product", "l2_distance"] | None,
|
|
1133
1319
|
top_k: int,
|
|
1134
|
-
filters:
|
|
1320
|
+
filters: dict[str, Any] | None = None,
|
|
1135
1321
|
) -> tuple[Composed, tuple]:
|
|
1136
1322
|
"""
|
|
1137
1323
|
Performs checks and builds the SQL query and the where parameters for embedding retrieval.
|
|
@@ -1171,7 +1357,7 @@ class PgvectorDocumentStore:
|
|
|
1171
1357
|
score=SQL(score_definition),
|
|
1172
1358
|
)
|
|
1173
1359
|
|
|
1174
|
-
sql_where_clause:
|
|
1360
|
+
sql_where_clause: Composed | SQL = SQL("")
|
|
1175
1361
|
params = ()
|
|
1176
1362
|
if filters:
|
|
1177
1363
|
sql_where_clause, params = _convert_filters_to_where_clause_and_params(filters)
|
|
@@ -1193,9 +1379,9 @@ class PgvectorDocumentStore:
|
|
|
1193
1379
|
self,
|
|
1194
1380
|
query_embedding: list[float],
|
|
1195
1381
|
*,
|
|
1196
|
-
filters:
|
|
1382
|
+
filters: dict[str, Any] | None = None,
|
|
1197
1383
|
top_k: int = 10,
|
|
1198
|
-
vector_function:
|
|
1384
|
+
vector_function: Literal["cosine_similarity", "inner_product", "l2_distance"] | None = None,
|
|
1199
1385
|
) -> list[Document]:
|
|
1200
1386
|
"""
|
|
1201
1387
|
Retrieves documents that are most similar to the query embedding using a vector similarity metric.
|
|
@@ -1227,9 +1413,9 @@ class PgvectorDocumentStore:
|
|
|
1227
1413
|
self,
|
|
1228
1414
|
query_embedding: list[float],
|
|
1229
1415
|
*,
|
|
1230
|
-
filters:
|
|
1416
|
+
filters: dict[str, Any] | None = None,
|
|
1231
1417
|
top_k: int = 10,
|
|
1232
|
-
vector_function:
|
|
1418
|
+
vector_function: Literal["cosine_similarity", "inner_product", "l2_distance"] | None = None,
|
|
1233
1419
|
) -> list[Document]:
|
|
1234
1420
|
"""
|
|
1235
1421
|
Asynchronously retrieves documents that are most similar to the query embedding using a
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
# SPDX-License-Identifier: Apache-2.0
|
|
4
4
|
from datetime import datetime
|
|
5
5
|
from itertools import chain
|
|
6
|
-
from typing import Any, Literal
|
|
6
|
+
from typing import Any, Literal
|
|
7
7
|
|
|
8
8
|
from haystack.errors import FilterError
|
|
9
9
|
from psycopg.sql import SQL, Composed
|
|
@@ -21,7 +21,7 @@ PYTHON_TYPES_TO_PG_TYPES = {
|
|
|
21
21
|
NO_VALUE = "no_value"
|
|
22
22
|
|
|
23
23
|
|
|
24
|
-
def _validate_filters(filters:
|
|
24
|
+
def _validate_filters(filters: dict[str, Any] | None = None) -> None:
|
|
25
25
|
"""
|
|
26
26
|
Validates the filters provided.
|
|
27
27
|
"""
|