pgvector-haystack 0.5.1__tar.gz → 1.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pgvector_haystack-0.5.1 → pgvector_haystack-1.1.0}/CHANGELOG.md +20 -0
- {pgvector_haystack-0.5.1 → pgvector_haystack-1.1.0}/PKG-INFO +2 -3
- {pgvector_haystack-0.5.1 → pgvector_haystack-1.1.0}/examples/embedding_retrieval.py +1 -0
- {pgvector_haystack-0.5.1 → pgvector_haystack-1.1.0}/examples/hybrid_retrieval.py +1 -0
- {pgvector_haystack-0.5.1 → pgvector_haystack-1.1.0}/pyproject.toml +14 -8
- {pgvector_haystack-0.5.1 → pgvector_haystack-1.1.0}/src/haystack_integrations/components/retrievers/pgvector/embedding_retriever.py +1 -0
- {pgvector_haystack-0.5.1 → pgvector_haystack-1.1.0}/src/haystack_integrations/components/retrievers/pgvector/keyword_retriever.py +1 -0
- {pgvector_haystack-0.5.1 → pgvector_haystack-1.1.0}/src/haystack_integrations/document_stores/pgvector/document_store.py +97 -29
- {pgvector_haystack-0.5.1 → pgvector_haystack-1.1.0}/tests/conftest.py +1 -0
- {pgvector_haystack-0.5.1 → pgvector_haystack-1.1.0}/tests/test_document_store.py +24 -1
- {pgvector_haystack-0.5.1 → pgvector_haystack-1.1.0}/tests/test_embedding_retrieval.py +2 -1
- {pgvector_haystack-0.5.1 → pgvector_haystack-1.1.0}/tests/test_filters.py +4 -3
- {pgvector_haystack-0.5.1 → pgvector_haystack-1.1.0}/tests/test_keyword_retrieval.py +1 -0
- {pgvector_haystack-0.5.1 → pgvector_haystack-1.1.0}/tests/test_retrievers.py +4 -1
- {pgvector_haystack-0.5.1 → pgvector_haystack-1.1.0}/.gitignore +0 -0
- {pgvector_haystack-0.5.1 → pgvector_haystack-1.1.0}/LICENSE.txt +0 -0
- {pgvector_haystack-0.5.1 → pgvector_haystack-1.1.0}/README.md +0 -0
- {pgvector_haystack-0.5.1 → pgvector_haystack-1.1.0}/pydoc/config.yml +0 -0
- {pgvector_haystack-0.5.1 → pgvector_haystack-1.1.0}/src/haystack_integrations/components/retrievers/pgvector/__init__.py +0 -0
- {pgvector_haystack-0.5.1 → pgvector_haystack-1.1.0}/src/haystack_integrations/document_stores/pgvector/__init__.py +0 -0
- {pgvector_haystack-0.5.1 → pgvector_haystack-1.1.0}/src/haystack_integrations/document_stores/pgvector/filters.py +0 -0
- {pgvector_haystack-0.5.1 → pgvector_haystack-1.1.0}/tests/__init__.py +0 -0
|
@@ -1,5 +1,25 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [integrations/pgvector-v1.0.0] - 2024-09-12
|
|
4
|
+
|
|
5
|
+
### 🚀 Features
|
|
6
|
+
|
|
7
|
+
- Add filter_policy to pgvector integration (#820)
|
|
8
|
+
|
|
9
|
+
### 🐛 Bug Fixes
|
|
10
|
+
|
|
11
|
+
- `PgVector` - Fallback to default filter policy when deserializing retrievers without the init parameter (#900)
|
|
12
|
+
|
|
13
|
+
### 🧪 Testing
|
|
14
|
+
|
|
15
|
+
- Do not retry tests in `hatch run test` command (#954)
|
|
16
|
+
|
|
17
|
+
### ⚙️ Miscellaneous Tasks
|
|
18
|
+
|
|
19
|
+
- Retry tests to reduce flakyness (#836)
|
|
20
|
+
- Update ruff invocation to include check parameter (#853)
|
|
21
|
+
- PgVector - remove legacy filter support (#1068)
|
|
22
|
+
|
|
3
23
|
## [integrations/pgvector-v0.4.0] - 2024-06-20
|
|
4
24
|
|
|
5
25
|
### 🚀 Features
|
|
@@ -1,13 +1,12 @@
|
|
|
1
1
|
Metadata-Version: 2.3
|
|
2
2
|
Name: pgvector-haystack
|
|
3
|
-
Version:
|
|
3
|
+
Version: 1.1.0
|
|
4
4
|
Summary: An integration of pgvector (vector search extension for Postgres) with Haystack
|
|
5
5
|
Project-URL: Source, https://github.com/deepset-ai/haystack-core-integrations
|
|
6
6
|
Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/blob/main/integrations/pgvector/README.md
|
|
7
7
|
Project-URL: Issues, https://github.com/deepset-ai/haystack-core-integrations/issues
|
|
8
8
|
Author-email: deepset GmbH <info@deepset.ai>
|
|
9
|
-
License
|
|
10
|
-
License-File: LICENSE.txt
|
|
9
|
+
License: Apache-2.0
|
|
11
10
|
Classifier: Development Status :: 4 - Beta
|
|
12
11
|
Classifier: License :: OSI Approved :: Apache Software License
|
|
13
12
|
Classifier: Programming Language :: Python
|
|
@@ -17,6 +17,7 @@ from haystack.components.converters import MarkdownToDocument
|
|
|
17
17
|
from haystack.components.embedders import SentenceTransformersDocumentEmbedder, SentenceTransformersTextEmbedder
|
|
18
18
|
from haystack.components.preprocessors import DocumentSplitter
|
|
19
19
|
from haystack.components.writers import DocumentWriter
|
|
20
|
+
|
|
20
21
|
from haystack_integrations.components.retrievers.pgvector import PgvectorEmbeddingRetriever
|
|
21
22
|
from haystack_integrations.document_stores.pgvector import PgvectorDocumentStore
|
|
22
23
|
|
|
@@ -18,6 +18,7 @@ from haystack.components.embedders import SentenceTransformersDocumentEmbedder,
|
|
|
18
18
|
from haystack.components.joiners import DocumentJoiner
|
|
19
19
|
from haystack.components.preprocessors import DocumentSplitter
|
|
20
20
|
from haystack.components.writers import DocumentWriter
|
|
21
|
+
|
|
21
22
|
from haystack_integrations.components.retrievers.pgvector import PgvectorEmbeddingRetriever, PgvectorKeywordRetriever
|
|
22
23
|
from haystack_integrations.document_stores.pgvector import PgvectorDocumentStore
|
|
23
24
|
|
|
@@ -42,6 +42,7 @@ root = "../.."
|
|
|
42
42
|
git_describe_command = 'git describe --tags --match="integrations/pgvector-v[0-9]*"'
|
|
43
43
|
|
|
44
44
|
[tool.hatch.envs.default]
|
|
45
|
+
installer = "uv"
|
|
45
46
|
dependencies = [
|
|
46
47
|
"coverage[toml]>=6.5",
|
|
47
48
|
"pytest",
|
|
@@ -50,22 +51,25 @@ dependencies = [
|
|
|
50
51
|
"haystack-pydoc-tools",
|
|
51
52
|
]
|
|
52
53
|
[tool.hatch.envs.default.scripts]
|
|
53
|
-
test = "pytest
|
|
54
|
-
test-cov = "coverage run -m pytest
|
|
54
|
+
test = "pytest {args:tests}"
|
|
55
|
+
test-cov = "coverage run -m pytest {args:tests}"
|
|
56
|
+
test-cov-retry = "test-cov --reruns 3 --reruns-delay 30 -x"
|
|
55
57
|
cov-report = ["- coverage combine", "coverage report"]
|
|
56
58
|
cov = ["test-cov", "cov-report"]
|
|
59
|
+
cov-retry = ["test-cov-retry", "cov-report"]
|
|
57
60
|
docs = ["pydoc-markdown pydoc/config.yml"]
|
|
58
61
|
|
|
59
62
|
[[tool.hatch.envs.all.matrix]]
|
|
60
63
|
python = ["3.8", "3.9", "3.10", "3.11", "3.12"]
|
|
61
64
|
|
|
62
65
|
[tool.hatch.envs.lint]
|
|
66
|
+
installer = "uv"
|
|
63
67
|
detached = true
|
|
64
|
-
dependencies = ["black>=23.1.0", "mypy>=1.0.0", "ruff>=0.0.243"]
|
|
68
|
+
dependencies = ["pip", "black>=23.1.0", "mypy>=1.0.0", "ruff>=0.0.243"]
|
|
65
69
|
[tool.hatch.envs.lint.scripts]
|
|
66
70
|
typing = "mypy --install-types --non-interactive --explicit-package-bases {args:src/ tests}"
|
|
67
71
|
style = ["ruff check {args:.}", "black --check --diff {args:.}"]
|
|
68
|
-
fmt = ["black {args:.}", "ruff --fix {args:.}", "style"]
|
|
72
|
+
fmt = ["black {args:.}", "ruff check --fix {args:.}", "style"]
|
|
69
73
|
all = ["style", "typing"]
|
|
70
74
|
|
|
71
75
|
[tool.black]
|
|
@@ -76,6 +80,8 @@ skip-string-normalization = true
|
|
|
76
80
|
[tool.ruff]
|
|
77
81
|
target-version = "py38"
|
|
78
82
|
line-length = 120
|
|
83
|
+
|
|
84
|
+
[tool.ruff.lint]
|
|
79
85
|
select = [
|
|
80
86
|
"A",
|
|
81
87
|
"ARG",
|
|
@@ -126,13 +132,13 @@ unfixable = [
|
|
|
126
132
|
"F401",
|
|
127
133
|
]
|
|
128
134
|
|
|
129
|
-
[tool.ruff.isort]
|
|
130
|
-
known-first-party = ["
|
|
135
|
+
[tool.ruff.lint.isort]
|
|
136
|
+
known-first-party = ["haystack_integrations"]
|
|
131
137
|
|
|
132
|
-
[tool.ruff.flake8-tidy-imports]
|
|
138
|
+
[tool.ruff.lint.flake8-tidy-imports]
|
|
133
139
|
ban-relative-imports = "parents"
|
|
134
140
|
|
|
135
|
-
[tool.ruff.per-file-ignores]
|
|
141
|
+
[tool.ruff.lint.per-file-ignores]
|
|
136
142
|
# Tests can use magic values, assertions, and relative imports
|
|
137
143
|
"tests/**/*" = ["PLR2004", "S101", "TID252"]
|
|
138
144
|
# examples can contain "print" commands
|
|
@@ -7,6 +7,7 @@ from haystack import component, default_from_dict, default_to_dict
|
|
|
7
7
|
from haystack.dataclasses import Document
|
|
8
8
|
from haystack.document_stores.types import FilterPolicy
|
|
9
9
|
from haystack.document_stores.types.filter_policy import apply_filter_policy
|
|
10
|
+
|
|
10
11
|
from haystack_integrations.document_stores.pgvector import PgvectorDocumentStore
|
|
11
12
|
from haystack_integrations.document_stores.pgvector.document_store import VALID_VECTOR_FUNCTIONS
|
|
12
13
|
|
|
@@ -7,6 +7,7 @@ from haystack import component, default_from_dict, default_to_dict
|
|
|
7
7
|
from haystack.dataclasses import Document
|
|
8
8
|
from haystack.document_stores.types import FilterPolicy
|
|
9
9
|
from haystack.document_stores.types.filter_policy import apply_filter_policy
|
|
10
|
+
|
|
10
11
|
from haystack_integrations.document_stores.pgvector import PgvectorDocumentStore
|
|
11
12
|
|
|
12
13
|
|
|
@@ -9,7 +9,6 @@ from haystack.dataclasses.document import ByteStream, Document
|
|
|
9
9
|
from haystack.document_stores.errors import DocumentStoreError, DuplicateDocumentError
|
|
10
10
|
from haystack.document_stores.types import DuplicatePolicy
|
|
11
11
|
from haystack.utils.auth import Secret, deserialize_secrets_inplace
|
|
12
|
-
from haystack.utils.filters import convert
|
|
13
12
|
from psycopg import Error, IntegrityError, connect
|
|
14
13
|
from psycopg.abc import Query
|
|
15
14
|
from psycopg.cursor import Cursor
|
|
@@ -25,7 +24,7 @@ from .filters import _convert_filters_to_where_clause_and_params
|
|
|
25
24
|
logger = logging.getLogger(__name__)
|
|
26
25
|
|
|
27
26
|
CREATE_TABLE_STATEMENT = """
|
|
28
|
-
CREATE TABLE IF NOT EXISTS {table_name} (
|
|
27
|
+
CREATE TABLE IF NOT EXISTS {schema_name}.{table_name} (
|
|
29
28
|
id VARCHAR(128) PRIMARY KEY,
|
|
30
29
|
embedding VECTOR({embedding_dimension}),
|
|
31
30
|
content TEXT,
|
|
@@ -37,7 +36,7 @@ meta JSONB)
|
|
|
37
36
|
"""
|
|
38
37
|
|
|
39
38
|
INSERT_STATEMENT = """
|
|
40
|
-
INSERT INTO {table_name}
|
|
39
|
+
INSERT INTO {schema_name}.{table_name}
|
|
41
40
|
(id, embedding, content, dataframe, blob_data, blob_meta, blob_mime_type, meta)
|
|
42
41
|
VALUES (%(id)s, %(embedding)s, %(content)s, %(dataframe)s, %(blob_data)s, %(blob_meta)s, %(blob_mime_type)s, %(meta)s)
|
|
43
42
|
"""
|
|
@@ -55,7 +54,7 @@ meta = EXCLUDED.meta
|
|
|
55
54
|
|
|
56
55
|
KEYWORD_QUERY = """
|
|
57
56
|
SELECT {table_name}.*, ts_rank_cd(to_tsvector({language}, content), query) AS score
|
|
58
|
-
FROM {table_name}, plainto_tsquery({language}, %s) query
|
|
57
|
+
FROM {schema_name}.{table_name}, plainto_tsquery({language}, %s) query
|
|
59
58
|
WHERE to_tsvector({language}, content) @@ query
|
|
60
59
|
"""
|
|
61
60
|
|
|
@@ -79,6 +78,7 @@ class PgvectorDocumentStore:
|
|
|
79
78
|
self,
|
|
80
79
|
*,
|
|
81
80
|
connection_string: Secret = Secret.from_env_var("PG_CONN_STR"),
|
|
81
|
+
schema_name: str = "public",
|
|
82
82
|
table_name: str = "haystack_documents",
|
|
83
83
|
language: str = "english",
|
|
84
84
|
embedding_dimension: int = 768,
|
|
@@ -97,7 +97,12 @@ class PgvectorDocumentStore:
|
|
|
97
97
|
A specific table to store Haystack documents will be created if it doesn't exist yet.
|
|
98
98
|
|
|
99
99
|
:param connection_string: The connection string to use to connect to the PostgreSQL database, defined as an
|
|
100
|
-
environment variable
|
|
100
|
+
environment variable. It can be provided in either URI format
|
|
101
|
+
e.g.: `PG_CONN_STR="postgresql://USER:PASSWORD@HOST:PORT/DB_NAME"`, or keyword/value format
|
|
102
|
+
e.g.: `PG_CONN_STR="host=HOST port=PORT dbname=DBNAME user=USER password=PASSWORD"`
|
|
103
|
+
See [PostgreSQL Documentation](https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING)
|
|
104
|
+
for more details.
|
|
105
|
+
:param schema_name: The name of the schema the table is created in. The schema must already exist.
|
|
101
106
|
:param table_name: The name of the table to use to store Haystack documents.
|
|
102
107
|
:param language: The language to be used to parse query and document content in keyword retrieval.
|
|
103
108
|
To see the list of available languages, you can run the following SQL query in your PostgreSQL database:
|
|
@@ -134,6 +139,7 @@ class PgvectorDocumentStore:
|
|
|
134
139
|
|
|
135
140
|
self.connection_string = connection_string
|
|
136
141
|
self.table_name = table_name
|
|
142
|
+
self.schema_name = schema_name
|
|
137
143
|
self.embedding_dimension = embedding_dimension
|
|
138
144
|
if vector_function not in VALID_VECTOR_FUNCTIONS:
|
|
139
145
|
msg = f"vector_function must be one of {VALID_VECTOR_FUNCTIONS}, but got {vector_function}"
|
|
@@ -150,29 +156,41 @@ class PgvectorDocumentStore:
|
|
|
150
156
|
self._connection = None
|
|
151
157
|
self._cursor = None
|
|
152
158
|
self._dict_cursor = None
|
|
159
|
+
self._table_initialized = False
|
|
153
160
|
|
|
154
161
|
@property
|
|
155
162
|
def cursor(self):
|
|
156
|
-
if self._cursor is None:
|
|
163
|
+
if self._cursor is None or not self._connection_is_valid(self._connection):
|
|
157
164
|
self._create_connection()
|
|
158
165
|
|
|
159
166
|
return self._cursor
|
|
160
167
|
|
|
161
168
|
@property
|
|
162
169
|
def dict_cursor(self):
|
|
163
|
-
if self._dict_cursor is None:
|
|
170
|
+
if self._dict_cursor is None or not self._connection_is_valid(self._connection):
|
|
164
171
|
self._create_connection()
|
|
165
172
|
|
|
166
173
|
return self._dict_cursor
|
|
167
174
|
|
|
168
175
|
@property
|
|
169
176
|
def connection(self):
|
|
170
|
-
if self._connection is None:
|
|
177
|
+
if self._connection is None or not self._connection_is_valid(self._connection):
|
|
171
178
|
self._create_connection()
|
|
172
179
|
|
|
173
180
|
return self._connection
|
|
174
181
|
|
|
175
182
|
def _create_connection(self):
|
|
183
|
+
"""
|
|
184
|
+
Internal method to create a connection to the PostgreSQL database.
|
|
185
|
+
"""
|
|
186
|
+
|
|
187
|
+
# close the connection if it already exists
|
|
188
|
+
if self._connection:
|
|
189
|
+
try:
|
|
190
|
+
self._connection.close()
|
|
191
|
+
except Error as e:
|
|
192
|
+
logger.debug("Failed to close connection: %s", str(e))
|
|
193
|
+
|
|
176
194
|
conn_str = self.connection_string.resolve_value() or ""
|
|
177
195
|
connection = connect(conn_str)
|
|
178
196
|
connection.autocommit = True
|
|
@@ -183,16 +201,40 @@ class PgvectorDocumentStore:
|
|
|
183
201
|
self._cursor = self._connection.cursor()
|
|
184
202
|
self._dict_cursor = self._connection.cursor(row_factory=dict_row)
|
|
185
203
|
|
|
186
|
-
|
|
204
|
+
if not self._table_initialized:
|
|
205
|
+
self._initialize_table()
|
|
206
|
+
|
|
207
|
+
return self._connection
|
|
208
|
+
|
|
209
|
+
def _initialize_table(self):
|
|
210
|
+
"""
|
|
211
|
+
Internal method to initialize the table.
|
|
212
|
+
"""
|
|
187
213
|
if self.recreate_table:
|
|
188
214
|
self.delete_table()
|
|
215
|
+
|
|
189
216
|
self._create_table_if_not_exists()
|
|
190
217
|
self._create_keyword_index_if_not_exists()
|
|
191
218
|
|
|
192
219
|
if self.search_strategy == "hnsw":
|
|
193
220
|
self._handle_hnsw()
|
|
194
221
|
|
|
195
|
-
|
|
222
|
+
self._table_initialized = True
|
|
223
|
+
|
|
224
|
+
@staticmethod
|
|
225
|
+
def _connection_is_valid(connection):
|
|
226
|
+
"""
|
|
227
|
+
Internal method to check if the connection is still valid.
|
|
228
|
+
"""
|
|
229
|
+
|
|
230
|
+
# implementation inspired to psycopg pool
|
|
231
|
+
# https://github.com/psycopg/psycopg/blob/d38cf7798b0c602ff43dac9f20bbab96237a9c38/psycopg_pool/psycopg_pool/pool.py#L528
|
|
232
|
+
|
|
233
|
+
try:
|
|
234
|
+
connection.execute("")
|
|
235
|
+
except Error:
|
|
236
|
+
return False
|
|
237
|
+
return True
|
|
196
238
|
|
|
197
239
|
def to_dict(self) -> Dict[str, Any]:
|
|
198
240
|
"""
|
|
@@ -204,6 +246,7 @@ class PgvectorDocumentStore:
|
|
|
204
246
|
return default_to_dict(
|
|
205
247
|
self,
|
|
206
248
|
connection_string=self.connection_string.to_dict(),
|
|
249
|
+
schema_name=self.schema_name,
|
|
207
250
|
table_name=self.table_name,
|
|
208
251
|
embedding_dimension=self.embedding_dimension,
|
|
209
252
|
vector_function=self.vector_function,
|
|
@@ -263,7 +306,9 @@ class PgvectorDocumentStore:
|
|
|
263
306
|
"""
|
|
264
307
|
|
|
265
308
|
create_sql = SQL(CREATE_TABLE_STATEMENT).format(
|
|
266
|
-
|
|
309
|
+
schema_name=Identifier(self.schema_name),
|
|
310
|
+
table_name=Identifier(self.table_name),
|
|
311
|
+
embedding_dimension=SQLLiteral(self.embedding_dimension),
|
|
267
312
|
)
|
|
268
313
|
|
|
269
314
|
self._execute_sql(create_sql, error_msg="Could not create table in PgvectorDocumentStore")
|
|
@@ -271,12 +316,18 @@ class PgvectorDocumentStore:
|
|
|
271
316
|
def delete_table(self):
|
|
272
317
|
"""
|
|
273
318
|
Deletes the table used to store Haystack documents.
|
|
274
|
-
The name of the
|
|
319
|
+
The name of the schema (`schema_name`) and the name of the table (`table_name`)
|
|
320
|
+
are defined when initializing the `PgvectorDocumentStore`.
|
|
275
321
|
"""
|
|
322
|
+
delete_sql = SQL("DROP TABLE IF EXISTS {schema_name}.{table_name}").format(
|
|
323
|
+
schema_name=Identifier(self.schema_name),
|
|
324
|
+
table_name=Identifier(self.table_name),
|
|
325
|
+
)
|
|
276
326
|
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
327
|
+
self._execute_sql(
|
|
328
|
+
delete_sql,
|
|
329
|
+
error_msg=f"Could not delete table {self.schema_name}.{self.table_name} in PgvectorDocumentStore",
|
|
330
|
+
)
|
|
280
331
|
|
|
281
332
|
def _create_keyword_index_if_not_exists(self):
|
|
282
333
|
"""
|
|
@@ -284,15 +335,16 @@ class PgvectorDocumentStore:
|
|
|
284
335
|
"""
|
|
285
336
|
index_exists = bool(
|
|
286
337
|
self._execute_sql(
|
|
287
|
-
"SELECT 1 FROM pg_indexes WHERE tablename = %s AND indexname = %s",
|
|
288
|
-
(self.table_name, self.keyword_index_name),
|
|
338
|
+
"SELECT 1 FROM pg_indexes WHERE schemaname = %s AND tablename = %s AND indexname = %s",
|
|
339
|
+
(self.schema_name, self.table_name, self.keyword_index_name),
|
|
289
340
|
"Could not check if keyword index exists",
|
|
290
341
|
).fetchone()
|
|
291
342
|
)
|
|
292
343
|
|
|
293
344
|
sql_create_index = SQL(
|
|
294
|
-
"CREATE INDEX {index_name} ON {table_name} USING GIN (to_tsvector({language}, content))"
|
|
345
|
+
"CREATE INDEX {index_name} ON {schema_name}.{table_name} USING GIN (to_tsvector({language}, content))"
|
|
295
346
|
).format(
|
|
347
|
+
schema_name=Identifier(self.schema_name),
|
|
296
348
|
index_name=Identifier(self.keyword_index_name),
|
|
297
349
|
table_name=Identifier(self.table_name),
|
|
298
350
|
language=SQLLiteral(self.language),
|
|
@@ -315,8 +367,8 @@ class PgvectorDocumentStore:
|
|
|
315
367
|
|
|
316
368
|
index_exists = bool(
|
|
317
369
|
self._execute_sql(
|
|
318
|
-
"SELECT 1 FROM pg_indexes WHERE tablename = %s AND indexname = %s",
|
|
319
|
-
(self.table_name, self.hnsw_index_name),
|
|
370
|
+
"SELECT 1 FROM pg_indexes WHERE schemaname = %s AND tablename = %s AND indexname = %s",
|
|
371
|
+
(self.schema_name, self.table_name, self.hnsw_index_name),
|
|
320
372
|
"Could not check if HNSW index exists",
|
|
321
373
|
).fetchone()
|
|
322
374
|
)
|
|
@@ -346,8 +398,13 @@ class PgvectorDocumentStore:
|
|
|
346
398
|
if key in HNSW_INDEX_CREATION_VALID_KWARGS
|
|
347
399
|
}
|
|
348
400
|
|
|
349
|
-
sql_create_index = SQL(
|
|
350
|
-
index_name
|
|
401
|
+
sql_create_index = SQL(
|
|
402
|
+
"CREATE INDEX {index_name} ON {schema_name}.{table_name} USING hnsw (embedding {ops}) "
|
|
403
|
+
).format(
|
|
404
|
+
schema_name=Identifier(self.schema_name),
|
|
405
|
+
index_name=Identifier(self.hnsw_index_name),
|
|
406
|
+
table_name=Identifier(self.table_name),
|
|
407
|
+
ops=SQL(pg_ops),
|
|
351
408
|
)
|
|
352
409
|
|
|
353
410
|
if actual_hnsw_index_creation_kwargs:
|
|
@@ -366,7 +423,9 @@ class PgvectorDocumentStore:
|
|
|
366
423
|
Returns how many documents are present in the document store.
|
|
367
424
|
"""
|
|
368
425
|
|
|
369
|
-
sql_count = SQL("SELECT COUNT(*) FROM {table_name}").format(
|
|
426
|
+
sql_count = SQL("SELECT COUNT(*) FROM {schema_name}.{table_name}").format(
|
|
427
|
+
schema_name=Identifier(self.schema_name), table_name=Identifier(self.table_name)
|
|
428
|
+
)
|
|
370
429
|
|
|
371
430
|
count = self._execute_sql(sql_count, error_msg="Could not count documents in PgvectorDocumentStore").fetchone()[
|
|
372
431
|
0
|
|
@@ -389,9 +448,12 @@ class PgvectorDocumentStore:
|
|
|
389
448
|
msg = "Filters must be a dictionary"
|
|
390
449
|
raise TypeError(msg)
|
|
391
450
|
if "operator" not in filters and "conditions" not in filters:
|
|
392
|
-
|
|
451
|
+
msg = "Invalid filter syntax. See https://docs.haystack.deepset.ai/docs/metadata-filtering for details."
|
|
452
|
+
raise ValueError(msg)
|
|
393
453
|
|
|
394
|
-
sql_filter = SQL("SELECT * FROM {table_name}").format(
|
|
454
|
+
sql_filter = SQL("SELECT * FROM {schema_name}.{table_name}").format(
|
|
455
|
+
schema_name=Identifier(self.schema_name), table_name=Identifier(self.table_name)
|
|
456
|
+
)
|
|
395
457
|
|
|
396
458
|
params = ()
|
|
397
459
|
if filters:
|
|
@@ -430,7 +492,9 @@ class PgvectorDocumentStore:
|
|
|
430
492
|
|
|
431
493
|
db_documents = self._from_haystack_to_pg_documents(documents)
|
|
432
494
|
|
|
433
|
-
sql_insert = SQL(INSERT_STATEMENT).format(
|
|
495
|
+
sql_insert = SQL(INSERT_STATEMENT).format(
|
|
496
|
+
schema_name=Identifier(self.schema_name), table_name=Identifier(self.table_name)
|
|
497
|
+
)
|
|
434
498
|
|
|
435
499
|
if policy == DuplicatePolicy.OVERWRITE:
|
|
436
500
|
sql_insert += SQL(UPDATE_STATEMENT)
|
|
@@ -539,8 +603,10 @@ class PgvectorDocumentStore:
|
|
|
539
603
|
|
|
540
604
|
document_ids_str = ", ".join(f"'{document_id}'" for document_id in document_ids)
|
|
541
605
|
|
|
542
|
-
delete_sql = SQL("DELETE FROM {table_name} WHERE id IN ({document_ids_str})").format(
|
|
543
|
-
|
|
606
|
+
delete_sql = SQL("DELETE FROM {schema_name}.{table_name} WHERE id IN ({document_ids_str})").format(
|
|
607
|
+
schema_name=Identifier(self.schema_name),
|
|
608
|
+
table_name=Identifier(self.table_name),
|
|
609
|
+
document_ids_str=SQL(document_ids_str),
|
|
544
610
|
)
|
|
545
611
|
|
|
546
612
|
self._execute_sql(delete_sql, error_msg="Could not delete documents from PgvectorDocumentStore")
|
|
@@ -566,6 +632,7 @@ class PgvectorDocumentStore:
|
|
|
566
632
|
raise ValueError(msg)
|
|
567
633
|
|
|
568
634
|
sql_select = SQL(KEYWORD_QUERY).format(
|
|
635
|
+
schema_name=Identifier(self.schema_name),
|
|
569
636
|
table_name=Identifier(self.table_name),
|
|
570
637
|
language=SQLLiteral(self.language),
|
|
571
638
|
query=SQLLiteral(query),
|
|
@@ -639,7 +706,8 @@ class PgvectorDocumentStore:
|
|
|
639
706
|
elif vector_function == "l2_distance":
|
|
640
707
|
score_definition = f"embedding <-> {query_embedding_for_postgres} AS score"
|
|
641
708
|
|
|
642
|
-
sql_select = SQL("SELECT *, {score} FROM {table_name}").format(
|
|
709
|
+
sql_select = SQL("SELECT *, {score} FROM {schema_name}.{table_name}").format(
|
|
710
|
+
schema_name=Identifier(self.schema_name),
|
|
643
711
|
table_name=Identifier(self.table_name),
|
|
644
712
|
score=SQL(score_definition),
|
|
645
713
|
)
|
|
@@ -11,9 +11,10 @@ from haystack.document_stores.errors import DuplicateDocumentError
|
|
|
11
11
|
from haystack.document_stores.types import DuplicatePolicy
|
|
12
12
|
from haystack.testing.document_store import CountDocumentsTest, DeleteDocumentsTest, WriteDocumentsTest
|
|
13
13
|
from haystack.utils import Secret
|
|
14
|
-
from haystack_integrations.document_stores.pgvector import PgvectorDocumentStore
|
|
15
14
|
from pandas import DataFrame
|
|
16
15
|
|
|
16
|
+
from haystack_integrations.document_stores.pgvector import PgvectorDocumentStore
|
|
17
|
+
|
|
17
18
|
|
|
18
19
|
@pytest.mark.integration
|
|
19
20
|
class TestDocumentStore(CountDocumentsTest, WriteDocumentsTest, DeleteDocumentsTest):
|
|
@@ -40,12 +41,32 @@ class TestDocumentStore(CountDocumentsTest, WriteDocumentsTest, DeleteDocumentsT
|
|
|
40
41
|
retrieved_docs = document_store.filter_documents()
|
|
41
42
|
assert retrieved_docs == docs
|
|
42
43
|
|
|
44
|
+
def test_connection_check_and_recreation(self, document_store: PgvectorDocumentStore):
|
|
45
|
+
original_connection = document_store.connection
|
|
46
|
+
|
|
47
|
+
with patch.object(PgvectorDocumentStore, "_connection_is_valid", return_value=False):
|
|
48
|
+
new_connection = document_store.connection
|
|
49
|
+
|
|
50
|
+
# verify that a new connection is created
|
|
51
|
+
assert new_connection is not original_connection
|
|
52
|
+
assert document_store._connection == new_connection
|
|
53
|
+
assert original_connection.closed
|
|
54
|
+
|
|
55
|
+
assert document_store._cursor is not None
|
|
56
|
+
assert document_store._dict_cursor is not None
|
|
57
|
+
|
|
58
|
+
# test with new connection
|
|
59
|
+
with patch.object(PgvectorDocumentStore, "_connection_is_valid", return_value=True):
|
|
60
|
+
same_connection = document_store.connection
|
|
61
|
+
assert same_connection is document_store._connection
|
|
62
|
+
|
|
43
63
|
|
|
44
64
|
@pytest.mark.usefixtures("patches_for_unit_tests")
|
|
45
65
|
def test_init(monkeypatch):
|
|
46
66
|
monkeypatch.setenv("PG_CONN_STR", "some_connection_string")
|
|
47
67
|
|
|
48
68
|
document_store = PgvectorDocumentStore(
|
|
69
|
+
schema_name="my_schema",
|
|
49
70
|
table_name="my_table",
|
|
50
71
|
embedding_dimension=512,
|
|
51
72
|
vector_function="l2_distance",
|
|
@@ -58,6 +79,7 @@ def test_init(monkeypatch):
|
|
|
58
79
|
keyword_index_name="my_keyword_index",
|
|
59
80
|
)
|
|
60
81
|
|
|
82
|
+
assert document_store.schema_name == "my_schema"
|
|
61
83
|
assert document_store.table_name == "my_table"
|
|
62
84
|
assert document_store.embedding_dimension == 512
|
|
63
85
|
assert document_store.vector_function == "l2_distance"
|
|
@@ -92,6 +114,7 @@ def test_to_dict(monkeypatch):
|
|
|
92
114
|
"init_parameters": {
|
|
93
115
|
"connection_string": {"env_vars": ["PG_CONN_STR"], "strict": True, "type": "env_var"},
|
|
94
116
|
"table_name": "my_table",
|
|
117
|
+
"schema_name": "public",
|
|
95
118
|
"embedding_dimension": 512,
|
|
96
119
|
"vector_function": "l2_distance",
|
|
97
120
|
"recreate_table": True,
|
|
@@ -6,9 +6,10 @@ from typing import List
|
|
|
6
6
|
|
|
7
7
|
import pytest
|
|
8
8
|
from haystack.dataclasses.document import Document
|
|
9
|
-
from haystack_integrations.document_stores.pgvector import PgvectorDocumentStore
|
|
10
9
|
from numpy.random import rand
|
|
11
10
|
|
|
11
|
+
from haystack_integrations.document_stores.pgvector import PgvectorDocumentStore
|
|
12
|
+
|
|
12
13
|
|
|
13
14
|
@pytest.mark.integration
|
|
14
15
|
class TestEmbeddingRetrieval:
|
|
@@ -3,6 +3,10 @@ from typing import List
|
|
|
3
3
|
import pytest
|
|
4
4
|
from haystack.dataclasses.document import Document
|
|
5
5
|
from haystack.testing.document_store import FilterDocumentsTest
|
|
6
|
+
from pandas import DataFrame
|
|
7
|
+
from psycopg.sql import SQL
|
|
8
|
+
from psycopg.types.json import Jsonb
|
|
9
|
+
|
|
6
10
|
from haystack_integrations.document_stores.pgvector.filters import (
|
|
7
11
|
FilterError,
|
|
8
12
|
_convert_filters_to_where_clause_and_params,
|
|
@@ -10,9 +14,6 @@ from haystack_integrations.document_stores.pgvector.filters import (
|
|
|
10
14
|
_parse_logical_condition,
|
|
11
15
|
_treat_meta_field,
|
|
12
16
|
)
|
|
13
|
-
from pandas import DataFrame
|
|
14
|
-
from psycopg.sql import SQL
|
|
15
|
-
from psycopg.types.json import Jsonb
|
|
16
17
|
|
|
17
18
|
|
|
18
19
|
@pytest.mark.integration
|
|
@@ -7,6 +7,7 @@ import pytest
|
|
|
7
7
|
from haystack.dataclasses import Document
|
|
8
8
|
from haystack.document_stores.types import FilterPolicy
|
|
9
9
|
from haystack.utils.auth import EnvVarSecret
|
|
10
|
+
|
|
10
11
|
from haystack_integrations.components.retrievers.pgvector import PgvectorEmbeddingRetriever, PgvectorKeywordRetriever
|
|
11
12
|
from haystack_integrations.document_stores.pgvector import PgvectorDocumentStore
|
|
12
13
|
|
|
@@ -49,6 +50,7 @@ class TestEmbeddingRetriever:
|
|
|
49
50
|
"type": "haystack_integrations.document_stores.pgvector.document_store.PgvectorDocumentStore",
|
|
50
51
|
"init_parameters": {
|
|
51
52
|
"connection_string": {"env_vars": ["PG_CONN_STR"], "strict": True, "type": "env_var"},
|
|
53
|
+
"schema_name": "public",
|
|
52
54
|
"table_name": "haystack",
|
|
53
55
|
"embedding_dimension": 768,
|
|
54
56
|
"vector_function": "cosine_similarity",
|
|
@@ -174,6 +176,7 @@ class TestKeywordRetriever:
|
|
|
174
176
|
"type": "haystack_integrations.document_stores.pgvector.document_store.PgvectorDocumentStore",
|
|
175
177
|
"init_parameters": {
|
|
176
178
|
"connection_string": {"env_vars": ["PG_CONN_STR"], "strict": True, "type": "env_var"},
|
|
179
|
+
"schema_name": "public",
|
|
177
180
|
"table_name": "haystack",
|
|
178
181
|
"embedding_dimension": 768,
|
|
179
182
|
"vector_function": "cosine_similarity",
|
|
@@ -311,7 +314,7 @@ class TestKeywordRetriever:
|
|
|
311
314
|
)
|
|
312
315
|
res = retriever.run(query="test query", filters={"field2": "value2"})
|
|
313
316
|
|
|
314
|
-
mock_store.
|
|
317
|
+
mock_store.mock_calls[0].assert_called_once_with(
|
|
315
318
|
query="test query", filters={"field": "value", "field2": "value2"}, top_k=10
|
|
316
319
|
)
|
|
317
320
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|