pgvector-haystack 3.0.0__py3-none-any.whl → 3.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -151,7 +151,8 @@ class PgvectorEmbeddingRetriever:
151
151
  :param top_k: Maximum number of Documents to return.
152
152
  :param vector_function: The similarity function to use when searching for similar embeddings.
153
153
 
154
- :returns: List of Documents similar to `query_embedding`.
154
+ :returns: A dictionary with the following keys:
155
+ - `documents`: List of `Document`s that are similar to `query_embedding`.
155
156
  """
156
157
  filters = apply_filter_policy(self.filter_policy, self.filters, filters)
157
158
  top_k = top_k or self.top_k
@@ -164,3 +165,36 @@ class PgvectorEmbeddingRetriever:
164
165
  vector_function=vector_function,
165
166
  )
166
167
  return {"documents": docs}
168
+
169
+ @component.output_types(documents=List[Document])
170
+ async def run_async(
171
+ self,
172
+ query_embedding: List[float],
173
+ filters: Optional[Dict[str, Any]] = None,
174
+ top_k: Optional[int] = None,
175
+ vector_function: Optional[Literal["cosine_similarity", "inner_product", "l2_distance"]] = None,
176
+ ):
177
+ """
178
+ Asynchronously retrieve documents from the `PgvectorDocumentStore`, based on their embeddings.
179
+
180
+ :param query_embedding: Embedding of the query.
181
+ :param filters: Filters applied to the retrieved Documents. The way runtime filters are applied depends on
182
+ the `filter_policy` chosen at retriever initialization. See init method docstring for more
183
+ details.
184
+ :param top_k: Maximum number of Documents to return.
185
+ :param vector_function: The similarity function to use when searching for similar embeddings.
186
+
187
+ :returns: A dictionary with the following keys:
188
+ - `documents`: List of `Document`s that are similar to `query_embedding`.
189
+ """
190
+ filters = apply_filter_policy(self.filter_policy, self.filters, filters)
191
+ top_k = top_k or self.top_k
192
+ vector_function = vector_function or self.vector_function
193
+
194
+ docs = await self.document_store._embedding_retrieval_async(
195
+ query_embedding=query_embedding,
196
+ filters=filters,
197
+ top_k=top_k,
198
+ vector_function=vector_function,
199
+ )
200
+ return {"documents": docs}
@@ -136,3 +136,33 @@ class PgvectorKeywordRetriever:
136
136
  top_k=top_k,
137
137
  )
138
138
  return {"documents": docs}
139
+
140
+ @component.output_types(documents=List[Document])
141
+ async def run_async(
142
+ self,
143
+ query: str,
144
+ filters: Optional[Dict[str, Any]] = None,
145
+ top_k: Optional[int] = None,
146
+ ):
147
+ """
148
+ Asynchronously retrieve documents from the `PgvectorDocumentStore`, based on keywords.
149
+
150
+ :param query: String to search in `Document`s' content.
151
+ :param filters: Filters applied to the retrieved Documents. The way runtime filters are applied depends on
152
+ the `filter_policy` chosen at retriever initialization. See init method docstring for more
153
+ details.
154
+ :param top_k: Maximum number of Documents to return.
155
+
156
+ :returns: A dictionary with the following keys:
157
+ - `documents`: List of `Document`s that match the query.
158
+ """
159
+ filters = apply_filter_policy(self.filter_policy, self.filters, filters)
160
+
161
+ top_k = top_k or self.top_k
162
+
163
+ docs = await self.document_store._keyword_retrieval_async(
164
+ query=query,
165
+ filters=filters,
166
+ top_k=top_k,
167
+ )
168
+ return {"documents": docs}
@@ -0,0 +1,65 @@
1
+ from typing import Any, Dict, List
2
+
3
+ from haystack import logging
4
+ from haystack.dataclasses import ByteStream, Document
5
+ from psycopg.types.json import Jsonb
6
+
7
+ logger = logging.getLogger(__name__)
8
+
9
+
10
+ def _from_haystack_to_pg_documents(documents: List[Document]) -> List[Dict[str, Any]]:
11
+ """
12
+ Internal method to convert a list of Haystack Documents to a list of dictionaries that can be used to insert
13
+ documents into the PgvectorDocumentStore.
14
+ """
15
+
16
+ db_documents = []
17
+ for document in documents:
18
+ db_document = {k: v for k, v in document.to_dict(flatten=False).items() if k not in ["score", "blob"]}
19
+
20
+ blob = document.blob
21
+ db_document["blob_data"] = blob.data if blob else None
22
+ db_document["blob_meta"] = Jsonb(blob.meta) if blob and blob.meta else None
23
+ db_document["blob_mime_type"] = blob.mime_type if blob and blob.mime_type else None
24
+ db_document["meta"] = Jsonb(db_document["meta"])
25
+
26
+ if "sparse_embedding" in db_document:
27
+ sparse_embedding = db_document.pop("sparse_embedding", None)
28
+ if sparse_embedding:
29
+ logger.warning(
30
+ "Document {doc_id} has the `sparse_embedding` field set,"
31
+ "but storing sparse embeddings in Pgvector is not currently supported."
32
+ "The `sparse_embedding` field will be ignored.",
33
+ doc_id=db_document["id"],
34
+ )
35
+
36
+ db_documents.append(db_document)
37
+
38
+ return db_documents
39
+
40
+
41
+ def _from_pg_to_haystack_documents(documents: List[Dict[str, Any]]) -> List[Document]:
42
+ """
43
+ Internal method to convert a list of dictionaries from pgvector to a list of Haystack Documents.
44
+ """
45
+
46
+ haystack_documents = []
47
+ for document in documents:
48
+ haystack_dict = dict(document)
49
+ blob_data = haystack_dict.pop("blob_data")
50
+ blob_meta = haystack_dict.pop("blob_meta")
51
+ blob_mime_type = haystack_dict.pop("blob_mime_type")
52
+
53
+ # convert the embedding to a list of floats
54
+ if document.get("embedding") is not None:
55
+ haystack_dict["embedding"] = document["embedding"].tolist()
56
+
57
+ haystack_document = Document.from_dict(haystack_dict)
58
+
59
+ if blob_data:
60
+ blob = ByteStream(data=blob_data, meta=blob_meta, mime_type=blob_mime_type)
61
+ haystack_document.blob = blob
62
+
63
+ haystack_documents.append(haystack_document)
64
+
65
+ return haystack_documents
@@ -4,26 +4,27 @@
4
4
  from typing import Any, Dict, List, Literal, Optional
5
5
 
6
6
  from haystack import default_from_dict, default_to_dict, logging
7
- from haystack.dataclasses.document import ByteStream, Document
7
+ from haystack.dataclasses.document import Document
8
8
  from haystack.document_stores.errors import DocumentStoreError, DuplicateDocumentError
9
9
  from haystack.document_stores.types import DuplicatePolicy
10
10
  from haystack.utils.auth import Secret, deserialize_secrets_inplace
11
- from psycopg import Error, IntegrityError, connect
11
+ from psycopg import AsyncConnection, Connection, Error, IntegrityError
12
12
  from psycopg.abc import Query
13
+ from psycopg.connection_async import AsyncCursor
13
14
  from psycopg.cursor import Cursor
14
15
  from psycopg.rows import dict_row
15
16
  from psycopg.sql import SQL, Identifier
16
17
  from psycopg.sql import Literal as SQLLiteral
17
- from psycopg.types.json import Jsonb
18
18
 
19
- from pgvector.psycopg import register_vector
19
+ from pgvector.psycopg import register_vector, register_vector_async
20
20
 
21
- from .filters import _convert_filters_to_where_clause_and_params
21
+ from .converters import _from_haystack_to_pg_documents, _from_pg_to_haystack_documents
22
+ from .filters import _convert_filters_to_where_clause_and_params, _validate_filters
22
23
 
23
24
  logger = logging.getLogger(__name__)
24
25
 
25
26
  CREATE_TABLE_STATEMENT = """
26
- CREATE TABLE IF NOT EXISTS {schema_name}.{table_name} (
27
+ CREATE TABLE {schema_name}.{table_name} (
27
28
  id VARCHAR(128) PRIMARY KEY,
28
29
  embedding VECTOR({embedding_dimension}),
29
30
  content TEXT,
@@ -156,90 +157,15 @@ class PgvectorDocumentStore:
156
157
  self.hnsw_ef_search = hnsw_ef_search
157
158
  self.keyword_index_name = keyword_index_name
158
159
  self.language = language
160
+
159
161
  self._connection = None
162
+ self._async_connection = None
160
163
  self._cursor = None
164
+ self._async_cursor = None
161
165
  self._dict_cursor = None
166
+ self._async_dict_cursor = None
162
167
  self._table_initialized = False
163
168
 
164
- @property
165
- def cursor(self):
166
- if self._cursor is None or not self._connection_is_valid(self._connection):
167
- self._create_connection()
168
-
169
- return self._cursor
170
-
171
- @property
172
- def dict_cursor(self):
173
- if self._dict_cursor is None or not self._connection_is_valid(self._connection):
174
- self._create_connection()
175
-
176
- return self._dict_cursor
177
-
178
- @property
179
- def connection(self):
180
- if self._connection is None or not self._connection_is_valid(self._connection):
181
- self._create_connection()
182
-
183
- return self._connection
184
-
185
- def _create_connection(self):
186
- """
187
- Internal method to create a connection to the PostgreSQL database.
188
- """
189
-
190
- # close the connection if it already exists
191
- if self._connection:
192
- try:
193
- self._connection.close()
194
- except Error as e:
195
- logger.debug("Failed to close connection: {e}", e=str(e))
196
-
197
- conn_str = self.connection_string.resolve_value() or ""
198
- connection = connect(conn_str)
199
- connection.autocommit = True
200
- if self.create_extension:
201
- connection.execute("CREATE EXTENSION IF NOT EXISTS vector")
202
- register_vector(connection) # Note: this must be called before creating the cursors.
203
-
204
- self._connection = connection
205
- self._cursor = self._connection.cursor()
206
- self._dict_cursor = self._connection.cursor(row_factory=dict_row)
207
-
208
- if not self._table_initialized:
209
- self._initialize_table()
210
-
211
- return self._connection
212
-
213
- def _initialize_table(self):
214
- """
215
- Internal method to initialize the table.
216
- """
217
- if self.recreate_table:
218
- self.delete_table()
219
-
220
- self._create_table_if_not_exists()
221
- self._create_keyword_index_if_not_exists()
222
-
223
- if self.search_strategy == "hnsw":
224
- self._handle_hnsw()
225
-
226
- self._table_initialized = True
227
-
228
- @staticmethod
229
- def _connection_is_valid(connection):
230
- """
231
- Internal method to check if the connection is still valid.
232
- """
233
-
234
- # implementation inspired to psycopg pool
235
- # https://github.com/psycopg/psycopg/blob/d38cf7798b0c602ff43dac9f20bbab96237a9c38/psycopg_pool/psycopg_pool/pool.py#L528
236
-
237
- try:
238
- connection.execute("")
239
- except Error:
240
- return False
241
- return True
242
-
243
169
  def to_dict(self) -> Dict[str, Any]:
244
170
  """
245
171
  Serializes the component to a dictionary.
@@ -278,6 +204,32 @@ class PgvectorDocumentStore:
278
204
  deserialize_secrets_inplace(data["init_parameters"], ["connection_string"])
279
205
  return default_from_dict(cls, data)
280
206
 
207
+ @staticmethod
208
+ def _connection_is_valid(connection):
209
+ """
210
+ Internal method to check if the connection is still valid.
211
+ """
212
+
213
+ # implementation inspired to psycopg pool
214
+ # https://github.com/psycopg/psycopg/blob/d38cf7798b0c602ff43dac9f20bbab96237a9c38/psycopg_pool/psycopg_pool/pool.py#L528
215
+
216
+ try:
217
+ connection.execute("")
218
+ except Error:
219
+ return False
220
+ return True
221
+
222
+ @staticmethod
223
+ async def _connection_is_valid_async(connection):
224
+ """
225
+ Internal method to check if the async connection is still valid.
226
+ """
227
+ try:
228
+ await connection.execute("")
229
+ except Error:
230
+ return False
231
+ return True
232
+
281
233
  def _execute_sql(
282
234
  self, sql_query: Query, params: Optional[tuple] = None, error_msg: str = "", cursor: Optional[Cursor] = None
283
235
  ):
@@ -287,11 +239,18 @@ class PgvectorDocumentStore:
287
239
  :param sql_query: The SQL query to execute.
288
240
  :param params: The parameters to pass to the SQL query.
289
241
  :param error_msg: The error message to use if an exception is raised.
290
- :param cursor: The cursor to use to execute the SQL query. Defaults to self.cursor.
242
+ :param cursor: The cursor to use to execute the SQL query. Defaults to self._cursor.
291
243
  """
292
244
 
293
245
  params = params or ()
294
- cursor = cursor or self.cursor
246
+ cursor = cursor or self._cursor
247
+
248
+ if cursor is None or self._connection is None:
249
+ message = (
250
+ "The cursor or the connection is not initialized. "
251
+ "Make sure to call _ensure_db_setup() before calling this method."
252
+ )
253
+ raise ValueError(message)
295
254
 
296
255
  sql_query_str = sql_query.as_string(cursor) if not isinstance(sql_query, str) else sql_query
297
256
  logger.debug("SQL query: {query}\nParameters: {parameters}", query=sql_query_str, parameters=params)
@@ -299,24 +258,215 @@ class PgvectorDocumentStore:
299
258
  try:
300
259
  result = cursor.execute(sql_query, params)
301
260
  except Error as e:
302
- self.connection.rollback()
261
+ self._connection.rollback()
303
262
  detailed_error_msg = f"{error_msg}.\nYou can find the SQL query and the parameters in the debug logs."
304
263
  raise DocumentStoreError(detailed_error_msg) from e
305
264
 
306
265
  return result
307
266
 
308
- def _create_table_if_not_exists(self):
267
+ async def _execute_sql_async(
268
+ self,
269
+ sql_query: Query,
270
+ params: Optional[tuple] = None,
271
+ error_msg: str = "",
272
+ cursor: Optional[AsyncCursor] = None,
273
+ ):
309
274
  """
310
- Creates the table to store Haystack documents if it doesn't exist yet.
275
+ Internal method to asynchronously execute SQL statements and handle exceptions.
276
+
277
+ :param sql_query: The SQL query to execute.
278
+ :param params: The parameters to pass to the SQL query.
279
+ :param error_msg: The error message to use if an exception is raised.
280
+ :param cursor: The cursor to use to execute the SQL query. Defaults to self._async_cursor.
311
281
  """
312
282
 
313
- create_sql = SQL(CREATE_TABLE_STATEMENT).format(
283
+ params = params or ()
284
+ cursor = cursor or self._async_cursor
285
+
286
+ if cursor is None or self._async_connection is None:
287
+ message = (
288
+ "The cursor or the connection is not initialized. "
289
+ "Make sure to call _ensure_db_setup_async() before calling this method."
290
+ )
291
+ raise ValueError(message)
292
+
293
+ sql_query_str = sql_query.as_string(cursor) if not isinstance(sql_query, str) else sql_query
294
+ logger.debug("SQL query: {query}\nParameters: {parameters}", query=sql_query_str, parameters=params)
295
+
296
+ try:
297
+ result = await cursor.execute(sql_query, params)
298
+ except Error as e:
299
+ await self._async_connection.rollback()
300
+ detailed_error_msg = f"{error_msg}.\nYou can find the SQL query and the parameters in the debug logs."
301
+ raise DocumentStoreError(detailed_error_msg) from e
302
+
303
+ return result
304
+
305
+ def _ensure_db_setup(self):
306
+ """
307
+ Ensures that the connection to the PostgreSQL database exists and is valid.
308
+ If not, connection and cursors are created.
309
+ If the table is not initialized, it will be set up.
310
+ """
311
+ if self._connection and self._cursor and self._dict_cursor and self._connection_is_valid(self._connection):
312
+ return
313
+
314
+ # close the connection if it already exists
315
+ if self._connection:
316
+ try:
317
+ self._connection.close()
318
+ except Error as e:
319
+ logger.debug("Failed to close connection: {e}", e=str(e))
320
+
321
+ conn_str = self.connection_string.resolve_value() or ""
322
+ connection = Connection.connect(conn_str)
323
+ connection.autocommit = True
324
+ if self.create_extension:
325
+ connection.execute("CREATE EXTENSION IF NOT EXISTS vector")
326
+ register_vector(connection) # Note: this must be called before creating the cursors.
327
+
328
+ self._connection = connection
329
+ self._cursor = self._connection.cursor()
330
+ self._dict_cursor = self._connection.cursor(row_factory=dict_row)
331
+
332
+ if not self._table_initialized:
333
+ self._initialize_table()
334
+
335
+ async def _ensure_db_setup_async(self):
336
+ """
337
+ Async internal method.
338
+ Ensures that the connection to the PostgreSQL database exists and is valid.
339
+ If not, connection and cursors are created.
340
+ If the table is not initialized, it will be set up.
341
+ """
342
+
343
+ if (
344
+ self._async_connection
345
+ and self._async_cursor
346
+ and self._async_dict_cursor
347
+ and await self._connection_is_valid_async(self._async_connection)
348
+ ):
349
+ return
350
+
351
+ # close the connection if it already exists
352
+ if self._async_connection:
353
+ await self._async_connection.close()
354
+
355
+ conn_str = self.connection_string.resolve_value() or ""
356
+ async_connection = await AsyncConnection.connect(conn_str)
357
+ await async_connection.set_autocommit(True)
358
+ if self.create_extension:
359
+ await async_connection.execute("CREATE EXTENSION IF NOT EXISTS vector")
360
+ await register_vector_async(async_connection) # Note: this must be called before creating the cursors.
361
+
362
+ self._async_connection = async_connection
363
+ self._async_cursor = self._async_connection.cursor()
364
+ self._async_dict_cursor = self._async_connection.cursor(row_factory=dict_row)
365
+
366
+ if not self._table_initialized:
367
+ await self._initialize_table_async()
368
+
369
+ def _build_table_creation_queries(self):
370
+ """
371
+ Internal method to build the SQL queries for table creation.
372
+ """
373
+
374
+ sql_table_exists = SQL("SELECT 1 FROM pg_tables WHERE schemaname = %s AND tablename = %s")
375
+ sql_create_table = SQL(CREATE_TABLE_STATEMENT).format(
314
376
  schema_name=Identifier(self.schema_name),
315
377
  table_name=Identifier(self.table_name),
316
378
  embedding_dimension=SQLLiteral(self.embedding_dimension),
317
379
  )
318
380
 
319
- self._execute_sql(create_sql, error_msg="Could not create table in PgvectorDocumentStore")
381
+ sql_keyword_index_exists = SQL(
382
+ "SELECT 1 FROM pg_indexes WHERE schemaname = %s AND tablename = %s AND indexname = %s"
383
+ )
384
+ sql_create_keyword_index = SQL(
385
+ "CREATE INDEX {index_name} ON {schema_name}.{table_name} USING GIN (to_tsvector({language}, content))"
386
+ ).format(
387
+ schema_name=Identifier(self.schema_name),
388
+ index_name=Identifier(self.keyword_index_name),
389
+ table_name=Identifier(self.table_name),
390
+ language=SQLLiteral(self.language),
391
+ )
392
+
393
+ return sql_table_exists, sql_create_table, sql_keyword_index_exists, sql_create_keyword_index
394
+
395
+ def _initialize_table(self):
396
+ """
397
+ Internal method to initialize the table.
398
+ """
399
+ if self.recreate_table:
400
+ self.delete_table()
401
+
402
+ sql_table_exists, sql_create_table, sql_keyword_index_exists, sql_create_keyword_index = (
403
+ self._build_table_creation_queries()
404
+ )
405
+
406
+ table_exists = bool(
407
+ self._execute_sql(
408
+ sql_table_exists, (self.schema_name, self.table_name), "Could not check if table exists"
409
+ ).fetchone()
410
+ )
411
+ if not table_exists:
412
+ self._execute_sql(sql_create_table, error_msg="Could not create table")
413
+
414
+ index_exists = bool(
415
+ self._execute_sql(
416
+ sql_keyword_index_exists,
417
+ (self.schema_name, self.table_name, self.keyword_index_name),
418
+ "Could not check if keyword index exists",
419
+ ).fetchone()
420
+ )
421
+ if not index_exists:
422
+ self._execute_sql(sql_create_keyword_index, error_msg="Could not create keyword index on table")
423
+
424
+ if self.search_strategy == "hnsw":
425
+ self._handle_hnsw()
426
+
427
+ self._table_initialized = True
428
+
429
+ async def _initialize_table_async(self):
430
+ """
431
+ Internal async method to initialize the table.
432
+ """
433
+ if self.recreate_table:
434
+ await self.delete_table_async()
435
+
436
+ sql_table_exists, sql_create_table, sql_keyword_index_exists, sql_create_keyword_index = (
437
+ self._build_table_creation_queries()
438
+ )
439
+
440
+ table_exists = bool(
441
+ await (
442
+ await self._execute_sql_async(
443
+ sql_table_exists,
444
+ (self.schema_name, self.table_name),
445
+ "Could not check if table exists",
446
+ self._async_cursor,
447
+ )
448
+ ).fetchone()
449
+ )
450
+ if not table_exists:
451
+ await self._execute_sql_async(sql_create_table, error_msg="Could not create table")
452
+
453
+ index_exists = bool(
454
+ await (
455
+ await self._execute_sql_async(
456
+ sql_keyword_index_exists,
457
+ (self.schema_name, self.table_name, self.keyword_index_name),
458
+ "Could not check if keyword index exists",
459
+ self._async_cursor,
460
+ )
461
+ ).fetchone()
462
+ )
463
+ if not index_exists:
464
+ await self._execute_sql_async(sql_create_keyword_index, error_msg="Could not create keyword index on table")
465
+
466
+ if self.search_strategy == "hnsw":
467
+ await self._handle_hnsw_async()
468
+
469
+ self._table_initialized = True
320
470
 
321
471
  def delete_table(self):
322
472
  """
@@ -334,29 +484,58 @@ class PgvectorDocumentStore:
334
484
  error_msg=f"Could not delete table {self.schema_name}.{self.table_name} in PgvectorDocumentStore",
335
485
  )
336
486
 
337
- def _create_keyword_index_if_not_exists(self):
487
+ async def delete_table_async(self):
338
488
  """
339
- Internal method to create the keyword index if not exists.
489
+ Async method to delete the table used to store Haystack documents.
340
490
  """
341
- index_exists = bool(
342
- self._execute_sql(
343
- "SELECT 1 FROM pg_indexes WHERE schemaname = %s AND tablename = %s AND indexname = %s",
344
- (self.schema_name, self.table_name, self.keyword_index_name),
345
- "Could not check if keyword index exists",
346
- ).fetchone()
491
+ delete_sql = SQL("DROP TABLE IF EXISTS {schema_name}.{table_name}").format(
492
+ schema_name=Identifier(self.schema_name),
493
+ table_name=Identifier(self.table_name),
347
494
  )
348
495
 
349
- sql_create_index = SQL(
350
- "CREATE INDEX {index_name} ON {schema_name}.{table_name} USING GIN (to_tsvector({language}, content))"
496
+ await self._execute_sql_async(
497
+ delete_sql,
498
+ error_msg=f"Could not delete table {self.schema_name}.{self.table_name} in PgvectorDocumentStore",
499
+ )
500
+
501
+ def _build_hnsw_queries(self):
502
+ """Common method to build all HNSW-related SQL queries"""
503
+
504
+ sql_set_hnsw_ef_search = (
505
+ SQL("SET hnsw.ef_search = {hnsw_ef_search}").format(hnsw_ef_search=SQLLiteral(self.hnsw_ef_search))
506
+ if self.hnsw_ef_search
507
+ else None
508
+ )
509
+
510
+ sql_hnsw_index_exists = SQL(
511
+ "SELECT 1 FROM pg_indexes WHERE schemaname = %s AND tablename = %s AND indexname = %s"
512
+ )
513
+
514
+ sql_drop_hnsw_index = SQL("DROP INDEX IF EXISTS {schema_name}.{index_name}").format(
515
+ schema_name=Identifier(self.schema_name),
516
+ index_name=Identifier(self.hnsw_index_name),
517
+ )
518
+
519
+ pg_ops = VECTOR_FUNCTION_TO_POSTGRESQL_OPS[self.vector_function]
520
+
521
+ sql_create_hnsw_index = SQL(
522
+ "CREATE INDEX {index_name} ON {schema_name}.{table_name} USING hnsw (embedding {ops})"
351
523
  ).format(
352
524
  schema_name=Identifier(self.schema_name),
353
- index_name=Identifier(self.keyword_index_name),
525
+ index_name=Identifier(self.hnsw_index_name),
354
526
  table_name=Identifier(self.table_name),
355
- language=SQLLiteral(self.language),
527
+ ops=SQL(pg_ops),
356
528
  )
357
529
 
358
- if not index_exists:
359
- self._execute_sql(sql_create_index, error_msg="Could not create keyword index on table")
530
+ # Add creation kwargs if any valid ones exist
531
+ valid_kwargs = {
532
+ k: v for k, v in self.hnsw_index_creation_kwargs.items() if k in HNSW_INDEX_CREATION_VALID_KWARGS
533
+ }
534
+ if valid_kwargs:
535
+ kwargs_str = ", ".join(f"{k} = {v}" for k, v in valid_kwargs.items())
536
+ sql_create_hnsw_index += SQL(" WITH ({})").format(SQL(kwargs_str))
537
+
538
+ return sql_set_hnsw_ef_search, sql_hnsw_index_exists, sql_drop_hnsw_index, sql_create_hnsw_index
360
539
 
361
540
  def _handle_hnsw(self):
362
541
  """
@@ -364,15 +543,16 @@ class PgvectorDocumentStore:
364
543
  It also sets the `hnsw.ef_search` parameter for queries if it is specified.
365
544
  """
366
545
 
546
+ sql_set_hnsw_ef_search, sql_hnsw_index_exists, sql_drop_hnsw_index, sql_create_hnsw_index = (
547
+ self._build_hnsw_queries()
548
+ )
549
+
367
550
  if self.hnsw_ef_search:
368
- sql_set_hnsw_ef_search = SQL("SET hnsw.ef_search = {hnsw_ef_search}").format(
369
- hnsw_ef_search=SQLLiteral(self.hnsw_ef_search)
370
- )
371
551
  self._execute_sql(sql_set_hnsw_ef_search, error_msg="Could not set hnsw.ef_search")
372
552
 
373
553
  index_exists = bool(
374
554
  self._execute_sql(
375
- "SELECT 1 FROM pg_indexes WHERE schemaname = %s AND tablename = %s AND indexname = %s",
555
+ sql_hnsw_index_exists,
376
556
  (self.schema_name, self.table_name, self.hnsw_index_name),
377
557
  "Could not check if HNSW index exists",
378
558
  ).fetchone()
@@ -386,59 +566,81 @@ class PgvectorDocumentStore:
386
566
  )
387
567
  return
388
568
 
389
- sql_drop_index = SQL("DROP INDEX IF EXISTS {schema_name}.{index_name}").format(
390
- schema_name=Identifier(self.schema_name), index_name=Identifier(self.hnsw_index_name)
391
- )
392
- self._execute_sql(sql_drop_index, error_msg="Could not drop HNSW index")
569
+ self._execute_sql(sql_drop_hnsw_index, error_msg="Could not drop HNSW index")
393
570
 
394
- self._create_hnsw_index()
571
+ self._execute_sql(sql_create_hnsw_index, error_msg="Could not create HNSW index")
395
572
 
396
- def _create_hnsw_index(self):
573
+ async def _handle_hnsw_async(self):
397
574
  """
398
- Internal method to create the HNSW index.
575
+ Internal async method to handle the HNSW index creation.
399
576
  """
400
577
 
401
- pg_ops = VECTOR_FUNCTION_TO_POSTGRESQL_OPS[self.vector_function]
402
- actual_hnsw_index_creation_kwargs = {
403
- key: value
404
- for key, value in self.hnsw_index_creation_kwargs.items()
405
- if key in HNSW_INDEX_CREATION_VALID_KWARGS
406
- }
578
+ sql_set_hnsw_ef_search, sql_hnsw_index_exists, sql_drop_hnsw_index, sql_create_hnsw_index = (
579
+ self._build_hnsw_queries()
580
+ )
407
581
 
408
- sql_create_index = SQL(
409
- "CREATE INDEX {index_name} ON {schema_name}.{table_name} USING hnsw (embedding {ops}) "
410
- ).format(
411
- schema_name=Identifier(self.schema_name),
412
- index_name=Identifier(self.hnsw_index_name),
413
- table_name=Identifier(self.table_name),
414
- ops=SQL(pg_ops),
582
+ if self.hnsw_ef_search:
583
+ await self._execute_sql_async(sql_set_hnsw_ef_search, error_msg="Could not set hnsw.ef_search")
584
+
585
+ index_exists = bool(
586
+ await (
587
+ await self._execute_sql_async(
588
+ sql_hnsw_index_exists,
589
+ (self.schema_name, self.table_name, self.hnsw_index_name),
590
+ "Could not check if HNSW index exists",
591
+ self._async_cursor,
592
+ )
593
+ ).fetchone()
415
594
  )
416
595
 
417
- if actual_hnsw_index_creation_kwargs:
418
- actual_hnsw_index_creation_kwargs_str = ", ".join(
419
- f"{key} = {value}" for key, value in actual_hnsw_index_creation_kwargs.items()
420
- )
421
- sql_add_creation_kwargs = SQL("WITH ({creation_kwargs_str})").format(
422
- creation_kwargs_str=SQL(actual_hnsw_index_creation_kwargs_str)
596
+ if index_exists and not self.hnsw_recreate_index_if_exists:
597
+ logger.warning(
598
+ "HNSW index already exists and won't be recreated. "
599
+ "If you want to recreate it, pass 'hnsw_recreate_index_if_exists=True' to the "
600
+ "Document Store constructor"
423
601
  )
424
- sql_create_index += sql_add_creation_kwargs
602
+ return
603
+
604
+ await self._execute_sql_async(sql_drop_hnsw_index, error_msg="Could not drop HNSW index")
425
605
 
426
- self._execute_sql(sql_create_index, error_msg="Could not create HNSW index")
606
+ await self._execute_sql_async(sql_create_hnsw_index, error_msg="Could not create HNSW index")
427
607
 
428
608
  def count_documents(self) -> int:
429
609
  """
430
610
  Returns how many documents are present in the document store.
431
- """
432
611
 
612
+ :returns:
613
+ Number of documents in the document store.
614
+ """
433
615
  sql_count = SQL("SELECT COUNT(*) FROM {schema_name}.{table_name}").format(
434
616
  schema_name=Identifier(self.schema_name), table_name=Identifier(self.table_name)
435
617
  )
436
618
 
619
+ self._ensure_db_setup()
437
620
  count = self._execute_sql(sql_count, error_msg="Could not count documents in PgvectorDocumentStore").fetchone()[
438
621
  0
439
622
  ]
440
623
  return count
441
624
 
625
+ async def count_documents_async(self) -> int:
626
+ """
627
+ Returns how many documents are present in the document store.
628
+
629
+ :returns:
630
+ Number of documents in the document store.
631
+ """
632
+ sql_count = SQL("SELECT COUNT(*) FROM {schema_name}.{table_name}").format(
633
+ schema_name=Identifier(self.schema_name), table_name=Identifier(self.table_name)
634
+ )
635
+
636
+ await self._ensure_db_setup_async()
637
+
638
+ result = await (
639
+ await self._execute_sql_async(sql_count, error_msg="Could not count documents in PgvectorDocumentStore")
640
+ ).fetchone()
641
+
642
+ return result[0]
643
+
442
644
  def filter_documents(self, filters: Optional[Dict[str, Any]] = None) -> List[Document]:
443
645
  """
444
646
  Returns the documents that match the filters provided.
@@ -448,15 +650,10 @@ class PgvectorDocumentStore:
448
650
 
449
651
  :param filters: The filters to apply to the document list.
450
652
  :raises TypeError: If `filters` is not a dictionary.
653
+ :raises ValueError: If `filters` syntax is invalid.
451
654
  :returns: A list of Documents that match the given filters.
452
655
  """
453
- if filters:
454
- if not isinstance(filters, dict):
455
- msg = "Filters must be a dictionary"
456
- raise TypeError(msg)
457
- if "operator" not in filters and "conditions" not in filters:
458
- msg = "Invalid filter syntax. See https://docs.haystack.deepset.ai/docs/metadata-filtering for details."
459
- raise ValueError(msg)
656
+ _validate_filters(filters)
460
657
 
461
658
  sql_filter = SQL("SELECT * FROM {schema_name}.{table_name}").format(
462
659
  schema_name=Identifier(self.schema_name), table_name=Identifier(self.table_name)
@@ -467,28 +664,83 @@ class PgvectorDocumentStore:
467
664
  sql_where_clause, params = _convert_filters_to_where_clause_and_params(filters)
468
665
  sql_filter += sql_where_clause
469
666
 
667
+ self._ensure_db_setup()
470
668
  result = self._execute_sql(
471
669
  sql_filter,
472
670
  params,
473
671
  error_msg="Could not filter documents from PgvectorDocumentStore.",
474
- cursor=self.dict_cursor,
672
+ cursor=self._dict_cursor,
475
673
  )
476
674
 
477
675
  records = result.fetchall()
478
- docs = self._from_pg_to_haystack_documents(records)
676
+ docs = _from_pg_to_haystack_documents(records)
479
677
  return docs
480
678
 
679
+ async def filter_documents_async(self, filters: Optional[Dict[str, Any]] = None) -> List[Document]:
680
+ """
681
+ Asynchronously returns the documents that match the filters provided.
682
+
683
+ For a detailed specification of the filters,
684
+ refer to the [documentation](https://docs.haystack.deepset.ai/v2.0/docs/metadata-filtering)
685
+
686
+ :param filters: The filters to apply to the document list.
687
+
688
+ :raises TypeError: If `filters` is not a dictionary.
689
+ :raises ValueError: If `filters` syntax is invalid.
690
+ :returns: A list of Documents that match the given filters.
691
+ """
692
+ _validate_filters(filters)
693
+
694
+ sql_filter = SQL("SELECT * FROM {schema_name}.{table_name}").format(
695
+ schema_name=Identifier(self.schema_name), table_name=Identifier(self.table_name)
696
+ )
697
+
698
+ params = ()
699
+ if filters:
700
+ sql_where_clause, params = _convert_filters_to_where_clause_and_params(filters)
701
+ sql_filter += sql_where_clause
702
+
703
+ await self._ensure_db_setup_async()
704
+ result = await self._execute_sql_async(
705
+ sql_filter,
706
+ params,
707
+ error_msg="Could not filter documents from PgvectorDocumentStore.",
708
+ cursor=self._async_dict_cursor,
709
+ )
710
+
711
+ records = await result.fetchall()
712
+ docs = _from_pg_to_haystack_documents(records)
713
+ return docs
714
+
715
+ def _build_insert_statement(self, policy: DuplicatePolicy):
716
+ """
717
+ Builds the SQL insert statement to write documents.
718
+ """
719
+ sql_insert = SQL(INSERT_STATEMENT).format(
720
+ schema_name=Identifier(self.schema_name), table_name=Identifier(self.table_name)
721
+ )
722
+
723
+ if policy == DuplicatePolicy.OVERWRITE:
724
+ sql_insert += SQL(UPDATE_STATEMENT)
725
+ elif policy == DuplicatePolicy.SKIP:
726
+ sql_insert += SQL("ON CONFLICT DO NOTHING")
727
+
728
+ sql_insert += SQL(" RETURNING id")
729
+
730
+ return sql_insert
731
+
481
732
  def write_documents(self, documents: List[Document], policy: DuplicatePolicy = DuplicatePolicy.NONE) -> int:
482
733
  """
483
734
  Writes documents to the document store.
484
735
 
485
736
  :param documents: A list of Documents to write to the document store.
486
737
  :param policy: The duplicate policy to use when writing documents.
738
+ :raises ValueError: If `documents` contains objects that are not of type `Document`.
487
739
  :raises DuplicateDocumentError: If a document with the same id already exists in the document store
488
740
  and the policy is set to `DuplicatePolicy.FAIL` (or not specified).
741
+ :raises DocumentStoreError: If the write operation fails for any other reason.
489
742
  :returns: The number of documents written to the document store.
490
743
  """
491
-
492
744
  if len(documents) > 0:
493
745
  if not isinstance(documents[0], Document):
494
746
  msg = "param 'documents' must contain a list of objects of type Document"
@@ -497,29 +749,24 @@ class PgvectorDocumentStore:
497
749
  if policy == DuplicatePolicy.NONE:
498
750
  policy = DuplicatePolicy.FAIL
499
751
 
500
- db_documents = self._from_haystack_to_pg_documents(documents)
752
+ db_documents = _from_haystack_to_pg_documents(documents)
501
753
 
502
- sql_insert = SQL(INSERT_STATEMENT).format(
503
- schema_name=Identifier(self.schema_name), table_name=Identifier(self.table_name)
504
- )
505
-
506
- if policy == DuplicatePolicy.OVERWRITE:
507
- sql_insert += SQL(UPDATE_STATEMENT)
508
- elif policy == DuplicatePolicy.SKIP:
509
- sql_insert += SQL("ON CONFLICT DO NOTHING")
754
+ sql_insert = self._build_insert_statement(policy)
510
755
 
511
- sql_insert += SQL(" RETURNING id")
756
+ self._ensure_db_setup()
757
+ assert self._cursor is not None # verified in _ensure_db_setup() but mypy doesn't know that
758
+ assert self._connection is not None # verified in _ensure_db_setup() but mypy doesn't know that
512
759
 
513
- sql_query_str = sql_insert.as_string(self.cursor) if not isinstance(sql_insert, str) else sql_insert
760
+ sql_query_str = sql_insert.as_string(self._cursor) if not isinstance(sql_insert, str) else sql_insert
514
761
  logger.debug("SQL query: {query}\nParameters: {parameters}", query=sql_query_str, parameters=db_documents)
515
762
 
516
763
  try:
517
- self.cursor.executemany(sql_insert, db_documents, returning=True)
764
+ self._cursor.executemany(sql_insert, db_documents, returning=True)
518
765
  except IntegrityError as ie:
519
- self.connection.rollback()
766
+ self._connection.rollback()
520
767
  raise DuplicateDocumentError from ie
521
768
  except Error as e:
522
- self.connection.rollback()
769
+ self._connection.rollback()
523
770
  error_msg = (
524
771
  "Could not write documents to PgvectorDocumentStore. \n"
525
772
  "You can find the SQL query and the parameters in the debug logs."
@@ -530,70 +777,64 @@ class PgvectorDocumentStore:
530
777
  # https://www.psycopg.org/psycopg3/docs/api/cursors.html#psycopg.Cursor.executemany
531
778
  written_docs = 0
532
779
  while True:
533
- if self.cursor.fetchone():
780
+ if self._cursor.fetchone():
534
781
  written_docs += 1
535
- if not self.cursor.nextset():
782
+ if not self._cursor.nextset():
536
783
  break
537
784
 
538
785
  return written_docs
539
786
 
540
- @staticmethod
541
- def _from_haystack_to_pg_documents(documents: List[Document]) -> List[Dict[str, Any]]:
542
- """
543
- Internal method to convert a list of Haystack Documents to a list of dictionaries that can be used to insert
544
- documents into the PgvectorDocumentStore.
787
+ async def write_documents_async(
788
+ self, documents: List[Document], policy: DuplicatePolicy = DuplicatePolicy.NONE
789
+ ) -> int:
545
790
  """
791
+ Asynchronously writes documents to the document store.
546
792
 
547
- db_documents = []
548
- for document in documents:
549
- db_document = {k: v for k, v in document.to_dict(flatten=False).items() if k not in ["score", "blob"]}
550
-
551
- blob = document.blob
552
- db_document["blob_data"] = blob.data if blob else None
553
- db_document["blob_meta"] = Jsonb(blob.meta) if blob and blob.meta else None
554
- db_document["blob_mime_type"] = blob.mime_type if blob and blob.mime_type else None
555
- db_document["meta"] = Jsonb(db_document["meta"])
793
+ :param documents: A list of Documents to write to the document store.
794
+ :param policy: The duplicate policy to use when writing documents.
795
+ :raises ValueError: If `documents` contains objects that are not of type `Document`.
796
+ :raises DuplicateDocumentError: If a document with the same id already exists in the document store
797
+ and the policy is set to `DuplicatePolicy.FAIL` (or not specified).
798
+ :raises DocumentStoreError: If the write operation fails for any other reason.
799
+ :returns: The number of documents written to the document store.
800
+ """
801
+ if len(documents) > 0:
802
+ if not isinstance(documents[0], Document):
803
+ msg = "param 'documents' must contain a list of objects of type Document"
804
+ raise ValueError(msg)
556
805
 
557
- if "sparse_embedding" in db_document:
558
- sparse_embedding = db_document.pop("sparse_embedding", None)
559
- if sparse_embedding:
560
- logger.warning(
561
- "Document {doc_id} has the `sparse_embedding` field set,"
562
- "but storing sparse embeddings in Pgvector is not currently supported."
563
- "The `sparse_embedding` field will be ignored.",
564
- doc_id=db_document["id"],
565
- )
806
+ if policy == DuplicatePolicy.NONE:
807
+ policy = DuplicatePolicy.FAIL
566
808
 
567
- db_documents.append(db_document)
809
+ db_documents = _from_haystack_to_pg_documents(documents)
568
810
 
569
- return db_documents
811
+ sql_insert = self._build_insert_statement(policy)
570
812
 
571
- @staticmethod
572
- def _from_pg_to_haystack_documents(documents: List[Dict[str, Any]]) -> List[Document]:
573
- """
574
- Internal method to convert a list of dictionaries from pgvector to a list of Haystack Documents.
575
- """
813
+ await self._ensure_db_setup_async()
814
+ assert self._async_cursor is not None # verified in _ensure_db_setup_async() but mypy doesn't know that
815
+ assert self._async_connection is not None # verified in _ensure_db_setup_async() but mypy doesn't know that
576
816
 
577
- haystack_documents = []
578
- for document in documents:
579
- haystack_dict = dict(document)
580
- blob_data = haystack_dict.pop("blob_data")
581
- blob_meta = haystack_dict.pop("blob_meta")
582
- blob_mime_type = haystack_dict.pop("blob_mime_type")
583
-
584
- # convert the embedding to a list of floats
585
- if document.get("embedding") is not None:
586
- haystack_dict["embedding"] = document["embedding"].tolist()
587
-
588
- haystack_document = Document.from_dict(haystack_dict)
817
+ sql_query_str = sql_insert.as_string(self._async_cursor) if not isinstance(sql_insert, str) else sql_insert
818
+ logger.debug("SQL query: {query}\nParameters: {parameters}", query=sql_query_str, parameters=db_documents)
589
819
 
590
- if blob_data:
591
- blob = ByteStream(data=blob_data, meta=blob_meta, mime_type=blob_mime_type)
592
- haystack_document.blob = blob
820
+ try:
821
+ await self._async_cursor.executemany(sql_insert, db_documents, returning=True)
822
+ except IntegrityError as ie:
823
+ await self._async_connection.rollback()
824
+ raise DuplicateDocumentError from ie
825
+ except Error as e:
826
+ await self._async_connection.rollback()
827
+ error_msg = (
828
+ "Could not write documents to PgvectorDocumentStore. \n"
829
+ "You can find the SQL query and the parameters in the debug logs."
830
+ )
831
+ raise DocumentStoreError(error_msg) from e
593
832
 
594
- haystack_documents.append(haystack_document)
833
+ written_docs = 0
834
+ async for _ in self._async_cursor:
835
+ written_docs += 1
595
836
 
596
- return haystack_documents
837
+ return written_docs
597
838
 
598
839
  def delete_documents(self, document_ids: List[str]) -> None:
599
840
  """
@@ -601,7 +842,6 @@ class PgvectorDocumentStore:
601
842
 
602
843
  :param document_ids: the document ids to delete
603
844
  """
604
-
605
845
  if not document_ids:
606
846
  return
607
847
 
@@ -613,28 +853,33 @@ class PgvectorDocumentStore:
613
853
  document_ids_str=SQL(document_ids_str),
614
854
  )
615
855
 
856
+ self._ensure_db_setup()
616
857
  self._execute_sql(delete_sql, error_msg="Could not delete documents from PgvectorDocumentStore")
617
858
 
618
- def _keyword_retrieval(
619
- self,
620
- query: str,
621
- *,
622
- filters: Optional[Dict[str, Any]] = None,
623
- top_k: int = 10,
624
- ) -> List[Document]:
859
+ async def delete_documents_async(self, document_ids: List[str]) -> None:
625
860
  """
626
- Retrieves documents that are most similar to the query using a full-text search.
861
+ Asynchronously deletes documents that match the provided `document_ids` from the document store.
627
862
 
628
- This method is not meant to be part of the public interface of
629
- `PgvectorDocumentStore` and it should not be called directly.
630
- `PgvectorKeywordRetriever` uses this method directly and is the public interface for it.
631
-
632
- :returns: List of Documents that are most similar to `query`
863
+ :param document_ids: the document ids to delete
633
864
  """
634
- if not query:
635
- msg = "query must be a non-empty string"
636
- raise ValueError(msg)
865
+ if not document_ids:
866
+ return
867
+
868
+ document_ids_str = ", ".join(f"'{document_id}'" for document_id in document_ids)
637
869
 
870
+ delete_sql = SQL("DELETE FROM {schema_name}.{table_name} WHERE id IN ({document_ids_str})").format(
871
+ schema_name=Identifier(self.schema_name),
872
+ table_name=Identifier(self.table_name),
873
+ document_ids_str=SQL(document_ids_str),
874
+ )
875
+
876
+ await self._ensure_db_setup_async()
877
+ await self._execute_sql_async(delete_sql, error_msg="Could not delete documents from PgvectorDocumentStore")
878
+
879
+ def _build_keyword_retrieval_query(self, query: str, top_k: int, filters: Optional[Dict[str, Any]] = None):
880
+ """
881
+ Builds the SQL query and the where parameters for keyword retrieval.
882
+ """
638
883
  sql_select = SQL(KEYWORD_QUERY).format(
639
884
  schema_name=Identifier(self.schema_name),
640
885
  table_name=Identifier(self.table_name),
@@ -653,33 +898,79 @@ class PgvectorDocumentStore:
653
898
 
654
899
  sql_query = sql_select + sql_where_clause + sql_sort
655
900
 
901
+ return sql_query, where_params
902
+
903
+ def _keyword_retrieval(
904
+ self,
905
+ query: str,
906
+ *,
907
+ filters: Optional[Dict[str, Any]] = None,
908
+ top_k: int = 10,
909
+ ) -> List[Document]:
910
+ """
911
+ Retrieves documents that are most similar to the query using a full-text search.
912
+
913
+ This method is not meant to be part of the public interface of
914
+ `PgvectorDocumentStore` and it should not be called directly.
915
+ `PgvectorKeywordRetriever` uses this method directly and is the public interface for it.
916
+
917
+ :returns: List of Documents that are most similar to `query`
918
+ """
919
+ if not query:
920
+ msg = "query must be a non-empty string"
921
+ raise ValueError(msg)
922
+
923
+ sql_query, where_params = self._build_keyword_retrieval_query(query=query, top_k=top_k, filters=filters)
924
+
925
+ self._ensure_db_setup()
656
926
  result = self._execute_sql(
657
927
  sql_query,
658
928
  (query, *where_params),
659
929
  error_msg="Could not retrieve documents from PgvectorDocumentStore.",
660
- cursor=self.dict_cursor,
930
+ cursor=self._dict_cursor,
661
931
  )
662
932
 
663
933
  records = result.fetchall()
664
- docs = self._from_pg_to_haystack_documents(records)
934
+ docs = _from_pg_to_haystack_documents(records)
665
935
  return docs
666
936
 
667
- def _embedding_retrieval(
937
+ async def _keyword_retrieval_async(
668
938
  self,
669
- query_embedding: List[float],
939
+ query: str,
670
940
  *,
671
941
  filters: Optional[Dict[str, Any]] = None,
672
942
  top_k: int = 10,
673
- vector_function: Optional[Literal["cosine_similarity", "inner_product", "l2_distance"]] = None,
674
943
  ) -> List[Document]:
675
944
  """
676
- Retrieves documents that are most similar to the query embedding using a vector similarity metric.
945
+ Retrieves documents that are most similar to the query using a full-text search asynchronously.
946
+ """
947
+ if not query:
948
+ msg = "query must be a non-empty string"
949
+ raise ValueError(msg)
677
950
 
678
- This method is not meant to be part of the public interface of
679
- `PgvectorDocumentStore` and it should not be called directly.
680
- `PgvectorEmbeddingRetriever` uses this method directly and is the public interface for it.
951
+ sql_query, where_params = self._build_keyword_retrieval_query(query=query, top_k=top_k, filters=filters)
681
952
 
682
- :returns: List of Documents that are most similar to `query_embedding`
953
+ await self._ensure_db_setup_async()
954
+ result = await self._execute_sql_async(
955
+ sql_query,
956
+ (query, *where_params),
957
+ error_msg="Could not retrieve documents from PgvectorDocumentStore.",
958
+ cursor=self._async_dict_cursor,
959
+ )
960
+
961
+ records = await result.fetchall()
962
+ docs = _from_pg_to_haystack_documents(records)
963
+ return docs
964
+
965
+ def _check_and_build_embedding_retrieval_query(
966
+ self,
967
+ query_embedding: List[float],
968
+ vector_function: Optional[Literal["cosine_similarity", "inner_product", "l2_distance"]],
969
+ top_k: int,
970
+ filters: Optional[Dict[str, Any]] = None,
971
+ ):
972
+ """
973
+ Performs checks and builds the SQL query and the where parameters for embedding retrieval.
683
974
  """
684
975
 
685
976
  if not query_embedding:
@@ -732,13 +1023,66 @@ class PgvectorDocumentStore:
732
1023
 
733
1024
  sql_query = sql_select + sql_where_clause + sql_sort
734
1025
 
1026
+ return sql_query, params
1027
+
1028
+ def _embedding_retrieval(
1029
+ self,
1030
+ query_embedding: List[float],
1031
+ *,
1032
+ filters: Optional[Dict[str, Any]] = None,
1033
+ top_k: int = 10,
1034
+ vector_function: Optional[Literal["cosine_similarity", "inner_product", "l2_distance"]] = None,
1035
+ ) -> List[Document]:
1036
+ """
1037
+ Retrieves documents that are most similar to the query embedding using a vector similarity metric.
1038
+
1039
+ This method is not meant to be part of the public interface of
1040
+ `PgvectorDocumentStore` and it should not be called directly.
1041
+ `PgvectorEmbeddingRetriever` uses this method directly and is the public interface for it.
1042
+
1043
+ :returns: List of Documents that are most similar to `query_embedding`
1044
+ """
1045
+
1046
+ sql_query, params = self._check_and_build_embedding_retrieval_query(
1047
+ query_embedding=query_embedding, vector_function=vector_function, top_k=top_k, filters=filters
1048
+ )
1049
+ self._ensure_db_setup()
735
1050
  result = self._execute_sql(
736
1051
  sql_query,
737
1052
  params,
738
1053
  error_msg="Could not retrieve documents from PgvectorDocumentStore.",
739
- cursor=self.dict_cursor,
1054
+ cursor=self._dict_cursor,
740
1055
  )
741
1056
 
742
1057
  records = result.fetchall()
743
- docs = self._from_pg_to_haystack_documents(records)
1058
+ docs = _from_pg_to_haystack_documents(records)
1059
+ return docs
1060
+
1061
+ async def _embedding_retrieval_async(
1062
+ self,
1063
+ query_embedding: List[float],
1064
+ *,
1065
+ filters: Optional[Dict[str, Any]] = None,
1066
+ top_k: int = 10,
1067
+ vector_function: Optional[Literal["cosine_similarity", "inner_product", "l2_distance"]] = None,
1068
+ ) -> List[Document]:
1069
+ """
1070
+ Asynchronously retrieves documents that are most similar to the query embedding using a
1071
+ vector similarity metric.
1072
+ """
1073
+
1074
+ sql_query, params = self._check_and_build_embedding_retrieval_query(
1075
+ query_embedding=query_embedding, vector_function=vector_function, top_k=top_k, filters=filters
1076
+ )
1077
+
1078
+ await self._ensure_db_setup_async()
1079
+ result = await self._execute_sql_async(
1080
+ sql_query,
1081
+ params,
1082
+ error_msg="Could not retrieve documents from PgvectorDocumentStore.",
1083
+ cursor=self._async_dict_cursor,
1084
+ )
1085
+
1086
+ records = await result.fetchall()
1087
+ docs = _from_pg_to_haystack_documents(records)
744
1088
  return docs
@@ -3,7 +3,7 @@
3
3
  # SPDX-License-Identifier: Apache-2.0
4
4
  from datetime import datetime
5
5
  from itertools import chain
6
- from typing import Any, Dict, List, Literal, Tuple
6
+ from typing import Any, Dict, List, Literal, Optional, Tuple
7
7
 
8
8
  from haystack.errors import FilterError
9
9
  from psycopg.sql import SQL
@@ -21,6 +21,19 @@ PYTHON_TYPES_TO_PG_TYPES = {
21
21
  NO_VALUE = "no_value"
22
22
 
23
23
 
24
+ def _validate_filters(filters: Optional[Dict[str, Any]] = None):
25
+ """
26
+ Validates the filters provided.
27
+ """
28
+ if filters:
29
+ if not isinstance(filters, dict):
30
+ msg = "Filters must be a dictionary"
31
+ raise TypeError(msg)
32
+ if "operator" not in filters and "conditions" not in filters:
33
+ msg = "Invalid filter syntax. See https://docs.haystack.deepset.ai/docs/metadata-filtering for details."
34
+ raise ValueError(msg)
35
+
36
+
24
37
  def _convert_filters_to_where_clause_and_params(
25
38
  filters: Dict[str, Any], operator: Literal["WHERE", "AND"] = "WHERE"
26
39
  ) -> Tuple[SQL, Tuple]:
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: pgvector-haystack
3
- Version: 3.0.0
3
+ Version: 3.1.0
4
4
  Summary: An integration of pgvector (vector search extension for Postgres) with Haystack
5
5
  Project-URL: Source, https://github.com/deepset-ai/haystack-core-integrations
6
6
  Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/blob/main/integrations/pgvector/README.md
@@ -15,6 +15,7 @@ Classifier: Programming Language :: Python :: 3.9
15
15
  Classifier: Programming Language :: Python :: 3.10
16
16
  Classifier: Programming Language :: Python :: 3.11
17
17
  Classifier: Programming Language :: Python :: 3.12
18
+ Classifier: Programming Language :: Python :: 3.13
18
19
  Classifier: Programming Language :: Python :: Implementation :: CPython
19
20
  Classifier: Programming Language :: Python :: Implementation :: PyPy
20
21
  Requires-Python: >=3.9
@@ -0,0 +1,11 @@
1
+ haystack_integrations/components/retrievers/pgvector/__init__.py,sha256=n4q4pUApJ9DZakJ_JL-egwOzlefdnI_ZxjU1Lu44Sqo,296
2
+ haystack_integrations/components/retrievers/pgvector/embedding_retriever.py,sha256=uK-ak1ZASjprix-5wgKoqP8rLnQfpQzExw_6X8fXUxg,9131
3
+ haystack_integrations/components/retrievers/pgvector/keyword_retriever.py,sha256=5NECAn5qlLqcFTvRf_c-x1crdqdgTCeLHsmCxtD3pHk,6784
4
+ haystack_integrations/document_stores/pgvector/__init__.py,sha256=Ge-7ggarfr3x8UqNdFHUuFGiu52u49sOVe3l16WEwZU,197
5
+ haystack_integrations/document_stores/pgvector/converters.py,sha256=ECNF04RcStH_xpjtY125xB00co0wN8OWNI1fFiv0fUA,2448
6
+ haystack_integrations/document_stores/pgvector/document_store.py,sha256=2Mphtj5K7yRVroEy_esvkxhzTN90glgl_WeAyWIrI5Q,44981
7
+ haystack_integrations/document_stores/pgvector/filters.py,sha256=1xfU-DJcqFUFQtBezZCA8UsYFN0D9znXtsick1KANoU,8998
8
+ pgvector_haystack-3.1.0.dist-info/METADATA,sha256=fhF5qTdD0UkVTtLRjgwVJui26CzrqASA6UNdhiwEhMY,2235
9
+ pgvector_haystack-3.1.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
10
+ pgvector_haystack-3.1.0.dist-info/licenses/LICENSE.txt,sha256=B05uMshqTA74s-0ltyHKI6yoPfJ3zYgQbvcXfDVGFf8,10280
11
+ pgvector_haystack-3.1.0.dist-info/RECORD,,
@@ -1,10 +0,0 @@
1
- haystack_integrations/components/retrievers/pgvector/__init__.py,sha256=n4q4pUApJ9DZakJ_JL-egwOzlefdnI_ZxjU1Lu44Sqo,296
2
- haystack_integrations/components/retrievers/pgvector/embedding_retriever.py,sha256=IEIdgKy_IWgmL5Qy_Hqkwa2mq2JxnQ_JOdjm38JPxmE,7530
3
- haystack_integrations/components/retrievers/pgvector/keyword_retriever.py,sha256=SYHqjTkDczobYZ2qq0z8yS9p_g8aINnzwA5AN_nKdHI,5632
4
- haystack_integrations/document_stores/pgvector/__init__.py,sha256=Ge-7ggarfr3x8UqNdFHUuFGiu52u49sOVe3l16WEwZU,197
5
- haystack_integrations/document_stores/pgvector/document_store.py,sha256=bs1MCwiQgoTw8Wrad8DaRRgIQokk8_t1fLmzxdufWS4,31267
6
- haystack_integrations/document_stores/pgvector/filters.py,sha256=M-_wmM1iKS1QdKhnRk27N8t2W1st2YZWAwVSFUyyTtk,8508
7
- pgvector_haystack-3.0.0.dist-info/METADATA,sha256=1rd0uO91j_NxG5hmxXIhn7RFPYJsDgO9w7XBqKMMw-Q,2184
8
- pgvector_haystack-3.0.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
9
- pgvector_haystack-3.0.0.dist-info/licenses/LICENSE.txt,sha256=B05uMshqTA74s-0ltyHKI6yoPfJ3zYgQbvcXfDVGFf8,10280
10
- pgvector_haystack-3.0.0.dist-info/RECORD,,