pgvector-haystack 3.0.1__py3-none-any.whl → 3.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -151,7 +151,8 @@ class PgvectorEmbeddingRetriever:
151
151
  :param top_k: Maximum number of Documents to return.
152
152
  :param vector_function: The similarity function to use when searching for similar embeddings.
153
153
 
154
- :returns: List of Documents similar to `query_embedding`.
154
+ :returns: A dictionary with the following keys:
155
+ - `documents`: List of `Document`s that are similar to `query_embedding`.
155
156
  """
156
157
  filters = apply_filter_policy(self.filter_policy, self.filters, filters)
157
158
  top_k = top_k or self.top_k
@@ -164,3 +165,36 @@ class PgvectorEmbeddingRetriever:
164
165
  vector_function=vector_function,
165
166
  )
166
167
  return {"documents": docs}
168
+
169
+ @component.output_types(documents=List[Document])
170
+ async def run_async(
171
+ self,
172
+ query_embedding: List[float],
173
+ filters: Optional[Dict[str, Any]] = None,
174
+ top_k: Optional[int] = None,
175
+ vector_function: Optional[Literal["cosine_similarity", "inner_product", "l2_distance"]] = None,
176
+ ):
177
+ """
178
+ Asynchronously retrieve documents from the `PgvectorDocumentStore`, based on their embeddings.
179
+
180
+ :param query_embedding: Embedding of the query.
181
+ :param filters: Filters applied to the retrieved Documents. The way runtime filters are applied depends on
182
+ the `filter_policy` chosen at retriever initialization. See init method docstring for more
183
+ details.
184
+ :param top_k: Maximum number of Documents to return.
185
+ :param vector_function: The similarity function to use when searching for similar embeddings.
186
+
187
+ :returns: A dictionary with the following keys:
188
+ - `documents`: List of `Document`s that are similar to `query_embedding`.
189
+ """
190
+ filters = apply_filter_policy(self.filter_policy, self.filters, filters)
191
+ top_k = top_k or self.top_k
192
+ vector_function = vector_function or self.vector_function
193
+
194
+ docs = await self.document_store._embedding_retrieval_async(
195
+ query_embedding=query_embedding,
196
+ filters=filters,
197
+ top_k=top_k,
198
+ vector_function=vector_function,
199
+ )
200
+ return {"documents": docs}
@@ -136,3 +136,33 @@ class PgvectorKeywordRetriever:
136
136
  top_k=top_k,
137
137
  )
138
138
  return {"documents": docs}
139
+
140
+ @component.output_types(documents=List[Document])
141
+ async def run_async(
142
+ self,
143
+ query: str,
144
+ filters: Optional[Dict[str, Any]] = None,
145
+ top_k: Optional[int] = None,
146
+ ):
147
+ """
148
+ Asynchronously retrieve documents from the `PgvectorDocumentStore`, based on keywords.
149
+
150
+ :param query: String to search in `Document`s' content.
151
+ :param filters: Filters applied to the retrieved Documents. The way runtime filters are applied depends on
152
+ the `filter_policy` chosen at retriever initialization. See init method docstring for more
153
+ details.
154
+ :param top_k: Maximum number of Documents to return.
155
+
156
+ :returns: A dictionary with the following keys:
157
+ - `documents`: List of `Document`s that match the query.
158
+ """
159
+ filters = apply_filter_policy(self.filter_policy, self.filters, filters)
160
+
161
+ top_k = top_k or self.top_k
162
+
163
+ docs = await self.document_store._keyword_retrieval_async(
164
+ query=query,
165
+ filters=filters,
166
+ top_k=top_k,
167
+ )
168
+ return {"documents": docs}
@@ -0,0 +1,65 @@
1
+ from typing import Any, Dict, List
2
+
3
+ from haystack import logging
4
+ from haystack.dataclasses import ByteStream, Document
5
+ from psycopg.types.json import Jsonb
6
+
7
+ logger = logging.getLogger(__name__)
8
+
9
+
10
+ def _from_haystack_to_pg_documents(documents: List[Document]) -> List[Dict[str, Any]]:
11
+ """
12
+ Internal method to convert a list of Haystack Documents to a list of dictionaries that can be used to insert
13
+ documents into the PgvectorDocumentStore.
14
+ """
15
+
16
+ db_documents = []
17
+ for document in documents:
18
+ db_document = {k: v for k, v in document.to_dict(flatten=False).items() if k not in ["score", "blob"]}
19
+
20
+ blob = document.blob
21
+ db_document["blob_data"] = blob.data if blob else None
22
+ db_document["blob_meta"] = Jsonb(blob.meta) if blob and blob.meta else None
23
+ db_document["blob_mime_type"] = blob.mime_type if blob and blob.mime_type else None
24
+ db_document["meta"] = Jsonb(db_document["meta"])
25
+
26
+ if "sparse_embedding" in db_document:
27
+ sparse_embedding = db_document.pop("sparse_embedding", None)
28
+ if sparse_embedding:
29
+ logger.warning(
30
+ "Document {doc_id} has the `sparse_embedding` field set,"
31
+ "but storing sparse embeddings in Pgvector is not currently supported."
32
+ "The `sparse_embedding` field will be ignored.",
33
+ doc_id=db_document["id"],
34
+ )
35
+
36
+ db_documents.append(db_document)
37
+
38
+ return db_documents
39
+
40
+
41
+ def _from_pg_to_haystack_documents(documents: List[Dict[str, Any]]) -> List[Document]:
42
+ """
43
+ Internal method to convert a list of dictionaries from pgvector to a list of Haystack Documents.
44
+ """
45
+
46
+ haystack_documents = []
47
+ for document in documents:
48
+ haystack_dict = dict(document)
49
+ blob_data = haystack_dict.pop("blob_data")
50
+ blob_meta = haystack_dict.pop("blob_meta")
51
+ blob_mime_type = haystack_dict.pop("blob_mime_type")
52
+
53
+ # convert the embedding to a list of floats
54
+ if document.get("embedding") is not None:
55
+ haystack_dict["embedding"] = document["embedding"].tolist()
56
+
57
+ haystack_document = Document.from_dict(haystack_dict)
58
+
59
+ if blob_data:
60
+ blob = ByteStream(data=blob_data, meta=blob_meta, mime_type=blob_mime_type)
61
+ haystack_document.blob = blob
62
+
63
+ haystack_documents.append(haystack_document)
64
+
65
+ return haystack_documents
@@ -4,21 +4,22 @@
4
4
  from typing import Any, Dict, List, Literal, Optional
5
5
 
6
6
  from haystack import default_from_dict, default_to_dict, logging
7
- from haystack.dataclasses.document import ByteStream, Document
7
+ from haystack.dataclasses.document import Document
8
8
  from haystack.document_stores.errors import DocumentStoreError, DuplicateDocumentError
9
9
  from haystack.document_stores.types import DuplicatePolicy
10
10
  from haystack.utils.auth import Secret, deserialize_secrets_inplace
11
- from psycopg import Error, IntegrityError, connect
11
+ from psycopg import AsyncConnection, Connection, Error, IntegrityError
12
12
  from psycopg.abc import Query
13
+ from psycopg.connection_async import AsyncCursor
13
14
  from psycopg.cursor import Cursor
14
15
  from psycopg.rows import dict_row
15
16
  from psycopg.sql import SQL, Identifier
16
17
  from psycopg.sql import Literal as SQLLiteral
17
- from psycopg.types.json import Jsonb
18
18
 
19
- from pgvector.psycopg import register_vector
19
+ from pgvector.psycopg import register_vector, register_vector_async
20
20
 
21
- from .filters import _convert_filters_to_where_clause_and_params
21
+ from .converters import _from_haystack_to_pg_documents, _from_pg_to_haystack_documents
22
+ from .filters import _convert_filters_to_where_clause_and_params, _validate_filters
22
23
 
23
24
  logger = logging.getLogger(__name__)
24
25
 
@@ -156,90 +157,15 @@ class PgvectorDocumentStore:
156
157
  self.hnsw_ef_search = hnsw_ef_search
157
158
  self.keyword_index_name = keyword_index_name
158
159
  self.language = language
160
+
159
161
  self._connection = None
162
+ self._async_connection = None
160
163
  self._cursor = None
164
+ self._async_cursor = None
161
165
  self._dict_cursor = None
166
+ self._async_dict_cursor = None
162
167
  self._table_initialized = False
163
168
 
164
- @property
165
- def cursor(self):
166
- if self._cursor is None or not self._connection_is_valid(self._connection):
167
- self._create_connection()
168
-
169
- return self._cursor
170
-
171
- @property
172
- def dict_cursor(self):
173
- if self._dict_cursor is None or not self._connection_is_valid(self._connection):
174
- self._create_connection()
175
-
176
- return self._dict_cursor
177
-
178
- @property
179
- def connection(self):
180
- if self._connection is None or not self._connection_is_valid(self._connection):
181
- self._create_connection()
182
-
183
- return self._connection
184
-
185
- def _create_connection(self):
186
- """
187
- Internal method to create a connection to the PostgreSQL database.
188
- """
189
-
190
- # close the connection if it already exists
191
- if self._connection:
192
- try:
193
- self._connection.close()
194
- except Error as e:
195
- logger.debug("Failed to close connection: {e}", e=str(e))
196
-
197
- conn_str = self.connection_string.resolve_value() or ""
198
- connection = connect(conn_str)
199
- connection.autocommit = True
200
- if self.create_extension:
201
- connection.execute("CREATE EXTENSION IF NOT EXISTS vector")
202
- register_vector(connection) # Note: this must be called before creating the cursors.
203
-
204
- self._connection = connection
205
- self._cursor = self._connection.cursor()
206
- self._dict_cursor = self._connection.cursor(row_factory=dict_row)
207
-
208
- if not self._table_initialized:
209
- self._initialize_table()
210
-
211
- return self._connection
212
-
213
- def _initialize_table(self):
214
- """
215
- Internal method to initialize the table.
216
- """
217
- if self.recreate_table:
218
- self.delete_table()
219
-
220
- self._create_table_if_not_exists()
221
- self._create_keyword_index_if_not_exists()
222
-
223
- if self.search_strategy == "hnsw":
224
- self._handle_hnsw()
225
-
226
- self._table_initialized = True
227
-
228
- @staticmethod
229
- def _connection_is_valid(connection):
230
- """
231
- Internal method to check if the connection is still valid.
232
- """
233
-
234
- # implementation inspired to psycopg pool
235
- # https://github.com/psycopg/psycopg/blob/d38cf7798b0c602ff43dac9f20bbab96237a9c38/psycopg_pool/psycopg_pool/pool.py#L528
236
-
237
- try:
238
- connection.execute("")
239
- except Error:
240
- return False
241
- return True
242
-
243
169
  def to_dict(self) -> Dict[str, Any]:
244
170
  """
245
171
  Serializes the component to a dictionary.
@@ -278,6 +204,32 @@ class PgvectorDocumentStore:
278
204
  deserialize_secrets_inplace(data["init_parameters"], ["connection_string"])
279
205
  return default_from_dict(cls, data)
280
206
 
207
+ @staticmethod
208
+ def _connection_is_valid(connection):
209
+ """
210
+ Internal method to check if the connection is still valid.
211
+ """
212
+
213
+ # implementation inspired to psycopg pool
214
+ # https://github.com/psycopg/psycopg/blob/d38cf7798b0c602ff43dac9f20bbab96237a9c38/psycopg_pool/psycopg_pool/pool.py#L528
215
+
216
+ try:
217
+ connection.execute("")
218
+ except Error:
219
+ return False
220
+ return True
221
+
222
+ @staticmethod
223
+ async def _connection_is_valid_async(connection):
224
+ """
225
+ Internal method to check if the async connection is still valid.
226
+ """
227
+ try:
228
+ await connection.execute("")
229
+ except Error:
230
+ return False
231
+ return True
232
+
281
233
  def _execute_sql(
282
234
  self, sql_query: Query, params: Optional[tuple] = None, error_msg: str = "", cursor: Optional[Cursor] = None
283
235
  ):
@@ -287,11 +239,18 @@ class PgvectorDocumentStore:
287
239
  :param sql_query: The SQL query to execute.
288
240
  :param params: The parameters to pass to the SQL query.
289
241
  :param error_msg: The error message to use if an exception is raised.
290
- :param cursor: The cursor to use to execute the SQL query. Defaults to self.cursor.
242
+ :param cursor: The cursor to use to execute the SQL query. Defaults to self._cursor.
291
243
  """
292
244
 
293
245
  params = params or ()
294
- cursor = cursor or self.cursor
246
+ cursor = cursor or self._cursor
247
+
248
+ if cursor is None or self._connection is None:
249
+ message = (
250
+ "The cursor or the connection is not initialized. "
251
+ "Make sure to call _ensure_db_setup() before calling this method."
252
+ )
253
+ raise ValueError(message)
295
254
 
296
255
  sql_query_str = sql_query.as_string(cursor) if not isinstance(sql_query, str) else sql_query
297
256
  logger.debug("SQL query: {query}\nParameters: {parameters}", query=sql_query_str, parameters=params)
@@ -299,33 +258,215 @@ class PgvectorDocumentStore:
299
258
  try:
300
259
  result = cursor.execute(sql_query, params)
301
260
  except Error as e:
302
- self.connection.rollback()
261
+ self._connection.rollback()
303
262
  detailed_error_msg = f"{error_msg}.\nYou can find the SQL query and the parameters in the debug logs."
304
263
  raise DocumentStoreError(detailed_error_msg) from e
305
264
 
306
265
  return result
307
266
 
308
- def _create_table_if_not_exists(self):
267
+ async def _execute_sql_async(
268
+ self,
269
+ sql_query: Query,
270
+ params: Optional[tuple] = None,
271
+ error_msg: str = "",
272
+ cursor: Optional[AsyncCursor] = None,
273
+ ):
274
+ """
275
+ Internal method to asynchronously execute SQL statements and handle exceptions.
276
+
277
+ :param sql_query: The SQL query to execute.
278
+ :param params: The parameters to pass to the SQL query.
279
+ :param error_msg: The error message to use if an exception is raised.
280
+ :param cursor: The cursor to use to execute the SQL query. Defaults to self._async_cursor.
281
+ """
282
+
283
+ params = params or ()
284
+ cursor = cursor or self._async_cursor
285
+
286
+ if cursor is None or self._async_connection is None:
287
+ message = (
288
+ "The cursor or the connection is not initialized. "
289
+ "Make sure to call _ensure_db_setup_async() before calling this method."
290
+ )
291
+ raise ValueError(message)
292
+
293
+ sql_query_str = sql_query.as_string(cursor) if not isinstance(sql_query, str) else sql_query
294
+ logger.debug("SQL query: {query}\nParameters: {parameters}", query=sql_query_str, parameters=params)
295
+
296
+ try:
297
+ result = await cursor.execute(sql_query, params)
298
+ except Error as e:
299
+ await self._async_connection.rollback()
300
+ detailed_error_msg = f"{error_msg}.\nYou can find the SQL query and the parameters in the debug logs."
301
+ raise DocumentStoreError(detailed_error_msg) from e
302
+
303
+ return result
304
+
305
+ def _ensure_db_setup(self):
306
+ """
307
+ Ensures that the connection to the PostgreSQL database exists and is valid.
308
+ If not, connection and cursors are created.
309
+ If the table is not initialized, it will be set up.
310
+ """
311
+ if self._connection and self._cursor and self._dict_cursor and self._connection_is_valid(self._connection):
312
+ return
313
+
314
+ # close the connection if it already exists
315
+ if self._connection:
316
+ try:
317
+ self._connection.close()
318
+ except Error as e:
319
+ logger.debug("Failed to close connection: {e}", e=str(e))
320
+
321
+ conn_str = self.connection_string.resolve_value() or ""
322
+ connection = Connection.connect(conn_str)
323
+ connection.autocommit = True
324
+ if self.create_extension:
325
+ connection.execute("CREATE EXTENSION IF NOT EXISTS vector")
326
+ register_vector(connection) # Note: this must be called before creating the cursors.
327
+
328
+ self._connection = connection
329
+ self._cursor = self._connection.cursor()
330
+ self._dict_cursor = self._connection.cursor(row_factory=dict_row)
331
+
332
+ if not self._table_initialized:
333
+ self._initialize_table()
334
+
335
+ async def _ensure_db_setup_async(self):
336
+ """
337
+ Async internal method.
338
+ Ensures that the connection to the PostgreSQL database exists and is valid.
339
+ If not, connection and cursors are created.
340
+ If the table is not initialized, it will be set up.
341
+ """
342
+
343
+ if (
344
+ self._async_connection
345
+ and self._async_cursor
346
+ and self._async_dict_cursor
347
+ and await self._connection_is_valid_async(self._async_connection)
348
+ ):
349
+ return
350
+
351
+ # close the connection if it already exists
352
+ if self._async_connection:
353
+ await self._async_connection.close()
354
+
355
+ conn_str = self.connection_string.resolve_value() or ""
356
+ async_connection = await AsyncConnection.connect(conn_str)
357
+ await async_connection.set_autocommit(True)
358
+ if self.create_extension:
359
+ await async_connection.execute("CREATE EXTENSION IF NOT EXISTS vector")
360
+ await register_vector_async(async_connection) # Note: this must be called before creating the cursors.
361
+
362
+ self._async_connection = async_connection
363
+ self._async_cursor = self._async_connection.cursor()
364
+ self._async_dict_cursor = self._async_connection.cursor(row_factory=dict_row)
365
+
366
+ if not self._table_initialized:
367
+ await self._initialize_table_async()
368
+
369
+ def _build_table_creation_queries(self):
309
370
  """
310
- Creates the table to store Haystack documents if it doesn't exist yet.
371
+ Internal method to build the SQL queries for table creation.
311
372
  """
312
373
 
374
+ sql_table_exists = SQL("SELECT 1 FROM pg_tables WHERE schemaname = %s AND tablename = %s")
375
+ sql_create_table = SQL(CREATE_TABLE_STATEMENT).format(
376
+ schema_name=Identifier(self.schema_name),
377
+ table_name=Identifier(self.table_name),
378
+ embedding_dimension=SQLLiteral(self.embedding_dimension),
379
+ )
380
+
381
+ sql_keyword_index_exists = SQL(
382
+ "SELECT 1 FROM pg_indexes WHERE schemaname = %s AND tablename = %s AND indexname = %s"
383
+ )
384
+ sql_create_keyword_index = SQL(
385
+ "CREATE INDEX {index_name} ON {schema_name}.{table_name} USING GIN (to_tsvector({language}, content))"
386
+ ).format(
387
+ schema_name=Identifier(self.schema_name),
388
+ index_name=Identifier(self.keyword_index_name),
389
+ table_name=Identifier(self.table_name),
390
+ language=SQLLiteral(self.language),
391
+ )
392
+
393
+ return sql_table_exists, sql_create_table, sql_keyword_index_exists, sql_create_keyword_index
394
+
395
+ def _initialize_table(self):
396
+ """
397
+ Internal method to initialize the table.
398
+ """
399
+ if self.recreate_table:
400
+ self.delete_table()
401
+
402
+ sql_table_exists, sql_create_table, sql_keyword_index_exists, sql_create_keyword_index = (
403
+ self._build_table_creation_queries()
404
+ )
405
+
313
406
  table_exists = bool(
314
407
  self._execute_sql(
315
- "SELECT 1 FROM pg_tables WHERE schemaname = %s AND tablename = %s",
316
- (self.schema_name, self.table_name),
317
- "Could not check if table exists",
408
+ sql_table_exists, (self.schema_name, self.table_name), "Could not check if table exists"
409
+ ).fetchone()
410
+ )
411
+ if not table_exists:
412
+ self._execute_sql(sql_create_table, error_msg="Could not create table")
413
+
414
+ index_exists = bool(
415
+ self._execute_sql(
416
+ sql_keyword_index_exists,
417
+ (self.schema_name, self.table_name, self.keyword_index_name),
418
+ "Could not check if keyword index exists",
318
419
  ).fetchone()
319
420
  )
421
+ if not index_exists:
422
+ self._execute_sql(sql_create_keyword_index, error_msg="Could not create keyword index on table")
423
+
424
+ if self.search_strategy == "hnsw":
425
+ self._handle_hnsw()
426
+
427
+ self._table_initialized = True
428
+
429
+ async def _initialize_table_async(self):
430
+ """
431
+ Internal async method to initialize the table.
432
+ """
433
+ if self.recreate_table:
434
+ await self.delete_table_async()
435
+
436
+ sql_table_exists, sql_create_table, sql_keyword_index_exists, sql_create_keyword_index = (
437
+ self._build_table_creation_queries()
438
+ )
320
439
 
440
+ table_exists = bool(
441
+ await (
442
+ await self._execute_sql_async(
443
+ sql_table_exists,
444
+ (self.schema_name, self.table_name),
445
+ "Could not check if table exists",
446
+ self._async_cursor,
447
+ )
448
+ ).fetchone()
449
+ )
321
450
  if not table_exists:
322
- create_sql = SQL(CREATE_TABLE_STATEMENT).format(
323
- schema_name=Identifier(self.schema_name),
324
- table_name=Identifier(self.table_name),
325
- embedding_dimension=SQLLiteral(self.embedding_dimension),
326
- )
451
+ await self._execute_sql_async(sql_create_table, error_msg="Could not create table")
452
+
453
+ index_exists = bool(
454
+ await (
455
+ await self._execute_sql_async(
456
+ sql_keyword_index_exists,
457
+ (self.schema_name, self.table_name, self.keyword_index_name),
458
+ "Could not check if keyword index exists",
459
+ self._async_cursor,
460
+ )
461
+ ).fetchone()
462
+ )
463
+ if not index_exists:
464
+ await self._execute_sql_async(sql_create_keyword_index, error_msg="Could not create keyword index on table")
327
465
 
328
- self._execute_sql(create_sql, error_msg="Could not create table in PgvectorDocumentStore")
466
+ if self.search_strategy == "hnsw":
467
+ await self._handle_hnsw_async()
468
+
469
+ self._table_initialized = True
329
470
 
330
471
  def delete_table(self):
331
472
  """
@@ -343,29 +484,58 @@ class PgvectorDocumentStore:
343
484
  error_msg=f"Could not delete table {self.schema_name}.{self.table_name} in PgvectorDocumentStore",
344
485
  )
345
486
 
346
- def _create_keyword_index_if_not_exists(self):
487
+ async def delete_table_async(self):
347
488
  """
348
- Internal method to create the keyword index if not exists.
489
+ Async method to delete the table used to store Haystack documents.
349
490
  """
350
- index_exists = bool(
351
- self._execute_sql(
352
- "SELECT 1 FROM pg_indexes WHERE schemaname = %s AND tablename = %s AND indexname = %s",
353
- (self.schema_name, self.table_name, self.keyword_index_name),
354
- "Could not check if keyword index exists",
355
- ).fetchone()
491
+ delete_sql = SQL("DROP TABLE IF EXISTS {schema_name}.{table_name}").format(
492
+ schema_name=Identifier(self.schema_name),
493
+ table_name=Identifier(self.table_name),
356
494
  )
357
495
 
358
- sql_create_index = SQL(
359
- "CREATE INDEX {index_name} ON {schema_name}.{table_name} USING GIN (to_tsvector({language}, content))"
496
+ await self._execute_sql_async(
497
+ delete_sql,
498
+ error_msg=f"Could not delete table {self.schema_name}.{self.table_name} in PgvectorDocumentStore",
499
+ )
500
+
501
+ def _build_hnsw_queries(self):
502
+ """Common method to build all HNSW-related SQL queries"""
503
+
504
+ sql_set_hnsw_ef_search = (
505
+ SQL("SET hnsw.ef_search = {hnsw_ef_search}").format(hnsw_ef_search=SQLLiteral(self.hnsw_ef_search))
506
+ if self.hnsw_ef_search
507
+ else None
508
+ )
509
+
510
+ sql_hnsw_index_exists = SQL(
511
+ "SELECT 1 FROM pg_indexes WHERE schemaname = %s AND tablename = %s AND indexname = %s"
512
+ )
513
+
514
+ sql_drop_hnsw_index = SQL("DROP INDEX IF EXISTS {schema_name}.{index_name}").format(
515
+ schema_name=Identifier(self.schema_name),
516
+ index_name=Identifier(self.hnsw_index_name),
517
+ )
518
+
519
+ pg_ops = VECTOR_FUNCTION_TO_POSTGRESQL_OPS[self.vector_function]
520
+
521
+ sql_create_hnsw_index = SQL(
522
+ "CREATE INDEX {index_name} ON {schema_name}.{table_name} USING hnsw (embedding {ops})"
360
523
  ).format(
361
524
  schema_name=Identifier(self.schema_name),
362
- index_name=Identifier(self.keyword_index_name),
525
+ index_name=Identifier(self.hnsw_index_name),
363
526
  table_name=Identifier(self.table_name),
364
- language=SQLLiteral(self.language),
527
+ ops=SQL(pg_ops),
365
528
  )
366
529
 
367
- if not index_exists:
368
- self._execute_sql(sql_create_index, error_msg="Could not create keyword index on table")
530
+ # Add creation kwargs if any valid ones exist
531
+ valid_kwargs = {
532
+ k: v for k, v in self.hnsw_index_creation_kwargs.items() if k in HNSW_INDEX_CREATION_VALID_KWARGS
533
+ }
534
+ if valid_kwargs:
535
+ kwargs_str = ", ".join(f"{k} = {v}" for k, v in valid_kwargs.items())
536
+ sql_create_hnsw_index += SQL(" WITH ({})").format(SQL(kwargs_str))
537
+
538
+ return sql_set_hnsw_ef_search, sql_hnsw_index_exists, sql_drop_hnsw_index, sql_create_hnsw_index
369
539
 
370
540
  def _handle_hnsw(self):
371
541
  """
@@ -373,15 +543,16 @@ class PgvectorDocumentStore:
373
543
  It also sets the `hnsw.ef_search` parameter for queries if it is specified.
374
544
  """
375
545
 
546
+ sql_set_hnsw_ef_search, sql_hnsw_index_exists, sql_drop_hnsw_index, sql_create_hnsw_index = (
547
+ self._build_hnsw_queries()
548
+ )
549
+
376
550
  if self.hnsw_ef_search:
377
- sql_set_hnsw_ef_search = SQL("SET hnsw.ef_search = {hnsw_ef_search}").format(
378
- hnsw_ef_search=SQLLiteral(self.hnsw_ef_search)
379
- )
380
551
  self._execute_sql(sql_set_hnsw_ef_search, error_msg="Could not set hnsw.ef_search")
381
552
 
382
553
  index_exists = bool(
383
554
  self._execute_sql(
384
- "SELECT 1 FROM pg_indexes WHERE schemaname = %s AND tablename = %s AND indexname = %s",
555
+ sql_hnsw_index_exists,
385
556
  (self.schema_name, self.table_name, self.hnsw_index_name),
386
557
  "Could not check if HNSW index exists",
387
558
  ).fetchone()
@@ -395,59 +566,81 @@ class PgvectorDocumentStore:
395
566
  )
396
567
  return
397
568
 
398
- sql_drop_index = SQL("DROP INDEX IF EXISTS {schema_name}.{index_name}").format(
399
- schema_name=Identifier(self.schema_name), index_name=Identifier(self.hnsw_index_name)
400
- )
401
- self._execute_sql(sql_drop_index, error_msg="Could not drop HNSW index")
569
+ self._execute_sql(sql_drop_hnsw_index, error_msg="Could not drop HNSW index")
402
570
 
403
- self._create_hnsw_index()
571
+ self._execute_sql(sql_create_hnsw_index, error_msg="Could not create HNSW index")
404
572
 
405
- def _create_hnsw_index(self):
573
+ async def _handle_hnsw_async(self):
406
574
  """
407
- Internal method to create the HNSW index.
575
+ Internal async method to handle the HNSW index creation.
408
576
  """
409
577
 
410
- pg_ops = VECTOR_FUNCTION_TO_POSTGRESQL_OPS[self.vector_function]
411
- actual_hnsw_index_creation_kwargs = {
412
- key: value
413
- for key, value in self.hnsw_index_creation_kwargs.items()
414
- if key in HNSW_INDEX_CREATION_VALID_KWARGS
415
- }
578
+ sql_set_hnsw_ef_search, sql_hnsw_index_exists, sql_drop_hnsw_index, sql_create_hnsw_index = (
579
+ self._build_hnsw_queries()
580
+ )
416
581
 
417
- sql_create_index = SQL(
418
- "CREATE INDEX {index_name} ON {schema_name}.{table_name} USING hnsw (embedding {ops}) "
419
- ).format(
420
- schema_name=Identifier(self.schema_name),
421
- index_name=Identifier(self.hnsw_index_name),
422
- table_name=Identifier(self.table_name),
423
- ops=SQL(pg_ops),
582
+ if self.hnsw_ef_search:
583
+ await self._execute_sql_async(sql_set_hnsw_ef_search, error_msg="Could not set hnsw.ef_search")
584
+
585
+ index_exists = bool(
586
+ await (
587
+ await self._execute_sql_async(
588
+ sql_hnsw_index_exists,
589
+ (self.schema_name, self.table_name, self.hnsw_index_name),
590
+ "Could not check if HNSW index exists",
591
+ self._async_cursor,
592
+ )
593
+ ).fetchone()
424
594
  )
425
595
 
426
- if actual_hnsw_index_creation_kwargs:
427
- actual_hnsw_index_creation_kwargs_str = ", ".join(
428
- f"{key} = {value}" for key, value in actual_hnsw_index_creation_kwargs.items()
429
- )
430
- sql_add_creation_kwargs = SQL("WITH ({creation_kwargs_str})").format(
431
- creation_kwargs_str=SQL(actual_hnsw_index_creation_kwargs_str)
596
+ if index_exists and not self.hnsw_recreate_index_if_exists:
597
+ logger.warning(
598
+ "HNSW index already exists and won't be recreated. "
599
+ "If you want to recreate it, pass 'hnsw_recreate_index_if_exists=True' to the "
600
+ "Document Store constructor"
432
601
  )
433
- sql_create_index += sql_add_creation_kwargs
602
+ return
434
603
 
435
- self._execute_sql(sql_create_index, error_msg="Could not create HNSW index")
604
+ await self._execute_sql_async(sql_drop_hnsw_index, error_msg="Could not drop HNSW index")
605
+
606
+ await self._execute_sql_async(sql_create_hnsw_index, error_msg="Could not create HNSW index")
436
607
 
437
608
  def count_documents(self) -> int:
438
609
  """
439
610
  Returns how many documents are present in the document store.
440
- """
441
611
 
612
+ :returns:
613
+ Number of documents in the document store.
614
+ """
442
615
  sql_count = SQL("SELECT COUNT(*) FROM {schema_name}.{table_name}").format(
443
616
  schema_name=Identifier(self.schema_name), table_name=Identifier(self.table_name)
444
617
  )
445
618
 
619
+ self._ensure_db_setup()
446
620
  count = self._execute_sql(sql_count, error_msg="Could not count documents in PgvectorDocumentStore").fetchone()[
447
621
  0
448
622
  ]
449
623
  return count
450
624
 
625
+ async def count_documents_async(self) -> int:
626
+ """
627
+ Returns how many documents are present in the document store.
628
+
629
+ :returns:
630
+ Number of documents in the document store.
631
+ """
632
+ sql_count = SQL("SELECT COUNT(*) FROM {schema_name}.{table_name}").format(
633
+ schema_name=Identifier(self.schema_name), table_name=Identifier(self.table_name)
634
+ )
635
+
636
+ await self._ensure_db_setup_async()
637
+
638
+ result = await (
639
+ await self._execute_sql_async(sql_count, error_msg="Could not count documents in PgvectorDocumentStore")
640
+ ).fetchone()
641
+
642
+ return result[0]
643
+
451
644
  def filter_documents(self, filters: Optional[Dict[str, Any]] = None) -> List[Document]:
452
645
  """
453
646
  Returns the documents that match the filters provided.
@@ -457,15 +650,10 @@ class PgvectorDocumentStore:
457
650
 
458
651
  :param filters: The filters to apply to the document list.
459
652
  :raises TypeError: If `filters` is not a dictionary.
653
+ :raises ValueError: If `filters` syntax is invalid.
460
654
  :returns: A list of Documents that match the given filters.
461
655
  """
462
- if filters:
463
- if not isinstance(filters, dict):
464
- msg = "Filters must be a dictionary"
465
- raise TypeError(msg)
466
- if "operator" not in filters and "conditions" not in filters:
467
- msg = "Invalid filter syntax. See https://docs.haystack.deepset.ai/docs/metadata-filtering for details."
468
- raise ValueError(msg)
656
+ _validate_filters(filters)
469
657
 
470
658
  sql_filter = SQL("SELECT * FROM {schema_name}.{table_name}").format(
471
659
  schema_name=Identifier(self.schema_name), table_name=Identifier(self.table_name)
@@ -476,28 +664,83 @@ class PgvectorDocumentStore:
476
664
  sql_where_clause, params = _convert_filters_to_where_clause_and_params(filters)
477
665
  sql_filter += sql_where_clause
478
666
 
667
+ self._ensure_db_setup()
479
668
  result = self._execute_sql(
480
669
  sql_filter,
481
670
  params,
482
671
  error_msg="Could not filter documents from PgvectorDocumentStore.",
483
- cursor=self.dict_cursor,
672
+ cursor=self._dict_cursor,
484
673
  )
485
674
 
486
675
  records = result.fetchall()
487
- docs = self._from_pg_to_haystack_documents(records)
676
+ docs = _from_pg_to_haystack_documents(records)
677
+ return docs
678
+
679
+ async def filter_documents_async(self, filters: Optional[Dict[str, Any]] = None) -> List[Document]:
680
+ """
681
+ Asynchronously returns the documents that match the filters provided.
682
+
683
+ For a detailed specification of the filters,
684
+ refer to the [documentation](https://docs.haystack.deepset.ai/v2.0/docs/metadata-filtering)
685
+
686
+ :param filters: The filters to apply to the document list.
687
+
688
+ :raises TypeError: If `filters` is not a dictionary.
689
+ :raises ValueError: If `filters` syntax is invalid.
690
+ :returns: A list of Documents that match the given filters.
691
+ """
692
+ _validate_filters(filters)
693
+
694
+ sql_filter = SQL("SELECT * FROM {schema_name}.{table_name}").format(
695
+ schema_name=Identifier(self.schema_name), table_name=Identifier(self.table_name)
696
+ )
697
+
698
+ params = ()
699
+ if filters:
700
+ sql_where_clause, params = _convert_filters_to_where_clause_and_params(filters)
701
+ sql_filter += sql_where_clause
702
+
703
+ await self._ensure_db_setup_async()
704
+ result = await self._execute_sql_async(
705
+ sql_filter,
706
+ params,
707
+ error_msg="Could not filter documents from PgvectorDocumentStore.",
708
+ cursor=self._async_dict_cursor,
709
+ )
710
+
711
+ records = await result.fetchall()
712
+ docs = _from_pg_to_haystack_documents(records)
488
713
  return docs
489
714
 
715
+ def _build_insert_statement(self, policy: DuplicatePolicy):
716
+ """
717
+ Builds the SQL insert statement to write documents.
718
+ """
719
+ sql_insert = SQL(INSERT_STATEMENT).format(
720
+ schema_name=Identifier(self.schema_name), table_name=Identifier(self.table_name)
721
+ )
722
+
723
+ if policy == DuplicatePolicy.OVERWRITE:
724
+ sql_insert += SQL(UPDATE_STATEMENT)
725
+ elif policy == DuplicatePolicy.SKIP:
726
+ sql_insert += SQL("ON CONFLICT DO NOTHING")
727
+
728
+ sql_insert += SQL(" RETURNING id")
729
+
730
+ return sql_insert
731
+
490
732
  def write_documents(self, documents: List[Document], policy: DuplicatePolicy = DuplicatePolicy.NONE) -> int:
491
733
  """
492
734
  Writes documents to the document store.
493
735
 
494
736
  :param documents: A list of Documents to write to the document store.
495
737
  :param policy: The duplicate policy to use when writing documents.
738
+ :raises ValueError: If `documents` contains objects that are not of type `Document`.
496
739
  :raises DuplicateDocumentError: If a document with the same id already exists in the document store
497
740
  and the policy is set to `DuplicatePolicy.FAIL` (or not specified).
741
+ :raises DocumentStoreError: If the write operation fails for any other reason.
498
742
  :returns: The number of documents written to the document store.
499
743
  """
500
-
501
744
  if len(documents) > 0:
502
745
  if not isinstance(documents[0], Document):
503
746
  msg = "param 'documents' must contain a list of objects of type Document"
@@ -506,29 +749,24 @@ class PgvectorDocumentStore:
506
749
  if policy == DuplicatePolicy.NONE:
507
750
  policy = DuplicatePolicy.FAIL
508
751
 
509
- db_documents = self._from_haystack_to_pg_documents(documents)
752
+ db_documents = _from_haystack_to_pg_documents(documents)
510
753
 
511
- sql_insert = SQL(INSERT_STATEMENT).format(
512
- schema_name=Identifier(self.schema_name), table_name=Identifier(self.table_name)
513
- )
754
+ sql_insert = self._build_insert_statement(policy)
514
755
 
515
- if policy == DuplicatePolicy.OVERWRITE:
516
- sql_insert += SQL(UPDATE_STATEMENT)
517
- elif policy == DuplicatePolicy.SKIP:
518
- sql_insert += SQL("ON CONFLICT DO NOTHING")
519
-
520
- sql_insert += SQL(" RETURNING id")
756
+ self._ensure_db_setup()
757
+ assert self._cursor is not None # verified in _ensure_db_setup() but mypy doesn't know that
758
+ assert self._connection is not None # verified in _ensure_db_setup() but mypy doesn't know that
521
759
 
522
- sql_query_str = sql_insert.as_string(self.cursor) if not isinstance(sql_insert, str) else sql_insert
760
+ sql_query_str = sql_insert.as_string(self._cursor) if not isinstance(sql_insert, str) else sql_insert
523
761
  logger.debug("SQL query: {query}\nParameters: {parameters}", query=sql_query_str, parameters=db_documents)
524
762
 
525
763
  try:
526
- self.cursor.executemany(sql_insert, db_documents, returning=True)
764
+ self._cursor.executemany(sql_insert, db_documents, returning=True)
527
765
  except IntegrityError as ie:
528
- self.connection.rollback()
766
+ self._connection.rollback()
529
767
  raise DuplicateDocumentError from ie
530
768
  except Error as e:
531
- self.connection.rollback()
769
+ self._connection.rollback()
532
770
  error_msg = (
533
771
  "Could not write documents to PgvectorDocumentStore. \n"
534
772
  "You can find the SQL query and the parameters in the debug logs."
@@ -539,70 +777,64 @@ class PgvectorDocumentStore:
539
777
  # https://www.psycopg.org/psycopg3/docs/api/cursors.html#psycopg.Cursor.executemany
540
778
  written_docs = 0
541
779
  while True:
542
- if self.cursor.fetchone():
780
+ if self._cursor.fetchone():
543
781
  written_docs += 1
544
- if not self.cursor.nextset():
782
+ if not self._cursor.nextset():
545
783
  break
546
784
 
547
785
  return written_docs
548
786
 
549
- @staticmethod
550
- def _from_haystack_to_pg_documents(documents: List[Document]) -> List[Dict[str, Any]]:
551
- """
552
- Internal method to convert a list of Haystack Documents to a list of dictionaries that can be used to insert
553
- documents into the PgvectorDocumentStore.
787
+ async def write_documents_async(
788
+ self, documents: List[Document], policy: DuplicatePolicy = DuplicatePolicy.NONE
789
+ ) -> int:
554
790
  """
791
+ Asynchronously writes documents to the document store.
555
792
 
556
- db_documents = []
557
- for document in documents:
558
- db_document = {k: v for k, v in document.to_dict(flatten=False).items() if k not in ["score", "blob"]}
559
-
560
- blob = document.blob
561
- db_document["blob_data"] = blob.data if blob else None
562
- db_document["blob_meta"] = Jsonb(blob.meta) if blob and blob.meta else None
563
- db_document["blob_mime_type"] = blob.mime_type if blob and blob.mime_type else None
564
- db_document["meta"] = Jsonb(db_document["meta"])
793
+ :param documents: A list of Documents to write to the document store.
794
+ :param policy: The duplicate policy to use when writing documents.
795
+ :raises ValueError: If `documents` contains objects that are not of type `Document`.
796
+ :raises DuplicateDocumentError: If a document with the same id already exists in the document store
797
+ and the policy is set to `DuplicatePolicy.FAIL` (or not specified).
798
+ :raises DocumentStoreError: If the write operation fails for any other reason.
799
+ :returns: The number of documents written to the document store.
800
+ """
801
+ if len(documents) > 0:
802
+ if not isinstance(documents[0], Document):
803
+ msg = "param 'documents' must contain a list of objects of type Document"
804
+ raise ValueError(msg)
565
805
 
566
- if "sparse_embedding" in db_document:
567
- sparse_embedding = db_document.pop("sparse_embedding", None)
568
- if sparse_embedding:
569
- logger.warning(
570
- "Document {doc_id} has the `sparse_embedding` field set,"
571
- "but storing sparse embeddings in Pgvector is not currently supported."
572
- "The `sparse_embedding` field will be ignored.",
573
- doc_id=db_document["id"],
574
- )
806
+ if policy == DuplicatePolicy.NONE:
807
+ policy = DuplicatePolicy.FAIL
575
808
 
576
- db_documents.append(db_document)
809
+ db_documents = _from_haystack_to_pg_documents(documents)
577
810
 
578
- return db_documents
811
+ sql_insert = self._build_insert_statement(policy)
579
812
 
580
- @staticmethod
581
- def _from_pg_to_haystack_documents(documents: List[Dict[str, Any]]) -> List[Document]:
582
- """
583
- Internal method to convert a list of dictionaries from pgvector to a list of Haystack Documents.
584
- """
813
+ await self._ensure_db_setup_async()
814
+ assert self._async_cursor is not None # verified in _ensure_db_setup_async() but mypy doesn't know that
815
+ assert self._async_connection is not None # verified in _ensure_db_setup_async() but mypy doesn't know that
585
816
 
586
- haystack_documents = []
587
- for document in documents:
588
- haystack_dict = dict(document)
589
- blob_data = haystack_dict.pop("blob_data")
590
- blob_meta = haystack_dict.pop("blob_meta")
591
- blob_mime_type = haystack_dict.pop("blob_mime_type")
592
-
593
- # convert the embedding to a list of floats
594
- if document.get("embedding") is not None:
595
- haystack_dict["embedding"] = document["embedding"].tolist()
596
-
597
- haystack_document = Document.from_dict(haystack_dict)
817
+ sql_query_str = sql_insert.as_string(self._async_cursor) if not isinstance(sql_insert, str) else sql_insert
818
+ logger.debug("SQL query: {query}\nParameters: {parameters}", query=sql_query_str, parameters=db_documents)
598
819
 
599
- if blob_data:
600
- blob = ByteStream(data=blob_data, meta=blob_meta, mime_type=blob_mime_type)
601
- haystack_document.blob = blob
820
+ try:
821
+ await self._async_cursor.executemany(sql_insert, db_documents, returning=True)
822
+ except IntegrityError as ie:
823
+ await self._async_connection.rollback()
824
+ raise DuplicateDocumentError from ie
825
+ except Error as e:
826
+ await self._async_connection.rollback()
827
+ error_msg = (
828
+ "Could not write documents to PgvectorDocumentStore. \n"
829
+ "You can find the SQL query and the parameters in the debug logs."
830
+ )
831
+ raise DocumentStoreError(error_msg) from e
602
832
 
603
- haystack_documents.append(haystack_document)
833
+ written_docs = 0
834
+ async for _ in self._async_cursor:
835
+ written_docs += 1
604
836
 
605
- return haystack_documents
837
+ return written_docs
606
838
 
607
839
  def delete_documents(self, document_ids: List[str]) -> None:
608
840
  """
@@ -610,7 +842,6 @@ class PgvectorDocumentStore:
610
842
 
611
843
  :param document_ids: the document ids to delete
612
844
  """
613
-
614
845
  if not document_ids:
615
846
  return
616
847
 
@@ -622,28 +853,33 @@ class PgvectorDocumentStore:
622
853
  document_ids_str=SQL(document_ids_str),
623
854
  )
624
855
 
856
+ self._ensure_db_setup()
625
857
  self._execute_sql(delete_sql, error_msg="Could not delete documents from PgvectorDocumentStore")
626
858
 
627
- def _keyword_retrieval(
628
- self,
629
- query: str,
630
- *,
631
- filters: Optional[Dict[str, Any]] = None,
632
- top_k: int = 10,
633
- ) -> List[Document]:
859
+ async def delete_documents_async(self, document_ids: List[str]) -> None:
634
860
  """
635
- Retrieves documents that are most similar to the query using a full-text search.
861
+ Asynchronously deletes documents that match the provided `document_ids` from the document store.
636
862
 
637
- This method is not meant to be part of the public interface of
638
- `PgvectorDocumentStore` and it should not be called directly.
639
- `PgvectorKeywordRetriever` uses this method directly and is the public interface for it.
640
-
641
- :returns: List of Documents that are most similar to `query`
863
+ :param document_ids: the document ids to delete
642
864
  """
643
- if not query:
644
- msg = "query must be a non-empty string"
645
- raise ValueError(msg)
865
+ if not document_ids:
866
+ return
867
+
868
+ document_ids_str = ", ".join(f"'{document_id}'" for document_id in document_ids)
646
869
 
870
+ delete_sql = SQL("DELETE FROM {schema_name}.{table_name} WHERE id IN ({document_ids_str})").format(
871
+ schema_name=Identifier(self.schema_name),
872
+ table_name=Identifier(self.table_name),
873
+ document_ids_str=SQL(document_ids_str),
874
+ )
875
+
876
+ await self._ensure_db_setup_async()
877
+ await self._execute_sql_async(delete_sql, error_msg="Could not delete documents from PgvectorDocumentStore")
878
+
879
+ def _build_keyword_retrieval_query(self, query: str, top_k: int, filters: Optional[Dict[str, Any]] = None):
880
+ """
881
+ Builds the SQL query and the where parameters for keyword retrieval.
882
+ """
647
883
  sql_select = SQL(KEYWORD_QUERY).format(
648
884
  schema_name=Identifier(self.schema_name),
649
885
  table_name=Identifier(self.table_name),
@@ -662,33 +898,79 @@ class PgvectorDocumentStore:
662
898
 
663
899
  sql_query = sql_select + sql_where_clause + sql_sort
664
900
 
901
+ return sql_query, where_params
902
+
903
+ def _keyword_retrieval(
904
+ self,
905
+ query: str,
906
+ *,
907
+ filters: Optional[Dict[str, Any]] = None,
908
+ top_k: int = 10,
909
+ ) -> List[Document]:
910
+ """
911
+ Retrieves documents that are most similar to the query using a full-text search.
912
+
913
+ This method is not meant to be part of the public interface of
914
+ `PgvectorDocumentStore` and it should not be called directly.
915
+ `PgvectorKeywordRetriever` uses this method directly and is the public interface for it.
916
+
917
+ :returns: List of Documents that are most similar to `query`
918
+ """
919
+ if not query:
920
+ msg = "query must be a non-empty string"
921
+ raise ValueError(msg)
922
+
923
+ sql_query, where_params = self._build_keyword_retrieval_query(query=query, top_k=top_k, filters=filters)
924
+
925
+ self._ensure_db_setup()
665
926
  result = self._execute_sql(
666
927
  sql_query,
667
928
  (query, *where_params),
668
929
  error_msg="Could not retrieve documents from PgvectorDocumentStore.",
669
- cursor=self.dict_cursor,
930
+ cursor=self._dict_cursor,
670
931
  )
671
932
 
672
933
  records = result.fetchall()
673
- docs = self._from_pg_to_haystack_documents(records)
934
+ docs = _from_pg_to_haystack_documents(records)
674
935
  return docs
675
936
 
676
- def _embedding_retrieval(
937
+ async def _keyword_retrieval_async(
677
938
  self,
678
- query_embedding: List[float],
939
+ query: str,
679
940
  *,
680
941
  filters: Optional[Dict[str, Any]] = None,
681
942
  top_k: int = 10,
682
- vector_function: Optional[Literal["cosine_similarity", "inner_product", "l2_distance"]] = None,
683
943
  ) -> List[Document]:
684
944
  """
685
- Retrieves documents that are most similar to the query embedding using a vector similarity metric.
945
+ Retrieves documents that are most similar to the query using a full-text search asynchronously.
946
+ """
947
+ if not query:
948
+ msg = "query must be a non-empty string"
949
+ raise ValueError(msg)
686
950
 
687
- This method is not meant to be part of the public interface of
688
- `PgvectorDocumentStore` and it should not be called directly.
689
- `PgvectorEmbeddingRetriever` uses this method directly and is the public interface for it.
951
+ sql_query, where_params = self._build_keyword_retrieval_query(query=query, top_k=top_k, filters=filters)
690
952
 
691
- :returns: List of Documents that are most similar to `query_embedding`
953
+ await self._ensure_db_setup_async()
954
+ result = await self._execute_sql_async(
955
+ sql_query,
956
+ (query, *where_params),
957
+ error_msg="Could not retrieve documents from PgvectorDocumentStore.",
958
+ cursor=self._async_dict_cursor,
959
+ )
960
+
961
+ records = await result.fetchall()
962
+ docs = _from_pg_to_haystack_documents(records)
963
+ return docs
964
+
965
+ def _check_and_build_embedding_retrieval_query(
966
+ self,
967
+ query_embedding: List[float],
968
+ vector_function: Optional[Literal["cosine_similarity", "inner_product", "l2_distance"]],
969
+ top_k: int,
970
+ filters: Optional[Dict[str, Any]] = None,
971
+ ):
972
+ """
973
+ Performs checks and builds the SQL query and the where parameters for embedding retrieval.
692
974
  """
693
975
 
694
976
  if not query_embedding:
@@ -741,13 +1023,66 @@ class PgvectorDocumentStore:
741
1023
 
742
1024
  sql_query = sql_select + sql_where_clause + sql_sort
743
1025
 
1026
+ return sql_query, params
1027
+
1028
+ def _embedding_retrieval(
1029
+ self,
1030
+ query_embedding: List[float],
1031
+ *,
1032
+ filters: Optional[Dict[str, Any]] = None,
1033
+ top_k: int = 10,
1034
+ vector_function: Optional[Literal["cosine_similarity", "inner_product", "l2_distance"]] = None,
1035
+ ) -> List[Document]:
1036
+ """
1037
+ Retrieves documents that are most similar to the query embedding using a vector similarity metric.
1038
+
1039
+ This method is not meant to be part of the public interface of
1040
+ `PgvectorDocumentStore` and it should not be called directly.
1041
+ `PgvectorEmbeddingRetriever` uses this method directly and is the public interface for it.
1042
+
1043
+ :returns: List of Documents that are most similar to `query_embedding`
1044
+ """
1045
+
1046
+ sql_query, params = self._check_and_build_embedding_retrieval_query(
1047
+ query_embedding=query_embedding, vector_function=vector_function, top_k=top_k, filters=filters
1048
+ )
1049
+ self._ensure_db_setup()
744
1050
  result = self._execute_sql(
745
1051
  sql_query,
746
1052
  params,
747
1053
  error_msg="Could not retrieve documents from PgvectorDocumentStore.",
748
- cursor=self.dict_cursor,
1054
+ cursor=self._dict_cursor,
749
1055
  )
750
1056
 
751
1057
  records = result.fetchall()
752
- docs = self._from_pg_to_haystack_documents(records)
1058
+ docs = _from_pg_to_haystack_documents(records)
1059
+ return docs
1060
+
1061
+ async def _embedding_retrieval_async(
1062
+ self,
1063
+ query_embedding: List[float],
1064
+ *,
1065
+ filters: Optional[Dict[str, Any]] = None,
1066
+ top_k: int = 10,
1067
+ vector_function: Optional[Literal["cosine_similarity", "inner_product", "l2_distance"]] = None,
1068
+ ) -> List[Document]:
1069
+ """
1070
+ Asynchronously retrieves documents that are most similar to the query embedding using a
1071
+ vector similarity metric.
1072
+ """
1073
+
1074
+ sql_query, params = self._check_and_build_embedding_retrieval_query(
1075
+ query_embedding=query_embedding, vector_function=vector_function, top_k=top_k, filters=filters
1076
+ )
1077
+
1078
+ await self._ensure_db_setup_async()
1079
+ result = await self._execute_sql_async(
1080
+ sql_query,
1081
+ params,
1082
+ error_msg="Could not retrieve documents from PgvectorDocumentStore.",
1083
+ cursor=self._async_dict_cursor,
1084
+ )
1085
+
1086
+ records = await result.fetchall()
1087
+ docs = _from_pg_to_haystack_documents(records)
753
1088
  return docs
@@ -3,7 +3,7 @@
3
3
  # SPDX-License-Identifier: Apache-2.0
4
4
  from datetime import datetime
5
5
  from itertools import chain
6
- from typing import Any, Dict, List, Literal, Tuple
6
+ from typing import Any, Dict, List, Literal, Optional, Tuple
7
7
 
8
8
  from haystack.errors import FilterError
9
9
  from psycopg.sql import SQL
@@ -21,6 +21,19 @@ PYTHON_TYPES_TO_PG_TYPES = {
21
21
  NO_VALUE = "no_value"
22
22
 
23
23
 
24
+ def _validate_filters(filters: Optional[Dict[str, Any]] = None):
25
+ """
26
+ Validates the filters provided.
27
+ """
28
+ if filters:
29
+ if not isinstance(filters, dict):
30
+ msg = "Filters must be a dictionary"
31
+ raise TypeError(msg)
32
+ if "operator" not in filters and "conditions" not in filters:
33
+ msg = "Invalid filter syntax. See https://docs.haystack.deepset.ai/docs/metadata-filtering for details."
34
+ raise ValueError(msg)
35
+
36
+
24
37
  def _convert_filters_to_where_clause_and_params(
25
38
  filters: Dict[str, Any], operator: Literal["WHERE", "AND"] = "WHERE"
26
39
  ) -> Tuple[SQL, Tuple]:
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: pgvector-haystack
3
- Version: 3.0.1
3
+ Version: 3.1.0
4
4
  Summary: An integration of pgvector (vector search extension for Postgres) with Haystack
5
5
  Project-URL: Source, https://github.com/deepset-ai/haystack-core-integrations
6
6
  Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/blob/main/integrations/pgvector/README.md
@@ -15,6 +15,7 @@ Classifier: Programming Language :: Python :: 3.9
15
15
  Classifier: Programming Language :: Python :: 3.10
16
16
  Classifier: Programming Language :: Python :: 3.11
17
17
  Classifier: Programming Language :: Python :: 3.12
18
+ Classifier: Programming Language :: Python :: 3.13
18
19
  Classifier: Programming Language :: Python :: Implementation :: CPython
19
20
  Classifier: Programming Language :: Python :: Implementation :: PyPy
20
21
  Requires-Python: >=3.9
@@ -0,0 +1,11 @@
1
+ haystack_integrations/components/retrievers/pgvector/__init__.py,sha256=n4q4pUApJ9DZakJ_JL-egwOzlefdnI_ZxjU1Lu44Sqo,296
2
+ haystack_integrations/components/retrievers/pgvector/embedding_retriever.py,sha256=uK-ak1ZASjprix-5wgKoqP8rLnQfpQzExw_6X8fXUxg,9131
3
+ haystack_integrations/components/retrievers/pgvector/keyword_retriever.py,sha256=5NECAn5qlLqcFTvRf_c-x1crdqdgTCeLHsmCxtD3pHk,6784
4
+ haystack_integrations/document_stores/pgvector/__init__.py,sha256=Ge-7ggarfr3x8UqNdFHUuFGiu52u49sOVe3l16WEwZU,197
5
+ haystack_integrations/document_stores/pgvector/converters.py,sha256=ECNF04RcStH_xpjtY125xB00co0wN8OWNI1fFiv0fUA,2448
6
+ haystack_integrations/document_stores/pgvector/document_store.py,sha256=2Mphtj5K7yRVroEy_esvkxhzTN90glgl_WeAyWIrI5Q,44981
7
+ haystack_integrations/document_stores/pgvector/filters.py,sha256=1xfU-DJcqFUFQtBezZCA8UsYFN0D9znXtsick1KANoU,8998
8
+ pgvector_haystack-3.1.0.dist-info/METADATA,sha256=fhF5qTdD0UkVTtLRjgwVJui26CzrqASA6UNdhiwEhMY,2235
9
+ pgvector_haystack-3.1.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
10
+ pgvector_haystack-3.1.0.dist-info/licenses/LICENSE.txt,sha256=B05uMshqTA74s-0ltyHKI6yoPfJ3zYgQbvcXfDVGFf8,10280
11
+ pgvector_haystack-3.1.0.dist-info/RECORD,,
@@ -1,10 +0,0 @@
1
- haystack_integrations/components/retrievers/pgvector/__init__.py,sha256=n4q4pUApJ9DZakJ_JL-egwOzlefdnI_ZxjU1Lu44Sqo,296
2
- haystack_integrations/components/retrievers/pgvector/embedding_retriever.py,sha256=IEIdgKy_IWgmL5Qy_Hqkwa2mq2JxnQ_JOdjm38JPxmE,7530
3
- haystack_integrations/components/retrievers/pgvector/keyword_retriever.py,sha256=SYHqjTkDczobYZ2qq0z8yS9p_g8aINnzwA5AN_nKdHI,5632
4
- haystack_integrations/document_stores/pgvector/__init__.py,sha256=Ge-7ggarfr3x8UqNdFHUuFGiu52u49sOVe3l16WEwZU,197
5
- haystack_integrations/document_stores/pgvector/document_store.py,sha256=gzyJJkDdGFfi6gdpPjKpTw9Ya2dEY28p9HstSjiJbIU,31590
6
- haystack_integrations/document_stores/pgvector/filters.py,sha256=M-_wmM1iKS1QdKhnRk27N8t2W1st2YZWAwVSFUyyTtk,8508
7
- pgvector_haystack-3.0.1.dist-info/METADATA,sha256=9xjXoMRnchq0NlQOflI_EQHdmn4fwZgwzLS7vkUVmIw,2184
8
- pgvector_haystack-3.0.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
9
- pgvector_haystack-3.0.1.dist-info/licenses/LICENSE.txt,sha256=B05uMshqTA74s-0ltyHKI6yoPfJ3zYgQbvcXfDVGFf8,10280
10
- pgvector_haystack-3.0.1.dist-info/RECORD,,