pgvector-haystack 6.2.0__py3-none-any.whl → 6.3.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- haystack_integrations/components/retrievers/pgvector/embedding_retriever.py +3 -1
- haystack_integrations/components/retrievers/pgvector/keyword_retriever.py +3 -1
- haystack_integrations/document_stores/pgvector/converters.py +3 -2
- haystack_integrations/document_stores/pgvector/document_store.py +47 -43
- haystack_integrations/document_stores/pgvector/filters.py +50 -58
- {pgvector_haystack-6.2.0.dist-info → pgvector_haystack-6.3.1.dist-info}/METADATA +3 -2
- {pgvector_haystack-6.2.0.dist-info → pgvector_haystack-6.3.1.dist-info}/RECORD +9 -9
- {pgvector_haystack-6.2.0.dist-info → pgvector_haystack-6.3.1.dist-info}/WHEEL +0 -0
- {pgvector_haystack-6.2.0.dist-info → pgvector_haystack-6.3.1.dist-info}/licenses/LICENSE.txt +0 -0
|
@@ -66,8 +66,10 @@ class PgvectorEmbeddingRetriever:
|
|
|
66
66
|
top_k: int = 10,
|
|
67
67
|
vector_function: Literal["cosine_similarity", "inner_product", "l2_distance"] | None = None,
|
|
68
68
|
filter_policy: str | FilterPolicy = FilterPolicy.REPLACE,
|
|
69
|
-
):
|
|
69
|
+
) -> None:
|
|
70
70
|
"""
|
|
71
|
+
Initialize the PgvectorEmbeddingRetriever.
|
|
72
|
+
|
|
71
73
|
:param document_store: An instance of `PgvectorDocumentStore`.
|
|
72
74
|
:param filters: Filters applied to the retrieved Documents.
|
|
73
75
|
:param top_k: Maximum number of Documents to return.
|
|
@@ -55,8 +55,10 @@ class PgvectorKeywordRetriever:
|
|
|
55
55
|
filters: dict[str, Any] | None = None,
|
|
56
56
|
top_k: int = 10,
|
|
57
57
|
filter_policy: str | FilterPolicy = FilterPolicy.REPLACE,
|
|
58
|
-
):
|
|
58
|
+
) -> None:
|
|
59
59
|
"""
|
|
60
|
+
Initialize the PgvectorKeywordRetriever.
|
|
61
|
+
|
|
60
62
|
:param document_store: An instance of `PgvectorDocumentStore`.
|
|
61
63
|
:param filters: Filters applied to the retrieved Documents.
|
|
62
64
|
:param top_k: Maximum number of Documents to return.
|
|
@@ -10,8 +10,9 @@ logger = logging.getLogger(__name__)
|
|
|
10
10
|
|
|
11
11
|
def _from_haystack_to_pg_documents(documents: list[Document]) -> list[dict[str, Any]]:
|
|
12
12
|
"""
|
|
13
|
-
Internal method to convert a list of Haystack Documents to a list of dictionaries
|
|
14
|
-
|
|
13
|
+
Internal method to convert a list of Haystack Documents to a list of dictionaries.
|
|
14
|
+
|
|
15
|
+
The resulting dictionaries can be used to insert documents into the PgvectorDocumentStore.
|
|
15
16
|
"""
|
|
16
17
|
|
|
17
18
|
db_documents = []
|
|
@@ -96,9 +96,10 @@ class PgvectorDocumentStore:
|
|
|
96
96
|
hnsw_index_name: str = "haystack_hnsw_index",
|
|
97
97
|
hnsw_ef_search: int | None = None,
|
|
98
98
|
keyword_index_name: str = "haystack_keyword_index",
|
|
99
|
-
):
|
|
99
|
+
) -> None:
|
|
100
100
|
"""
|
|
101
101
|
Creates a new PgvectorDocumentStore instance.
|
|
102
|
+
|
|
102
103
|
It is meant to be connected to a PostgreSQL database with the pgvector extension installed.
|
|
103
104
|
A specific table to store Haystack documents will be created if it doesn't exist yet.
|
|
104
105
|
|
|
@@ -223,7 +224,7 @@ class PgvectorDocumentStore:
|
|
|
223
224
|
return default_from_dict(cls, data)
|
|
224
225
|
|
|
225
226
|
@staticmethod
|
|
226
|
-
def _connection_is_valid(connection):
|
|
227
|
+
def _connection_is_valid(connection: Connection) -> bool:
|
|
227
228
|
"""
|
|
228
229
|
Internal method to check if the connection is still valid.
|
|
229
230
|
"""
|
|
@@ -238,7 +239,7 @@ class PgvectorDocumentStore:
|
|
|
238
239
|
return True
|
|
239
240
|
|
|
240
241
|
@staticmethod
|
|
241
|
-
async def _connection_is_valid_async(connection):
|
|
242
|
+
async def _connection_is_valid_async(connection: AsyncConnection) -> bool:
|
|
242
243
|
"""
|
|
243
244
|
Internal method to check if the async connection is still valid.
|
|
244
245
|
"""
|
|
@@ -250,18 +251,18 @@ class PgvectorDocumentStore:
|
|
|
250
251
|
|
|
251
252
|
@overload
|
|
252
253
|
def _execute_sql(
|
|
253
|
-
self, cursor: Cursor, sql_query: Composed, params: tuple | None = None, error_msg: str = ""
|
|
254
|
+
self, cursor: Cursor, sql_query: SQL | Composed, params: tuple | None = None, error_msg: str = ""
|
|
254
255
|
) -> Cursor: ...
|
|
255
256
|
|
|
256
257
|
@overload
|
|
257
258
|
def _execute_sql(
|
|
258
|
-
self, cursor: Cursor[DictRow], sql_query: Composed, params: tuple | None = None, error_msg: str = ""
|
|
259
|
+
self, cursor: Cursor[DictRow], sql_query: SQL | Composed, params: tuple | None = None, error_msg: str = ""
|
|
259
260
|
) -> Cursor[DictRow]: ...
|
|
260
261
|
|
|
261
262
|
def _execute_sql(
|
|
262
263
|
self,
|
|
263
264
|
cursor: Cursor | Cursor[DictRow],
|
|
264
|
-
sql_query: Composed,
|
|
265
|
+
sql_query: SQL | Composed,
|
|
265
266
|
params: tuple | None = None,
|
|
266
267
|
error_msg: str = "",
|
|
267
268
|
) -> Cursor | Cursor[DictRow]:
|
|
@@ -299,18 +300,18 @@ class PgvectorDocumentStore:
|
|
|
299
300
|
|
|
300
301
|
@overload
|
|
301
302
|
async def _execute_sql_async(
|
|
302
|
-
self, cursor: AsyncCursor, sql_query: Composed, params: tuple | None = None, error_msg: str = ""
|
|
303
|
+
self, cursor: AsyncCursor, sql_query: SQL | Composed, params: tuple | None = None, error_msg: str = ""
|
|
303
304
|
) -> AsyncCursor: ...
|
|
304
305
|
|
|
305
306
|
@overload
|
|
306
307
|
async def _execute_sql_async(
|
|
307
|
-
self, cursor: AsyncCursor[DictRow], sql_query: Composed, params: tuple | None = None, error_msg: str = ""
|
|
308
|
+
self, cursor: AsyncCursor[DictRow], sql_query: SQL | Composed, params: tuple | None = None, error_msg: str = ""
|
|
308
309
|
) -> AsyncCursor[DictRow]: ...
|
|
309
310
|
|
|
310
311
|
async def _execute_sql_async(
|
|
311
312
|
self,
|
|
312
313
|
cursor: AsyncCursor | AsyncCursor[DictRow],
|
|
313
|
-
sql_query: Composed,
|
|
314
|
+
sql_query: SQL | Composed,
|
|
314
315
|
params: tuple | None = None,
|
|
315
316
|
error_msg: str = "",
|
|
316
317
|
) -> AsyncCursor | AsyncCursor[DictRow]:
|
|
@@ -346,9 +347,10 @@ class PgvectorDocumentStore:
|
|
|
346
347
|
|
|
347
348
|
return result
|
|
348
349
|
|
|
349
|
-
def _ensure_db_setup(self):
|
|
350
|
+
def _ensure_db_setup(self) -> None:
|
|
350
351
|
"""
|
|
351
352
|
Ensures that the connection to the PostgreSQL database exists and is valid.
|
|
353
|
+
|
|
352
354
|
If not, connection and cursors are created.
|
|
353
355
|
If the table is not initialized, it will be set up.
|
|
354
356
|
"""
|
|
@@ -384,9 +386,10 @@ class PgvectorDocumentStore:
|
|
|
384
386
|
if not self._table_initialized:
|
|
385
387
|
self._initialize_table()
|
|
386
388
|
|
|
387
|
-
async def _ensure_db_setup_async(self):
|
|
389
|
+
async def _ensure_db_setup_async(self) -> None:
|
|
388
390
|
"""
|
|
389
391
|
Async internal method.
|
|
392
|
+
|
|
390
393
|
Ensures that the connection to the PostgreSQL database exists and is valid.
|
|
391
394
|
If not, connection and cursors are created.
|
|
392
395
|
If the table is not initialized, it will be set up.
|
|
@@ -426,7 +429,7 @@ class PgvectorDocumentStore:
|
|
|
426
429
|
if not self._table_initialized:
|
|
427
430
|
await self._initialize_table_async()
|
|
428
431
|
|
|
429
|
-
def _build_table_creation_queries(self):
|
|
432
|
+
def _build_table_creation_queries(self) -> tuple[SQL, Composed, SQL, Composed]:
|
|
430
433
|
"""
|
|
431
434
|
Internal method to build the SQL queries for table creation.
|
|
432
435
|
"""
|
|
@@ -453,7 +456,7 @@ class PgvectorDocumentStore:
|
|
|
453
456
|
|
|
454
457
|
return sql_table_exists, sql_create_table, sql_keyword_index_exists, sql_create_keyword_index
|
|
455
458
|
|
|
456
|
-
def _initialize_table(self):
|
|
459
|
+
def _initialize_table(self) -> None:
|
|
457
460
|
"""
|
|
458
461
|
Internal method to initialize the table.
|
|
459
462
|
"""
|
|
@@ -497,7 +500,7 @@ class PgvectorDocumentStore:
|
|
|
497
500
|
|
|
498
501
|
self._table_initialized = True
|
|
499
502
|
|
|
500
|
-
async def _initialize_table_async(self):
|
|
503
|
+
async def _initialize_table_async(self) -> None:
|
|
501
504
|
"""
|
|
502
505
|
Internal async method to initialize the table.
|
|
503
506
|
"""
|
|
@@ -547,9 +550,10 @@ class PgvectorDocumentStore:
|
|
|
547
550
|
|
|
548
551
|
self._table_initialized = True
|
|
549
552
|
|
|
550
|
-
def delete_table(self):
|
|
553
|
+
def delete_table(self) -> None:
|
|
551
554
|
"""
|
|
552
555
|
Deletes the table used to store Haystack documents.
|
|
556
|
+
|
|
553
557
|
The name of the schema (`schema_name`) and the name of the table (`table_name`)
|
|
554
558
|
are defined when initializing the `PgvectorDocumentStore`.
|
|
555
559
|
"""
|
|
@@ -567,7 +571,7 @@ class PgvectorDocumentStore:
|
|
|
567
571
|
error_msg=f"Could not delete table {self.schema_name}.{self.table_name} in PgvectorDocumentStore",
|
|
568
572
|
)
|
|
569
573
|
|
|
570
|
-
async def delete_table_async(self):
|
|
574
|
+
async def delete_table_async(self) -> None:
|
|
571
575
|
"""
|
|
572
576
|
Async method to delete the table used to store Haystack documents.
|
|
573
577
|
"""
|
|
@@ -585,7 +589,7 @@ class PgvectorDocumentStore:
|
|
|
585
589
|
error_msg=f"Could not delete table {self.schema_name}.{self.table_name} in PgvectorDocumentStore",
|
|
586
590
|
)
|
|
587
591
|
|
|
588
|
-
def _build_hnsw_queries(self):
|
|
592
|
+
def _build_hnsw_queries(self) -> tuple[Composed | None, SQL, Composed, Composed]:
|
|
589
593
|
"""Common method to build all HNSW-related SQL queries"""
|
|
590
594
|
|
|
591
595
|
sql_set_hnsw_ef_search = (
|
|
@@ -633,9 +637,10 @@ class PgvectorDocumentStore:
|
|
|
633
637
|
|
|
634
638
|
return sql_set_hnsw_ef_search, sql_hnsw_index_exists, sql_drop_hnsw_index, sql_create_hnsw_index
|
|
635
639
|
|
|
636
|
-
def _handle_hnsw(self):
|
|
640
|
+
def _handle_hnsw(self) -> None:
|
|
637
641
|
"""
|
|
638
642
|
Internal method to handle the HNSW index creation.
|
|
643
|
+
|
|
639
644
|
It also sets the `hnsw.ef_search` parameter for queries if it is specified.
|
|
640
645
|
"""
|
|
641
646
|
|
|
@@ -645,7 +650,7 @@ class PgvectorDocumentStore:
|
|
|
645
650
|
|
|
646
651
|
assert self._cursor is not None
|
|
647
652
|
|
|
648
|
-
if
|
|
653
|
+
if sql_set_hnsw_ef_search is not None:
|
|
649
654
|
self._execute_sql(
|
|
650
655
|
cursor=self._cursor, sql_query=sql_set_hnsw_ef_search, error_msg="Could not set hnsw.ef_search"
|
|
651
656
|
)
|
|
@@ -671,7 +676,7 @@ class PgvectorDocumentStore:
|
|
|
671
676
|
|
|
672
677
|
self._execute_sql(cursor=self._cursor, sql_query=sql_create_hnsw_index, error_msg="Could not create HNSW index")
|
|
673
678
|
|
|
674
|
-
async def _handle_hnsw_async(self):
|
|
679
|
+
async def _handle_hnsw_async(self) -> None:
|
|
675
680
|
"""
|
|
676
681
|
Internal async method to handle the HNSW index creation.
|
|
677
682
|
"""
|
|
@@ -682,7 +687,7 @@ class PgvectorDocumentStore:
|
|
|
682
687
|
|
|
683
688
|
assert self._async_cursor is not None
|
|
684
689
|
|
|
685
|
-
if
|
|
690
|
+
if sql_set_hnsw_ef_search is not None:
|
|
686
691
|
await self._execute_sql_async(
|
|
687
692
|
cursor=self._async_cursor, sql_query=sql_set_hnsw_ef_search, error_msg="Could not set hnsw.ef_search"
|
|
688
693
|
)
|
|
@@ -1346,10 +1351,13 @@ class PgvectorDocumentStore:
|
|
|
1346
1351
|
# cosine_similarity and inner_product are modified from the result of the operator
|
|
1347
1352
|
if vector_function == "cosine_similarity":
|
|
1348
1353
|
score_definition = f"1 - (embedding <=> {query_embedding_for_postgres}) AS score"
|
|
1354
|
+
order_by_definition = f"embedding <=> {query_embedding_for_postgres}"
|
|
1349
1355
|
elif vector_function == "inner_product":
|
|
1350
1356
|
score_definition = f"(embedding <#> {query_embedding_for_postgres}) * -1 AS score"
|
|
1357
|
+
order_by_definition = f"embedding <#> {query_embedding_for_postgres}"
|
|
1351
1358
|
elif vector_function == "l2_distance":
|
|
1352
1359
|
score_definition = f"embedding <-> {query_embedding_for_postgres} AS score"
|
|
1360
|
+
order_by_definition = f"embedding <-> {query_embedding_for_postgres}"
|
|
1353
1361
|
|
|
1354
1362
|
sql_select = SQL("SELECT *, {score} FROM {schema_name}.{table_name}").format(
|
|
1355
1363
|
schema_name=Identifier(self.schema_name),
|
|
@@ -1362,13 +1370,9 @@ class PgvectorDocumentStore:
|
|
|
1362
1370
|
if filters:
|
|
1363
1371
|
sql_where_clause, params = _convert_filters_to_where_clause_and_params(filters)
|
|
1364
1372
|
|
|
1365
|
-
|
|
1366
|
-
# so when using l2_distance, the sort order must be ASC
|
|
1367
|
-
sort_order = "ASC" if vector_function == "l2_distance" else "DESC"
|
|
1368
|
-
|
|
1369
|
-
sql_sort = SQL(" ORDER BY score {sort_order} LIMIT {top_k}").format(
|
|
1373
|
+
sql_sort = SQL(" ORDER BY {order_by} ASC LIMIT {top_k}").format(
|
|
1370
1374
|
top_k=SQLLiteral(top_k),
|
|
1371
|
-
|
|
1375
|
+
order_by=SQL(order_by_definition),
|
|
1372
1376
|
)
|
|
1373
1377
|
|
|
1374
1378
|
sql_query = sql_select + sql_where_clause + sql_sort
|
|
@@ -1418,8 +1422,9 @@ class PgvectorDocumentStore:
|
|
|
1418
1422
|
vector_function: Literal["cosine_similarity", "inner_product", "l2_distance"] | None = None,
|
|
1419
1423
|
) -> list[Document]:
|
|
1420
1424
|
"""
|
|
1421
|
-
Asynchronously retrieves documents that are most similar to the query embedding
|
|
1422
|
-
|
|
1425
|
+
Asynchronously retrieves documents that are most similar to the query embedding.
|
|
1426
|
+
|
|
1427
|
+
Uses a vector similarity metric for comparison.
|
|
1423
1428
|
"""
|
|
1424
1429
|
|
|
1425
1430
|
sql_query, params = self._check_and_build_embedding_retrieval_query(
|
|
@@ -1439,7 +1444,7 @@ class PgvectorDocumentStore:
|
|
|
1439
1444
|
docs = _from_pg_to_haystack_documents(records)
|
|
1440
1445
|
return docs
|
|
1441
1446
|
|
|
1442
|
-
def _prepare_filters_count_documents(self, filters):
|
|
1447
|
+
def _prepare_filters_count_documents(self, filters: dict[str, Any] | None) -> tuple[tuple[Any, ...], Composed]:
|
|
1443
1448
|
_validate_filters(filters)
|
|
1444
1449
|
sql_count = SQL("SELECT COUNT(*) FROM {schema_name}.{table_name}").format(
|
|
1445
1450
|
schema_name=Identifier(self.schema_name), table_name=Identifier(self.table_name)
|
|
@@ -1571,8 +1576,9 @@ class PgvectorDocumentStore:
|
|
|
1571
1576
|
|
|
1572
1577
|
def count_unique_metadata_by_filter(self, filters: dict[str, Any], metadata_fields: list[str]) -> dict[str, int]:
|
|
1573
1578
|
"""
|
|
1574
|
-
Returns the count of unique values for each specified metadata field
|
|
1575
|
-
|
|
1579
|
+
Returns the count of unique values for each specified metadata field.
|
|
1580
|
+
|
|
1581
|
+
Considers only documents that match the provided filters.
|
|
1576
1582
|
|
|
1577
1583
|
:param filters: The filters to apply to select documents.
|
|
1578
1584
|
For filter syntax, see [Haystack metadata filtering](https://docs.haystack.deepset.ai/docs/metadata-filtering)
|
|
@@ -1604,8 +1610,9 @@ class PgvectorDocumentStore:
|
|
|
1604
1610
|
self, filters: dict[str, Any], metadata_fields: list[str]
|
|
1605
1611
|
) -> dict[str, int]:
|
|
1606
1612
|
"""
|
|
1607
|
-
Asynchronously returns the count of unique values for each specified metadata field
|
|
1608
|
-
|
|
1613
|
+
Asynchronously returns the count of unique values for each specified metadata field.
|
|
1614
|
+
|
|
1615
|
+
Considers only documents that match the provided filters.
|
|
1609
1616
|
|
|
1610
1617
|
:param filters: The filters to apply to select documents.
|
|
1611
1618
|
For filter syntax, see [Haystack metadata filtering](https://docs.haystack.deepset.ai/docs/metadata-filtering)
|
|
@@ -1661,7 +1668,7 @@ class PgvectorDocumentStore:
|
|
|
1661
1668
|
:param records: List of database records containing 'meta' field.
|
|
1662
1669
|
:returns: A dictionary mapping field names to their type information.
|
|
1663
1670
|
"""
|
|
1664
|
-
fields_info: dict[str, dict[str, str]] = {
|
|
1671
|
+
fields_info: dict[str, dict[str, str]] = {}
|
|
1665
1672
|
|
|
1666
1673
|
# Analyze metadata from all documents
|
|
1667
1674
|
for record in records:
|
|
@@ -1681,7 +1688,7 @@ class PgvectorDocumentStore:
|
|
|
1681
1688
|
|
|
1682
1689
|
return fields_info
|
|
1683
1690
|
|
|
1684
|
-
def _analyze_metadata_from_docs_query(self):
|
|
1691
|
+
def _analyze_metadata_from_docs_query(self) -> Composed:
|
|
1685
1692
|
# query all documents to analyze metadata structure
|
|
1686
1693
|
sql_query = SQL("SELECT meta FROM {schema_name}.{table_name} WHERE meta IS NOT NULL").format(
|
|
1687
1694
|
schema_name=Identifier(self.schema_name), table_name=Identifier(self.table_name)
|
|
@@ -1698,7 +1705,6 @@ class PgvectorDocumentStore:
|
|
|
1698
1705
|
Example return:
|
|
1699
1706
|
```python
|
|
1700
1707
|
{
|
|
1701
|
-
'content': {'type': 'text'},
|
|
1702
1708
|
'category': {'type': 'text'},
|
|
1703
1709
|
'status': {'type': 'text'},
|
|
1704
1710
|
'priority': {'type': 'integer'},
|
|
@@ -1838,15 +1844,14 @@ class PgvectorDocumentStore:
|
|
|
1838
1844
|
:returns: A dictionary with 'min' and 'max' keys containing the minimum and maximum values.
|
|
1839
1845
|
For numeric fields (integer, real), returns numeric min/max.
|
|
1840
1846
|
For text fields, returns lexicographic min/max based on database collation.
|
|
1841
|
-
|
|
1847
|
+
Returns `{"min": None, "max": None}` when the field has no values or the store is empty.
|
|
1842
1848
|
"""
|
|
1843
1849
|
normalized_field = PgvectorDocumentStore._normalize_metadata_field_name(metadata_field)
|
|
1844
1850
|
|
|
1845
1851
|
# Get field type information from metadata fields info
|
|
1846
1852
|
fields_info = self.get_metadata_fields_info()
|
|
1847
1853
|
if normalized_field not in fields_info:
|
|
1848
|
-
|
|
1849
|
-
raise ValueError(msg)
|
|
1854
|
+
return {"min": None, "max": None}
|
|
1850
1855
|
|
|
1851
1856
|
field_type = fields_info[normalized_field]["type"]
|
|
1852
1857
|
sql_query = self._build_min_max_query(normalized_field, field_type)
|
|
@@ -1871,15 +1876,14 @@ class PgvectorDocumentStore:
|
|
|
1871
1876
|
:returns: A dictionary with 'min' and 'max' keys containing the minimum and maximum values.
|
|
1872
1877
|
For numeric fields (integer, real), returns numeric min/max.
|
|
1873
1878
|
For text fields, returns lexicographic min/max based on database collation.
|
|
1874
|
-
|
|
1879
|
+
Returns ``{"min": None, "max": None}`` when the field has no values or the store is empty.
|
|
1875
1880
|
"""
|
|
1876
1881
|
normalized_field = PgvectorDocumentStore._normalize_metadata_field_name(metadata_field)
|
|
1877
1882
|
|
|
1878
1883
|
# Get field type information from metadata fields info
|
|
1879
1884
|
fields_info = await self.get_metadata_fields_info_async()
|
|
1880
1885
|
if normalized_field not in fields_info:
|
|
1881
|
-
|
|
1882
|
-
raise ValueError(msg)
|
|
1886
|
+
return {"min": None, "max": None}
|
|
1883
1887
|
|
|
1884
1888
|
field_type = fields_info[normalized_field]["type"]
|
|
1885
1889
|
sql_query = self._build_min_max_query(normalized_field, field_type)
|
|
@@ -2,13 +2,13 @@
|
|
|
2
2
|
#
|
|
3
3
|
# SPDX-License-Identifier: Apache-2.0
|
|
4
4
|
|
|
5
|
-
import re
|
|
6
5
|
from datetime import datetime
|
|
7
6
|
from itertools import chain
|
|
8
7
|
from typing import Any, Literal
|
|
9
8
|
|
|
10
9
|
from haystack.errors import FilterError
|
|
11
|
-
from psycopg.sql import SQL, Composed
|
|
10
|
+
from psycopg.sql import SQL, Composable, Composed, Identifier
|
|
11
|
+
from psycopg.sql import Literal as SQLLiteral
|
|
12
12
|
from psycopg.types.json import Jsonb
|
|
13
13
|
|
|
14
14
|
# we need this mapping to cast meta values to the correct type,
|
|
@@ -21,7 +21,6 @@ PYTHON_TYPES_TO_PG_TYPES = {
|
|
|
21
21
|
}
|
|
22
22
|
|
|
23
23
|
NO_VALUE = "no_value"
|
|
24
|
-
SAFE_META_FIELD_RE = re.compile(r"^[a-zA-Z0-9_-]+$")
|
|
25
24
|
|
|
26
25
|
|
|
27
26
|
def _validate_filters(filters: dict[str, Any] | None = None) -> None:
|
|
@@ -48,13 +47,13 @@ def _convert_filters_to_where_clause_and_params(
|
|
|
48
47
|
else:
|
|
49
48
|
query, values = _parse_logical_condition(filters)
|
|
50
49
|
|
|
51
|
-
where_clause = SQL(f" {operator} ") +
|
|
50
|
+
where_clause = SQL(f" {operator} ") + query
|
|
52
51
|
params = tuple(value for value in values if value != NO_VALUE)
|
|
53
52
|
|
|
54
53
|
return where_clause, params
|
|
55
54
|
|
|
56
55
|
|
|
57
|
-
def _parse_logical_condition(condition: dict[str, Any]) -> tuple[
|
|
56
|
+
def _parse_logical_condition(condition: dict[str, Any]) -> tuple[Composed, list[Any]]:
|
|
58
57
|
if "operator" not in condition:
|
|
59
58
|
msg = f"'operator' key missing in {condition}"
|
|
60
59
|
raise FilterError(msg)
|
|
@@ -84,17 +83,14 @@ def _parse_logical_condition(condition: dict[str, Any]) -> tuple[str, list[Any]]
|
|
|
84
83
|
values = list(chain.from_iterable(values))
|
|
85
84
|
|
|
86
85
|
if operator == "AND":
|
|
87
|
-
sql_query =
|
|
88
|
-
|
|
89
|
-
sql_query =
|
|
90
|
-
else:
|
|
91
|
-
msg = f"Unknown logical operator '{operator}'"
|
|
92
|
-
raise FilterError(msg)
|
|
86
|
+
sql_query = SQL("(") + SQL(" AND ").join(query_parts) + SQL(")")
|
|
87
|
+
else: # operator == "OR"
|
|
88
|
+
sql_query = SQL("(") + SQL(" OR ").join(query_parts) + SQL(")")
|
|
93
89
|
|
|
94
90
|
return sql_query, values
|
|
95
91
|
|
|
96
92
|
|
|
97
|
-
def _parse_comparison_condition(condition: dict[str, Any]) -> tuple[
|
|
93
|
+
def _parse_comparison_condition(condition: dict[str, Any]) -> tuple[Composed, list[Any]]:
|
|
98
94
|
field: str = condition["field"]
|
|
99
95
|
if "operator" not in condition:
|
|
100
96
|
msg = f"'operator' key missing in {condition}"
|
|
@@ -110,34 +106,36 @@ def _parse_comparison_condition(condition: dict[str, Any]) -> tuple[str, list[An
|
|
|
110
106
|
value: Any = condition["value"]
|
|
111
107
|
|
|
112
108
|
if field.startswith("meta."):
|
|
113
|
-
|
|
109
|
+
sql_field: Composable = _treat_meta_field(field, value)
|
|
110
|
+
else:
|
|
111
|
+
sql_field = Identifier(field)
|
|
114
112
|
|
|
115
|
-
|
|
116
|
-
return
|
|
113
|
+
sql_expr, value = COMPARISON_OPERATORS[operator](sql_field, value)
|
|
114
|
+
return sql_expr, [value]
|
|
117
115
|
|
|
118
116
|
|
|
119
|
-
def _treat_meta_field(field: str, value: Any) ->
|
|
117
|
+
def _treat_meta_field(field: str, value: Any) -> Composed:
|
|
120
118
|
"""
|
|
121
|
-
Internal method that
|
|
122
|
-
|
|
119
|
+
Internal method that returns a psycopg Composed object to make the meta JSONB field queryable safely.
|
|
120
|
+
|
|
121
|
+
Uses psycopg.sql.Literal to embed the field name, preventing SQL injection
|
|
122
|
+
via metadata field names without requiring regex validation.
|
|
123
|
+
|
|
124
|
+
Use the ->> operator to access keys in the meta JSONB field.
|
|
123
125
|
|
|
124
126
|
Examples:
|
|
125
127
|
>>> _treat_meta_field(field="meta.number", value=9)
|
|
126
|
-
|
|
128
|
+
Composed([SQL('(meta->>'), Literal('number'), SQL(')::integer')])
|
|
127
129
|
|
|
128
130
|
>>> _treat_meta_field(field="meta.name", value="my_name")
|
|
129
|
-
|
|
130
|
-
"""
|
|
131
|
+
Composed([SQL('meta->>'), Literal('name')])
|
|
131
132
|
|
|
132
|
-
|
|
133
|
+
"""
|
|
133
134
|
field_name = field.split(".", 1)[-1]
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
)
|
|
139
|
-
raise FilterError(msg)
|
|
140
|
-
field = f"meta->>'{field_name}'"
|
|
135
|
+
|
|
136
|
+
# Use SQLLiteral to safely embed the field name as a SQL string literal,
|
|
137
|
+
# preventing SQL injection via metadata field names.
|
|
138
|
+
composed: Composed = SQL("meta->>") + SQLLiteral(field_name)
|
|
141
139
|
|
|
142
140
|
# meta fields are stored as strings in the JSONB field,
|
|
143
141
|
# so we need to cast them to the correct type
|
|
@@ -146,25 +144,24 @@ def _treat_meta_field(field: str, value: Any) -> str:
|
|
|
146
144
|
type_value = PYTHON_TYPES_TO_PG_TYPES.get(type(value[0]))
|
|
147
145
|
|
|
148
146
|
if type_value:
|
|
149
|
-
|
|
147
|
+
composed = SQL("(") + composed + SQL(f")::{type_value}")
|
|
150
148
|
|
|
151
|
-
return
|
|
149
|
+
return composed
|
|
152
150
|
|
|
153
151
|
|
|
154
|
-
def _equal(field:
|
|
152
|
+
def _equal(field: Composable, value: Any) -> tuple[Composed, Any]:
|
|
155
153
|
if value is None:
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
return f"{field} = %s", value
|
|
154
|
+
return SQL("{} IS NULL").format(field), NO_VALUE
|
|
155
|
+
return SQL("{} = %s").format(field), value
|
|
159
156
|
|
|
160
157
|
|
|
161
|
-
def _not_equal(field:
|
|
158
|
+
def _not_equal(field: Composable, value: Any) -> tuple[Composed, Any]:
|
|
162
159
|
# we use IS DISTINCT FROM to correctly handle NULL values
|
|
163
160
|
# (not handled by !=)
|
|
164
|
-
return
|
|
161
|
+
return SQL("{} IS DISTINCT FROM %s").format(field), value
|
|
165
162
|
|
|
166
163
|
|
|
167
|
-
def _greater_than(field:
|
|
164
|
+
def _greater_than(field: Composable, value: Any) -> tuple[Composed, Any]:
|
|
168
165
|
if isinstance(value, str):
|
|
169
166
|
try:
|
|
170
167
|
datetime.fromisoformat(value)
|
|
@@ -177,11 +174,10 @@ def _greater_than(field: str, value: Any) -> tuple[str, Any]:
|
|
|
177
174
|
if type(value) in [list, Jsonb]:
|
|
178
175
|
msg = f"Filter value can't be of type {type(value)} using operators '>', '>=', '<', '<='"
|
|
179
176
|
raise FilterError(msg)
|
|
177
|
+
return SQL("{} > %s").format(field), value
|
|
180
178
|
|
|
181
|
-
return f"{field} > %s", value
|
|
182
179
|
|
|
183
|
-
|
|
184
|
-
def _greater_than_equal(field: str, value: Any) -> tuple[str, Any]:
|
|
180
|
+
def _greater_than_equal(field: Composable, value: Any) -> tuple[Composed, Any]:
|
|
185
181
|
if isinstance(value, str):
|
|
186
182
|
try:
|
|
187
183
|
datetime.fromisoformat(value)
|
|
@@ -194,11 +190,10 @@ def _greater_than_equal(field: str, value: Any) -> tuple[str, Any]:
|
|
|
194
190
|
if type(value) in [list, Jsonb]:
|
|
195
191
|
msg = f"Filter value can't be of type {type(value)} using operators '>', '>=', '<', '<='"
|
|
196
192
|
raise FilterError(msg)
|
|
197
|
-
|
|
198
|
-
return f"{field} >= %s", value
|
|
193
|
+
return SQL("{} >= %s").format(field), value
|
|
199
194
|
|
|
200
195
|
|
|
201
|
-
def _less_than(field:
|
|
196
|
+
def _less_than(field: Composable, value: Any) -> tuple[Composed, Any]:
|
|
202
197
|
if isinstance(value, str):
|
|
203
198
|
try:
|
|
204
199
|
datetime.fromisoformat(value)
|
|
@@ -211,11 +206,10 @@ def _less_than(field: str, value: Any) -> tuple[str, Any]:
|
|
|
211
206
|
if type(value) in [list, Jsonb]:
|
|
212
207
|
msg = f"Filter value can't be of type {type(value)} using operators '>', '>=', '<', '<='"
|
|
213
208
|
raise FilterError(msg)
|
|
209
|
+
return SQL("{} < %s").format(field), value
|
|
214
210
|
|
|
215
|
-
return f"{field} < %s", value
|
|
216
211
|
|
|
217
|
-
|
|
218
|
-
def _less_than_equal(field: str, value: Any) -> tuple[str, Any]:
|
|
212
|
+
def _less_than_equal(field: Composable, value: Any) -> tuple[Composed, Any]:
|
|
219
213
|
if isinstance(value, str):
|
|
220
214
|
try:
|
|
221
215
|
datetime.fromisoformat(value)
|
|
@@ -228,39 +222,37 @@ def _less_than_equal(field: str, value: Any) -> tuple[str, Any]:
|
|
|
228
222
|
if type(value) in [list, Jsonb]:
|
|
229
223
|
msg = f"Filter value can't be of type {type(value)} using operators '>', '>=', '<', '<='"
|
|
230
224
|
raise FilterError(msg)
|
|
231
|
-
|
|
232
|
-
return f"{field} <= %s", value
|
|
225
|
+
return SQL("{} <= %s").format(field), value
|
|
233
226
|
|
|
234
227
|
|
|
235
|
-
def _not_in(field:
|
|
228
|
+
def _not_in(field: Composable, value: Any) -> tuple[Composed, list]:
|
|
236
229
|
if not isinstance(value, list):
|
|
237
230
|
msg = f"{field}'s value must be a list when using 'not in' comparator in Pinecone"
|
|
238
231
|
raise FilterError(msg)
|
|
239
|
-
|
|
240
|
-
return f"{field} IS NULL OR {field} != ALL(%s)", [value]
|
|
232
|
+
return SQL("{} IS NULL OR {} != ALL(%s)").format(field, field), [value]
|
|
241
233
|
|
|
242
234
|
|
|
243
|
-
def _in(field:
|
|
235
|
+
def _in(field: Composable, value: Any) -> tuple[Composed, list]:
|
|
244
236
|
if not isinstance(value, list):
|
|
245
237
|
msg = f"{field}'s value must be a list when using 'in' comparator in Pinecone"
|
|
246
238
|
raise FilterError(msg)
|
|
247
239
|
|
|
248
240
|
# see https://www.psycopg.org/psycopg3/docs/basic/adapt.html#lists-adaptation
|
|
249
|
-
return
|
|
241
|
+
return SQL("{} = ANY(%s)").format(field), [value]
|
|
250
242
|
|
|
251
243
|
|
|
252
|
-
def _like(field:
|
|
244
|
+
def _like(field: Composable, value: Any) -> tuple[Composed, Any]:
|
|
253
245
|
if not isinstance(value, str):
|
|
254
246
|
msg = f"{field}'s value must be a str when using 'LIKE' "
|
|
255
247
|
raise FilterError(msg)
|
|
256
|
-
return
|
|
248
|
+
return SQL("{} LIKE %s").format(field), value
|
|
257
249
|
|
|
258
250
|
|
|
259
|
-
def _not_like(field:
|
|
251
|
+
def _not_like(field: Composable, value: Any) -> tuple[Composed, Any]:
|
|
260
252
|
if not isinstance(value, str):
|
|
261
253
|
msg = f"{field}'s value must be a str when using 'LIKE' "
|
|
262
254
|
raise FilterError(msg)
|
|
263
|
-
return
|
|
255
|
+
return SQL("{} NOT LIKE %s").format(field), value
|
|
264
256
|
|
|
265
257
|
|
|
266
258
|
COMPARISON_OPERATORS = {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: pgvector-haystack
|
|
3
|
-
Version: 6.
|
|
3
|
+
Version: 6.3.1
|
|
4
4
|
Summary: An integration of pgvector (vector search extension for Postgres) with Haystack
|
|
5
5
|
Project-URL: Source, https://github.com/deepset-ai/haystack-core-integrations
|
|
6
6
|
Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/blob/main/integrations/pgvector/README.md
|
|
@@ -15,10 +15,11 @@ Classifier: Programming Language :: Python :: 3.10
|
|
|
15
15
|
Classifier: Programming Language :: Python :: 3.11
|
|
16
16
|
Classifier: Programming Language :: Python :: 3.12
|
|
17
17
|
Classifier: Programming Language :: Python :: 3.13
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
18
19
|
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
19
20
|
Classifier: Programming Language :: Python :: Implementation :: PyPy
|
|
20
21
|
Requires-Python: >=3.10
|
|
21
|
-
Requires-Dist: haystack-ai>=2.
|
|
22
|
+
Requires-Dist: haystack-ai>=2.28.0
|
|
22
23
|
Requires-Dist: pgvector>=0.3.0
|
|
23
24
|
Requires-Dist: psycopg[binary]
|
|
24
25
|
Description-Content-Type: text/markdown
|
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
haystack_integrations/components/retrievers/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
2
|
haystack_integrations/components/retrievers/pgvector/__init__.py,sha256=n4q4pUApJ9DZakJ_JL-egwOzlefdnI_ZxjU1Lu44Sqo,296
|
|
3
|
-
haystack_integrations/components/retrievers/pgvector/embedding_retriever.py,sha256=
|
|
4
|
-
haystack_integrations/components/retrievers/pgvector/keyword_retriever.py,sha256=
|
|
3
|
+
haystack_integrations/components/retrievers/pgvector/embedding_retriever.py,sha256=WnKXvRgzAyKQ2KfqnkiCTjy0WtiMNL4mK3_ZxJ16Lfk,9190
|
|
4
|
+
haystack_integrations/components/retrievers/pgvector/keyword_retriever.py,sha256=tks3Wror_cfrUPxObzWzPmii_5ryNU1KEq3Q-DWnJ8U,6850
|
|
5
5
|
haystack_integrations/document_stores/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
6
6
|
haystack_integrations/document_stores/pgvector/__init__.py,sha256=Ge-7ggarfr3x8UqNdFHUuFGiu52u49sOVe3l16WEwZU,197
|
|
7
|
-
haystack_integrations/document_stores/pgvector/converters.py,sha256=
|
|
8
|
-
haystack_integrations/document_stores/pgvector/document_store.py,sha256=
|
|
9
|
-
haystack_integrations/document_stores/pgvector/filters.py,sha256=
|
|
10
|
-
pgvector_haystack-6.
|
|
11
|
-
pgvector_haystack-6.
|
|
12
|
-
pgvector_haystack-6.
|
|
13
|
-
pgvector_haystack-6.
|
|
7
|
+
haystack_integrations/document_stores/pgvector/converters.py,sha256=ZJ28EC7ianclAdkjlGsGLwqCqUv2H-1UYz9PqnLX_aU,3205
|
|
8
|
+
haystack_integrations/document_stores/pgvector/document_store.py,sha256=SdXnbOSns1pUDCSy9pifK5WrPDsvrCdbAiTIneIXF0Q,84725
|
|
9
|
+
haystack_integrations/document_stores/pgvector/filters.py,sha256=91TiRZULHiKWCIIGXP7ppgiHJeDEV6oS2kR8EkQE-uI,9659
|
|
10
|
+
pgvector_haystack-6.3.1.dist-info/METADATA,sha256=0OzM0VaPjJsTe8CbbHIZK-O1WINU2BQRaHYs7o1x6LE,2146
|
|
11
|
+
pgvector_haystack-6.3.1.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
|
|
12
|
+
pgvector_haystack-6.3.1.dist-info/licenses/LICENSE.txt,sha256=B05uMshqTA74s-0ltyHKI6yoPfJ3zYgQbvcXfDVGFf8,10280
|
|
13
|
+
pgvector_haystack-6.3.1.dist-info/RECORD,,
|
|
File without changes
|
{pgvector_haystack-6.2.0.dist-info → pgvector_haystack-6.3.1.dist-info}/licenses/LICENSE.txt
RENAMED
|
File without changes
|