pgvector-haystack 0.3.0__py3-none-any.whl → 0.5.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- haystack_integrations/components/retrievers/pgvector/embedding_retriever.py +14 -4
- haystack_integrations/components/retrievers/pgvector/keyword_retriever.py +15 -4
- haystack_integrations/document_stores/pgvector/document_store.py +13 -9
- {pgvector_haystack-0.3.0.dist-info → pgvector_haystack-0.5.0.dist-info}/METADATA +1 -1
- {pgvector_haystack-0.3.0.dist-info → pgvector_haystack-0.5.0.dist-info}/RECORD +7 -7
- {pgvector_haystack-0.3.0.dist-info → pgvector_haystack-0.5.0.dist-info}/WHEEL +1 -1
- {pgvector_haystack-0.3.0.dist-info → pgvector_haystack-0.5.0.dist-info}/licenses/LICENSE.txt +0 -0
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
# SPDX-FileCopyrightText: 2023-present deepset GmbH <info@deepset.ai>
|
|
2
2
|
#
|
|
3
3
|
# SPDX-License-Identifier: Apache-2.0
|
|
4
|
-
from typing import Any, Dict, List, Literal, Optional
|
|
4
|
+
from typing import Any, Dict, List, Literal, Optional, Union
|
|
5
5
|
|
|
6
6
|
from haystack import component, default_from_dict, default_to_dict
|
|
7
7
|
from haystack.dataclasses import Document
|
|
8
|
+
from haystack.document_stores.types import FilterPolicy
|
|
9
|
+
from haystack.document_stores.types.filter_policy import apply_filter_policy
|
|
8
10
|
from haystack_integrations.document_stores.pgvector import PgvectorDocumentStore
|
|
9
11
|
from haystack_integrations.document_stores.pgvector.document_store import VALID_VECTOR_FUNCTIONS
|
|
10
12
|
|
|
@@ -62,6 +64,7 @@ class PgvectorEmbeddingRetriever:
|
|
|
62
64
|
filters: Optional[Dict[str, Any]] = None,
|
|
63
65
|
top_k: int = 10,
|
|
64
66
|
vector_function: Optional[Literal["cosine_similarity", "inner_product", "l2_distance"]] = None,
|
|
67
|
+
filter_policy: Union[str, FilterPolicy] = FilterPolicy.REPLACE,
|
|
65
68
|
):
|
|
66
69
|
"""
|
|
67
70
|
:param document_store: An instance of `PgvectorDocumentStore`.
|
|
@@ -75,7 +78,7 @@ class PgvectorEmbeddingRetriever:
|
|
|
75
78
|
and the most similar documents are the ones with the smallest score.
|
|
76
79
|
**Important**: if the document store is using the `"hnsw"` search strategy, the vector function
|
|
77
80
|
should match the one utilized during index creation to take advantage of the index.
|
|
78
|
-
|
|
81
|
+
:param filter_policy: Policy to determine how filters are applied.
|
|
79
82
|
:raises ValueError: If `document_store` is not an instance of `PgvectorDocumentStore` or if `vector_function`
|
|
80
83
|
is not one of the valid options.
|
|
81
84
|
"""
|
|
@@ -91,6 +94,9 @@ class PgvectorEmbeddingRetriever:
|
|
|
91
94
|
self.filters = filters or {}
|
|
92
95
|
self.top_k = top_k
|
|
93
96
|
self.vector_function = vector_function or document_store.vector_function
|
|
97
|
+
self.filter_policy = (
|
|
98
|
+
filter_policy if isinstance(filter_policy, FilterPolicy) else FilterPolicy.from_str(filter_policy)
|
|
99
|
+
)
|
|
94
100
|
|
|
95
101
|
def to_dict(self) -> Dict[str, Any]:
|
|
96
102
|
"""
|
|
@@ -104,6 +110,7 @@ class PgvectorEmbeddingRetriever:
|
|
|
104
110
|
filters=self.filters,
|
|
105
111
|
top_k=self.top_k,
|
|
106
112
|
vector_function=self.vector_function,
|
|
113
|
+
filter_policy=self.filter_policy.value,
|
|
107
114
|
document_store=self.document_store.to_dict(),
|
|
108
115
|
)
|
|
109
116
|
|
|
@@ -119,6 +126,7 @@ class PgvectorEmbeddingRetriever:
|
|
|
119
126
|
"""
|
|
120
127
|
doc_store_params = data["init_parameters"]["document_store"]
|
|
121
128
|
data["init_parameters"]["document_store"] = PgvectorDocumentStore.from_dict(doc_store_params)
|
|
129
|
+
data["init_parameters"]["filter_policy"] = FilterPolicy.from_str(data["init_parameters"]["filter_policy"])
|
|
122
130
|
return default_from_dict(cls, data)
|
|
123
131
|
|
|
124
132
|
@component.output_types(documents=List[Document])
|
|
@@ -133,13 +141,15 @@ class PgvectorEmbeddingRetriever:
|
|
|
133
141
|
Retrieve documents from the `PgvectorDocumentStore`, based on their embeddings.
|
|
134
142
|
|
|
135
143
|
:param query_embedding: Embedding of the query.
|
|
136
|
-
:param filters: Filters applied to the retrieved Documents.
|
|
144
|
+
:param filters: Filters applied to the retrieved Documents. The way runtime filters are applied depends on
|
|
145
|
+
the `filter_policy` chosen at retriever initialization. See init method docstring for more
|
|
146
|
+
details.
|
|
137
147
|
:param top_k: Maximum number of Documents to return.
|
|
138
148
|
:param vector_function: The similarity function to use when searching for similar embeddings.
|
|
139
149
|
|
|
140
150
|
:returns: List of Documents similar to `query_embedding`.
|
|
141
151
|
"""
|
|
142
|
-
filters =
|
|
152
|
+
filters = apply_filter_policy(self.filter_policy, self.filters, filters)
|
|
143
153
|
top_k = top_k or self.top_k
|
|
144
154
|
vector_function = vector_function or self.vector_function
|
|
145
155
|
|
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
# SPDX-FileCopyrightText: 2023-present deepset GmbH <info@deepset.ai>
|
|
2
2
|
#
|
|
3
3
|
# SPDX-License-Identifier: Apache-2.0
|
|
4
|
-
from typing import Any, Dict, List, Optional
|
|
4
|
+
from typing import Any, Dict, List, Optional, Union
|
|
5
5
|
|
|
6
6
|
from haystack import component, default_from_dict, default_to_dict
|
|
7
7
|
from haystack.dataclasses import Document
|
|
8
|
+
from haystack.document_stores.types import FilterPolicy
|
|
9
|
+
from haystack.document_stores.types.filter_policy import apply_filter_policy
|
|
8
10
|
from haystack_integrations.document_stores.pgvector import PgvectorDocumentStore
|
|
9
11
|
|
|
10
12
|
|
|
@@ -51,12 +53,13 @@ class PgvectorKeywordRetriever:
|
|
|
51
53
|
document_store: PgvectorDocumentStore,
|
|
52
54
|
filters: Optional[Dict[str, Any]] = None,
|
|
53
55
|
top_k: int = 10,
|
|
56
|
+
filter_policy: Union[str, FilterPolicy] = FilterPolicy.REPLACE,
|
|
54
57
|
):
|
|
55
58
|
"""
|
|
56
59
|
:param document_store: An instance of `PgvectorDocumentStore`.
|
|
57
60
|
:param filters: Filters applied to the retrieved Documents.
|
|
58
61
|
:param top_k: Maximum number of Documents to return.
|
|
59
|
-
|
|
62
|
+
:param filter_policy: Policy to determine how filters are applied.
|
|
60
63
|
:raises ValueError: If `document_store` is not an instance of `PgvectorDocumentStore`.
|
|
61
64
|
"""
|
|
62
65
|
if not isinstance(document_store, PgvectorDocumentStore):
|
|
@@ -66,6 +69,9 @@ class PgvectorKeywordRetriever:
|
|
|
66
69
|
self.document_store = document_store
|
|
67
70
|
self.filters = filters or {}
|
|
68
71
|
self.top_k = top_k
|
|
72
|
+
self.filter_policy = (
|
|
73
|
+
filter_policy if isinstance(filter_policy, FilterPolicy) else FilterPolicy.from_str(filter_policy)
|
|
74
|
+
)
|
|
69
75
|
|
|
70
76
|
def to_dict(self) -> Dict[str, Any]:
|
|
71
77
|
"""
|
|
@@ -78,6 +84,7 @@ class PgvectorKeywordRetriever:
|
|
|
78
84
|
self,
|
|
79
85
|
filters=self.filters,
|
|
80
86
|
top_k=self.top_k,
|
|
87
|
+
filter_policy=self.filter_policy.value,
|
|
81
88
|
document_store=self.document_store.to_dict(),
|
|
82
89
|
)
|
|
83
90
|
|
|
@@ -93,6 +100,7 @@ class PgvectorKeywordRetriever:
|
|
|
93
100
|
"""
|
|
94
101
|
doc_store_params = data["init_parameters"]["document_store"]
|
|
95
102
|
data["init_parameters"]["document_store"] = PgvectorDocumentStore.from_dict(doc_store_params)
|
|
103
|
+
data["init_parameters"]["filter_policy"] = FilterPolicy.from_str(data["init_parameters"]["filter_policy"])
|
|
96
104
|
return default_from_dict(cls, data)
|
|
97
105
|
|
|
98
106
|
@component.output_types(documents=List[Document])
|
|
@@ -106,13 +114,16 @@ class PgvectorKeywordRetriever:
|
|
|
106
114
|
Retrieve documents from the `PgvectorDocumentStore`, based on keywords.
|
|
107
115
|
|
|
108
116
|
:param query: String to search in `Document`s' content.
|
|
109
|
-
:param filters: Filters applied to the retrieved Documents.
|
|
117
|
+
:param filters: Filters applied to the retrieved Documents. The way runtime filters are applied depends on
|
|
118
|
+
the `filter_policy` chosen at retriever initialization. See init method docstring for more
|
|
119
|
+
details.
|
|
110
120
|
:param top_k: Maximum number of Documents to return.
|
|
111
121
|
|
|
112
122
|
:returns: A dictionary with the following keys:
|
|
113
123
|
- `documents`: List of `Document`s that match the query.
|
|
114
124
|
"""
|
|
115
|
-
filters =
|
|
125
|
+
filters = apply_filter_policy(self.filter_policy, self.filters, filters)
|
|
126
|
+
|
|
116
127
|
top_k = top_k or self.top_k
|
|
117
128
|
|
|
118
129
|
docs = self.document_store._keyword_retrieval(
|
|
@@ -69,10 +69,6 @@ VECTOR_FUNCTION_TO_POSTGRESQL_OPS = {
|
|
|
69
69
|
|
|
70
70
|
HNSW_INDEX_CREATION_VALID_KWARGS = ["m", "ef_construction"]
|
|
71
71
|
|
|
72
|
-
HNSW_INDEX_NAME = "haystack_hnsw_index"
|
|
73
|
-
|
|
74
|
-
KEYWORD_INDEX_NAME = "haystack_keyword_index"
|
|
75
|
-
|
|
76
72
|
|
|
77
73
|
class PgvectorDocumentStore:
|
|
78
74
|
"""
|
|
@@ -91,7 +87,9 @@ class PgvectorDocumentStore:
|
|
|
91
87
|
search_strategy: Literal["exact_nearest_neighbor", "hnsw"] = "exact_nearest_neighbor",
|
|
92
88
|
hnsw_recreate_index_if_exists: bool = False,
|
|
93
89
|
hnsw_index_creation_kwargs: Optional[Dict[str, int]] = None,
|
|
90
|
+
hnsw_index_name: str = "haystack_hnsw_index",
|
|
94
91
|
hnsw_ef_search: Optional[int] = None,
|
|
92
|
+
keyword_index_name: str = "haystack_keyword_index",
|
|
95
93
|
):
|
|
96
94
|
"""
|
|
97
95
|
Creates a new PgvectorDocumentStore instance.
|
|
@@ -127,9 +125,11 @@ class PgvectorDocumentStore:
|
|
|
127
125
|
:param hnsw_index_creation_kwargs: Additional keyword arguments to pass to the HNSW index creation.
|
|
128
126
|
Only used if search_strategy is set to `"hnsw"`. You can find the list of valid arguments in the
|
|
129
127
|
[pgvector documentation](https://github.com/pgvector/pgvector?tab=readme-ov-file#hnsw)
|
|
128
|
+
:param hnsw_index_name: Index name for the HNSW index.
|
|
130
129
|
:param hnsw_ef_search: The `ef_search` parameter to use at query time. Only used if search_strategy is set to
|
|
131
130
|
`"hnsw"`. You can find more information about this parameter in the
|
|
132
131
|
[pgvector documentation](https://github.com/pgvector/pgvector?tab=readme-ov-file#hnsw).
|
|
132
|
+
:param keyword_index_name: Index name for the Keyword index.
|
|
133
133
|
"""
|
|
134
134
|
|
|
135
135
|
self.connection_string = connection_string
|
|
@@ -143,7 +143,9 @@ class PgvectorDocumentStore:
|
|
|
143
143
|
self.search_strategy = search_strategy
|
|
144
144
|
self.hnsw_recreate_index_if_exists = hnsw_recreate_index_if_exists
|
|
145
145
|
self.hnsw_index_creation_kwargs = hnsw_index_creation_kwargs or {}
|
|
146
|
+
self.hnsw_index_name = hnsw_index_name
|
|
146
147
|
self.hnsw_ef_search = hnsw_ef_search
|
|
148
|
+
self.keyword_index_name = keyword_index_name
|
|
147
149
|
self.language = language
|
|
148
150
|
self._connection = None
|
|
149
151
|
self._cursor = None
|
|
@@ -209,7 +211,9 @@ class PgvectorDocumentStore:
|
|
|
209
211
|
search_strategy=self.search_strategy,
|
|
210
212
|
hnsw_recreate_index_if_exists=self.hnsw_recreate_index_if_exists,
|
|
211
213
|
hnsw_index_creation_kwargs=self.hnsw_index_creation_kwargs,
|
|
214
|
+
hnsw_index_name=self.hnsw_index_name,
|
|
212
215
|
hnsw_ef_search=self.hnsw_ef_search,
|
|
216
|
+
keyword_index_name=self.keyword_index_name,
|
|
213
217
|
language=self.language,
|
|
214
218
|
)
|
|
215
219
|
|
|
@@ -281,7 +285,7 @@ class PgvectorDocumentStore:
|
|
|
281
285
|
index_exists = bool(
|
|
282
286
|
self._execute_sql(
|
|
283
287
|
"SELECT 1 FROM pg_indexes WHERE tablename = %s AND indexname = %s",
|
|
284
|
-
(self.table_name,
|
|
288
|
+
(self.table_name, self.keyword_index_name),
|
|
285
289
|
"Could not check if keyword index exists",
|
|
286
290
|
).fetchone()
|
|
287
291
|
)
|
|
@@ -289,7 +293,7 @@ class PgvectorDocumentStore:
|
|
|
289
293
|
sql_create_index = SQL(
|
|
290
294
|
"CREATE INDEX {index_name} ON {table_name} USING GIN (to_tsvector({language}, content))"
|
|
291
295
|
).format(
|
|
292
|
-
index_name=Identifier(
|
|
296
|
+
index_name=Identifier(self.keyword_index_name),
|
|
293
297
|
table_name=Identifier(self.table_name),
|
|
294
298
|
language=SQLLiteral(self.language),
|
|
295
299
|
)
|
|
@@ -312,7 +316,7 @@ class PgvectorDocumentStore:
|
|
|
312
316
|
index_exists = bool(
|
|
313
317
|
self._execute_sql(
|
|
314
318
|
"SELECT 1 FROM pg_indexes WHERE tablename = %s AND indexname = %s",
|
|
315
|
-
(self.table_name,
|
|
319
|
+
(self.table_name, self.hnsw_index_name),
|
|
316
320
|
"Could not check if HNSW index exists",
|
|
317
321
|
).fetchone()
|
|
318
322
|
)
|
|
@@ -325,7 +329,7 @@ class PgvectorDocumentStore:
|
|
|
325
329
|
)
|
|
326
330
|
return
|
|
327
331
|
|
|
328
|
-
sql_drop_index = SQL("DROP INDEX IF EXISTS {index_name}").format(index_name=Identifier(
|
|
332
|
+
sql_drop_index = SQL("DROP INDEX IF EXISTS {index_name}").format(index_name=Identifier(self.hnsw_index_name))
|
|
329
333
|
self._execute_sql(sql_drop_index, error_msg="Could not drop HNSW index")
|
|
330
334
|
|
|
331
335
|
self._create_hnsw_index()
|
|
@@ -343,7 +347,7 @@ class PgvectorDocumentStore:
|
|
|
343
347
|
}
|
|
344
348
|
|
|
345
349
|
sql_create_index = SQL("CREATE INDEX {index_name} ON {table_name} USING hnsw (embedding {ops}) ").format(
|
|
346
|
-
index_name=Identifier(
|
|
350
|
+
index_name=Identifier(self.hnsw_index_name), table_name=Identifier(self.table_name), ops=SQL(pg_ops)
|
|
347
351
|
)
|
|
348
352
|
|
|
349
353
|
if actual_hnsw_index_creation_kwargs:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.3
|
|
2
2
|
Name: pgvector-haystack
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.5.0
|
|
4
4
|
Summary: An integration of pgvector (vector search extension for Postgres) with Haystack
|
|
5
5
|
Project-URL: Source, https://github.com/deepset-ai/haystack-core-integrations
|
|
6
6
|
Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/blob/main/integrations/pgvector/README.md
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
haystack_integrations/components/retrievers/pgvector/__init__.py,sha256=n4q4pUApJ9DZakJ_JL-egwOzlefdnI_ZxjU1Lu44Sqo,296
|
|
2
|
-
haystack_integrations/components/retrievers/pgvector/embedding_retriever.py,sha256=
|
|
3
|
-
haystack_integrations/components/retrievers/pgvector/keyword_retriever.py,sha256=
|
|
2
|
+
haystack_integrations/components/retrievers/pgvector/embedding_retriever.py,sha256=P5xx5M7q8gtW6P6mde2lV7MXKBrkRlpEXjE7G6BtbIA,7362
|
|
3
|
+
haystack_integrations/components/retrievers/pgvector/keyword_retriever.py,sha256=udOpbP4wxe3JyLG3on9J72mYRoq2iyhBeAtz3jI-m5A,5464
|
|
4
4
|
haystack_integrations/document_stores/pgvector/__init__.py,sha256=Ge-7ggarfr3x8UqNdFHUuFGiu52u49sOVe3l16WEwZU,197
|
|
5
|
-
haystack_integrations/document_stores/pgvector/document_store.py,sha256=
|
|
5
|
+
haystack_integrations/document_stores/pgvector/document_store.py,sha256=bvN4tYQeNNZ0s07Ur8VtJNDL_XtFQqHq0IYAUbkmTho,28018
|
|
6
6
|
haystack_integrations/document_stores/pgvector/filters.py,sha256=751_Z1QnXJCG0GWC2jcfeZ25NNYfJewQEIUCeJS70PA,8212
|
|
7
|
-
pgvector_haystack-0.
|
|
8
|
-
pgvector_haystack-0.
|
|
9
|
-
pgvector_haystack-0.
|
|
10
|
-
pgvector_haystack-0.
|
|
7
|
+
pgvector_haystack-0.5.0.dist-info/METADATA,sha256=paARhQ7BWHNKEmhKbvazKC4H3ickTtuxh1jNOohS0Os,2219
|
|
8
|
+
pgvector_haystack-0.5.0.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
|
|
9
|
+
pgvector_haystack-0.5.0.dist-info/licenses/LICENSE.txt,sha256=B05uMshqTA74s-0ltyHKI6yoPfJ3zYgQbvcXfDVGFf8,10280
|
|
10
|
+
pgvector_haystack-0.5.0.dist-info/RECORD,,
|
{pgvector_haystack-0.3.0.dist-info → pgvector_haystack-0.5.0.dist-info}/licenses/LICENSE.txt
RENAMED
|
File without changes
|