pgvector-haystack 0.3.0__py3-none-any.whl → 0.5.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,10 +1,12 @@
1
1
  # SPDX-FileCopyrightText: 2023-present deepset GmbH <info@deepset.ai>
2
2
  #
3
3
  # SPDX-License-Identifier: Apache-2.0
4
- from typing import Any, Dict, List, Literal, Optional
4
+ from typing import Any, Dict, List, Literal, Optional, Union
5
5
 
6
6
  from haystack import component, default_from_dict, default_to_dict
7
7
  from haystack.dataclasses import Document
8
+ from haystack.document_stores.types import FilterPolicy
9
+ from haystack.document_stores.types.filter_policy import apply_filter_policy
8
10
  from haystack_integrations.document_stores.pgvector import PgvectorDocumentStore
9
11
  from haystack_integrations.document_stores.pgvector.document_store import VALID_VECTOR_FUNCTIONS
10
12
 
@@ -62,6 +64,7 @@ class PgvectorEmbeddingRetriever:
62
64
  filters: Optional[Dict[str, Any]] = None,
63
65
  top_k: int = 10,
64
66
  vector_function: Optional[Literal["cosine_similarity", "inner_product", "l2_distance"]] = None,
67
+ filter_policy: Union[str, FilterPolicy] = FilterPolicy.REPLACE,
65
68
  ):
66
69
  """
67
70
  :param document_store: An instance of `PgvectorDocumentStore`.
@@ -75,7 +78,7 @@ class PgvectorEmbeddingRetriever:
75
78
  and the most similar documents are the ones with the smallest score.
76
79
  **Important**: if the document store is using the `"hnsw"` search strategy, the vector function
77
80
  should match the one utilized during index creation to take advantage of the index.
78
-
81
+ :param filter_policy: Policy to determine how filters are applied.
79
82
  :raises ValueError: If `document_store` is not an instance of `PgvectorDocumentStore` or if `vector_function`
80
83
  is not one of the valid options.
81
84
  """
@@ -91,6 +94,9 @@ class PgvectorEmbeddingRetriever:
91
94
  self.filters = filters or {}
92
95
  self.top_k = top_k
93
96
  self.vector_function = vector_function or document_store.vector_function
97
+ self.filter_policy = (
98
+ filter_policy if isinstance(filter_policy, FilterPolicy) else FilterPolicy.from_str(filter_policy)
99
+ )
94
100
 
95
101
  def to_dict(self) -> Dict[str, Any]:
96
102
  """
@@ -104,6 +110,7 @@ class PgvectorEmbeddingRetriever:
104
110
  filters=self.filters,
105
111
  top_k=self.top_k,
106
112
  vector_function=self.vector_function,
113
+ filter_policy=self.filter_policy.value,
107
114
  document_store=self.document_store.to_dict(),
108
115
  )
109
116
 
@@ -119,6 +126,7 @@ class PgvectorEmbeddingRetriever:
119
126
  """
120
127
  doc_store_params = data["init_parameters"]["document_store"]
121
128
  data["init_parameters"]["document_store"] = PgvectorDocumentStore.from_dict(doc_store_params)
129
+ data["init_parameters"]["filter_policy"] = FilterPolicy.from_str(data["init_parameters"]["filter_policy"])
122
130
  return default_from_dict(cls, data)
123
131
 
124
132
  @component.output_types(documents=List[Document])
@@ -133,13 +141,15 @@ class PgvectorEmbeddingRetriever:
133
141
  Retrieve documents from the `PgvectorDocumentStore`, based on their embeddings.
134
142
 
135
143
  :param query_embedding: Embedding of the query.
136
- :param filters: Filters applied to the retrieved Documents.
144
+ :param filters: Filters applied to the retrieved Documents. The way runtime filters are applied depends on
145
+ the `filter_policy` chosen at retriever initialization. See init method docstring for more
146
+ details.
137
147
  :param top_k: Maximum number of Documents to return.
138
148
  :param vector_function: The similarity function to use when searching for similar embeddings.
139
149
 
140
150
  :returns: List of Documents similar to `query_embedding`.
141
151
  """
142
- filters = filters or self.filters
152
+ filters = apply_filter_policy(self.filter_policy, self.filters, filters)
143
153
  top_k = top_k or self.top_k
144
154
  vector_function = vector_function or self.vector_function
145
155
 
@@ -1,10 +1,12 @@
1
1
  # SPDX-FileCopyrightText: 2023-present deepset GmbH <info@deepset.ai>
2
2
  #
3
3
  # SPDX-License-Identifier: Apache-2.0
4
- from typing import Any, Dict, List, Optional
4
+ from typing import Any, Dict, List, Optional, Union
5
5
 
6
6
  from haystack import component, default_from_dict, default_to_dict
7
7
  from haystack.dataclasses import Document
8
+ from haystack.document_stores.types import FilterPolicy
9
+ from haystack.document_stores.types.filter_policy import apply_filter_policy
8
10
  from haystack_integrations.document_stores.pgvector import PgvectorDocumentStore
9
11
 
10
12
 
@@ -51,12 +53,13 @@ class PgvectorKeywordRetriever:
51
53
  document_store: PgvectorDocumentStore,
52
54
  filters: Optional[Dict[str, Any]] = None,
53
55
  top_k: int = 10,
56
+ filter_policy: Union[str, FilterPolicy] = FilterPolicy.REPLACE,
54
57
  ):
55
58
  """
56
59
  :param document_store: An instance of `PgvectorDocumentStore`.
57
60
  :param filters: Filters applied to the retrieved Documents.
58
61
  :param top_k: Maximum number of Documents to return.
59
-
62
+ :param filter_policy: Policy to determine how filters are applied.
60
63
  :raises ValueError: If `document_store` is not an instance of `PgvectorDocumentStore`.
61
64
  """
62
65
  if not isinstance(document_store, PgvectorDocumentStore):
@@ -66,6 +69,9 @@ class PgvectorKeywordRetriever:
66
69
  self.document_store = document_store
67
70
  self.filters = filters or {}
68
71
  self.top_k = top_k
72
+ self.filter_policy = (
73
+ filter_policy if isinstance(filter_policy, FilterPolicy) else FilterPolicy.from_str(filter_policy)
74
+ )
69
75
 
70
76
  def to_dict(self) -> Dict[str, Any]:
71
77
  """
@@ -78,6 +84,7 @@ class PgvectorKeywordRetriever:
78
84
  self,
79
85
  filters=self.filters,
80
86
  top_k=self.top_k,
87
+ filter_policy=self.filter_policy.value,
81
88
  document_store=self.document_store.to_dict(),
82
89
  )
83
90
 
@@ -93,6 +100,7 @@ class PgvectorKeywordRetriever:
93
100
  """
94
101
  doc_store_params = data["init_parameters"]["document_store"]
95
102
  data["init_parameters"]["document_store"] = PgvectorDocumentStore.from_dict(doc_store_params)
103
+ data["init_parameters"]["filter_policy"] = FilterPolicy.from_str(data["init_parameters"]["filter_policy"])
96
104
  return default_from_dict(cls, data)
97
105
 
98
106
  @component.output_types(documents=List[Document])
@@ -106,13 +114,16 @@ class PgvectorKeywordRetriever:
106
114
  Retrieve documents from the `PgvectorDocumentStore`, based on keywords.
107
115
 
108
116
  :param query: String to search in `Document`s' content.
109
- :param filters: Filters applied to the retrieved Documents.
117
+ :param filters: Filters applied to the retrieved Documents. The way runtime filters are applied depends on
118
+ the `filter_policy` chosen at retriever initialization. See init method docstring for more
119
+ details.
110
120
  :param top_k: Maximum number of Documents to return.
111
121
 
112
122
  :returns: A dictionary with the following keys:
113
123
  - `documents`: List of `Document`s that match the query.
114
124
  """
115
- filters = filters or self.filters
125
+ filters = apply_filter_policy(self.filter_policy, self.filters, filters)
126
+
116
127
  top_k = top_k or self.top_k
117
128
 
118
129
  docs = self.document_store._keyword_retrieval(
@@ -69,10 +69,6 @@ VECTOR_FUNCTION_TO_POSTGRESQL_OPS = {
69
69
 
70
70
  HNSW_INDEX_CREATION_VALID_KWARGS = ["m", "ef_construction"]
71
71
 
72
- HNSW_INDEX_NAME = "haystack_hnsw_index"
73
-
74
- KEYWORD_INDEX_NAME = "haystack_keyword_index"
75
-
76
72
 
77
73
  class PgvectorDocumentStore:
78
74
  """
@@ -91,7 +87,9 @@ class PgvectorDocumentStore:
91
87
  search_strategy: Literal["exact_nearest_neighbor", "hnsw"] = "exact_nearest_neighbor",
92
88
  hnsw_recreate_index_if_exists: bool = False,
93
89
  hnsw_index_creation_kwargs: Optional[Dict[str, int]] = None,
90
+ hnsw_index_name: str = "haystack_hnsw_index",
94
91
  hnsw_ef_search: Optional[int] = None,
92
+ keyword_index_name: str = "haystack_keyword_index",
95
93
  ):
96
94
  """
97
95
  Creates a new PgvectorDocumentStore instance.
@@ -127,9 +125,11 @@ class PgvectorDocumentStore:
127
125
  :param hnsw_index_creation_kwargs: Additional keyword arguments to pass to the HNSW index creation.
128
126
  Only used if search_strategy is set to `"hnsw"`. You can find the list of valid arguments in the
129
127
  [pgvector documentation](https://github.com/pgvector/pgvector?tab=readme-ov-file#hnsw)
128
+ :param hnsw_index_name: Index name for the HNSW index.
130
129
  :param hnsw_ef_search: The `ef_search` parameter to use at query time. Only used if search_strategy is set to
131
130
  `"hnsw"`. You can find more information about this parameter in the
132
131
  [pgvector documentation](https://github.com/pgvector/pgvector?tab=readme-ov-file#hnsw).
132
+ :param keyword_index_name: Index name for the Keyword index.
133
133
  """
134
134
 
135
135
  self.connection_string = connection_string
@@ -143,7 +143,9 @@ class PgvectorDocumentStore:
143
143
  self.search_strategy = search_strategy
144
144
  self.hnsw_recreate_index_if_exists = hnsw_recreate_index_if_exists
145
145
  self.hnsw_index_creation_kwargs = hnsw_index_creation_kwargs or {}
146
+ self.hnsw_index_name = hnsw_index_name
146
147
  self.hnsw_ef_search = hnsw_ef_search
148
+ self.keyword_index_name = keyword_index_name
147
149
  self.language = language
148
150
  self._connection = None
149
151
  self._cursor = None
@@ -209,7 +211,9 @@ class PgvectorDocumentStore:
209
211
  search_strategy=self.search_strategy,
210
212
  hnsw_recreate_index_if_exists=self.hnsw_recreate_index_if_exists,
211
213
  hnsw_index_creation_kwargs=self.hnsw_index_creation_kwargs,
214
+ hnsw_index_name=self.hnsw_index_name,
212
215
  hnsw_ef_search=self.hnsw_ef_search,
216
+ keyword_index_name=self.keyword_index_name,
213
217
  language=self.language,
214
218
  )
215
219
 
@@ -281,7 +285,7 @@ class PgvectorDocumentStore:
281
285
  index_exists = bool(
282
286
  self._execute_sql(
283
287
  "SELECT 1 FROM pg_indexes WHERE tablename = %s AND indexname = %s",
284
- (self.table_name, KEYWORD_INDEX_NAME),
288
+ (self.table_name, self.keyword_index_name),
285
289
  "Could not check if keyword index exists",
286
290
  ).fetchone()
287
291
  )
@@ -289,7 +293,7 @@ class PgvectorDocumentStore:
289
293
  sql_create_index = SQL(
290
294
  "CREATE INDEX {index_name} ON {table_name} USING GIN (to_tsvector({language}, content))"
291
295
  ).format(
292
- index_name=Identifier(KEYWORD_INDEX_NAME),
296
+ index_name=Identifier(self.keyword_index_name),
293
297
  table_name=Identifier(self.table_name),
294
298
  language=SQLLiteral(self.language),
295
299
  )
@@ -312,7 +316,7 @@ class PgvectorDocumentStore:
312
316
  index_exists = bool(
313
317
  self._execute_sql(
314
318
  "SELECT 1 FROM pg_indexes WHERE tablename = %s AND indexname = %s",
315
- (self.table_name, HNSW_INDEX_NAME),
319
+ (self.table_name, self.hnsw_index_name),
316
320
  "Could not check if HNSW index exists",
317
321
  ).fetchone()
318
322
  )
@@ -325,7 +329,7 @@ class PgvectorDocumentStore:
325
329
  )
326
330
  return
327
331
 
328
- sql_drop_index = SQL("DROP INDEX IF EXISTS {index_name}").format(index_name=Identifier(HNSW_INDEX_NAME))
332
+ sql_drop_index = SQL("DROP INDEX IF EXISTS {index_name}").format(index_name=Identifier(self.hnsw_index_name))
329
333
  self._execute_sql(sql_drop_index, error_msg="Could not drop HNSW index")
330
334
 
331
335
  self._create_hnsw_index()
@@ -343,7 +347,7 @@ class PgvectorDocumentStore:
343
347
  }
344
348
 
345
349
  sql_create_index = SQL("CREATE INDEX {index_name} ON {table_name} USING hnsw (embedding {ops}) ").format(
346
- index_name=Identifier(HNSW_INDEX_NAME), table_name=Identifier(self.table_name), ops=SQL(pg_ops)
350
+ index_name=Identifier(self.hnsw_index_name), table_name=Identifier(self.table_name), ops=SQL(pg_ops)
347
351
  )
348
352
 
349
353
  if actual_hnsw_index_creation_kwargs:
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.3
2
2
  Name: pgvector-haystack
3
- Version: 0.3.0
3
+ Version: 0.5.0
4
4
  Summary: An integration of pgvector (vector search extension for Postgres) with Haystack
5
5
  Project-URL: Source, https://github.com/deepset-ai/haystack-core-integrations
6
6
  Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/blob/main/integrations/pgvector/README.md
@@ -1,10 +1,10 @@
1
1
  haystack_integrations/components/retrievers/pgvector/__init__.py,sha256=n4q4pUApJ9DZakJ_JL-egwOzlefdnI_ZxjU1Lu44Sqo,296
2
- haystack_integrations/components/retrievers/pgvector/embedding_retriever.py,sha256=6LuQmF7JmPOCU_I10ksPcbDC6QMu-2SOq0--BQxBxpg,6523
3
- haystack_integrations/components/retrievers/pgvector/keyword_retriever.py,sha256=fAJ3M2l_Pt8zKqHES1l1L4gM16BOHeZWq8xc18Bhq7A,4624
2
+ haystack_integrations/components/retrievers/pgvector/embedding_retriever.py,sha256=P5xx5M7q8gtW6P6mde2lV7MXKBrkRlpEXjE7G6BtbIA,7362
3
+ haystack_integrations/components/retrievers/pgvector/keyword_retriever.py,sha256=udOpbP4wxe3JyLG3on9J72mYRoq2iyhBeAtz3jI-m5A,5464
4
4
  haystack_integrations/document_stores/pgvector/__init__.py,sha256=Ge-7ggarfr3x8UqNdFHUuFGiu52u49sOVe3l16WEwZU,197
5
- haystack_integrations/document_stores/pgvector/document_store.py,sha256=eXfHx1hAQu_s4CN8rBMK-3WYDVYFQOIAOazBqA3wG_s,27629
5
+ haystack_integrations/document_stores/pgvector/document_store.py,sha256=bvN4tYQeNNZ0s07Ur8VtJNDL_XtFQqHq0IYAUbkmTho,28018
6
6
  haystack_integrations/document_stores/pgvector/filters.py,sha256=751_Z1QnXJCG0GWC2jcfeZ25NNYfJewQEIUCeJS70PA,8212
7
- pgvector_haystack-0.3.0.dist-info/METADATA,sha256=7TIp_Uibizl86fDfdGDqbUCnxSR9IUepEepNJQE7Fm4,2219
8
- pgvector_haystack-0.3.0.dist-info/WHEEL,sha256=zEMcRr9Kr03x1ozGwg5v9NQBKn3kndp6LSoSlVg-jhU,87
9
- pgvector_haystack-0.3.0.dist-info/licenses/LICENSE.txt,sha256=B05uMshqTA74s-0ltyHKI6yoPfJ3zYgQbvcXfDVGFf8,10280
10
- pgvector_haystack-0.3.0.dist-info/RECORD,,
7
+ pgvector_haystack-0.5.0.dist-info/METADATA,sha256=paARhQ7BWHNKEmhKbvazKC4H3ickTtuxh1jNOohS0Os,2219
8
+ pgvector_haystack-0.5.0.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
9
+ pgvector_haystack-0.5.0.dist-info/licenses/LICENSE.txt,sha256=B05uMshqTA74s-0ltyHKI6yoPfJ3zYgQbvcXfDVGFf8,10280
10
+ pgvector_haystack-0.5.0.dist-info/RECORD,,
@@ -1,4 +1,4 @@
1
1
  Wheel-Version: 1.0
2
- Generator: hatchling 1.24.2
2
+ Generator: hatchling 1.25.0
3
3
  Root-Is-Purelib: true
4
4
  Tag: py3-none-any