pgvector-haystack 6.4.0__tar.gz → 6.5.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (27) hide show
  1. {pgvector_haystack-6.4.0 → pgvector_haystack-6.5.0}/CHANGELOG.md +24 -0
  2. {pgvector_haystack-6.4.0 → pgvector_haystack-6.5.0}/PKG-INFO +1 -1
  3. {pgvector_haystack-6.4.0 → pgvector_haystack-6.5.0}/pyproject.toml +1 -0
  4. {pgvector_haystack-6.4.0 → pgvector_haystack-6.5.0}/src/haystack_integrations/document_stores/pgvector/document_store.py +22 -20
  5. {pgvector_haystack-6.4.0 → pgvector_haystack-6.5.0}/src/haystack_integrations/document_stores/pgvector/filters.py +74 -4
  6. {pgvector_haystack-6.4.0 → pgvector_haystack-6.5.0}/tests/test_document_store.py +69 -11
  7. {pgvector_haystack-6.4.0 → pgvector_haystack-6.5.0}/tests/test_document_store_async.py +16 -0
  8. {pgvector_haystack-6.4.0 → pgvector_haystack-6.5.0}/tests/test_filters.py +196 -1
  9. {pgvector_haystack-6.4.0 → pgvector_haystack-6.5.0}/.gitignore +0 -0
  10. {pgvector_haystack-6.4.0 → pgvector_haystack-6.5.0}/LICENSE.txt +0 -0
  11. {pgvector_haystack-6.4.0 → pgvector_haystack-6.5.0}/README.md +0 -0
  12. {pgvector_haystack-6.4.0 → pgvector_haystack-6.5.0}/examples/embedding_retrieval.py +0 -0
  13. {pgvector_haystack-6.4.0 → pgvector_haystack-6.5.0}/examples/hybrid_retrieval.py +0 -0
  14. {pgvector_haystack-6.4.0 → pgvector_haystack-6.5.0}/pydoc/config_docusaurus.yml +0 -0
  15. {pgvector_haystack-6.4.0 → pgvector_haystack-6.5.0}/src/haystack_integrations/components/retrievers/pgvector/__init__.py +0 -0
  16. {pgvector_haystack-6.4.0 → pgvector_haystack-6.5.0}/src/haystack_integrations/components/retrievers/pgvector/embedding_retriever.py +0 -0
  17. {pgvector_haystack-6.4.0 → pgvector_haystack-6.5.0}/src/haystack_integrations/components/retrievers/pgvector/keyword_retriever.py +0 -0
  18. {pgvector_haystack-6.4.0 → pgvector_haystack-6.5.0}/src/haystack_integrations/components/retrievers/py.typed +0 -0
  19. {pgvector_haystack-6.4.0 → pgvector_haystack-6.5.0}/src/haystack_integrations/document_stores/pgvector/__init__.py +0 -0
  20. {pgvector_haystack-6.4.0 → pgvector_haystack-6.5.0}/src/haystack_integrations/document_stores/pgvector/converters.py +0 -0
  21. {pgvector_haystack-6.4.0 → pgvector_haystack-6.5.0}/src/haystack_integrations/document_stores/py.typed +0 -0
  22. {pgvector_haystack-6.4.0 → pgvector_haystack-6.5.0}/tests/__init__.py +0 -0
  23. {pgvector_haystack-6.4.0 → pgvector_haystack-6.5.0}/tests/conftest.py +0 -0
  24. {pgvector_haystack-6.4.0 → pgvector_haystack-6.5.0}/tests/test_converters.py +0 -0
  25. {pgvector_haystack-6.4.0 → pgvector_haystack-6.5.0}/tests/test_retrieval.py +0 -0
  26. {pgvector_haystack-6.4.0 → pgvector_haystack-6.5.0}/tests/test_retrieval_async.py +0 -0
  27. {pgvector_haystack-6.4.0 → pgvector_haystack-6.5.0}/tests/test_retrievers.py +0 -0
@@ -1,5 +1,29 @@
1
1
  # Changelog
2
2
 
3
+ ## [integrations/pgvector-v6.4.1] - 2026-07-23
4
+
5
+ ### 🐛 Bug Fixes
6
+
7
+ - *(pgvector)* Parenthesize `not in` clause so it does not escape an AND filter (#3586)
8
+ - `PGVectorDocumentStore` match `search_term` against metadata field value, not content (#3648)
9
+ - Fix (pgvector): fix error messages to say pgvector instead of pinecone (#3654)
10
+
11
+
12
+ ## [integrations/pgvector-v6.4.0] - 2026-07-23
13
+
14
+ ### 🚀 Features
15
+
16
+ - Pgvector - add closing methods (#3650)
17
+
18
+ ### 🐛 Bug Fixes
19
+
20
+ - *(pgvector)* NO_VALUE sentinel collides with the literal string 'no_value' (#3593)
21
+
22
+ ### 📚 Documentation
23
+
24
+ - Replace old haystack core imports with haystack_integrations paths (#3545)
25
+
26
+
3
27
  ## [integrations/pgvector-v6.3.1] - 2026-06-01
4
28
 
5
29
  ### 🐛 Bug Fixes
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: pgvector-haystack
3
- Version: 6.4.0
3
+ Version: 6.5.0
4
4
  Summary: An integration of pgvector (vector search extension for Postgres) with Haystack
5
5
  Project-URL: Source, https://github.com/deepset-ai/haystack-core-integrations
6
6
  Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/blob/main/integrations/pgvector/README.md
@@ -143,6 +143,7 @@ ignore = [
143
143
  "PLR0912",
144
144
  "PLR0913",
145
145
  "PLR0915",
146
+ "PLR0917",
146
147
  # ignore function-call-in-default-argument
147
148
  "B008",
148
149
  # ignore assertions
@@ -1932,7 +1932,8 @@ class PgvectorDocumentStore:
1932
1932
  Builds SQL queries for getting unique metadata field values.
1933
1933
 
1934
1934
  :param normalized_field: The normalized metadata field name.
1935
- :param search_term: Optional search term to filter documents by content.
1935
+ :param search_term: Optional search term to filter unique values by a case-insensitive substring match
1936
+ against the metadata field's own value.
1936
1937
  :param from_: The offset for pagination (0-based).
1937
1938
  :param size: The number of unique values to return.
1938
1939
  :returns: A tuple containing (count_query, select_query, params).
@@ -1940,7 +1941,10 @@ class PgvectorDocumentStore:
1940
1941
  field_literal = SQLLiteral(normalized_field)
1941
1942
 
1942
1943
  # base query components
1943
- sql_select = SQL("SELECT DISTINCT meta->>{} AS value").format(field_literal)
1944
+ # Use the `->` (jsonb) operator rather than `->>` (text) so DISTINCT/ORDER BY operate on the
1945
+ # value's original JSON type (numbers sort numerically, not lexicographically) and psycopg
1946
+ # decodes the result to its native Python type instead of a string.
1947
+ sql_select = SQL("SELECT DISTINCT meta->{} AS value").format(field_literal)
1944
1948
  sql_from = SQL(" FROM {schema_name}.{table_name}").format(
1945
1949
  schema_name=Identifier(self.schema_name), table_name=Identifier(self.table_name)
1946
1950
  )
@@ -1948,14 +1952,12 @@ class PgvectorDocumentStore:
1948
1952
 
1949
1953
  params: tuple = ()
1950
1954
  if search_term:
1951
- # Use full-text search with word boundaries (similar to keyword retrieval)
1952
- sql_where += SQL(" AND to_tsvector({language}, content) @@ plainto_tsquery({language}, %s)").format(
1953
- language=SQLLiteral(self.language)
1954
- )
1955
- params = (search_term,)
1955
+ # Case-insensitive substring match against the metadata field's own (text) value.
1956
+ sql_where += SQL(" AND meta->>{} ILIKE %s").format(field_literal)
1957
+ params = (f"%{search_term}%",)
1956
1958
 
1957
1959
  # count query
1958
- sql_count = SQL("SELECT COUNT(DISTINCT meta->>{} ) AS total").format(field_literal)
1960
+ sql_count = SQL("SELECT COUNT(DISTINCT meta->{} ) AS total").format(field_literal)
1959
1961
  sql_count += sql_from + sql_where
1960
1962
 
1961
1963
  # paginated select query
@@ -1969,7 +1971,7 @@ class PgvectorDocumentStore:
1969
1971
  @staticmethod
1970
1972
  def _process_unique_values_result(
1971
1973
  count_result: dict[str, Any] | None, records: list[dict[str, Any]]
1972
- ) -> tuple[list[str], int]:
1974
+ ) -> tuple[list[Any], int]:
1973
1975
  """
1974
1976
  Processes the results from unique values queries.
1975
1977
 
@@ -1978,22 +1980,22 @@ class PgvectorDocumentStore:
1978
1980
  :returns: A tuple containing (unique_values, total_count).
1979
1981
  """
1980
1982
  total_count = count_result.get("total", 0) if count_result else 0
1981
- unique_values = [str(record.get("value", "")) for record in records if record.get("value") is not None]
1983
+ unique_values = [record.get("value") for record in records if record.get("value") is not None]
1982
1984
  return unique_values, total_count
1983
1985
 
1984
1986
  def get_metadata_field_unique_values(
1985
- self, metadata_field: str, search_term: str | None, from_: int, size: int
1986
- ) -> tuple[list[str], int]:
1987
+ self, metadata_field: str, search_term: str | None = None, from_: int = 0, size: int = 10
1988
+ ) -> tuple[list[Any], int]:
1987
1989
  """
1988
1990
  Returns unique values for a given metadata field, optionally filtered by a search term.
1989
1991
 
1990
1992
  :param metadata_field: The name of the metadata field. Can include or omit the "meta." prefix.
1991
- :param search_term: Optional search term to filter documents by content before extracting unique values.
1992
- If None, all documents are considered.
1993
+ :param search_term: Optional search term to filter unique values by a case-insensitive substring
1994
+ match against the metadata field's own value. If None, all values are considered.
1993
1995
  :param from_: The offset for pagination (0-based).
1994
1996
  :param size: The number of unique values to return.
1995
1997
  :returns: A tuple containing:
1996
- - A list of unique values (as strings)
1998
+ - A list of unique values in their original type
1997
1999
  - The total count of unique values
1998
2000
  """
1999
2001
  normalized_field = PgvectorDocumentStore._normalize_metadata_field_name(metadata_field)
@@ -2020,18 +2022,18 @@ class PgvectorDocumentStore:
2020
2022
  return PgvectorDocumentStore._process_unique_values_result(count_result, records)
2021
2023
 
2022
2024
  async def get_metadata_field_unique_values_async(
2023
- self, metadata_field: str, search_term: str | None, from_: int, size: int
2024
- ) -> tuple[list[str], int]:
2025
+ self, metadata_field: str, search_term: str | None = None, from_: int = 0, size: int = 10
2026
+ ) -> tuple[list[Any], int]:
2025
2027
  """
2026
2028
  Asynchronously returns unique values for a given metadata field, optionally filtered by a search term.
2027
2029
 
2028
2030
  :param metadata_field: The name of the metadata field. Can include or omit the "meta." prefix.
2029
- :param search_term: Optional search term to filter documents by content before extracting unique values.
2030
- If None, all documents are considered.
2031
+ :param search_term: Optional search term to filter unique values by a case-insensitive substring
2032
+ match against the metadata field's own value. If None, all values are considered.
2031
2033
  :param from_: The offset for pagination (0-based).
2032
2034
  :param size: The number of unique values to return.
2033
2035
  :returns: A tuple containing:
2034
- - A list of unique values (as strings)
2036
+ - A list of unique values in their original type
2035
2037
  - The total count of unique values
2036
2038
  """
2037
2039
  normalized_field = PgvectorDocumentStore._normalize_metadata_field_name(metadata_field)
@@ -22,6 +22,11 @@ PYTHON_TYPES_TO_PG_TYPES = {
22
22
 
23
23
  NO_VALUE = object() # unique sentinel; can never equal a user-supplied filter value
24
24
 
25
+ # Operators that match against a JSONB array stored in the meta field.
26
+ # They need the JSONB value (via the `->` accessor) as their left operand,
27
+ # unlike the scalar operators which access meta as text (via `->>`).
28
+ ARRAY_OPERATORS = ("array_contains", "array_overlaps")
29
+
25
30
 
26
31
  def _validate_filters(filters: dict[str, Any] | None = None) -> None:
27
32
  """
@@ -106,7 +111,18 @@ def _parse_comparison_condition(condition: dict[str, Any]) -> tuple[Composed, li
106
111
  value: Any = condition["value"]
107
112
 
108
113
  if field.startswith("meta."):
109
- sql_field: Composable = _treat_meta_field(field, value)
114
+ # array operators need the JSONB value (->), the other operators access meta as text (->>)
115
+ sql_field: Composable = (
116
+ _treat_meta_field_as_jsonb(field) if operator in ARRAY_OPERATORS else _treat_meta_field(field, value)
117
+ )
118
+ elif operator in ARRAY_OPERATORS:
119
+ # top-level columns are never JSONB arrays, so these operators would render SQL that the
120
+ # database rejects with an opaque UndefinedFunction error. Fail with a FilterError instead.
121
+ msg = (
122
+ f"Operator '{operator}' is only supported on meta fields, but got field '{field}'. "
123
+ f"Prefix the field name with 'meta.' to filter on an array stored in a document's metadata."
124
+ )
125
+ raise FilterError(msg)
110
126
  else:
111
127
  sql_field = Identifier(field)
112
128
 
@@ -149,6 +165,24 @@ def _treat_meta_field(field: str, value: Any) -> Composed:
149
165
  return composed
150
166
 
151
167
 
168
+ def _treat_meta_field_as_jsonb(field: str) -> Composed:
169
+ """
170
+ Internal method that returns a psycopg Composed object accessing a meta key as a JSONB value.
171
+
172
+ Unlike `_treat_meta_field`, this uses the `->` operator (which returns a JSONB value) instead of
173
+ `->>` (which returns text) and applies no type cast. The array operators need a JSONB left operand
174
+ for their JSONB operators (`@>`, `?|`) to work.
175
+
176
+ Uses psycopg.sql.Literal to embed the field name, preventing SQL injection via metadata field names.
177
+
178
+ Example:
179
+ >>> _treat_meta_field_as_jsonb(field="meta.tags")
180
+ Composed([SQL('meta->'), Literal('tags')])
181
+ """
182
+ field_name = field.split(".", 1)[-1]
183
+ return SQL("meta->") + SQLLiteral(field_name)
184
+
185
+
152
186
  def _equal(field: Composable, value: Any) -> tuple[Composed, Any]:
153
187
  if value is None:
154
188
  return SQL("{} IS NULL").format(field), NO_VALUE
@@ -227,14 +261,14 @@ def _less_than_equal(field: Composable, value: Any) -> tuple[Composed, Any]:
227
261
 
228
262
  def _not_in(field: Composable, value: Any) -> tuple[Composed, list]:
229
263
  if not isinstance(value, list):
230
- msg = f"{field}'s value must be a list when using 'not in' comparator in Pinecone"
264
+ msg = f"{field}'s value must be a list when using 'not in' comparator in PgVector"
231
265
  raise FilterError(msg)
232
- return SQL("{} IS NULL OR {} != ALL(%s)").format(field, field), [value]
266
+ return SQL("({} IS NULL OR {} != ALL(%s))").format(field, field), [value]
233
267
 
234
268
 
235
269
  def _in(field: Composable, value: Any) -> tuple[Composed, list]:
236
270
  if not isinstance(value, list):
237
- msg = f"{field}'s value must be a list when using 'in' comparator in Pinecone"
271
+ msg = f"{field}'s value must be a list when using 'in' comparator in PgVector"
238
272
  raise FilterError(msg)
239
273
 
240
274
  # see https://www.psycopg.org/psycopg3/docs/basic/adapt.html#lists-adaptation
@@ -255,6 +289,40 @@ def _not_like(field: Composable, value: Any) -> tuple[Composed, Any]:
255
289
  return SQL("{} NOT LIKE %s").format(field), value
256
290
 
257
291
 
292
+ def _validate_array_operator_value(field: Composable, value: Any, operator: str) -> None:
293
+ if not isinstance(value, list):
294
+ msg = f"{field}'s value must be a list when using '{operator}' comparator in PgVector"
295
+ raise FilterError(msg)
296
+ # an empty list has no useful match semantics: `@> '[]'` matches every array and `?| '{}'` matches
297
+ # nothing, so the same empty filter would mean "everything" for one operator and "nothing" for the
298
+ # other. Callers building the list dynamically should omit the condition instead.
299
+ if not value:
300
+ msg = f"{field}'s value must be a non-empty list when using '{operator}' comparator in PgVector"
301
+ raise FilterError(msg)
302
+
303
+
304
+ def _array_contains(field: Composable, value: Any) -> tuple[Composed, Jsonb]:
305
+ _validate_array_operator_value(field, value, "array_contains")
306
+ # `jsonb @> jsonb` is true when the left array contains every element of the right array,
307
+ # so the meta array must contain all of the given values. Containment compares elements by value,
308
+ # so a non-array meta value (a bare string, an object) never matches a list of values.
309
+ return SQL("{} @> %s").format(field), Jsonb(value)
310
+
311
+
312
+ def _array_overlaps(field: Composable, value: Any) -> tuple[Composed, list]:
313
+ _validate_array_operator_value(field, value, "array_overlaps")
314
+ # `jsonb ?| text[]` only matches string elements, so we reject non-string values to fail fast
315
+ # instead of silently returning no results.
316
+ if not all(isinstance(item, str) for item in value):
317
+ msg = f"{field}'s value must be a list of strings when using 'array_overlaps' comparator in PgVector"
318
+ raise FilterError(msg)
319
+ # `?|` is true when a given string exists as a top-level key OR array element, so on its own it
320
+ # also matches objects (by key) and bare strings. Restricting it to arrays with jsonb_typeof keeps
321
+ # it in agreement with array_contains. The parentheses stop this AND from being split by an
322
+ # enclosing OR (see _not_in for the same reason).
323
+ return SQL("(jsonb_typeof({}) = 'array' AND {} ?| %s)").format(field, field), [value]
324
+
325
+
258
326
  COMPARISON_OPERATORS = {
259
327
  "==": _equal,
260
328
  "!=": _not_equal,
@@ -266,4 +334,6 @@ COMPARISON_OPERATORS = {
266
334
  "not in": _not_in,
267
335
  "like": _like,
268
336
  "not like": _not_like,
337
+ "array_contains": _array_contains,
338
+ "array_overlaps": _array_overlaps,
269
339
  }
@@ -662,17 +662,30 @@ def test_get_metadata_field_unique_values(document_store: PgvectorDocumentStore)
662
662
  # All three pages should be different
663
663
  assert len(set(unique_values_single1 + unique_values_single2 + unique_values_single3)) == 3
664
664
 
665
- # Test with search term - filter by content matching "Python"
665
+ # Test with search term - case-insensitive substring match against the metadata field's OWN value
666
+ # (not against document content).
666
667
  unique_values_filtered, total_filtered = document_store.get_metadata_field_unique_values(
667
- "meta.category", "Python", 0, 10
668
+ "meta.language", "Python", 0, 10
668
669
  )
669
- assert set(unique_values_filtered) == {"A"} # Only category A has documents with "Python" in content
670
+ assert set(unique_values_filtered) == {"Python"}
670
671
  assert total_filtered == 1
671
672
 
672
- # Test with search term - filter by content matching "Java"
673
- unique_values_java, total_java = document_store.get_metadata_field_unique_values("meta.category", "Java", 0, 10)
674
- assert set(unique_values_java) == {"B"} # Only category B has documents with "Java" in content
675
- assert total_java == 1
673
+ # Case-insensitivity: lowercase search term should still match "Python"
674
+ unique_values_lower, total_lower = document_store.get_metadata_field_unique_values("meta.language", "python", 0, 10)
675
+ assert set(unique_values_lower) == {"Python"}
676
+ assert total_lower == 1
677
+
678
+ # Substring matching: "Java" is a substring of both "Java" and "JavaScript"
679
+ unique_values_java, total_java = document_store.get_metadata_field_unique_values("meta.language", "Java", 0, 10)
680
+ assert set(unique_values_java) == {"Java", "JavaScript"}
681
+ assert total_java == 2
682
+
683
+ # Substring matching in the middle/end of a value
684
+ unique_values_script, total_script = document_store.get_metadata_field_unique_values(
685
+ "meta.language", "Script", 0, 10
686
+ )
687
+ assert set(unique_values_script) == {"JavaScript"}
688
+ assert total_script == 1
676
689
 
677
690
  # Test pagination with search term
678
691
  unique_values_search_page1, total_search = document_store.get_metadata_field_unique_values(
@@ -689,6 +702,17 @@ def test_get_metadata_field_unique_values(document_store: PgvectorDocumentStore)
689
702
  assert len(unique_values_search_empty) == 0
690
703
  assert total_search_empty == 1
691
704
 
705
+ # Test that search_term matches metadata VALUE, not content
706
+ content_vs_metadata_docs = [
707
+ Document(content="This document mentions Python explicitly", meta={"topic": "cooking"}),
708
+ Document(content="Unrelated content about recipes", meta={"topic": "python-tutorial"}),
709
+ ]
710
+ document_store.write_documents(content_vs_metadata_docs)
711
+
712
+ unique_topics, total_topics = document_store.get_metadata_field_unique_values("meta.topic", "python", 0, 10)
713
+ assert set(unique_topics) == {"python-tutorial"}
714
+ assert total_topics == 1
715
+
692
716
  # Test with integer values
693
717
  int_docs = [
694
718
  Document(content="Doc 1", meta={"priority": 1}),
@@ -698,12 +722,46 @@ def test_get_metadata_field_unique_values(document_store: PgvectorDocumentStore)
698
722
  ]
699
723
  document_store.write_documents(int_docs)
700
724
  unique_priorities, total_priorities = document_store.get_metadata_field_unique_values("meta.priority", None, 0, 10)
701
- assert set(unique_priorities) == {"1", "2", "3"}
725
+ assert set(unique_priorities) == {1, 2, 3}
702
726
  assert total_priorities == 3
703
727
 
704
- # Test with search term on integer field
728
+ # Test with search term on integer field - substring match against the field's own (stringified)
729
+ # value, e.g. "Doc 1" (content) no longer matches; "1" (the value itself) does. The returned
730
+ # values themselves keep their original type (int here), only the match is textual.
705
731
  unique_priorities_filtered, total_priorities_filtered = document_store.get_metadata_field_unique_values(
706
- "meta.priority", "Doc 1", 0, 10
732
+ "meta.priority", "1", 0, 10
707
733
  )
708
- assert set(unique_priorities_filtered) == {"1"}
734
+ assert set(unique_priorities_filtered) == {1}
709
735
  assert total_priorities_filtered == 1
736
+
737
+
738
+ @pytest.mark.integration
739
+ def test_get_metadata_field_unique_values_no_search_term(document_store: PgvectorDocumentStore):
740
+ docs = [
741
+ Document(content="mentions tsvector and content in the body", meta={"category": "A"}),
742
+ Document(content="another document", meta={"category": "B"}),
743
+ ]
744
+ document_store.write_documents(docs)
745
+
746
+ values, total = document_store.get_metadata_field_unique_values("meta.category")
747
+
748
+ assert set(values) == {"A", "B"}
749
+ assert total == 2
750
+
751
+
752
+ @pytest.mark.integration
753
+ def test_get_metadata_field_unique_values_search_term_matches_field_value_not_content(
754
+ document_store: PgvectorDocumentStore,
755
+ ):
756
+ docs = [
757
+ # "python" appears in the content but NOT in the category value -> must be EXCLUDED
758
+ Document(content="Python programming guide", meta={"category": "Backend"}),
759
+ # "python" appears in the category value but NOT in the content -> must be INCLUDED
760
+ Document(content="General purpose scripting language", meta={"category": "Python-based"}),
761
+ ]
762
+ document_store.write_documents(docs)
763
+
764
+ values, total = document_store.get_metadata_field_unique_values("meta.category", search_term="python")
765
+
766
+ assert values == ["Python-based"]
767
+ assert total == 1
@@ -124,6 +124,22 @@ class TestDocumentStoreAsync(
124
124
  filters={"field": "meta.category", "operator": "==", "value": "A"}, meta={}
125
125
  )
126
126
 
127
+ async def test_get_metadata_field_unique_values_async_preserves_non_string_types(
128
+ self, document_store: PgvectorDocumentStore
129
+ ):
130
+ """Non-string metadata values (e.g. ints) are returned in their original type, not stringified."""
131
+ docs = [
132
+ Document(content="Doc 1", meta={"priority": 1}),
133
+ Document(content="Doc 2", meta={"priority": 2}),
134
+ Document(content="Doc 3", meta={"priority": 1}),
135
+ ]
136
+ await document_store.write_documents_async(docs)
137
+
138
+ values, total = await document_store.get_metadata_field_unique_values_async("meta.priority")
139
+
140
+ assert set(values) == {1, 2}
141
+ assert total == 2
142
+
127
143
  async def test_delete_table_async_first_call(self, document_store: PgvectorDocumentStore):
128
144
  """
129
145
  Test that delete_table_async can be executed as the initial operation on the Document Store
@@ -10,6 +10,7 @@ from haystack.testing.document_store import FilterDocumentsTest
10
10
  from psycopg.adapt import Transformer
11
11
  from psycopg.sql import SQL, Composed
12
12
  from psycopg.sql import Literal as SQLLiteral
13
+ from psycopg.types.json import Jsonb
13
14
 
14
15
  from haystack_integrations.document_stores.pgvector.filters import (
15
16
  FilterError,
@@ -17,6 +18,7 @@ from haystack_integrations.document_stores.pgvector.filters import (
17
18
  _parse_comparison_condition,
18
19
  _parse_logical_condition,
19
20
  _treat_meta_field,
21
+ _treat_meta_field_as_jsonb,
20
22
  _validate_filters,
21
23
  )
22
24
 
@@ -141,6 +143,86 @@ class TestFilters(FilterDocumentsTest):
141
143
  ],
142
144
  )
143
145
 
146
+ @staticmethod
147
+ def _docs_with_array_meta():
148
+ return [
149
+ Document(content="doc with tag1 and tag2", meta={"tags": ["tag1", "tag2"]}),
150
+ Document(content="doc with tag1 and tag3", meta={"tags": ["tag1", "tag3"]}),
151
+ Document(content="doc with tag4", meta={"tags": ["tag4"]}),
152
+ ]
153
+
154
+ def test_array_contains_operator(self, document_store):
155
+ docs = self._docs_with_array_meta()
156
+ document_store.write_documents(docs)
157
+
158
+ # array_contains requires ALL listed values to be present in the meta array
159
+ result = document_store.filter_documents(
160
+ filters={"field": "meta.tags", "operator": "array_contains", "value": ["tag1", "tag2"]}
161
+ )
162
+ assert {doc.content for doc in result} == {"doc with tag1 and tag2"}
163
+
164
+ def test_array_contains_operator_single_value(self, document_store):
165
+ docs = self._docs_with_array_meta()
166
+ document_store.write_documents(docs)
167
+
168
+ result = document_store.filter_documents(
169
+ filters={"field": "meta.tags", "operator": "array_contains", "value": ["tag1"]}
170
+ )
171
+ assert {doc.content for doc in result} == {"doc with tag1 and tag2", "doc with tag1 and tag3"}
172
+
173
+ def test_array_overlaps_operator(self, document_store):
174
+ docs = self._docs_with_array_meta()
175
+ document_store.write_documents(docs)
176
+
177
+ # array_overlaps requires AT LEAST ONE listed value to be present in the meta array
178
+ result = document_store.filter_documents(
179
+ filters={"field": "meta.tags", "operator": "array_overlaps", "value": ["tag1", "tag3"]}
180
+ )
181
+ assert {doc.content for doc in result} == {"doc with tag1 and tag2", "doc with tag1 and tag3"}
182
+
183
+ def test_array_overlaps_operator_no_match(self, document_store):
184
+ docs = self._docs_with_array_meta()
185
+ document_store.write_documents(docs)
186
+
187
+ result = document_store.filter_documents(
188
+ filters={"field": "meta.tags", "operator": "array_overlaps", "value": ["nonexistent"]}
189
+ )
190
+ assert result == []
191
+
192
+ def test_array_operators_combined(self, document_store):
193
+ docs = self._docs_with_array_meta()
194
+ document_store.write_documents(docs)
195
+
196
+ result = document_store.filter_documents(
197
+ filters={
198
+ "operator": "AND",
199
+ "conditions": [
200
+ {"field": "meta.tags", "operator": "array_contains", "value": ["tag1"]},
201
+ {"field": "meta.tags", "operator": "array_overlaps", "value": ["tag3"]},
202
+ ],
203
+ }
204
+ )
205
+ assert {doc.content for doc in result} == {"doc with tag1 and tag3"}
206
+
207
+ @pytest.mark.parametrize("operator", ["array_contains", "array_overlaps"])
208
+ def test_array_operators_ignore_non_array_meta(self, document_store, operator):
209
+ # meta shapes vary across sources, so the same key can hold an array in one document and a
210
+ # bare string or an object in another. `?|` matches a top-level key or array element, so
211
+ # without the jsonb_typeof guard the object and the string below would both match.
212
+ document_store.write_documents(
213
+ [
214
+ Document(content="array meta", meta={"tags": ["tag1", "tag2"]}),
215
+ Document(content="string meta", meta={"tags": "tag1"}),
216
+ Document(content="object meta", meta={"tags": {"tag1": "value"}}),
217
+ Document(content="missing key", meta={"other": "tag1"}),
218
+ ]
219
+ )
220
+
221
+ result = document_store.filter_documents(
222
+ filters={"field": "meta.tags", "operator": operator, "value": ["tag1"]}
223
+ )
224
+ assert {doc.content for doc in result} == {"array meta"}
225
+
144
226
 
145
227
  def test_treat_meta_field():
146
228
  cast_integer = SQL("(") + SQL("meta->>") + SQLLiteral("number") + SQL(")::integer")
@@ -165,6 +247,92 @@ def test_treat_meta_field_sql_injection_is_safely_escaped():
165
247
  assert result == SQL("meta->>") + SQLLiteral("name' OR 1=1 --")
166
248
 
167
249
 
250
+ def test_treat_meta_field_as_jsonb():
251
+ # array operators access the meta value as JSONB (->) without a cast, unlike _treat_meta_field (->>)
252
+ assert _treat_meta_field_as_jsonb(field="meta.tags") == SQL("meta->") + SQLLiteral("tags")
253
+
254
+
255
+ def test_treat_meta_field_as_jsonb_sql_injection_is_safely_escaped():
256
+ result = _treat_meta_field_as_jsonb(field="meta.tags' OR 1=1 --")
257
+ assert isinstance(result, Composed)
258
+ assert result == SQL("meta->") + SQLLiteral("tags' OR 1=1 --")
259
+
260
+
261
+ def test_array_contains_condition():
262
+ condition = {"field": "meta.tags", "operator": "array_contains", "value": ["tag1", "tag2"]}
263
+ query, values = _parse_comparison_condition(condition)
264
+
265
+ assert _render(query) == "meta->'tags' @> %s"
266
+ # the value is wrapped as a single Jsonb param holding the original list
267
+ assert len(values) == 1
268
+ assert isinstance(values[0], Jsonb)
269
+ assert values[0].obj == ["tag1", "tag2"]
270
+
271
+
272
+ def test_array_overlaps_condition():
273
+ condition = {"field": "meta.tags", "operator": "array_overlaps", "value": ["tag1", "tag2"]}
274
+ query, values = _parse_comparison_condition(condition)
275
+
276
+ # the jsonb_typeof guard keeps `?|` from matching objects (by key) or bare strings
277
+ assert _render(query) == "(jsonb_typeof(meta->'tags') = 'array' AND meta->'tags' ?| %s)"
278
+ # the list is wrapped twice (once here, once by _parse_comparison_condition), exactly like 'in',
279
+ # so psycopg adapts it to a nested Postgres text[]; `?|` deconstructs it to the same elements
280
+ assert values == [[["tag1", "tag2"]]]
281
+
282
+
283
+ def test_array_contains_non_list_value_raises():
284
+ condition = {"field": "meta.tags", "operator": "array_contains", "value": "tag1"}
285
+ with pytest.raises(FilterError):
286
+ _parse_comparison_condition(condition)
287
+
288
+
289
+ def test_array_overlaps_non_list_value_raises():
290
+ condition = {"field": "meta.tags", "operator": "array_overlaps", "value": "tag1"}
291
+ with pytest.raises(FilterError):
292
+ _parse_comparison_condition(condition)
293
+
294
+
295
+ def test_array_overlaps_non_string_elements_raises():
296
+ condition = {"field": "meta.tags", "operator": "array_overlaps", "value": [1, 2]}
297
+ with pytest.raises(FilterError):
298
+ _parse_comparison_condition(condition)
299
+
300
+
301
+ @pytest.mark.parametrize("operator", ["array_contains", "array_overlaps"])
302
+ def test_array_operators_empty_list_raises(operator):
303
+ # `@> '[]'` matches every array and `?| '{}'` matches nothing, so an empty list would mean
304
+ # "everything" for one operator and "nothing" for the other. Reject it instead.
305
+ condition = {"field": "meta.tags", "operator": operator, "value": []}
306
+ with pytest.raises(FilterError, match="non-empty list"):
307
+ _parse_comparison_condition(condition)
308
+
309
+
310
+ @pytest.mark.parametrize("operator", ["array_contains", "array_overlaps"])
311
+ def test_array_operators_on_non_meta_field_raises(operator):
312
+ # top-level columns are never JSONB arrays, so this must fail before reaching the database
313
+ condition = {"field": "content", "operator": operator, "value": ["tag1"]}
314
+ with pytest.raises(FilterError, match="only supported on meta fields"):
315
+ _parse_comparison_condition(condition)
316
+
317
+
318
+ def test_convert_filters_array_contains():
319
+ filters = {"field": "meta.tags", "operator": "array_contains", "value": ["tag1", "tag2"]}
320
+ where_clause, params = _convert_filters_to_where_clause_and_params(filters)
321
+
322
+ assert _render(where_clause) == " WHERE meta->'tags' @> %s"
323
+ assert len(params) == 1
324
+ assert isinstance(params[0], Jsonb)
325
+ assert params[0].obj == ["tag1", "tag2"]
326
+
327
+
328
+ def test_convert_filters_array_overlaps():
329
+ filters = {"field": "meta.tags", "operator": "array_overlaps", "value": ["tag1", "tag2"]}
330
+ where_clause, params = _convert_filters_to_where_clause_and_params(filters)
331
+
332
+ assert _render(where_clause) == " WHERE (jsonb_typeof(meta->'tags') = 'array' AND meta->'tags' ?| %s)"
333
+ assert params == ([["tag1", "tag2"]],)
334
+
335
+
168
336
  def test_comparison_condition_missing_operator():
169
337
  condition = {"field": "meta.type", "value": "article"}
170
338
  with pytest.raises(FilterError):
@@ -183,6 +351,14 @@ def test_comparison_condition_unknown_operator():
183
351
  _parse_comparison_condition(condition)
184
352
 
185
353
 
354
+ @pytest.mark.parametrize("operator", ["like", "not like"])
355
+ def test_comparison_condition_like_operator_requires_str_value(operator):
356
+ # pgvector-specific: LIKE / NOT LIKE require a string operand.
357
+ condition = {"field": "meta.name", "operator": operator, "value": 123}
358
+ with pytest.raises(FilterError, match="must be a str when using 'LIKE'"):
359
+ _parse_comparison_condition(condition)
360
+
361
+
186
362
  def test_logical_condition_missing_operator():
187
363
  condition = {"conditions": []}
188
364
  with pytest.raises(FilterError):
@@ -228,13 +404,32 @@ def test_logical_condition_nested():
228
404
  "("
229
405
  "(meta->>'domain' IS DISTINCT FROM %s OR meta->>'chapter' = ANY(%s))"
230
406
  " AND "
231
- "((meta->>'number')::integer >= %s OR meta->>'author' IS NULL OR meta->>'author' != ALL(%s))"
407
+ "((meta->>'number')::integer >= %s OR (meta->>'author' IS NULL OR meta->>'author' != ALL(%s)))"
232
408
  ")"
233
409
  )
234
410
  assert _render(query) == expected_sql
235
411
  assert values == ["science", [["intro", "conclusion"]], 90, [["John", "Jane"]]]
236
412
 
237
413
 
414
+ def test_logical_condition_not_in_is_parenthesized_when_and_combined():
415
+ # `not in` renders an internal `OR` (`IS NULL OR != ALL`). Without surrounding
416
+ # parentheses that `OR` escapes an enclosing `AND` (AND binds tighter than OR in
417
+ # SQL), so an AND-combined `not in` would wrongly match documents.
418
+ condition = {
419
+ "operator": "AND",
420
+ "conditions": [
421
+ {"field": "meta.number", "operator": "==", "value": 5},
422
+ {"field": "meta.author", "operator": "not in", "value": ["John", "Jane"]},
423
+ ],
424
+ }
425
+ query, values = _parse_logical_condition(condition)
426
+ assert isinstance(query, Composed)
427
+
428
+ expected_sql = "((meta->>'number')::integer = %s AND (meta->>'author' IS NULL OR meta->>'author' != ALL(%s)))"
429
+ assert _render(query) == expected_sql
430
+ assert values == [5, [["John", "Jane"]]]
431
+
432
+
238
433
  def test_convert_filters_to_where_clause_and_params():
239
434
  filters = {
240
435
  "operator": "AND",