pgvector-haystack 6.4.1__tar.gz → 6.6.0__tar.gz

{pgvector_haystack-6.4.1 → pgvector_haystack-6.6.0}/CHANGELOG.md

@@ -1,5 +1,26 @@
 # Changelog
 
+## [integrations/pgvector-v6.5.0] - 2026-08-03
+
+### 🚀 Features
+
+- Add array_contains and array_overlaps operators to pgvector filters (#3459)
+
+### 🐛 Bug Fixes
+
+- Fix new issues raised by ruff 0.16.0 (#3670)
+- `PGVectorDocumentStore` get_metadata_field_unique_values setting list of values to return to `Any` (#3709)
+
+
+## [integrations/pgvector-v6.4.1] - 2026-07-23
+
+### 🐛 Bug Fixes
+
+- *(pgvector)* Parenthesize `not in` clause so it does not escape an AND filter (#3586)
+- `PGVectorDocumentStore` match `search_term` against metadata field value, not content (#3648)
+- Fix (pgvector): fix error messages to say pgvector instead of pinecone (#3654)
+
+
 ## [integrations/pgvector-v6.4.0] - 2026-07-23
 
 ### 🚀 Features

{pgvector_haystack-6.4.1 → pgvector_haystack-6.6.0}/LICENSE.txt

@@ -58,7 +58,7 @@ APPENDIX: How to apply the Apache License to your work.
 
 To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!)  The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives.
 
-Copyright [yyyy] [name of copyright owner]
+Copyright 2023-present deepset GmbH
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

{pgvector_haystack-6.4.1 → pgvector_haystack-6.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: pgvector-haystack
-Version: 6.4.1
+Version: 6.6.0
 Summary: An integration of pgvector (vector search extension for Postgres) with Haystack
 Project-URL: Source, https://github.com/deepset-ai/haystack-core-integrations
 Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/blob/main/integrations/pgvector/README.md
@@ -34,6 +34,8 @@ Description-Content-Type: text/markdown
 
 ---
 
+## Contributing
+
 Refer to the general [Contribution Guidelines](https://github.com/deepset-ai/haystack-core-integrations/blob/main/CONTRIBUTING.md).
 
 To run integration tests locally, you need to have a PostgreSQL DB running with the `pgvector` extension.

{pgvector_haystack-6.4.1 → pgvector_haystack-6.6.0}/README.md

@@ -8,6 +8,8 @@
 
 ---
 
+## Contributing
+
 Refer to the general [Contribution Guidelines](https://github.com/deepset-ai/haystack-core-integrations/blob/main/CONTRIBUTING.md).
 
 To run integration tests locally, you need to have a PostgreSQL DB running with the `pgvector` extension.

{pgvector_haystack-6.4.1 → pgvector_haystack-6.6.0}/pyproject.toml

@@ -143,6 +143,7 @@ ignore = [
   "PLR0912",
   "PLR0913",
   "PLR0915",
+  "PLR0917",
   # ignore function-call-in-default-argument
   "B008",
   # ignore assertions

{pgvector_haystack-6.4.1 → pgvector_haystack-6.6.0}/src/haystack_integrations/document_stores/pgvector/document_store.py

@@ -1926,7 +1926,12 @@ class PgvectorDocumentStore:
         return {"min": min_value, "max": max_value}
 
     def _build_unique_values_queries(
-        self, normalized_field: str, search_term: str | None, from_: int, size: int
+        self,
+        normalized_field: str,
+        search_term: str | None,
+        from_: int,
+        size: int,
+        filters: dict[str, Any] | None = None,
     ) -> tuple[Composed, Composed, tuple]:
         """
         Builds SQL queries for getting unique metadata field values.
@@ -1936,25 +1941,38 @@ class PgvectorDocumentStore:
             against the metadata field's own value.
         :param from_: The offset for pagination (0-based).
         :param size: The number of unique values to return.
+        :param filters: Optional filters to restrict the documents considered.
         :returns: A tuple containing (count_query, select_query, params).
         """
         field_literal = SQLLiteral(normalized_field)
 
         # base query components
-        sql_select = SQL("SELECT DISTINCT meta->>{} AS value").format(field_literal)
+        # Use the `->` (jsonb) operator rather than `->>` (text) so DISTINCT/ORDER BY operate on the
+        # value's original JSON type (numbers sort numerically, not lexicographically) and psycopg
+        # decodes the result to its native Python type instead of a string.
+        #
+        # Not fixable here: JSONB equality treats a whole-number float (1.0) as identical to a
+        # numerically equal int (1), so DISTINCT still collapses them - see this method's docstring.
+        sql_select = SQL("SELECT DISTINCT meta->{} AS value").format(field_literal)
         sql_from = SQL(" FROM {schema_name}.{table_name}").format(
             schema_name=Identifier(self.schema_name), table_name=Identifier(self.table_name)
         )
         sql_where = SQL(" WHERE meta->>{} IS NOT NULL").format(field_literal)
 
         params: tuple = ()
+        if filters:
+            _validate_filters(filters)
+            filters_where_clause, filters_params = _convert_filters_to_where_clause_and_params(filters, operator="AND")
+            sql_where += filters_where_clause
+            params += filters_params
+
         if search_term:
-            # Case-insensitive substring match against the metadata field's own value.
+            # Case-insensitive substring match against the metadata field's own (text) value.
             sql_where += SQL(" AND meta->>{} ILIKE %s").format(field_literal)
-            params = (f"%{search_term}%",)
+            params += (f"%{search_term}%",)
 
         # count query
-        sql_count = SQL("SELECT COUNT(DISTINCT meta->>{} ) AS total").format(field_literal)
+        sql_count = SQL("SELECT COUNT(DISTINCT meta->{} ) AS total").format(field_literal)
         sql_count += sql_from + sql_where
 
         # paginated select query
@@ -1968,7 +1986,7 @@ class PgvectorDocumentStore:
     @staticmethod
     def _process_unique_values_result(
         count_result: dict[str, Any] | None, records: list[dict[str, Any]]
-    ) -> tuple[list[str], int]:
+    ) -> tuple[list[Any], int]:
         """
         Processes the results from unique values queries.
 
@@ -1977,26 +1995,40 @@ class PgvectorDocumentStore:
         :returns: A tuple containing (unique_values, total_count).
         """
         total_count = count_result.get("total", 0) if count_result else 0
-        unique_values = [str(record.get("value", "")) for record in records if record.get("value") is not None]
+        unique_values = [record.get("value") for record in records if record.get("value") is not None]
         return unique_values, total_count
 
     def get_metadata_field_unique_values(
-        self, metadata_field: str, search_term: str | None = None, from_: int = 0, size: int = 10
-    ) -> tuple[list[str], int]:
+        self,
+        metadata_field: str,
+        search_term: str | None = None,
+        from_: int = 0,
+        size: int = 10,
+        filters: dict[str, Any] | None = None,
+    ) -> tuple[list[Any], int]:
         """
         Returns unique values for a given metadata field, optionally filtered by a search term.
 
+        **Note**: values of different JSON type categories are kept distinct - a string, a number and
+        a boolean never collapse into each other, even when they share a textual form (e.g. the string
+        `"1"` and the number `1`). One exception: the ``meta`` column is JSONB, whose equality treats a
+        whole-number float (`1.0`) as identical to a numerically equal int (`1`), so those two collapse
+        into a single value. Floats with a fractional part (e.g. `1.5`) are unaffected.
+
         :param metadata_field: The name of the metadata field. Can include or omit the "meta." prefix.
         :param search_term: Optional search term to filter unique values by a case-insensitive substring
             match against the metadata field's own value. If None, all values are considered.
         :param from_: The offset for pagination (0-based).
         :param size: The number of unique values to return.
+        :param filters: Optional filters to restrict the documents considered.
         :returns: A tuple containing:
-            - A list of unique values (as strings)
+            - A list of unique values in their original type
             - The total count of unique values
         """
         normalized_field = PgvectorDocumentStore._normalize_metadata_field_name(metadata_field)
-        sql_count, sql_query, params = self._build_unique_values_queries(normalized_field, search_term, from_, size)
+        sql_count, sql_query, params = self._build_unique_values_queries(
+            normalized_field, search_term, from_, size, filters
+        )
 
         self._ensure_db_setup()
         assert self._dict_cursor is not None
@@ -2019,22 +2051,36 @@ class PgvectorDocumentStore:
         return PgvectorDocumentStore._process_unique_values_result(count_result, records)
 
     async def get_metadata_field_unique_values_async(
-        self, metadata_field: str, search_term: str | None = None, from_: int = 0, size: int = 10
-    ) -> tuple[list[str], int]:
+        self,
+        metadata_field: str,
+        search_term: str | None = None,
+        from_: int = 0,
+        size: int = 10,
+        filters: dict[str, Any] | None = None,
+    ) -> tuple[list[Any], int]:
         """
         Asynchronously returns unique values for a given metadata field, optionally filtered by a search term.
 
+        **Note**: values of different JSON type categories are kept distinct - a string, a number and
+        a boolean never collapse into each other, even when they share a textual form (e.g. the string
+        `"1"` and the number `1`). One exception: the ``meta`` column is JSONB, whose equality treats a
+        whole-number float (`1.0`) as identical to a numerically equal int (`1`), so those two collapse
+        into a single value. Floats with a fractional part (e.g. `1.5`) are unaffected.
+
         :param metadata_field: The name of the metadata field. Can include or omit the "meta." prefix.
         :param search_term: Optional search term to filter unique values by a case-insensitive substring
             match against the metadata field's own value. If None, all values are considered.
         :param from_: The offset for pagination (0-based).
         :param size: The number of unique values to return.
+        :param filters: Optional filters to restrict the documents considered.
         :returns: A tuple containing:
-            - A list of unique values (as strings)
+            - A list of unique values in their original type
             - The total count of unique values
         """
         normalized_field = PgvectorDocumentStore._normalize_metadata_field_name(metadata_field)
-        sql_count, sql_query, params = self._build_unique_values_queries(normalized_field, search_term, from_, size)
+        sql_count, sql_query, params = self._build_unique_values_queries(
+            normalized_field, search_term, from_, size, filters
+        )
 
         await self._ensure_db_setup_async()
         assert self._async_dict_cursor is not None

{pgvector_haystack-6.4.1 → pgvector_haystack-6.6.0}/src/haystack_integrations/document_stores/pgvector/filters.py

@@ -22,6 +22,11 @@ PYTHON_TYPES_TO_PG_TYPES = {
 
 NO_VALUE = object()  # unique sentinel; can never equal a user-supplied filter value
 
+# Operators that match against a JSONB array stored in the meta field.
+# They need the JSONB value (via the `->` accessor) as their left operand,
+# unlike the scalar operators which access meta as text (via `->>`).
+ARRAY_OPERATORS = ("array_contains", "array_overlaps")
+
 
 def _validate_filters(filters: dict[str, Any] | None = None) -> None:
     """
@@ -106,7 +111,18 @@ def _parse_comparison_condition(condition: dict[str, Any]) -> tuple[Composed, li
     value: Any = condition["value"]
 
     if field.startswith("meta."):
-        sql_field: Composable = _treat_meta_field(field, value)
+        # array operators need the JSONB value (->), the other operators access meta as text (->>)
+        sql_field: Composable = (
+            _treat_meta_field_as_jsonb(field) if operator in ARRAY_OPERATORS else _treat_meta_field(field, value)
+        )
+    elif operator in ARRAY_OPERATORS:
+        # top-level columns are never JSONB arrays, so these operators would render SQL that the
+        # database rejects with an opaque UndefinedFunction error. Fail with a FilterError instead.
+        msg = (
+            f"Operator '{operator}' is only supported on meta fields, but got field '{field}'. "
+            f"Prefix the field name with 'meta.' to filter on an array stored in a document's metadata."
+        )
+        raise FilterError(msg)
     else:
         sql_field = Identifier(field)
 
@@ -149,6 +165,24 @@ def _treat_meta_field(field: str, value: Any) -> Composed:
     return composed
 
 
+def _treat_meta_field_as_jsonb(field: str) -> Composed:
+    """
+    Internal method that returns a psycopg Composed object accessing a meta key as a JSONB value.
+
+    Unlike `_treat_meta_field`, this uses the `->` operator (which returns a JSONB value) instead of
+    `->>` (which returns text) and applies no type cast. The array operators need a JSONB left operand
+    for their JSONB operators (`@>`, `?|`) to work.
+
+    Uses psycopg.sql.Literal to embed the field name, preventing SQL injection via metadata field names.
+
+    Example:
+    >>> _treat_meta_field_as_jsonb(field="meta.tags")
+    Composed([SQL('meta->'), Literal('tags')])
+    """
+    field_name = field.split(".", 1)[-1]
+    return SQL("meta->") + SQLLiteral(field_name)
+
+
 def _equal(field: Composable, value: Any) -> tuple[Composed, Any]:
     if value is None:
         return SQL("{} IS NULL").format(field), NO_VALUE
@@ -255,6 +289,40 @@ def _not_like(field: Composable, value: Any) -> tuple[Composed, Any]:
     return SQL("{} NOT LIKE %s").format(field), value
 
 
+def _validate_array_operator_value(field: Composable, value: Any, operator: str) -> None:
+    if not isinstance(value, list):
+        msg = f"{field}'s value must be a list when using '{operator}' comparator in PgVector"
+        raise FilterError(msg)
+    # an empty list has no useful match semantics: `@> '[]'` matches every array and `?| '{}'` matches
+    # nothing, so the same empty filter would mean "everything" for one operator and "nothing" for the
+    # other. Callers building the list dynamically should omit the condition instead.
+    if not value:
+        msg = f"{field}'s value must be a non-empty list when using '{operator}' comparator in PgVector"
+        raise FilterError(msg)
+
+
+def _array_contains(field: Composable, value: Any) -> tuple[Composed, Jsonb]:
+    _validate_array_operator_value(field, value, "array_contains")
+    # `jsonb @> jsonb` is true when the left array contains every element of the right array,
+    # so the meta array must contain all of the given values. Containment compares elements by value,
+    # so a non-array meta value (a bare string, an object) never matches a list of values.
+    return SQL("{} @> %s").format(field), Jsonb(value)
+
+
+def _array_overlaps(field: Composable, value: Any) -> tuple[Composed, list]:
+    _validate_array_operator_value(field, value, "array_overlaps")
+    # `jsonb ?| text[]` only matches string elements, so we reject non-string values to fail fast
+    # instead of silently returning no results.
+    if not all(isinstance(item, str) for item in value):
+        msg = f"{field}'s value must be a list of strings when using 'array_overlaps' comparator in PgVector"
+        raise FilterError(msg)
+    # `?|` is true when a given string exists as a top-level key OR array element, so on its own it
+    # also matches objects (by key) and bare strings. Restricting it to arrays with jsonb_typeof keeps
+    # it in agreement with array_contains. The parentheses stop this AND from being split by an
+    # enclosing OR (see _not_in for the same reason).
+    return SQL("(jsonb_typeof({}) = 'array' AND {} ?| %s)").format(field, field), [value]
+
+
 COMPARISON_OPERATORS = {
     "==": _equal,
     "!=": _not_equal,
@@ -266,4 +334,6 @@ COMPARISON_OPERATORS = {
     "not in": _not_in,
     "like": _like,
     "not like": _not_like,
+    "array_contains": _array_contains,
+    "array_overlaps": _array_overlaps,
 }

{pgvector_haystack-6.4.1 → pgvector_haystack-6.6.0}/tests/test_document_store.py

@@ -66,6 +66,41 @@ class TestDocumentStore(
         with pytest.raises(DuplicateDocumentError):
             document_store.write_documents(docs, DuplicatePolicy.FAIL)
 
+    def test_get_metadata_field_unique_values_distinct_types(self, document_store: PgvectorDocumentStore):
+        """
+        Override: the base mixin test stores int, float, str and bool under the *same* metadata field
+        name and expects all four back as distinct values. This store's ``meta`` column is JSONB, and
+        PostgreSQL's JSONB equality treats a whole-number float (e.g. 1.0) and a numerically equal int
+        (1) as the same value, so ``SELECT DISTINCT meta->'field'`` collapses them regardless of which
+        other values share that field.
+
+        This adapts the same intent - int, float, str and bool must come back as distinct, unmangled
+        types via get_metadata_field_unique_values() - using one field per type instead of one shared
+        field, which is what this store can actually support.
+
+        The float value is a non-whole number (1.5, not 1.0): a whole-number float would still collapse
+        with an int under PostgreSQL's JSONB numeric equality even in its own field, so a fractional
+        value is used to sidestep that ambiguity entirely.
+        """
+        docs = [
+            Document(content="Doc 1", meta={"priority_int": 1}),
+            Document(content="Doc 2", meta={"priority_str": "1"}),
+            Document(content="Doc 3", meta={"priority_float": 1.5}),
+            Document(content="Doc 4", meta={"priority_bool": True}),
+        ]
+        document_store.write_documents(docs)
+
+        int_values, int_count = document_store.get_metadata_field_unique_values(metadata_field="priority_int")
+        str_values, str_count = document_store.get_metadata_field_unique_values(metadata_field="priority_str")
+        float_values, float_count = document_store.get_metadata_field_unique_values(metadata_field="priority_float")
+        bool_values, bool_count = document_store.get_metadata_field_unique_values(metadata_field="priority_bool")
+
+        assert (int_count, str_count, float_count, bool_count) == (1, 1, 1, 1)
+        assert int_values == [1] and type(int_values[0]) is int
+        assert str_values == ["1"] and type(str_values[0]) is str
+        assert float_values == [1.5] and type(float_values[0]) is float
+        assert bool_values == [True] and type(bool_values[0]) is bool
+
     def test_write_blob(self, document_store: PgvectorDocumentStore):
         bytestream = ByteStream(b"test", meta={"meta_key": "meta_value"}, mime_type="mime_type")
         docs = [Document(id="1", blob=bytestream)]
@@ -596,171 +631,30 @@ def test_get_metadata_field_min_max(document_store: PgvectorDocumentStore):
 
 
 @pytest.mark.integration
-def test_get_metadata_field_unique_values(document_store: PgvectorDocumentStore):
-    # Test with string values
+def test_get_metadata_field_unique_values_pagination_edge_cases(document_store: PgvectorDocumentStore):
+    """
+    Edge cases not covered by the shared GetMetadataFieldUniqueValuesTest mixin's pagination test:
+    size larger than the total count, from_ beyond the total count, and single-item-per-page paging.
+    """
     docs = [
-        Document(content="Python programming", meta={"category": "A", "language": "Python"}),
-        Document(content="Java programming", meta={"category": "B", "language": "Java"}),
-        Document(content="Python scripting", meta={"category": "A", "language": "Python"}),
-        Document(content="JavaScript development", meta={"category": "C", "language": "JavaScript"}),
-        Document(content="Python data science", meta={"category": "A", "language": "Python"}),
-        Document(content="Java backend", meta={"category": "B", "language": "Java"}),
+        Document(content="Python programming", meta={"category": "A"}),
+        Document(content="Java programming", meta={"category": "B"}),
+        Document(content="JavaScript development", meta={"category": "C"}),
     ]
     document_store.write_documents(docs)
 
-    # Test getting all unique values without search term
-    unique_values, total_count = document_store.get_metadata_field_unique_values("meta.category", None, 0, 10)
-    assert set(unique_values) == {"A", "B", "C"}
-    assert total_count == 3
-
-    # Test with "meta." prefix
-    unique_languages, total_languages = document_store.get_metadata_field_unique_values("meta.language", None, 0, 10)
-    assert set(unique_languages) == {"Python", "Java", "JavaScript"}
-    assert total_languages == 3
-
-    # Test pagination - first page
-    unique_values_page1, total_count_page1 = document_store.get_metadata_field_unique_values(
-        "meta.category", None, 0, 2
-    )
-    assert len(unique_values_page1) == 2
-    assert all(val in ["A", "B", "C"] for val in unique_values_page1)
-    assert total_count_page1 == 3
-
-    # Test pagination - second page
-    unique_values_page2, total_count_page2 = document_store.get_metadata_field_unique_values(
-        "meta.category", None, 2, 2
-    )
-    assert len(unique_values_page2) == 1
-    assert unique_values_page2[0] in ["A", "B", "C"]
-    assert total_count_page2 == 3
-
-    # Test pagination - verify pages don't overlap
-    assert not set(unique_values_page1).intersection(set(unique_values_page2))
-
-    # Test pagination - verify all values are covered
-    all_values = set(unique_values_page1) | set(unique_values_page2)
-    assert all_values == {"A", "B", "C"}
-
-    # Test pagination - size larger than total count
     unique_values_large, total_large = document_store.get_metadata_field_unique_values("meta.category", None, 0, 100)
-    assert len(unique_values_large) == 3
     assert set(unique_values_large) == {"A", "B", "C"}
     assert total_large == 3
 
-    # Test pagination - from_ beyond total count (should return empty)
     unique_values_beyond, total_beyond = document_store.get_metadata_field_unique_values("meta.category", None, 10, 10)
     assert len(unique_values_beyond) == 0
     assert total_beyond == 3
 
-    # Test pagination - single item per page
     unique_values_single1, _ = document_store.get_metadata_field_unique_values("meta.category", None, 0, 1)
     unique_values_single2, _ = document_store.get_metadata_field_unique_values("meta.category", None, 1, 1)
     unique_values_single3, _ = document_store.get_metadata_field_unique_values("meta.category", None, 2, 1)
     assert len(unique_values_single1) == 1
     assert len(unique_values_single2) == 1
     assert len(unique_values_single3) == 1
-    # All three pages should be different
     assert len(set(unique_values_single1 + unique_values_single2 + unique_values_single3)) == 3
-
-    # Test with search term - case-insensitive substring match against the metadata field's OWN value
-    # (not against document content).
-    unique_values_filtered, total_filtered = document_store.get_metadata_field_unique_values(
-        "meta.language", "Python", 0, 10
-    )
-    assert set(unique_values_filtered) == {"Python"}
-    assert total_filtered == 1
-
-    # Case-insensitivity: lowercase search term should still match "Python"
-    unique_values_lower, total_lower = document_store.get_metadata_field_unique_values("meta.language", "python", 0, 10)
-    assert set(unique_values_lower) == {"Python"}
-    assert total_lower == 1
-
-    # Substring matching: "Java" is a substring of both "Java" and "JavaScript"
-    unique_values_java, total_java = document_store.get_metadata_field_unique_values("meta.language", "Java", 0, 10)
-    assert set(unique_values_java) == {"Java", "JavaScript"}
-    assert total_java == 2
-
-    # Substring matching in the middle/end of a value
-    unique_values_script, total_script = document_store.get_metadata_field_unique_values(
-        "meta.language", "Script", 0, 10
-    )
-    assert set(unique_values_script) == {"JavaScript"}
-    assert total_script == 1
-
-    # Test pagination with search term
-    unique_values_search_page1, total_search = document_store.get_metadata_field_unique_values(
-        "meta.language", "Python", 0, 1
-    )
-    assert len(unique_values_search_page1) == 1
-    assert unique_values_search_page1[0] == "Python"
-    assert total_search == 1
-
-    # Test pagination with search term - beyond results
-    unique_values_search_empty, total_search_empty = document_store.get_metadata_field_unique_values(
-        "meta.language", "Python", 10, 10
-    )
-    assert len(unique_values_search_empty) == 0
-    assert total_search_empty == 1
-
-    # Test that search_term matches metadata VALUE, not content
-    content_vs_metadata_docs = [
-        Document(content="This document mentions Python explicitly", meta={"topic": "cooking"}),
-        Document(content="Unrelated content about recipes", meta={"topic": "python-tutorial"}),
-    ]
-    document_store.write_documents(content_vs_metadata_docs)
-
-    unique_topics, total_topics = document_store.get_metadata_field_unique_values("meta.topic", "python", 0, 10)
-    assert set(unique_topics) == {"python-tutorial"}
-    assert total_topics == 1
-
-    # Test with integer values
-    int_docs = [
-        Document(content="Doc 1", meta={"priority": 1}),
-        Document(content="Doc 2", meta={"priority": 2}),
-        Document(content="Doc 3", meta={"priority": 1}),
-        Document(content="Doc 4", meta={"priority": 3}),
-    ]
-    document_store.write_documents(int_docs)
-    unique_priorities, total_priorities = document_store.get_metadata_field_unique_values("meta.priority", None, 0, 10)
-    assert set(unique_priorities) == {"1", "2", "3"}
-    assert total_priorities == 3
-
-    # Test with search term on integer field - substring match against the field's own (stringified)
-    # value, e.g. "Doc 1" (content) no longer matches; "1" (the value itself) does.
-    unique_priorities_filtered, total_priorities_filtered = document_store.get_metadata_field_unique_values(
-        "meta.priority", "1", 0, 10
-    )
-    assert set(unique_priorities_filtered) == {"1"}
-    assert total_priorities_filtered == 1
-
-
-@pytest.mark.integration
-def test_get_metadata_field_unique_values_no_search_term(document_store: PgvectorDocumentStore):
-    docs = [
-        Document(content="mentions tsvector and content in the body", meta={"category": "A"}),
-        Document(content="another document", meta={"category": "B"}),
-    ]
-    document_store.write_documents(docs)
-
-    values, total = document_store.get_metadata_field_unique_values("meta.category")
-
-    assert set(values) == {"A", "B"}
-    assert total == 2
-
-
-@pytest.mark.integration
-def test_get_metadata_field_unique_values_search_term_matches_field_value_not_content(
-    document_store: PgvectorDocumentStore,
-):
-    docs = [
-        # "python" appears in the content but NOT in the category value -> must be EXCLUDED
-        Document(content="Python programming guide", meta={"category": "Backend"}),
-        # "python" appears in the category value but NOT in the content -> must be INCLUDED
-        Document(content="General purpose scripting language", meta={"category": "Python-based"}),
-    ]
-    document_store.write_documents(docs)
-
-    values, total = document_store.get_metadata_field_unique_values("meta.category", search_term="python")
-
-    assert values == ["Python-based"]
-    assert total == 1

{pgvector_haystack-6.4.1 → pgvector_haystack-6.6.0}/tests/test_document_store_async.py

@@ -71,6 +71,49 @@ class TestDocumentStoreAsync(
         with pytest.raises(DuplicateDocumentError):
             await document_store.write_documents_async(docs, DuplicatePolicy.FAIL)
 
+    async def test_get_metadata_field_unique_values_distinct_types_async(self, document_store: PgvectorDocumentStore):
+        """
+        Override: the base mixin test stores int, float, str and bool under the *same* metadata field
+        name and expects all four back as distinct values. This store's ``meta`` column is JSONB, and
+        PostgreSQL's JSONB equality treats a whole-number float (e.g. 1.0) and a numerically equal int
+        (1) as the same value, so ``SELECT DISTINCT meta->'field'`` collapses them regardless of which
+        other values share that field.
+
+        This adapts the same intent - int, float, str and bool must come back as distinct, unmangled
+        types via get_metadata_field_unique_values_async() - using one field per type instead of one
+        shared field, which is what this store can actually support.
+
+        The float value is a non-whole number (1.5, not 1.0): a whole-number float would still collapse
+        with an int under PostgreSQL's JSONB numeric equality even in its own field, so a fractional
+        value is used to sidestep that ambiguity entirely.
+        """
+        docs = [
+            Document(content="Doc 1", meta={"priority_int": 1}),
+            Document(content="Doc 2", meta={"priority_str": "1"}),
+            Document(content="Doc 3", meta={"priority_float": 1.5}),
+            Document(content="Doc 4", meta={"priority_bool": True}),
+        ]
+        await document_store.write_documents_async(docs)
+
+        int_values, int_count = await document_store.get_metadata_field_unique_values_async(
+            metadata_field="priority_int"
+        )
+        str_values, str_count = await document_store.get_metadata_field_unique_values_async(
+            metadata_field="priority_str"
+        )
+        float_values, float_count = await document_store.get_metadata_field_unique_values_async(
+            metadata_field="priority_float"
+        )
+        bool_values, bool_count = await document_store.get_metadata_field_unique_values_async(
+            metadata_field="priority_bool"
+        )
+
+        assert (int_count, str_count, float_count, bool_count) == (1, 1, 1, 1)
+        assert int_values == [1] and type(int_values[0]) is int
+        assert str_values == ["1"] and type(str_values[0]) is str
+        assert float_values == [1.5] and type(float_values[0]) is float
+        assert bool_values == [True] and type(bool_values[0]) is bool
+
     async def test_count_not_empty_async(self, document_store: PgvectorDocumentStore):
         """Override: mixin method is missing 'self', causing fixture injection to fail."""
         await document_store.write_documents_async(

{pgvector_haystack-6.4.1 → pgvector_haystack-6.6.0}/tests/test_filters.py

@@ -10,6 +10,7 @@ from haystack.testing.document_store import FilterDocumentsTest
 from psycopg.adapt import Transformer
 from psycopg.sql import SQL, Composed
 from psycopg.sql import Literal as SQLLiteral
+from psycopg.types.json import Jsonb
 
 from haystack_integrations.document_stores.pgvector.filters import (
     FilterError,
@@ -17,6 +18,7 @@ from haystack_integrations.document_stores.pgvector.filters import (
     _parse_comparison_condition,
     _parse_logical_condition,
     _treat_meta_field,
+    _treat_meta_field_as_jsonb,
     _validate_filters,
 )
 
@@ -141,6 +143,86 @@ class TestFilters(FilterDocumentsTest):
             ],
         )
 
+    @staticmethod
+    def _docs_with_array_meta():
+        return [
+            Document(content="doc with tag1 and tag2", meta={"tags": ["tag1", "tag2"]}),
+            Document(content="doc with tag1 and tag3", meta={"tags": ["tag1", "tag3"]}),
+            Document(content="doc with tag4", meta={"tags": ["tag4"]}),
+        ]
+
+    def test_array_contains_operator(self, document_store):
+        docs = self._docs_with_array_meta()
+        document_store.write_documents(docs)
+
+        # array_contains requires ALL listed values to be present in the meta array
+        result = document_store.filter_documents(
+            filters={"field": "meta.tags", "operator": "array_contains", "value": ["tag1", "tag2"]}
+        )
+        assert {doc.content for doc in result} == {"doc with tag1 and tag2"}
+
+    def test_array_contains_operator_single_value(self, document_store):
+        docs = self._docs_with_array_meta()
+        document_store.write_documents(docs)
+
+        result = document_store.filter_documents(
+            filters={"field": "meta.tags", "operator": "array_contains", "value": ["tag1"]}
+        )
+        assert {doc.content for doc in result} == {"doc with tag1 and tag2", "doc with tag1 and tag3"}
+
+    def test_array_overlaps_operator(self, document_store):
+        docs = self._docs_with_array_meta()
+        document_store.write_documents(docs)
+
+        # array_overlaps requires AT LEAST ONE listed value to be present in the meta array
+        result = document_store.filter_documents(
+            filters={"field": "meta.tags", "operator": "array_overlaps", "value": ["tag1", "tag3"]}
+        )
+        assert {doc.content for doc in result} == {"doc with tag1 and tag2", "doc with tag1 and tag3"}
+
+    def test_array_overlaps_operator_no_match(self, document_store):
+        docs = self._docs_with_array_meta()
+        document_store.write_documents(docs)
+
+        result = document_store.filter_documents(
+            filters={"field": "meta.tags", "operator": "array_overlaps", "value": ["nonexistent"]}
+        )
+        assert result == []
+
+    def test_array_operators_combined(self, document_store):
+        docs = self._docs_with_array_meta()
+        document_store.write_documents(docs)
+
+        result = document_store.filter_documents(
+            filters={
+                "operator": "AND",
+                "conditions": [
+                    {"field": "meta.tags", "operator": "array_contains", "value": ["tag1"]},
+                    {"field": "meta.tags", "operator": "array_overlaps", "value": ["tag3"]},
+                ],
+            }
+        )
+        assert {doc.content for doc in result} == {"doc with tag1 and tag3"}
+
+    @pytest.mark.parametrize("operator", ["array_contains", "array_overlaps"])
+    def test_array_operators_ignore_non_array_meta(self, document_store, operator):
+        # meta shapes vary across sources, so the same key can hold an array in one document and a
+        # bare string or an object in another. `?|` matches a top-level key or array element, so
+        # without the jsonb_typeof guard the object and the string below would both match.
+        document_store.write_documents(
+            [
+                Document(content="array meta", meta={"tags": ["tag1", "tag2"]}),
+                Document(content="string meta", meta={"tags": "tag1"}),
+                Document(content="object meta", meta={"tags": {"tag1": "value"}}),
+                Document(content="missing key", meta={"other": "tag1"}),
+            ]
+        )
+
+        result = document_store.filter_documents(
+            filters={"field": "meta.tags", "operator": operator, "value": ["tag1"]}
+        )
+        assert {doc.content for doc in result} == {"array meta"}
+
 
 def test_treat_meta_field():
     cast_integer = SQL("(") + SQL("meta->>") + SQLLiteral("number") + SQL(")::integer")
@@ -165,6 +247,92 @@ def test_treat_meta_field_sql_injection_is_safely_escaped():
     assert result == SQL("meta->>") + SQLLiteral("name' OR 1=1 --")
 
 
+def test_treat_meta_field_as_jsonb():
+    # array operators access the meta value as JSONB (->) without a cast, unlike _treat_meta_field (->>)
+    assert _treat_meta_field_as_jsonb(field="meta.tags") == SQL("meta->") + SQLLiteral("tags")
+
+
+def test_treat_meta_field_as_jsonb_sql_injection_is_safely_escaped():
+    result = _treat_meta_field_as_jsonb(field="meta.tags' OR 1=1 --")
+    assert isinstance(result, Composed)
+    assert result == SQL("meta->") + SQLLiteral("tags' OR 1=1 --")
+
+
+def test_array_contains_condition():
+    condition = {"field": "meta.tags", "operator": "array_contains", "value": ["tag1", "tag2"]}
+    query, values = _parse_comparison_condition(condition)
+
+    assert _render(query) == "meta->'tags' @> %s"
+    # the value is wrapped as a single Jsonb param holding the original list
+    assert len(values) == 1
+    assert isinstance(values[0], Jsonb)
+    assert values[0].obj == ["tag1", "tag2"]
+
+
+def test_array_overlaps_condition():
+    condition = {"field": "meta.tags", "operator": "array_overlaps", "value": ["tag1", "tag2"]}
+    query, values = _parse_comparison_condition(condition)
+
+    # the jsonb_typeof guard keeps `?|` from matching objects (by key) or bare strings
+    assert _render(query) == "(jsonb_typeof(meta->'tags') = 'array' AND meta->'tags' ?| %s)"
+    # the list is wrapped twice (once here, once by _parse_comparison_condition), exactly like 'in',
+    # so psycopg adapts it to a nested Postgres text[]; `?|` deconstructs it to the same elements
+    assert values == [[["tag1", "tag2"]]]
+
+
+def test_array_contains_non_list_value_raises():
+    condition = {"field": "meta.tags", "operator": "array_contains", "value": "tag1"}
+    with pytest.raises(FilterError):
+        _parse_comparison_condition(condition)
+
+
+def test_array_overlaps_non_list_value_raises():
+    condition = {"field": "meta.tags", "operator": "array_overlaps", "value": "tag1"}
+    with pytest.raises(FilterError):
+        _parse_comparison_condition(condition)
+
+
+def test_array_overlaps_non_string_elements_raises():
+    condition = {"field": "meta.tags", "operator": "array_overlaps", "value": [1, 2]}
+    with pytest.raises(FilterError):
+        _parse_comparison_condition(condition)
+
+
+@pytest.mark.parametrize("operator", ["array_contains", "array_overlaps"])
+def test_array_operators_empty_list_raises(operator):
+    # `@> '[]'` matches every array and `?| '{}'` matches nothing, so an empty list would mean
+    # "everything" for one operator and "nothing" for the other. Reject it instead.
+    condition = {"field": "meta.tags", "operator": operator, "value": []}
+    with pytest.raises(FilterError, match="non-empty list"):
+        _parse_comparison_condition(condition)
+
+
+@pytest.mark.parametrize("operator", ["array_contains", "array_overlaps"])
+def test_array_operators_on_non_meta_field_raises(operator):
+    # top-level columns are never JSONB arrays, so this must fail before reaching the database
+    condition = {"field": "content", "operator": operator, "value": ["tag1"]}
+    with pytest.raises(FilterError, match="only supported on meta fields"):
+        _parse_comparison_condition(condition)
+
+
+def test_convert_filters_array_contains():
+    filters = {"field": "meta.tags", "operator": "array_contains", "value": ["tag1", "tag2"]}
+    where_clause, params = _convert_filters_to_where_clause_and_params(filters)
+
+    assert _render(where_clause) == " WHERE meta->'tags' @> %s"
+    assert len(params) == 1
+    assert isinstance(params[0], Jsonb)
+    assert params[0].obj == ["tag1", "tag2"]
+
+
+def test_convert_filters_array_overlaps():
+    filters = {"field": "meta.tags", "operator": "array_overlaps", "value": ["tag1", "tag2"]}
+    where_clause, params = _convert_filters_to_where_clause_and_params(filters)
+
+    assert _render(where_clause) == " WHERE (jsonb_typeof(meta->'tags') = 'array' AND meta->'tags' ?| %s)"
+    assert params == ([["tag1", "tag2"]],)
+
+
 def test_comparison_condition_missing_operator():
     condition = {"field": "meta.type", "value": "article"}
     with pytest.raises(FilterError):