pgvector-haystack 6.4.1__tar.gz → 6.5.0__tar.gz

{pgvector_haystack-6.4.1 → pgvector_haystack-6.5.0}/CHANGELOG.md

@@ -1,5 +1,14 @@
 # Changelog
 
+## [integrations/pgvector-v6.4.1] - 2026-07-23
+
+### 🐛 Bug Fixes
+
+- *(pgvector)* Parenthesize `not in` clause so it does not escape an AND filter (#3586)
+- `PGVectorDocumentStore` match `search_term` against metadata field value, not content (#3648)
+- Fix (pgvector): fix error messages to say pgvector instead of pinecone (#3654)
+
+
 ## [integrations/pgvector-v6.4.0] - 2026-07-23
 
 ### 🚀 Features

{pgvector_haystack-6.4.1 → pgvector_haystack-6.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pgvector-haystack
-Version: 6.4.1
+Version: 6.5.0
 Summary: An integration of pgvector (vector search extension for Postgres) with Haystack
 Project-URL: Source, https://github.com/deepset-ai/haystack-core-integrations
 Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/blob/main/integrations/pgvector/README.md

{pgvector_haystack-6.4.1 → pgvector_haystack-6.5.0}/pyproject.toml

@@ -143,6 +143,7 @@ ignore = [
   "PLR0912",
   "PLR0913",
   "PLR0915",
+  "PLR0917",
   # ignore function-call-in-default-argument
   "B008",
   # ignore assertions

{pgvector_haystack-6.4.1 → pgvector_haystack-6.5.0}/src/haystack_integrations/document_stores/pgvector/document_store.py

@@ -1941,7 +1941,10 @@ class PgvectorDocumentStore:
         field_literal = SQLLiteral(normalized_field)
 
         # base query components
-        sql_select = SQL("SELECT DISTINCT meta->>{} AS value").format(field_literal)
+        # Use the `->` (jsonb) operator rather than `->>` (text) so DISTINCT/ORDER BY operate on the
+        # value's original JSON type (numbers sort numerically, not lexicographically) and psycopg
+        # decodes the result to its native Python type instead of a string.
+        sql_select = SQL("SELECT DISTINCT meta->{} AS value").format(field_literal)
         sql_from = SQL(" FROM {schema_name}.{table_name}").format(
             schema_name=Identifier(self.schema_name), table_name=Identifier(self.table_name)
         )
@@ -1949,12 +1952,12 @@ class PgvectorDocumentStore:
 
         params: tuple = ()
         if search_term:
-            # Case-insensitive substring match against the metadata field's own value.
+            # Case-insensitive substring match against the metadata field's own (text) value.
             sql_where += SQL(" AND meta->>{} ILIKE %s").format(field_literal)
             params = (f"%{search_term}%",)
 
         # count query
-        sql_count = SQL("SELECT COUNT(DISTINCT meta->>{} ) AS total").format(field_literal)
+        sql_count = SQL("SELECT COUNT(DISTINCT meta->{} ) AS total").format(field_literal)
         sql_count += sql_from + sql_where
 
         # paginated select query
@@ -1968,7 +1971,7 @@ class PgvectorDocumentStore:
     @staticmethod
     def _process_unique_values_result(
         count_result: dict[str, Any] | None, records: list[dict[str, Any]]
-    ) -> tuple[list[str], int]:
+    ) -> tuple[list[Any], int]:
         """
         Processes the results from unique values queries.
 
@@ -1977,12 +1980,12 @@ class PgvectorDocumentStore:
         :returns: A tuple containing (unique_values, total_count).
         """
         total_count = count_result.get("total", 0) if count_result else 0
-        unique_values = [str(record.get("value", "")) for record in records if record.get("value") is not None]
+        unique_values = [record.get("value") for record in records if record.get("value") is not None]
         return unique_values, total_count
 
     def get_metadata_field_unique_values(
         self, metadata_field: str, search_term: str | None = None, from_: int = 0, size: int = 10
-    ) -> tuple[list[str], int]:
+    ) -> tuple[list[Any], int]:
         """
         Returns unique values for a given metadata field, optionally filtered by a search term.
 
@@ -1992,7 +1995,7 @@ class PgvectorDocumentStore:
         :param from_: The offset for pagination (0-based).
         :param size: The number of unique values to return.
         :returns: A tuple containing:
-            - A list of unique values (as strings)
+            - A list of unique values in their original type
             - The total count of unique values
         """
         normalized_field = PgvectorDocumentStore._normalize_metadata_field_name(metadata_field)
@@ -2020,7 +2023,7 @@ class PgvectorDocumentStore:
 
     async def get_metadata_field_unique_values_async(
         self, metadata_field: str, search_term: str | None = None, from_: int = 0, size: int = 10
-    ) -> tuple[list[str], int]:
+    ) -> tuple[list[Any], int]:
         """
         Asynchronously returns unique values for a given metadata field, optionally filtered by a search term.
 
@@ -2030,7 +2033,7 @@ class PgvectorDocumentStore:
         :param from_: The offset for pagination (0-based).
         :param size: The number of unique values to return.
         :returns: A tuple containing:
-            - A list of unique values (as strings)
+            - A list of unique values in their original type
             - The total count of unique values
         """
         normalized_field = PgvectorDocumentStore._normalize_metadata_field_name(metadata_field)

{pgvector_haystack-6.4.1 → pgvector_haystack-6.5.0}/src/haystack_integrations/document_stores/pgvector/filters.py

@@ -22,6 +22,11 @@ PYTHON_TYPES_TO_PG_TYPES = {
 
 NO_VALUE = object()  # unique sentinel; can never equal a user-supplied filter value
 
+# Operators that match against a JSONB array stored in the meta field.
+# They need the JSONB value (via the `->` accessor) as their left operand,
+# unlike the scalar operators which access meta as text (via `->>`).
+ARRAY_OPERATORS = ("array_contains", "array_overlaps")
+
 
 def _validate_filters(filters: dict[str, Any] | None = None) -> None:
     """
@@ -106,7 +111,18 @@ def _parse_comparison_condition(condition: dict[str, Any]) -> tuple[Composed, li
     value: Any = condition["value"]
 
     if field.startswith("meta."):
-        sql_field: Composable = _treat_meta_field(field, value)
+        # array operators need the JSONB value (->), the other operators access meta as text (->>)
+        sql_field: Composable = (
+            _treat_meta_field_as_jsonb(field) if operator in ARRAY_OPERATORS else _treat_meta_field(field, value)
+        )
+    elif operator in ARRAY_OPERATORS:
+        # top-level columns are never JSONB arrays, so these operators would render SQL that the
+        # database rejects with an opaque UndefinedFunction error. Fail with a FilterError instead.
+        msg = (
+            f"Operator '{operator}' is only supported on meta fields, but got field '{field}'. "
+            f"Prefix the field name with 'meta.' to filter on an array stored in a document's metadata."
+        )
+        raise FilterError(msg)
     else:
         sql_field = Identifier(field)
 
@@ -149,6 +165,24 @@ def _treat_meta_field(field: str, value: Any) -> Composed:
     return composed
 
 
+def _treat_meta_field_as_jsonb(field: str) -> Composed:
+    """
+    Internal method that returns a psycopg Composed object accessing a meta key as a JSONB value.
+
+    Unlike `_treat_meta_field`, this uses the `->` operator (which returns a JSONB value) instead of
+    `->>` (which returns text) and applies no type cast. The array operators need a JSONB left operand
+    for their JSONB operators (`@>`, `?|`) to work.
+
+    Uses psycopg.sql.Literal to embed the field name, preventing SQL injection via metadata field names.
+
+    Example:
+    >>> _treat_meta_field_as_jsonb(field="meta.tags")
+    Composed([SQL('meta->'), Literal('tags')])
+    """
+    field_name = field.split(".", 1)[-1]
+    return SQL("meta->") + SQLLiteral(field_name)
+
+
 def _equal(field: Composable, value: Any) -> tuple[Composed, Any]:
     if value is None:
         return SQL("{} IS NULL").format(field), NO_VALUE
@@ -255,6 +289,40 @@ def _not_like(field: Composable, value: Any) -> tuple[Composed, Any]:
     return SQL("{} NOT LIKE %s").format(field), value
 
 
+def _validate_array_operator_value(field: Composable, value: Any, operator: str) -> None:
+    if not isinstance(value, list):
+        msg = f"{field}'s value must be a list when using '{operator}' comparator in PgVector"
+        raise FilterError(msg)
+    # an empty list has no useful match semantics: `@> '[]'` matches every array and `?| '{}'` matches
+    # nothing, so the same empty filter would mean "everything" for one operator and "nothing" for the
+    # other. Callers building the list dynamically should omit the condition instead.
+    if not value:
+        msg = f"{field}'s value must be a non-empty list when using '{operator}' comparator in PgVector"
+        raise FilterError(msg)
+
+
+def _array_contains(field: Composable, value: Any) -> tuple[Composed, Jsonb]:
+    _validate_array_operator_value(field, value, "array_contains")
+    # `jsonb @> jsonb` is true when the left array contains every element of the right array,
+    # so the meta array must contain all of the given values. Containment compares elements by value,
+    # so a non-array meta value (a bare string, an object) never matches a list of values.
+    return SQL("{} @> %s").format(field), Jsonb(value)
+
+
+def _array_overlaps(field: Composable, value: Any) -> tuple[Composed, list]:
+    _validate_array_operator_value(field, value, "array_overlaps")
+    # `jsonb ?| text[]` only matches string elements, so we reject non-string values to fail fast
+    # instead of silently returning no results.
+    if not all(isinstance(item, str) for item in value):
+        msg = f"{field}'s value must be a list of strings when using 'array_overlaps' comparator in PgVector"
+        raise FilterError(msg)
+    # `?|` is true when a given string exists as a top-level key OR array element, so on its own it
+    # also matches objects (by key) and bare strings. Restricting it to arrays with jsonb_typeof keeps
+    # it in agreement with array_contains. The parentheses stop this AND from being split by an
+    # enclosing OR (see _not_in for the same reason).
+    return SQL("(jsonb_typeof({}) = 'array' AND {} ?| %s)").format(field, field), [value]
+
+
 COMPARISON_OPERATORS = {
     "==": _equal,
     "!=": _not_equal,
@@ -266,4 +334,6 @@ COMPARISON_OPERATORS = {
     "not in": _not_in,
     "like": _like,
     "not like": _not_like,
+    "array_contains": _array_contains,
+    "array_overlaps": _array_overlaps,
 }

{pgvector_haystack-6.4.1 → pgvector_haystack-6.5.0}/tests/test_document_store.py

@@ -722,15 +722,16 @@ def test_get_metadata_field_unique_values(document_store: PgvectorDocumentStore)
     ]
     document_store.write_documents(int_docs)
     unique_priorities, total_priorities = document_store.get_metadata_field_unique_values("meta.priority", None, 0, 10)
-    assert set(unique_priorities) == {"1", "2", "3"}
+    assert set(unique_priorities) == {1, 2, 3}
     assert total_priorities == 3
 
     # Test with search term on integer field - substring match against the field's own (stringified)
-    # value, e.g. "Doc 1" (content) no longer matches; "1" (the value itself) does.
+    # value, e.g. "Doc 1" (content) no longer matches; "1" (the value itself) does. The returned
+    # values themselves keep their original type (int here), only the match is textual.
     unique_priorities_filtered, total_priorities_filtered = document_store.get_metadata_field_unique_values(
         "meta.priority", "1", 0, 10
     )
-    assert set(unique_priorities_filtered) == {"1"}
+    assert set(unique_priorities_filtered) == {1}
     assert total_priorities_filtered == 1
 
 

{pgvector_haystack-6.4.1 → pgvector_haystack-6.5.0}/tests/test_document_store_async.py

@@ -124,6 +124,22 @@ class TestDocumentStoreAsync(
                 filters={"field": "meta.category", "operator": "==", "value": "A"}, meta={}
             )
 
+    async def test_get_metadata_field_unique_values_async_preserves_non_string_types(
+        self, document_store: PgvectorDocumentStore
+    ):
+        """Non-string metadata values (e.g. ints) are returned in their original type, not stringified."""
+        docs = [
+            Document(content="Doc 1", meta={"priority": 1}),
+            Document(content="Doc 2", meta={"priority": 2}),
+            Document(content="Doc 3", meta={"priority": 1}),
+        ]
+        await document_store.write_documents_async(docs)
+
+        values, total = await document_store.get_metadata_field_unique_values_async("meta.priority")
+
+        assert set(values) == {1, 2}
+        assert total == 2
+
     async def test_delete_table_async_first_call(self, document_store: PgvectorDocumentStore):
         """
         Test that delete_table_async can be executed as the initial operation on the Document Store

{pgvector_haystack-6.4.1 → pgvector_haystack-6.5.0}/tests/test_filters.py

@@ -10,6 +10,7 @@ from haystack.testing.document_store import FilterDocumentsTest
 from psycopg.adapt import Transformer
 from psycopg.sql import SQL, Composed
 from psycopg.sql import Literal as SQLLiteral
+from psycopg.types.json import Jsonb
 
 from haystack_integrations.document_stores.pgvector.filters import (
     FilterError,
@@ -17,6 +18,7 @@ from haystack_integrations.document_stores.pgvector.filters import (
     _parse_comparison_condition,
     _parse_logical_condition,
     _treat_meta_field,
+    _treat_meta_field_as_jsonb,
     _validate_filters,
 )
 
@@ -141,6 +143,86 @@ class TestFilters(FilterDocumentsTest):
             ],
         )
 
+    @staticmethod
+    def _docs_with_array_meta():
+        return [
+            Document(content="doc with tag1 and tag2", meta={"tags": ["tag1", "tag2"]}),
+            Document(content="doc with tag1 and tag3", meta={"tags": ["tag1", "tag3"]}),
+            Document(content="doc with tag4", meta={"tags": ["tag4"]}),
+        ]
+
+    def test_array_contains_operator(self, document_store):
+        docs = self._docs_with_array_meta()
+        document_store.write_documents(docs)
+
+        # array_contains requires ALL listed values to be present in the meta array
+        result = document_store.filter_documents(
+            filters={"field": "meta.tags", "operator": "array_contains", "value": ["tag1", "tag2"]}
+        )
+        assert {doc.content for doc in result} == {"doc with tag1 and tag2"}
+
+    def test_array_contains_operator_single_value(self, document_store):
+        docs = self._docs_with_array_meta()
+        document_store.write_documents(docs)
+
+        result = document_store.filter_documents(
+            filters={"field": "meta.tags", "operator": "array_contains", "value": ["tag1"]}
+        )
+        assert {doc.content for doc in result} == {"doc with tag1 and tag2", "doc with tag1 and tag3"}
+
+    def test_array_overlaps_operator(self, document_store):
+        docs = self._docs_with_array_meta()
+        document_store.write_documents(docs)
+
+        # array_overlaps requires AT LEAST ONE listed value to be present in the meta array
+        result = document_store.filter_documents(
+            filters={"field": "meta.tags", "operator": "array_overlaps", "value": ["tag1", "tag3"]}
+        )
+        assert {doc.content for doc in result} == {"doc with tag1 and tag2", "doc with tag1 and tag3"}
+
+    def test_array_overlaps_operator_no_match(self, document_store):
+        docs = self._docs_with_array_meta()
+        document_store.write_documents(docs)
+
+        result = document_store.filter_documents(
+            filters={"field": "meta.tags", "operator": "array_overlaps", "value": ["nonexistent"]}
+        )
+        assert result == []
+
+    def test_array_operators_combined(self, document_store):
+        docs = self._docs_with_array_meta()
+        document_store.write_documents(docs)
+
+        result = document_store.filter_documents(
+            filters={
+                "operator": "AND",
+                "conditions": [
+                    {"field": "meta.tags", "operator": "array_contains", "value": ["tag1"]},
+                    {"field": "meta.tags", "operator": "array_overlaps", "value": ["tag3"]},
+                ],
+            }
+        )
+        assert {doc.content for doc in result} == {"doc with tag1 and tag3"}
+
+    @pytest.mark.parametrize("operator", ["array_contains", "array_overlaps"])
+    def test_array_operators_ignore_non_array_meta(self, document_store, operator):
+        # meta shapes vary across sources, so the same key can hold an array in one document and a
+        # bare string or an object in another. `?|` matches a top-level key or array element, so
+        # without the jsonb_typeof guard the object and the string below would both match.
+        document_store.write_documents(
+            [
+                Document(content="array meta", meta={"tags": ["tag1", "tag2"]}),
+                Document(content="string meta", meta={"tags": "tag1"}),
+                Document(content="object meta", meta={"tags": {"tag1": "value"}}),
+                Document(content="missing key", meta={"other": "tag1"}),
+            ]
+        )
+
+        result = document_store.filter_documents(
+            filters={"field": "meta.tags", "operator": operator, "value": ["tag1"]}
+        )
+        assert {doc.content for doc in result} == {"array meta"}
+
 
 def test_treat_meta_field():
     cast_integer = SQL("(") + SQL("meta->>") + SQLLiteral("number") + SQL(")::integer")
@@ -165,6 +247,92 @@ def test_treat_meta_field_sql_injection_is_safely_escaped():
     assert result == SQL("meta->>") + SQLLiteral("name' OR 1=1 --")
 
 
+def test_treat_meta_field_as_jsonb():
+    # array operators access the meta value as JSONB (->) without a cast, unlike _treat_meta_field (->>)
+    assert _treat_meta_field_as_jsonb(field="meta.tags") == SQL("meta->") + SQLLiteral("tags")
+
+
+def test_treat_meta_field_as_jsonb_sql_injection_is_safely_escaped():
+    result = _treat_meta_field_as_jsonb(field="meta.tags' OR 1=1 --")
+    assert isinstance(result, Composed)
+    assert result == SQL("meta->") + SQLLiteral("tags' OR 1=1 --")
+
+
+def test_array_contains_condition():
+    condition = {"field": "meta.tags", "operator": "array_contains", "value": ["tag1", "tag2"]}
+    query, values = _parse_comparison_condition(condition)
+
+    assert _render(query) == "meta->'tags' @> %s"
+    # the value is wrapped as a single Jsonb param holding the original list
+    assert len(values) == 1
+    assert isinstance(values[0], Jsonb)
+    assert values[0].obj == ["tag1", "tag2"]
+
+
+def test_array_overlaps_condition():
+    condition = {"field": "meta.tags", "operator": "array_overlaps", "value": ["tag1", "tag2"]}
+    query, values = _parse_comparison_condition(condition)
+
+    # the jsonb_typeof guard keeps `?|` from matching objects (by key) or bare strings
+    assert _render(query) == "(jsonb_typeof(meta->'tags') = 'array' AND meta->'tags' ?| %s)"
+    # the list is wrapped twice (once here, once by _parse_comparison_condition), exactly like 'in',
+    # so psycopg adapts it to a nested Postgres text[]; `?|` deconstructs it to the same elements
+    assert values == [[["tag1", "tag2"]]]
+
+
+def test_array_contains_non_list_value_raises():
+    condition = {"field": "meta.tags", "operator": "array_contains", "value": "tag1"}
+    with pytest.raises(FilterError):
+        _parse_comparison_condition(condition)
+
+
+def test_array_overlaps_non_list_value_raises():
+    condition = {"field": "meta.tags", "operator": "array_overlaps", "value": "tag1"}
+    with pytest.raises(FilterError):
+        _parse_comparison_condition(condition)
+
+
+def test_array_overlaps_non_string_elements_raises():
+    condition = {"field": "meta.tags", "operator": "array_overlaps", "value": [1, 2]}
+    with pytest.raises(FilterError):
+        _parse_comparison_condition(condition)
+
+
+@pytest.mark.parametrize("operator", ["array_contains", "array_overlaps"])
+def test_array_operators_empty_list_raises(operator):
+    # `@> '[]'` matches every array and `?| '{}'` matches nothing, so an empty list would mean
+    # "everything" for one operator and "nothing" for the other. Reject it instead.
+    condition = {"field": "meta.tags", "operator": operator, "value": []}
+    with pytest.raises(FilterError, match="non-empty list"):
+        _parse_comparison_condition(condition)
+
+
+@pytest.mark.parametrize("operator", ["array_contains", "array_overlaps"])
+def test_array_operators_on_non_meta_field_raises(operator):
+    # top-level columns are never JSONB arrays, so this must fail before reaching the database
+    condition = {"field": "content", "operator": operator, "value": ["tag1"]}
+    with pytest.raises(FilterError, match="only supported on meta fields"):
+        _parse_comparison_condition(condition)
+
+
+def test_convert_filters_array_contains():
+    filters = {"field": "meta.tags", "operator": "array_contains", "value": ["tag1", "tag2"]}
+    where_clause, params = _convert_filters_to_where_clause_and_params(filters)
+
+    assert _render(where_clause) == " WHERE meta->'tags' @> %s"
+    assert len(params) == 1
+    assert isinstance(params[0], Jsonb)
+    assert params[0].obj == ["tag1", "tag2"]
+
+
+def test_convert_filters_array_overlaps():
+    filters = {"field": "meta.tags", "operator": "array_overlaps", "value": ["tag1", "tag2"]}
+    where_clause, params = _convert_filters_to_where_clause_and_params(filters)
+
+    assert _render(where_clause) == " WHERE (jsonb_typeof(meta->'tags') = 'array' AND meta->'tags' ?| %s)"
+    assert params == ([["tag1", "tag2"]],)
+
+
 def test_comparison_condition_missing_operator():
     condition = {"field": "meta.type", "value": "article"}
     with pytest.raises(FilterError):