pgvector-haystack 1.2.1__tar.gz → 2.0.0__tar.gz

{pgvector_haystack-1.2.1 → pgvector_haystack-2.0.0}/CHANGELOG.md

@@ -1,5 +1,23 @@
 # Changelog
 
+## [integrations/pgvector-v1.3.0] - 2025-02-03
+
+### 🚀 Features
+
+- Pgvector - add like and not like filters (#1341)
+
+### 🧹 Chores
+
+- Inherit from `FilterDocumentsTestWithDataframe` in Document Stores (#1290)
+
+
+## [integrations/pgvector-v1.2.1] - 2025-01-10
+
+### 🐛 Bug Fixes
+
+- PgvectorDocumentStore - use appropriate schema name if dropping index (#1277)
+
+
 ## [integrations/pgvector-v1.2.0] - 2024-11-22
 
 ### 🚀 Features

{pgvector_haystack-1.2.1 → pgvector_haystack-2.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pgvector-haystack
-Version: 1.2.1
+Version: 2.0.0
 Summary: An integration of pgvector (vector search extension for Postgres) with Haystack
 Project-URL: Source, https://github.com/deepset-ai/haystack-core-integrations
 Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/blob/main/integrations/pgvector/README.md

{pgvector_haystack-1.2.1 → pgvector_haystack-2.0.0}/pyproject.toml

@@ -163,6 +163,7 @@ markers = ["integration: integration tests"]
 module = [
   "haystack.*",
   "haystack_integrations.*",
+  "numpy.*",
   "pgvector.*",
   "psycopg.*",
   "pytest.*",

{pgvector_haystack-1.2.1 → pgvector_haystack-2.0.0}/src/haystack_integrations/document_stores/pgvector/document_store.py

@@ -28,7 +28,6 @@ CREATE TABLE IF NOT EXISTS {schema_name}.{table_name} (
 id VARCHAR(128) PRIMARY KEY,
 embedding VECTOR({embedding_dimension}),
 content TEXT,
-dataframe JSONB,
 blob_data BYTEA,
 blob_meta JSONB,
 blob_mime_type VARCHAR(255),
@@ -37,15 +36,14 @@ meta JSONB)
 
 INSERT_STATEMENT = """
 INSERT INTO {schema_name}.{table_name}
-(id, embedding, content, dataframe, blob_data, blob_meta, blob_mime_type, meta)
-VALUES (%(id)s, %(embedding)s, %(content)s, %(dataframe)s, %(blob_data)s, %(blob_meta)s, %(blob_mime_type)s, %(meta)s)
+(id, embedding, content, blob_data, blob_meta, blob_mime_type, meta)
+VALUES (%(id)s, %(embedding)s, %(content)s, %(blob_data)s, %(blob_meta)s, %(blob_mime_type)s, %(meta)s)
 """
 
 UPDATE_STATEMENT = """
 ON CONFLICT (id) DO UPDATE SET
 embedding = EXCLUDED.embedding,
 content = EXCLUDED.content,
-dataframe = EXCLUDED.dataframe,
 blob_data = EXCLUDED.blob_data,
 blob_meta = EXCLUDED.blob_meta,
 blob_mime_type = EXCLUDED.blob_mime_type,
@@ -555,10 +553,18 @@ class PgvectorDocumentStore:
             db_document["blob_data"] = blob.data if blob else None
             db_document["blob_meta"] = Jsonb(blob.meta) if blob and blob.meta else None
             db_document["blob_mime_type"] = blob.mime_type if blob and blob.mime_type else None
-
-            db_document["dataframe"] = Jsonb(db_document["dataframe"]) if db_document["dataframe"] else None
             db_document["meta"] = Jsonb(db_document["meta"])
 
+            if "dataframe" in db_document:
+                dataframe = db_document.pop("dataframe", None)
+                if dataframe:
+                    logger.warning(
+                        "Document %s has the `dataframe` field set. "
+                        "PgvectorDocumentStore no longer supports dataframes and this field will be ignored. "
+                        "The `dataframe` field will soon be removed from Haystack Document.",
+                        db_document["id"],
+                    )
+
             if "sparse_embedding" in db_document:
                 sparse_embedding = db_document.pop("sparse_embedding", None)
                 if sparse_embedding:
@@ -585,12 +591,20 @@ class PgvectorDocumentStore:
             blob_data = haystack_dict.pop("blob_data")
             blob_meta = haystack_dict.pop("blob_meta")
             blob_mime_type = haystack_dict.pop("blob_mime_type")
+            dataframe = haystack_dict.pop("dataframe", None)
 
-            # postgresql returns the embedding as a string
-            # so we need to convert it to a list of floats
+            # convert the embedding to a list of floats
             if document.get("embedding") is not None:
                 haystack_dict["embedding"] = document["embedding"].tolist()
 
+            if dataframe:
+                logger.warning(
+                    "Document %s has the `dataframe` field set. "
+                    "PgvectorDocumentStore no longer supports dataframes and this field will be ignored. "
+                    "The `dataframe` field will soon be removed from Haystack Document.",
+                    haystack_dict["id"],
+                )
+
             haystack_document = Document.from_dict(haystack_dict)
 
             if blob_data:

{pgvector_haystack-1.2.1 → pgvector_haystack-2.0.0}/src/haystack_integrations/document_stores/pgvector/filters.py

@@ -6,7 +6,6 @@ from itertools import chain
 from typing import Any, Dict, List, Literal, Tuple
 
 from haystack.errors import FilterError
-from pandas import DataFrame
 from psycopg.sql import SQL
 from psycopg.types.json import Jsonb
 
@@ -93,10 +92,6 @@ def _parse_comparison_condition(condition: Dict[str, Any]) -> Tuple[str, List[An
         raise FilterError(msg)
 
     value: Any = condition["value"]
-    if isinstance(value, DataFrame):
-        # DataFrames are stored as JSONB and we query them as such
-        value = Jsonb(value.to_json())
-        field = f"({field})::jsonb"
 
     if field.startswith("meta."):
         field = _treat_meta_field(field, value)
@@ -232,6 +227,20 @@ def _in(field: str, value: Any) -> Tuple[str, List]:
     return f"{field} = ANY(%s)", [value]
 
 
+def _like(field: str, value: Any) -> Tuple[str, Any]:
+    if not isinstance(value, str):
+        msg = f"{field}'s value must be a str when using 'LIKE' "
+        raise FilterError(msg)
+    return f"{field} LIKE %s", value
+
+
+def _not_like(field: str, value: Any) -> Tuple[str, Any]:
+    if not isinstance(value, str):
+        msg = f"{field}'s value must be a str when using 'LIKE' "
+        raise FilterError(msg)
+    return f"{field} NOT LIKE %s", value
+
+
 COMPARISON_OPERATORS = {
     "==": _equal,
     "!=": _not_equal,
@@ -241,4 +250,6 @@ COMPARISON_OPERATORS = {
     "<=": _less_than_equal,
     "in": _in,
     "not in": _not_in,
+    "like": _like,
+    "not like": _not_like,
 }

{pgvector_haystack-1.2.1 → pgvector_haystack-2.0.0}/tests/test_document_store.py

@@ -12,7 +12,6 @@ from haystack.document_stores.errors import DuplicateDocumentError
 from haystack.document_stores.types import DuplicatePolicy
 from haystack.testing.document_store import CountDocumentsTest, DeleteDocumentsTest, WriteDocumentsTest
 from haystack.utils import Secret
-from pandas import DataFrame
 
 from haystack_integrations.document_stores.pgvector import PgvectorDocumentStore
 
@@ -33,15 +32,6 @@ class TestDocumentStore(CountDocumentsTest, WriteDocumentsTest, DeleteDocumentsT
         retrieved_docs = document_store.filter_documents()
         assert retrieved_docs == docs
 
-    def test_write_dataframe(self, document_store: PgvectorDocumentStore):
-        dataframe = DataFrame({"col1": [1, 2], "col2": [3, 4]})
-        docs = [Document(id="1", dataframe=dataframe)]
-
-        document_store.write_documents(docs)
-
-        retrieved_docs = document_store.filter_documents()
-        assert retrieved_docs == docs
-
     def test_connection_check_and_recreation(self, document_store: PgvectorDocumentStore):
         original_connection = document_store.connection
 
@@ -145,7 +135,7 @@ def test_from_haystack_to_pg_documents():
         ),
         Document(
             id="2",
-            dataframe=DataFrame({"col1": [1, 2], "col2": [3, 4]}),
+            content="This is another text",
             meta={"meta_key": "meta_value"},
             embedding=[0.4, 0.5, 0.6],
             score=0.6,
@@ -169,30 +159,30 @@ def test_from_haystack_to_pg_documents():
 
     assert pg_docs[0]["id"] == "1"
     assert pg_docs[0]["content"] == "This is a text"
-    assert pg_docs[0]["dataframe"] is None
     assert pg_docs[0]["blob_data"] is None
     assert pg_docs[0]["blob_meta"] is None
     assert pg_docs[0]["blob_mime_type"] is None
+    assert "dataframe" not in pg_docs[0]
     assert pg_docs[0]["meta"].obj == {"meta_key": "meta_value"}
     assert pg_docs[0]["embedding"] == [0.1, 0.2, 0.3]
     assert "score" not in pg_docs[0]
 
     assert pg_docs[1]["id"] == "2"
-    assert pg_docs[1]["content"] is None
-    assert pg_docs[1]["dataframe"].obj == DataFrame({"col1": [1, 2], "col2": [3, 4]}).to_json()
+    assert pg_docs[1]["content"] == "This is another text"
     assert pg_docs[1]["blob_data"] is None
     assert pg_docs[1]["blob_meta"] is None
     assert pg_docs[1]["blob_mime_type"] is None
+    assert "dataframe" not in pg_docs[1]
     assert pg_docs[1]["meta"].obj == {"meta_key": "meta_value"}
     assert pg_docs[1]["embedding"] == [0.4, 0.5, 0.6]
     assert "score" not in pg_docs[1]
 
     assert pg_docs[2]["id"] == "3"
     assert pg_docs[2]["content"] is None
-    assert pg_docs[2]["dataframe"] is None
     assert pg_docs[2]["blob_data"] == b"test"
     assert pg_docs[2]["blob_meta"].obj == {"blob_meta_key": "blob_meta_value"}
     assert pg_docs[2]["blob_mime_type"] == "mime_type"
+    assert "dataframe" not in pg_docs[2]
     assert pg_docs[2]["meta"].obj == {"meta_key": "meta_value"}
     assert pg_docs[2]["embedding"] == [0.7, 0.8, 0.9]
     assert "score" not in pg_docs[2]
@@ -203,7 +193,6 @@ def test_from_pg_to_haystack_documents():
         {
             "id": "1",
             "content": "This is a text",
-            "dataframe": None,
             "blob_data": None,
             "blob_meta": None,
             "blob_mime_type": None,
@@ -212,8 +201,7 @@ def test_from_pg_to_haystack_documents():
         },
         {
             "id": "2",
-            "content": None,
-            "dataframe": DataFrame({"col1": [1, 2], "col2": [3, 4]}).to_json(),
+            "content": "This is another text",
             "blob_data": None,
             "blob_meta": None,
             "blob_mime_type": None,
@@ -223,7 +211,6 @@ def test_from_pg_to_haystack_documents():
         {
             "id": "3",
             "content": None,
-            "dataframe": None,
             "blob_data": b"test",
             "blob_meta": {"blob_meta_key": "blob_meta_value"},
             "blob_mime_type": "mime_type",
@@ -237,15 +224,13 @@ def test_from_pg_to_haystack_documents():
 
     assert haystack_docs[0].id == "1"
     assert haystack_docs[0].content == "This is a text"
-    assert haystack_docs[0].dataframe is None
     assert haystack_docs[0].blob is None
     assert haystack_docs[0].meta == {"meta_key": "meta_value"}
     assert haystack_docs[0].embedding == [0.1, 0.2, 0.3]
     assert haystack_docs[0].score is None
 
     assert haystack_docs[1].id == "2"
-    assert haystack_docs[1].content is None
-    assert haystack_docs[1].dataframe.equals(DataFrame({"col1": [1, 2], "col2": [3, 4]}))
+    assert haystack_docs[1].content == "This is another text"
     assert haystack_docs[1].blob is None
     assert haystack_docs[1].meta == {"meta_key": "meta_value"}
     assert haystack_docs[1].embedding == [0.4, 0.5, 0.6]
@@ -253,7 +238,6 @@ def test_from_pg_to_haystack_documents():
 
     assert haystack_docs[2].id == "3"
     assert haystack_docs[2].content is None
-    assert haystack_docs[2].dataframe is None
     assert haystack_docs[2].blob.data == b"test"
     assert haystack_docs[2].blob.meta == {"blob_meta_key": "blob_meta_value"}
     assert haystack_docs[2].blob.mime_type == "mime_type"

{pgvector_haystack-1.2.1 → pgvector_haystack-2.0.0}/tests/test_filters.py

@@ -3,9 +3,7 @@ from typing import List
 import pytest
 from haystack.dataclasses.document import Document
 from haystack.testing.document_store import FilterDocumentsTest
-from pandas import DataFrame
 from psycopg.sql import SQL
-from psycopg.types.json import Jsonb
 
 from haystack_integrations.document_stores.pgvector.filters import (
     FilterError,
@@ -40,6 +38,62 @@ class TestFilters(FilterDocumentsTest):
     @pytest.mark.skip(reason="NOT operator is not supported in PgvectorDocumentStore")
     def test_not_operator(self, document_store, filterable_docs): ...
 
+    def test_like_operator(self, document_store, filterable_docs):
+        document_store.write_documents(filterable_docs)
+        filters = {"field": "content", "operator": "like", "value": "%Foo%"}
+        result = document_store.filter_documents(filters=filters)
+
+        self.assert_documents_are_equal(result, [d for d in filterable_docs if "Foo" in d.content])
+
+    def test_like_operator_startswith(self, document_store, filterable_docs):
+        document_store.write_documents(filterable_docs)
+        filters = {"field": "content", "operator": "like", "value": "Foo%"}
+        result = document_store.filter_documents(filters=filters)
+
+        self.assert_documents_are_equal(result, [d for d in filterable_docs if d.content.startswith("Foo")])
+
+    def test_like_operator_nb_chars(self, document_store, filterable_docs):
+        document_store.write_documents(filterable_docs)
+        filters = {"field": "content", "operator": "like", "value": "A Foobar Document__"}
+        result = document_store.filter_documents(filters=filters)
+
+        self.assert_documents_are_equal(
+            result,
+            [
+                d
+                for d in filterable_docs
+                if (d.content.startswith("A Foobar Document") and len(d.content) == (len("A Foobar Document") + 2))
+            ],
+        )
+
+    def test_not_like_operator(self, document_store, filterable_docs):
+        document_store.write_documents(filterable_docs)
+        filters = {"field": "content", "operator": "not like", "value": "%Foo%"}
+        result = document_store.filter_documents(filters=filters)
+
+        self.assert_documents_are_equal(result, [d for d in filterable_docs if "Foo" not in d.content])
+
+    def test_not_like_operator_startswith(self, document_store, filterable_docs):
+        document_store.write_documents(filterable_docs)
+        filters = {"field": "content", "operator": "not like", "value": "Foo%"}
+        result = document_store.filter_documents(filters=filters)
+
+        self.assert_documents_are_equal(result, [d for d in filterable_docs if not d.content.startswith("Foo")])
+
+    def test_not_like_operator_nb_chars(self, document_store, filterable_docs):
+        document_store.write_documents(filterable_docs)
+        filters = {"field": "content", "operator": "not like", "value": "A Foobar Document__"}
+        result = document_store.filter_documents(filters=filters)
+
+        self.assert_documents_are_equal(
+            result,
+            [
+                d
+                for d in filterable_docs
+                if not (d.content.startswith("A Foobar Document") and len(d.content) == (len("A Foobar Document") + 2))
+            ],
+        )
+
     def test_complex_filter(self, document_store, filterable_docs):
         document_store.write_documents(filterable_docs)
         filters = {
@@ -91,17 +145,6 @@ def test_treat_meta_field():
     assert _treat_meta_field(field="meta.name", value=None) == "meta->>'name'"
 
 
-def test_comparison_condition_dataframe_jsonb_conversion():
-    dataframe = DataFrame({"a": [1, 2, 3], "b": ["a", "b", "c"]})
-    condition = {"field": "meta.df", "operator": "==", "value": dataframe}
-    field, values = _parse_comparison_condition(condition)
-    assert field == "(meta.df)::jsonb = %s"
-
-    # we check each slot of the Jsonb object because it does not implement __eq__
-    assert values[0].obj == Jsonb(dataframe.to_json()).obj
-    assert values[0].dumps == Jsonb(dataframe.to_json()).dumps
-
-
 def test_comparison_condition_missing_operator():
     condition = {"field": "meta.type", "value": "article"}
     with pytest.raises(FilterError):