pgvector-haystack 1.3.0__tar.gz → 3.0.0__tar.gz

{pgvector_haystack-1.3.0 → pgvector_haystack-3.0.0}/CHANGELOG.md

@@ -1,5 +1,22 @@
 # Changelog
 
+## [integrations/pgvector-v2.0.0] - 2025-02-13
+
+### 🧹 Chores
+
+- Pgvector - remove support for dataframe (#1370)
+
+## [integrations/pgvector-v1.3.0] - 2025-02-03
+
+### 🚀 Features
+
+- Pgvector - add like and not like filters (#1341)
+
+### 🧹 Chores
+
+- Inherit from `FilterDocumentsTestWithDataframe` in Document Stores (#1290)
+
+
 ## [integrations/pgvector-v1.2.1] - 2025-01-10
 
 ### 🐛 Bug Fixes

{pgvector_haystack-1.3.0 → pgvector_haystack-3.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pgvector-haystack
-Version: 1.3.0
+Version: 3.0.0
 Summary: An integration of pgvector (vector search extension for Postgres) with Haystack
 Project-URL: Source, https://github.com/deepset-ai/haystack-core-integrations
 Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/blob/main/integrations/pgvector/README.md
@@ -11,15 +11,14 @@ License-File: LICENSE.txt
 Classifier: Development Status :: 4 - Beta
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python
-Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy
-Requires-Python: >=3.8
-Requires-Dist: haystack-ai
+Requires-Python: >=3.9
+Requires-Dist: haystack-ai>=2.11.0
 Requires-Dist: pgvector
 Requires-Dist: psycopg[binary]
 Description-Content-Type: text/markdown

{pgvector_haystack-1.3.0 → pgvector_haystack-3.0.0}/pyproject.toml

@@ -7,7 +7,7 @@ name = "pgvector-haystack"
 dynamic = ["version"]
 description = "An integration of pgvector (vector search extension for Postgres) with Haystack"
 readme = "README.md"
-requires-python = ">=3.8"
+requires-python = ">=3.9"
 license = "Apache-2.0"
 keywords = []
 authors = [{ name = "deepset GmbH", email = "info@deepset.ai" }]
@@ -15,7 +15,6 @@ classifiers = [
   "License :: OSI Approved :: Apache Software License",
   "Development Status :: 4 - Beta",
   "Programming Language :: Python",
-  "Programming Language :: Python :: 3.8",
   "Programming Language :: Python :: 3.9",
   "Programming Language :: Python :: 3.10",
   "Programming Language :: Python :: 3.11",
@@ -23,7 +22,7 @@ classifiers = [
   "Programming Language :: Python :: Implementation :: CPython",
   "Programming Language :: Python :: Implementation :: PyPy",
 ]
-dependencies = ["haystack-ai", "pgvector", "psycopg[binary]"]
+dependencies = ["haystack-ai>=2.11.0", "pgvector", "psycopg[binary]"]
 
 [project.urls]
 Source = "https://github.com/deepset-ai/haystack-core-integrations"
@@ -60,7 +59,7 @@ cov-retry = ["test-cov-retry", "cov-report"]
 docs = ["pydoc-markdown pydoc/config.yml"]
 
 [[tool.hatch.envs.all.matrix]]
-python = ["3.8", "3.9", "3.10", "3.11", "3.12"]
+python = [ "3.9", "3.10", "3.11", "3.12"]
 
 [tool.hatch.envs.lint]
 installer = "uv"
@@ -163,6 +162,7 @@ markers = ["integration: integration tests"]
 module = [
   "haystack.*",
   "haystack_integrations.*",
+  "numpy.*",
   "pgvector.*",
   "psycopg.*",
   "pytest.*",

{pgvector_haystack-1.3.0 → pgvector_haystack-3.0.0}/src/haystack_integrations/document_stores/pgvector/document_store.py

@@ -1,10 +1,9 @@
 # SPDX-FileCopyrightText: 2023-present deepset GmbH <info@deepset.ai>
 #
 # SPDX-License-Identifier: Apache-2.0
-import logging
 from typing import Any, Dict, List, Literal, Optional
 
-from haystack import default_from_dict, default_to_dict
+from haystack import default_from_dict, default_to_dict, logging
 from haystack.dataclasses.document import ByteStream, Document
 from haystack.document_stores.errors import DocumentStoreError, DuplicateDocumentError
 from haystack.document_stores.types import DuplicatePolicy
@@ -28,7 +27,6 @@ CREATE TABLE IF NOT EXISTS {schema_name}.{table_name} (
 id VARCHAR(128) PRIMARY KEY,
 embedding VECTOR({embedding_dimension}),
 content TEXT,
-dataframe JSONB,
 blob_data BYTEA,
 blob_meta JSONB,
 blob_mime_type VARCHAR(255),
@@ -37,15 +35,14 @@ meta JSONB)
 
 INSERT_STATEMENT = """
 INSERT INTO {schema_name}.{table_name}
-(id, embedding, content, dataframe, blob_data, blob_meta, blob_mime_type, meta)
-VALUES (%(id)s, %(embedding)s, %(content)s, %(dataframe)s, %(blob_data)s, %(blob_meta)s, %(blob_mime_type)s, %(meta)s)
+(id, embedding, content, blob_data, blob_meta, blob_mime_type, meta)
+VALUES (%(id)s, %(embedding)s, %(content)s, %(blob_data)s, %(blob_meta)s, %(blob_mime_type)s, %(meta)s)
 """
 
 UPDATE_STATEMENT = """
 ON CONFLICT (id) DO UPDATE SET
 embedding = EXCLUDED.embedding,
 content = EXCLUDED.content,
-dataframe = EXCLUDED.dataframe,
 blob_data = EXCLUDED.blob_data,
 blob_meta = EXCLUDED.blob_meta,
 blob_mime_type = EXCLUDED.blob_mime_type,
@@ -195,7 +192,7 @@ class PgvectorDocumentStore:
             try:
                 self._connection.close()
             except Error as e:
-                logger.debug("Failed to close connection: %s", str(e))
+                logger.debug("Failed to close connection: {e}", e=str(e))
 
         conn_str = self.connection_string.resolve_value() or ""
         connection = connect(conn_str)
@@ -297,7 +294,7 @@ class PgvectorDocumentStore:
         cursor = cursor or self.cursor
 
         sql_query_str = sql_query.as_string(cursor) if not isinstance(sql_query, str) else sql_query
-        logger.debug("SQL query: %s\nParameters: %s", sql_query_str, params)
+        logger.debug("SQL query: {query}\nParameters: {parameters}", query=sql_query_str, parameters=params)
 
         try:
             result = cursor.execute(sql_query, params)
@@ -514,7 +511,7 @@ class PgvectorDocumentStore:
         sql_insert += SQL(" RETURNING id")
 
         sql_query_str = sql_insert.as_string(self.cursor) if not isinstance(sql_insert, str) else sql_insert
-        logger.debug("SQL query: %s\nParameters: %s", sql_query_str, db_documents)
+        logger.debug("SQL query: {query}\nParameters: {parameters}", query=sql_query_str, parameters=db_documents)
 
         try:
             self.cursor.executemany(sql_insert, db_documents, returning=True)
@@ -555,18 +552,16 @@ class PgvectorDocumentStore:
             db_document["blob_data"] = blob.data if blob else None
             db_document["blob_meta"] = Jsonb(blob.meta) if blob and blob.meta else None
             db_document["blob_mime_type"] = blob.mime_type if blob and blob.mime_type else None
-
-            db_document["dataframe"] = Jsonb(db_document["dataframe"]) if db_document["dataframe"] else None
             db_document["meta"] = Jsonb(db_document["meta"])
 
             if "sparse_embedding" in db_document:
                 sparse_embedding = db_document.pop("sparse_embedding", None)
                 if sparse_embedding:
                     logger.warning(
-                        "Document %s has the `sparse_embedding` field set,"
+                        "Document {doc_id} has the `sparse_embedding` field set,"
                         "but storing sparse embeddings in Pgvector is not currently supported."
                         "The `sparse_embedding` field will be ignored.",
-                        db_document["id"],
+                        doc_id=db_document["id"],
                     )
 
             db_documents.append(db_document)
@@ -586,8 +581,7 @@ class PgvectorDocumentStore:
             blob_meta = haystack_dict.pop("blob_meta")
             blob_mime_type = haystack_dict.pop("blob_mime_type")
 
-            # postgresql returns the embedding as a string
-            # so we need to convert it to a list of floats
+            # convert the embedding to a list of floats
             if document.get("embedding") is not None:
                 haystack_dict["embedding"] = document["embedding"].tolist()
 

{pgvector_haystack-1.3.0 → pgvector_haystack-3.0.0}/src/haystack_integrations/document_stores/pgvector/filters.py

@@ -6,7 +6,6 @@ from itertools import chain
 from typing import Any, Dict, List, Literal, Tuple
 
 from haystack.errors import FilterError
-from pandas import DataFrame
 from psycopg.sql import SQL
 from psycopg.types.json import Jsonb
 
@@ -93,10 +92,6 @@ def _parse_comparison_condition(condition: Dict[str, Any]) -> Tuple[str, List[An
         raise FilterError(msg)
 
     value: Any = condition["value"]
-    if isinstance(value, DataFrame):
-        # DataFrames are stored as JSONB and we query them as such
-        value = Jsonb(value.to_json())
-        field = f"({field})::jsonb"
 
     if field.startswith("meta."):
         field = _treat_meta_field(field, value)

{pgvector_haystack-1.3.0 → pgvector_haystack-3.0.0}/tests/test_document_store.py

@@ -12,7 +12,6 @@ from haystack.document_stores.errors import DuplicateDocumentError
 from haystack.document_stores.types import DuplicatePolicy
 from haystack.testing.document_store import CountDocumentsTest, DeleteDocumentsTest, WriteDocumentsTest
 from haystack.utils import Secret
-from pandas import DataFrame
 
 from haystack_integrations.document_stores.pgvector import PgvectorDocumentStore
 
@@ -33,15 +32,6 @@ class TestDocumentStore(CountDocumentsTest, WriteDocumentsTest, DeleteDocumentsT
         retrieved_docs = document_store.filter_documents()
         assert retrieved_docs == docs
 
-    def test_write_dataframe(self, document_store: PgvectorDocumentStore):
-        dataframe = DataFrame({"col1": [1, 2], "col2": [3, 4]})
-        docs = [Document(id="1", dataframe=dataframe)]
-
-        document_store.write_documents(docs)
-
-        retrieved_docs = document_store.filter_documents()
-        assert retrieved_docs == docs
-
     def test_connection_check_and_recreation(self, document_store: PgvectorDocumentStore):
         original_connection = document_store.connection
 
@@ -145,7 +135,7 @@ def test_from_haystack_to_pg_documents():
         ),
         Document(
             id="2",
-            dataframe=DataFrame({"col1": [1, 2], "col2": [3, 4]}),
+            content="This is another text",
             meta={"meta_key": "meta_value"},
             embedding=[0.4, 0.5, 0.6],
             score=0.6,
@@ -169,30 +159,30 @@ def test_from_haystack_to_pg_documents():
 
     assert pg_docs[0]["id"] == "1"
     assert pg_docs[0]["content"] == "This is a text"
-    assert pg_docs[0]["dataframe"] is None
     assert pg_docs[0]["blob_data"] is None
     assert pg_docs[0]["blob_meta"] is None
     assert pg_docs[0]["blob_mime_type"] is None
+    assert "dataframe" not in pg_docs[0]
     assert pg_docs[0]["meta"].obj == {"meta_key": "meta_value"}
     assert pg_docs[0]["embedding"] == [0.1, 0.2, 0.3]
     assert "score" not in pg_docs[0]
 
     assert pg_docs[1]["id"] == "2"
-    assert pg_docs[1]["content"] is None
-    assert pg_docs[1]["dataframe"].obj == DataFrame({"col1": [1, 2], "col2": [3, 4]}).to_json()
+    assert pg_docs[1]["content"] == "This is another text"
     assert pg_docs[1]["blob_data"] is None
     assert pg_docs[1]["blob_meta"] is None
     assert pg_docs[1]["blob_mime_type"] is None
+    assert "dataframe" not in pg_docs[1]
     assert pg_docs[1]["meta"].obj == {"meta_key": "meta_value"}
     assert pg_docs[1]["embedding"] == [0.4, 0.5, 0.6]
     assert "score" not in pg_docs[1]
 
     assert pg_docs[2]["id"] == "3"
     assert pg_docs[2]["content"] is None
-    assert pg_docs[2]["dataframe"] is None
     assert pg_docs[2]["blob_data"] == b"test"
     assert pg_docs[2]["blob_meta"].obj == {"blob_meta_key": "blob_meta_value"}
     assert pg_docs[2]["blob_mime_type"] == "mime_type"
+    assert "dataframe" not in pg_docs[2]
     assert pg_docs[2]["meta"].obj == {"meta_key": "meta_value"}
     assert pg_docs[2]["embedding"] == [0.7, 0.8, 0.9]
     assert "score" not in pg_docs[2]
@@ -203,7 +193,6 @@ def test_from_pg_to_haystack_documents():
         {
             "id": "1",
             "content": "This is a text",
-            "dataframe": None,
             "blob_data": None,
             "blob_meta": None,
             "blob_mime_type": None,
@@ -212,8 +201,7 @@ def test_from_pg_to_haystack_documents():
         },
         {
             "id": "2",
-            "content": None,
-            "dataframe": DataFrame({"col1": [1, 2], "col2": [3, 4]}).to_json(),
+            "content": "This is another text",
             "blob_data": None,
             "blob_meta": None,
             "blob_mime_type": None,
@@ -223,7 +211,6 @@ def test_from_pg_to_haystack_documents():
         {
             "id": "3",
             "content": None,
-            "dataframe": None,
             "blob_data": b"test",
             "blob_meta": {"blob_meta_key": "blob_meta_value"},
             "blob_mime_type": "mime_type",
@@ -237,15 +224,13 @@ def test_from_pg_to_haystack_documents():
 
     assert haystack_docs[0].id == "1"
     assert haystack_docs[0].content == "This is a text"
-    assert haystack_docs[0].dataframe is None
     assert haystack_docs[0].blob is None
     assert haystack_docs[0].meta == {"meta_key": "meta_value"}
     assert haystack_docs[0].embedding == [0.1, 0.2, 0.3]
     assert haystack_docs[0].score is None
 
     assert haystack_docs[1].id == "2"
-    assert haystack_docs[1].content is None
-    assert haystack_docs[1].dataframe.equals(DataFrame({"col1": [1, 2], "col2": [3, 4]}))
+    assert haystack_docs[1].content == "This is another text"
     assert haystack_docs[1].blob is None
     assert haystack_docs[1].meta == {"meta_key": "meta_value"}
     assert haystack_docs[1].embedding == [0.4, 0.5, 0.6]
@@ -253,7 +238,6 @@ def test_from_pg_to_haystack_documents():
 
     assert haystack_docs[2].id == "3"
     assert haystack_docs[2].content is None
-    assert haystack_docs[2].dataframe is None
     assert haystack_docs[2].blob.data == b"test"
     assert haystack_docs[2].blob.meta == {"blob_meta_key": "blob_meta_value"}
     assert haystack_docs[2].blob.mime_type == "mime_type"

{pgvector_haystack-1.3.0 → pgvector_haystack-3.0.0}/tests/test_filters.py

@@ -2,10 +2,8 @@ from typing import List
 
 import pytest
 from haystack.dataclasses.document import Document
-from haystack.testing.document_store import FilterDocumentsTest, FilterDocumentsTestWithDataframe
-from pandas import DataFrame
+from haystack.testing.document_store import FilterDocumentsTest
 from psycopg.sql import SQL
-from psycopg.types.json import Jsonb
 
 from haystack_integrations.document_stores.pgvector.filters import (
     FilterError,
@@ -17,7 +15,7 @@ from haystack_integrations.document_stores.pgvector.filters import (
 
 
 @pytest.mark.integration
-class TestFilters(FilterDocumentsTest, FilterDocumentsTestWithDataframe):
+class TestFilters(FilterDocumentsTest):
     def assert_documents_are_equal(self, received: List[Document], expected: List[Document]):
         """
         This overrides the default assert_documents_are_equal from FilterDocumentsTest.
@@ -147,17 +145,6 @@ def test_treat_meta_field():
     assert _treat_meta_field(field="meta.name", value=None) == "meta->>'name'"
 
 
-def test_comparison_condition_dataframe_jsonb_conversion():
-    dataframe = DataFrame({"a": [1, 2, 3], "b": ["a", "b", "c"]})
-    condition = {"field": "meta.df", "operator": "==", "value": dataframe}
-    field, values = _parse_comparison_condition(condition)
-    assert field == "(meta.df)::jsonb = %s"
-
-    # we check each slot of the Jsonb object because it does not implement __eq__
-    assert values[0].obj == Jsonb(dataframe.to_json()).obj
-    assert values[0].dumps == Jsonb(dataframe.to_json()).dumps
-
-
 def test_comparison_condition_missing_operator():
     condition = {"field": "meta.type", "value": "article"}
     with pytest.raises(FilterError):