pgvector-haystack 3.1.0__tar.gz → 3.3.0__tar.gz

{pgvector_haystack-3.1.0 → pgvector_haystack-3.3.0}/CHANGELOG.md

@@ -1,13 +1,33 @@
 # Changelog
 
+## [integrations/pgvector-v3.2.0] - 2025-04-11
+
+### 🚀 Features
+
+- Add halfvec support for storing high-dimensional embeddings in half-precision (#1607)
+
+
+## [integrations/pgvector-v3.1.0] - 2025-03-20
+
+### 🚀 Features
+
+- Pgvector - async support (+ refactoring) (#1547)
+
+
+### ⚙️ CI
+
+- Review testing workflows (#1541)
+
 ## [integrations/pgvector-v3.0.1] - 2025-03-12
 
+
 ### 🌀 Miscellaneous
 
 - Fix: pgvector - improve `_create_table_if_not_exists` to be used without admin rights (#1490)
 
 ## [integrations/pgvector-v3.0.0] - 2025-03-11
 
+
 ### 🧹 Chores
 
 - Remove Python 3.8 support (#1421)
@@ -20,7 +40,6 @@
 
 - Pgvector - remove support for dataframe (#1370)
 
-### 🌀 Miscellaneous
 
 ## [integrations/pgvector-v1.3.0] - 2025-02-03
 

{pgvector_haystack-3.1.0 → pgvector_haystack-3.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pgvector-haystack
-Version: 3.1.0
+Version: 3.3.0
 Summary: An integration of pgvector (vector search extension for Postgres) with Haystack
 Project-URL: Source, https://github.com/deepset-ai/haystack-core-integrations
 Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/blob/main/integrations/pgvector/README.md

{pgvector_haystack-3.1.0 → pgvector_haystack-3.3.0}/src/haystack_integrations/document_stores/pgvector/converters.py

@@ -51,8 +51,12 @@ def _from_pg_to_haystack_documents(documents: List[Dict[str, Any]]) -> List[Docu
         blob_mime_type = haystack_dict.pop("blob_mime_type")
 
         # convert the embedding to a list of floats
+        # for strange reasons, halfvec and vector have different methods to convert the embedding to a list
         if document.get("embedding") is not None:
-            haystack_dict["embedding"] = document["embedding"].tolist()
+            if hasattr(document["embedding"], "tolist"):  # vector
+                haystack_dict["embedding"] = document["embedding"].tolist()
+            else:  # halfvec
+                haystack_dict["embedding"] = document["embedding"].to_list()
 
         haystack_document = Document.from_dict(haystack_dict)
 

{pgvector_haystack-3.1.0 → pgvector_haystack-3.3.0}/src/haystack_integrations/document_stores/pgvector/document_store.py

@@ -26,7 +26,7 @@ logger = logging.getLogger(__name__)
 CREATE_TABLE_STATEMENT = """
 CREATE TABLE {schema_name}.{table_name} (
 id VARCHAR(128) PRIMARY KEY,
-embedding VECTOR({embedding_dimension}),
+embedding {embedding_col_type}({embedding_dimension}),
 content TEXT,
 blob_data BYTEA,
 blob_meta JSONB,
@@ -64,6 +64,12 @@ VECTOR_FUNCTION_TO_POSTGRESQL_OPS = {
     "l2_distance": "vector_l2_ops",
 }
 
+HALF_VECTOR_FUNCTION_TO_POSTGRESQL_OPS = {
+    "cosine_similarity": "halfvec_cosine_ops",
+    "inner_product": "halfvec_ip_ops",
+    "l2_distance": "halfvec_l2_ops",
+}
+
 HNSW_INDEX_CREATION_VALID_KWARGS = ["m", "ef_construction"]
 
 
@@ -81,6 +87,7 @@ class PgvectorDocumentStore:
         table_name: str = "haystack_documents",
         language: str = "english",
         embedding_dimension: int = 768,
+        vector_type: Literal["vector", "halfvec"] = "vector",
         vector_function: Literal["cosine_similarity", "inner_product", "l2_distance"] = "cosine_similarity",
         recreate_table: bool = False,
         search_strategy: Literal["exact_nearest_neighbor", "hnsw"] = "exact_nearest_neighbor",
@@ -112,6 +119,11 @@ class PgvectorDocumentStore:
             `SELECT cfgname FROM pg_ts_config;`.
             More information can be found in this [StackOverflow answer](https://stackoverflow.com/a/39752553).
         :param embedding_dimension: The dimension of the embedding.
+        :param vector_type: The type of vector used for embedding storage.
+            "vector" is the default.
+            "halfvec" stores embeddings in half-precision, which is particularly useful for high-dimensional embeddings
+            (dimension greater than 2,000 and up to 4,000). Requires pgvector versions 0.7.0 or later. For more
+            information, see the [pgvector documentation](https://github.com/pgvector/pgvector?tab=readme-ov-file).
         :param vector_function: The similarity function to use when searching for similar embeddings.
             `"cosine_similarity"` and `"inner_product"` are similarity functions and
             higher scores indicate greater similarity between the documents.
@@ -145,6 +157,10 @@ class PgvectorDocumentStore:
         self.table_name = table_name
         self.schema_name = schema_name
         self.embedding_dimension = embedding_dimension
+        if vector_type not in ["vector", "halfvec"]:
+            msg = "vector_type must be one of ['vector', 'halfvec']"
+            raise ValueError(msg)
+        self.vector_type = vector_type
         if vector_function not in VALID_VECTOR_FUNCTIONS:
             msg = f"vector_function must be one of {VALID_VECTOR_FUNCTIONS}, but got {vector_function}"
             raise ValueError(msg)
@@ -180,6 +196,7 @@ class PgvectorDocumentStore:
             schema_name=self.schema_name,
             table_name=self.table_name,
             embedding_dimension=self.embedding_dimension,
+            vector_type=self.vector_type,
             vector_function=self.vector_function,
             recreate_table=self.recreate_table,
             search_strategy=self.search_strategy,
@@ -259,7 +276,9 @@ class PgvectorDocumentStore:
             result = cursor.execute(sql_query, params)
         except Error as e:
             self._connection.rollback()
-            detailed_error_msg = f"{error_msg}.\nYou can find the SQL query and the parameters in the debug logs."
+            detailed_error_msg = (
+                f"{error_msg}. Error: {e!r}. \nYou can find the SQL query and the parameters in the debug logs."
+            )
             raise DocumentStoreError(detailed_error_msg) from e
 
         return result
@@ -297,7 +316,9 @@ class PgvectorDocumentStore:
             result = await cursor.execute(sql_query, params)
         except Error as e:
             await self._async_connection.rollback()
-            detailed_error_msg = f"{error_msg}.\nYou can find the SQL query and the parameters in the debug logs."
+            detailed_error_msg = (
+                f"{error_msg}. Error: {e!r}. \nYou can find the SQL query and the parameters in the debug logs."
+            )
             raise DocumentStoreError(detailed_error_msg) from e
 
         return result
@@ -376,6 +397,7 @@ class PgvectorDocumentStore:
             schema_name=Identifier(self.schema_name),
             table_name=Identifier(self.table_name),
             embedding_dimension=SQLLiteral(self.embedding_dimension),
+            embedding_col_type=SQL(self.vector_type),
         )
 
         sql_keyword_index_exists = SQL(
@@ -516,7 +538,16 @@ class PgvectorDocumentStore:
             index_name=Identifier(self.hnsw_index_name),
         )
 
-        pg_ops = VECTOR_FUNCTION_TO_POSTGRESQL_OPS[self.vector_function]
+        if self.vector_type == "halfvec":
+            if self.vector_function not in HALF_VECTOR_FUNCTION_TO_POSTGRESQL_OPS:
+                msg = f"Unsupported vector_function '{self.vector_function}' for halfvec type."
+                raise ValueError(msg)
+            pg_ops = HALF_VECTOR_FUNCTION_TO_POSTGRESQL_OPS[self.vector_function]
+        else:
+            if self.vector_function not in VECTOR_FUNCTION_TO_POSTGRESQL_OPS:
+                msg = f"Unsupported vector_function '{self.vector_function}' for vector type."
+                raise ValueError(msg)
+            pg_ops = VECTOR_FUNCTION_TO_POSTGRESQL_OPS[self.vector_function]
 
         sql_create_hnsw_index = SQL(
             "CREATE INDEX {index_name} ON {schema_name}.{table_name} USING hnsw (embedding {ops})"
@@ -768,7 +799,7 @@ class PgvectorDocumentStore:
         except Error as e:
             self._connection.rollback()
             error_msg = (
-                "Could not write documents to PgvectorDocumentStore. \n"
+                f"Could not write documents to PgvectorDocumentStore. Error: {e!r}. \n"
                 "You can find the SQL query and the parameters in the debug logs."
             )
             raise DocumentStoreError(error_msg) from e
@@ -825,7 +856,7 @@ class PgvectorDocumentStore:
         except Error as e:
             await self._async_connection.rollback()
             error_msg = (
-                "Could not write documents to PgvectorDocumentStore. \n"
+                f"Could not write documents to PgvectorDocumentStore. Error: {e!r}. \n"
                 "You can find the SQL query and the parameters in the debug logs."
             )
             raise DocumentStoreError(error_msg) from e

{pgvector_haystack-3.1.0 → pgvector_haystack-3.3.0}/tests/conftest.py

@@ -54,6 +54,30 @@ def document_store_w_hnsw_index(request, monkeypatch):
     store.delete_table()
 
 
+@pytest.fixture
+def document_store_w_halfvec_hnsw_index(request, monkeypatch):
+    monkeypatch.setenv("PG_CONN_STR", "postgresql://postgres:postgres@localhost:5432/postgres")
+    table_name = f"haystack_halfvec_hnsw_{request.node.name}"
+    embedding_dimension = 2500
+    vector_function = "cosine_similarity"
+    recreate_table = True
+
+    store = PgvectorDocumentStore(
+        table_name=table_name,
+        embedding_dimension=embedding_dimension,
+        vector_function=vector_function,
+        recreate_table=recreate_table,
+        search_strategy="hnsw",
+        vector_type="halfvec",
+    )
+    yield store
+
+    # Ensure connection just for table deletion.
+    # During test execution, the tested methods are expected to call _ensure_db_setup() themselves.
+    store._ensure_db_setup()
+    store.delete_table()
+
+
 @pytest.fixture
 def patches_for_unit_tests():
     with patch("haystack_integrations.document_stores.pgvector.document_store.register_vector") as mock_register, patch(

{pgvector_haystack-3.1.0 → pgvector_haystack-3.3.0}/tests/test_document_store.py

@@ -95,6 +95,7 @@ def test_to_dict(monkeypatch):
         create_extension=False,
         table_name="my_table",
         embedding_dimension=512,
+        vector_type="halfvec",
         vector_function="l2_distance",
         recreate_table=True,
         search_strategy="hnsw",
@@ -113,6 +114,7 @@ def test_to_dict(monkeypatch):
             "table_name": "my_table",
             "schema_name": "public",
             "embedding_dimension": 512,
+            "vector_type": "halfvec",
             "vector_function": "l2_distance",
             "recreate_table": True,
             "search_strategy": "hnsw",
@@ -126,6 +128,26 @@ def test_to_dict(monkeypatch):
     }
 
 
+@pytest.mark.integration
+def test_halfvec_hnsw_write_documents(document_store_w_halfvec_hnsw_index: PgvectorDocumentStore):
+    documents = [
+        Document(id="1", content="Hello, world!", embedding=[0.1] * 2500),
+        Document(id="2", content="Hello, mum!", embedding=[0.3] * 2500),
+        Document(id="3", content="Hello, dad!", embedding=[0.2] * 2500),
+    ]
+    document_store_w_halfvec_hnsw_index.write_documents(documents)
+
+    retrieved_docs = document_store_w_halfvec_hnsw_index.filter_documents()
+    retrieved_docs.sort(key=lambda x: x.id)
+
+    for original_doc, retrieved_doc in zip(documents, retrieved_docs):
+        assert original_doc.id == retrieved_doc.id
+        assert original_doc.content == retrieved_doc.content
+        assert len(original_doc.embedding) == len(retrieved_doc.embedding)
+        # these embeddings are in half precision, so we increase the tolerance
+        assert original_doc.embedding == pytest.approx(retrieved_doc.embedding, abs=5e-5)
+
+
 @pytest.mark.integration
 def test_hnsw_index_recreation():
     def get_index_oid(document_store, schema_name, index_name):

{pgvector_haystack-3.1.0 → pgvector_haystack-3.3.0}/tests/test_retrievers.py

@@ -54,6 +54,7 @@ class TestEmbeddingRetriever:
                         "schema_name": "public",
                         "table_name": "haystack",
                         "embedding_dimension": 768,
+                        "vector_type": "vector",
                         "vector_function": "cosine_similarity",
                         "recreate_table": True,
                         "search_strategy": "exact_nearest_neighbor",
@@ -198,6 +199,7 @@ class TestKeywordRetriever:
                         "schema_name": "public",
                         "table_name": "haystack",
                         "embedding_dimension": 768,
+                        "vector_type": "vector",
                         "vector_function": "cosine_similarity",
                         "recreate_table": True,
                         "search_strategy": "exact_nearest_neighbor",