pgvector-haystack 0.2.0__tar.gz → 0.4.0__tar.gz

pgvector_haystack-0.4.0/CHANGELOG.md

@@ -0,0 +1,35 @@
+# Changelog
+
+## [unreleased]
+
+### 🚀 Features
+
+- Defer the database connection to when it's needed (#773)
+
+## [integrations/pgvector-v0.2.0] - 2024-05-08
+
+### 🚀 Features
+
+- `MongoDBAtlasEmbeddingRetriever` (#427)
+- Implement keyword retrieval for pgvector integration (#644)
+
+### 🐛 Bug Fixes
+
+- Fix order of API docs (#447)
+
+This PR will also push the docs to Readme
+
+### 📚 Documentation
+
+- Update category slug (#442)
+- Disable-class-def (#556)
+
+## [integrations/pgvector-v0.1.0] - 2024-02-14
+
+### 🐛 Bug Fixes
+
+- Fix linting (#328)
+
+
+
+<!-- generated by git-cliff -->

{pgvector_haystack-0.2.0 → pgvector_haystack-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: pgvector-haystack
-Version: 0.2.0
+Version: 0.4.0
 Summary: An integration of pgvector (vector search extension for Postgres) with Haystack
 Project-URL: Source, https://github.com/deepset-ai/haystack-core-integrations
 Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/blob/main/integrations/pgvector/README.md

pgvector_haystack-0.4.0/examples/hybrid_retrieval.py

@@ -0,0 +1,69 @@
+# Before running this example, ensure you have PostgreSQL installed with the pgvector extension.
+# For a quick setup using Docker:
+# docker run -d -p 5432:5432 -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres
+# -e POSTGRES_DB=postgres ankane/pgvector
+
+# Install required packages for this example, including pgvector-haystack and other libraries needed
+# for Markdown conversion and embeddings generation. Use the following command:
+# pip install pgvector-haystack markdown-it-py mdit_plain "sentence-transformers>=2.2.0"
+
+# Download some Markdown files to index.
+# git clone https://github.com/anakin87/neural-search-pills
+
+import glob
+
+from haystack import Pipeline
+from haystack.components.converters import MarkdownToDocument
+from haystack.components.embedders import SentenceTransformersDocumentEmbedder, SentenceTransformersTextEmbedder
+from haystack.components.joiners import DocumentJoiner
+from haystack.components.preprocessors import DocumentSplitter
+from haystack.components.writers import DocumentWriter
+from haystack_integrations.components.retrievers.pgvector import PgvectorEmbeddingRetriever, PgvectorKeywordRetriever
+from haystack_integrations.document_stores.pgvector import PgvectorDocumentStore
+
+# Set an environment variable `PG_CONN_STR` with the connection string to your PostgreSQL database.
+# e.g., "postgresql://USER:PASSWORD@HOST:PORT/DB_NAME"
+
+# Initialize PgvectorDocumentStore
+document_store = PgvectorDocumentStore(
+    table_name="haystack_test",
+    embedding_dimension=768,
+    vector_function="cosine_similarity",
+    recreate_table=True,
+    search_strategy="hnsw",
+)
+
+# Create the indexing Pipeline and index some documents
+file_paths = glob.glob("neural-search-pills/pills/*.md")
+
+
+indexing = Pipeline()
+indexing.add_component("converter", MarkdownToDocument())
+indexing.add_component("splitter", DocumentSplitter(split_by="sentence", split_length=2))
+indexing.add_component("document_embedder", SentenceTransformersDocumentEmbedder())
+indexing.add_component("writer", DocumentWriter(document_store))
+indexing.connect("converter", "splitter")
+indexing.connect("splitter", "document_embedder")
+indexing.connect("document_embedder", "writer")
+
+indexing.run({"converter": {"sources": file_paths}})
+
+# Create the querying Pipeline and try a query
+querying = Pipeline()
+querying.add_component("text_embedder", SentenceTransformersTextEmbedder())
+querying.add_component("retriever", PgvectorEmbeddingRetriever(document_store=document_store, top_k=3))
+querying.add_component("keyword_retriever", PgvectorKeywordRetriever(document_store=document_store, top_k=3))
+querying.add_component(
+    "joiner",
+    DocumentJoiner(join_mode="reciprocal_rank_fusion", top_k=3),
+)
+querying.connect("text_embedder", "retriever")
+querying.connect("keyword_retriever", "joiner")
+querying.connect("retriever", "joiner")
+
+query = "cross-encoder"
+results = querying.run({"text_embedder": {"text": query}, "keyword_retriever": {"query": query}})
+
+for doc in results["joiner"]["documents"]:
+    print(doc)
+    print("-" * 10)

{pgvector_haystack-0.2.0 → pgvector_haystack-0.4.0}/pydoc/config.yml

@@ -3,6 +3,7 @@ loaders:
     search_path: [../src]
     modules: [
       "haystack_integrations.components.retrievers.pgvector.embedding_retriever",
+      "haystack_integrations.components.retrievers.pgvector.keyword_retriever",
       "haystack_integrations.document_stores.pgvector.document_store",
     ]
     ignore_when_discovered: ["__init__"]

{pgvector_haystack-0.2.0 → pgvector_haystack-0.4.0}/src/haystack_integrations/document_stores/pgvector/document_store.py

@@ -69,10 +69,6 @@ VECTOR_FUNCTION_TO_POSTGRESQL_OPS = {
 
 HNSW_INDEX_CREATION_VALID_KWARGS = ["m", "ef_construction"]
 
-HNSW_INDEX_NAME = "haystack_hnsw_index"
-
-KEYWORD_INDEX_NAME = "haystack_keyword_index"
-
 
 class PgvectorDocumentStore:
     """
@@ -91,7 +87,9 @@ class PgvectorDocumentStore:
         search_strategy: Literal["exact_nearest_neighbor", "hnsw"] = "exact_nearest_neighbor",
         hnsw_recreate_index_if_exists: bool = False,
         hnsw_index_creation_kwargs: Optional[Dict[str, int]] = None,
+        hnsw_index_name: str = "haystack_hnsw_index",
         hnsw_ef_search: Optional[int] = None,
+        keyword_index_name: str = "haystack_keyword_index",
     ):
         """
         Creates a new PgvectorDocumentStore instance.
@@ -127,9 +125,11 @@ class PgvectorDocumentStore:
         :param hnsw_index_creation_kwargs: Additional keyword arguments to pass to the HNSW index creation.
             Only used if search_strategy is set to `"hnsw"`. You can find the list of valid arguments in the
             [pgvector documentation](https://github.com/pgvector/pgvector?tab=readme-ov-file#hnsw)
+        :param hnsw_index_name: Index name for the HNSW index.
         :param hnsw_ef_search: The `ef_search` parameter to use at query time. Only used if search_strategy is set to
             `"hnsw"`. You can find more information about this parameter in the
             [pgvector documentation](https://github.com/pgvector/pgvector?tab=readme-ov-file#hnsw).
+        :param keyword_index_name: Index name for the Keyword index.
         """
 
         self.connection_string = connection_string
@@ -143,28 +143,57 @@ class PgvectorDocumentStore:
         self.search_strategy = search_strategy
         self.hnsw_recreate_index_if_exists = hnsw_recreate_index_if_exists
         self.hnsw_index_creation_kwargs = hnsw_index_creation_kwargs or {}
+        self.hnsw_index_name = hnsw_index_name
         self.hnsw_ef_search = hnsw_ef_search
+        self.keyword_index_name = keyword_index_name
         self.language = language
+        self._connection = None
+        self._cursor = None
+        self._dict_cursor = None
 
-        connection = connect(self.connection_string.resolve_value())
-        connection.autocommit = True
-        self._connection = connection
+    @property
+    def cursor(self):
+        if self._cursor is None:
+            self._create_connection()
+
+        return self._cursor
+
+    @property
+    def dict_cursor(self):
+        if self._dict_cursor is None:
+            self._create_connection()
 
-        # we create a generic cursor and another one that returns dictionaries
-        self._cursor = connection.cursor()
-        self._dict_cursor = connection.cursor(row_factory=dict_row)
+        return self._dict_cursor
 
+    @property
+    def connection(self):
+        if self._connection is None:
+            self._create_connection()
+
+        return self._connection
+
+    def _create_connection(self):
+        conn_str = self.connection_string.resolve_value() or ""
+        connection = connect(conn_str)
+        connection.autocommit = True
         connection.execute("CREATE EXTENSION IF NOT EXISTS vector")
-        register_vector(connection)
+        register_vector(connection)  # Note: this must be called before creating the cursors.
 
-        if recreate_table:
+        self._connection = connection
+        self._cursor = self._connection.cursor()
+        self._dict_cursor = self._connection.cursor(row_factory=dict_row)
+
+        # Init schema
+        if self.recreate_table:
             self.delete_table()
         self._create_table_if_not_exists()
         self._create_keyword_index_if_not_exists()
 
-        if search_strategy == "hnsw":
+        if self.search_strategy == "hnsw":
             self._handle_hnsw()
 
+        return self._connection
+
     def to_dict(self) -> Dict[str, Any]:
         """
         Serializes the component to a dictionary.
@@ -182,7 +211,9 @@ class PgvectorDocumentStore:
             search_strategy=self.search_strategy,
             hnsw_recreate_index_if_exists=self.hnsw_recreate_index_if_exists,
             hnsw_index_creation_kwargs=self.hnsw_index_creation_kwargs,
+            hnsw_index_name=self.hnsw_index_name,
             hnsw_ef_search=self.hnsw_ef_search,
+            keyword_index_name=self.keyword_index_name,
             language=self.language,
         )
 
@@ -208,11 +239,11 @@ class PgvectorDocumentStore:
         :param sql_query: The SQL query to execute.
         :param params: The parameters to pass to the SQL query.
         :param error_msg: The error message to use if an exception is raised.
-        :param cursor: The cursor to use to execute the SQL query. Defaults to self._cursor.
+        :param cursor: The cursor to use to execute the SQL query. Defaults to self.cursor.
         """
 
         params = params or ()
-        cursor = cursor or self._cursor
+        cursor = cursor or self.cursor
 
         sql_query_str = sql_query.as_string(cursor) if not isinstance(sql_query, str) else sql_query
         logger.debug("SQL query: %s\nParameters: %s", sql_query_str, params)
@@ -220,7 +251,7 @@ class PgvectorDocumentStore:
         try:
             result = cursor.execute(sql_query, params)
         except Error as e:
-            self._connection.rollback()
+            self.connection.rollback()
             detailed_error_msg = f"{error_msg}.\nYou can find the SQL query and the parameters in the debug logs."
             raise DocumentStoreError(detailed_error_msg) from e
 
@@ -254,7 +285,7 @@ class PgvectorDocumentStore:
         index_exists = bool(
             self._execute_sql(
                 "SELECT 1 FROM pg_indexes WHERE tablename = %s AND indexname = %s",
-                (self.table_name, KEYWORD_INDEX_NAME),
+                (self.table_name, self.keyword_index_name),
                 "Could not check if keyword index exists",
             ).fetchone()
         )
@@ -262,7 +293,7 @@ class PgvectorDocumentStore:
         sql_create_index = SQL(
             "CREATE INDEX {index_name} ON {table_name} USING GIN (to_tsvector({language}, content))"
         ).format(
-            index_name=Identifier(KEYWORD_INDEX_NAME),
+            index_name=Identifier(self.keyword_index_name),
             table_name=Identifier(self.table_name),
             language=SQLLiteral(self.language),
         )
@@ -285,7 +316,7 @@ class PgvectorDocumentStore:
         index_exists = bool(
             self._execute_sql(
                 "SELECT 1 FROM pg_indexes WHERE tablename = %s AND indexname = %s",
-                (self.table_name, HNSW_INDEX_NAME),
+                (self.table_name, self.hnsw_index_name),
                 "Could not check if HNSW index exists",
             ).fetchone()
         )
@@ -298,7 +329,7 @@ class PgvectorDocumentStore:
             )
             return
 
-        sql_drop_index = SQL("DROP INDEX IF EXISTS {index_name}").format(index_name=Identifier(HNSW_INDEX_NAME))
+        sql_drop_index = SQL("DROP INDEX IF EXISTS {index_name}").format(index_name=Identifier(self.hnsw_index_name))
         self._execute_sql(sql_drop_index, error_msg="Could not drop HNSW index")
 
         self._create_hnsw_index()
@@ -316,7 +347,7 @@ class PgvectorDocumentStore:
         }
 
         sql_create_index = SQL("CREATE INDEX {index_name} ON {table_name} USING hnsw (embedding {ops}) ").format(
-            index_name=Identifier(HNSW_INDEX_NAME), table_name=Identifier(self.table_name), ops=SQL(pg_ops)
+            index_name=Identifier(self.hnsw_index_name), table_name=Identifier(self.table_name), ops=SQL(pg_ops)
         )
 
         if actual_hnsw_index_creation_kwargs:
@@ -371,7 +402,7 @@ class PgvectorDocumentStore:
             sql_filter,
             params,
             error_msg="Could not filter documents from PgvectorDocumentStore.",
-            cursor=self._dict_cursor,
+            cursor=self.dict_cursor,
         )
 
         records = result.fetchall()
@@ -408,16 +439,16 @@ class PgvectorDocumentStore:
 
         sql_insert += SQL(" RETURNING id")
 
-        sql_query_str = sql_insert.as_string(self._cursor) if not isinstance(sql_insert, str) else sql_insert
+        sql_query_str = sql_insert.as_string(self.cursor) if not isinstance(sql_insert, str) else sql_insert
         logger.debug("SQL query: %s\nParameters: %s", sql_query_str, db_documents)
 
         try:
-            self._cursor.executemany(sql_insert, db_documents, returning=True)
+            self.cursor.executemany(sql_insert, db_documents, returning=True)
         except IntegrityError as ie:
-            self._connection.rollback()
+            self.connection.rollback()
             raise DuplicateDocumentError from ie
         except Error as e:
-            self._connection.rollback()
+            self.connection.rollback()
             error_msg = (
                 "Could not write documents to PgvectorDocumentStore. \n"
                 "You can find the SQL query and the parameters in the debug logs."
@@ -428,9 +459,9 @@ class PgvectorDocumentStore:
         # https://www.psycopg.org/psycopg3/docs/api/cursors.html#psycopg.Cursor.executemany
         written_docs = 0
         while True:
-            if self._cursor.fetchone():
+            if self.cursor.fetchone():
                 written_docs += 1
-            if not self._cursor.nextset():
+            if not self.cursor.nextset():
                 break
 
         return written_docs
@@ -483,8 +514,8 @@ class PgvectorDocumentStore:
 
             # postgresql returns the embedding as a string
             # so we need to convert it to a list of floats
-            if document.get("embedding"):
-                haystack_dict["embedding"] = [float(el) for el in document["embedding"].strip("[]").split(",")]
+            if document.get("embedding") is not None:
+                haystack_dict["embedding"] = document["embedding"].tolist()
 
             haystack_document = Document.from_dict(haystack_dict)
 
@@ -555,7 +586,7 @@ class PgvectorDocumentStore:
             sql_query,
             (query, *where_params),
             error_msg="Could not retrieve documents from PgvectorDocumentStore.",
-            cursor=self._dict_cursor,
+            cursor=self.dict_cursor,
         )
 
         records = result.fetchall()
@@ -633,7 +664,7 @@ class PgvectorDocumentStore:
             sql_query,
             params,
             error_msg="Could not retrieve documents from PgvectorDocumentStore.",
-            cursor=self._dict_cursor,
+            cursor=self.dict_cursor,
         )
 
         records = result.fetchall()

{pgvector_haystack-0.2.0 → pgvector_haystack-0.4.0}/tests/conftest.py

@@ -40,7 +40,6 @@ def patches_for_unit_tests():
     ) as mock_create_kw_index, patch(
         "haystack_integrations.document_stores.pgvector.document_store.PgvectorDocumentStore._handle_hnsw"
     ) as mock_hnsw:
-
         yield mock_connect, mock_register, mock_delete, mock_create, mock_create_kw_index, mock_hnsw
 
 

{pgvector_haystack-0.2.0 → pgvector_haystack-0.4.0}/tests/test_document_store.py

@@ -4,11 +4,13 @@
 
 from unittest.mock import patch
 
+import numpy as np
 import pytest
 from haystack.dataclasses.document import ByteStream, Document
 from haystack.document_stores.errors import DuplicateDocumentError
 from haystack.document_stores.types import DuplicatePolicy
 from haystack.testing.document_store import CountDocumentsTest, DeleteDocumentsTest, WriteDocumentsTest
+from haystack.utils import Secret
 from haystack_integrations.document_stores.pgvector import PgvectorDocumentStore
 from pandas import DataFrame
 
@@ -51,7 +53,9 @@ def test_init(monkeypatch):
         search_strategy="hnsw",
         hnsw_recreate_index_if_exists=True,
         hnsw_index_creation_kwargs={"m": 32, "ef_construction": 128},
+        hnsw_index_name="my_hnsw_index",
         hnsw_ef_search=50,
+        keyword_index_name="my_keyword_index",
     )
 
     assert document_store.table_name == "my_table"
@@ -61,7 +65,9 @@ def test_init(monkeypatch):
     assert document_store.search_strategy == "hnsw"
     assert document_store.hnsw_recreate_index_if_exists
     assert document_store.hnsw_index_creation_kwargs == {"m": 32, "ef_construction": 128}
+    assert document_store.hnsw_index_name == "my_hnsw_index"
     assert document_store.hnsw_ef_search == 50
+    assert document_store.keyword_index_name == "my_keyword_index"
 
 
 @pytest.mark.usefixtures("patches_for_unit_tests")
@@ -76,7 +82,9 @@ def test_to_dict(monkeypatch):
         search_strategy="hnsw",
         hnsw_recreate_index_if_exists=True,
         hnsw_index_creation_kwargs={"m": 32, "ef_construction": 128},
+        hnsw_index_name="my_hnsw_index",
         hnsw_ef_search=50,
+        keyword_index_name="my_keyword_index",
     )
 
     assert document_store.to_dict() == {
@@ -91,7 +99,9 @@ def test_to_dict(monkeypatch):
             "hnsw_recreate_index_if_exists": True,
             "language": "english",
             "hnsw_index_creation_kwargs": {"m": 32, "ef_construction": 128},
+            "hnsw_index_name": "my_hnsw_index",
             "hnsw_ef_search": 50,
+            "keyword_index_name": "my_keyword_index",
         },
     }
 
@@ -170,7 +180,7 @@ def test_from_pg_to_haystack_documents():
             "blob_meta": None,
             "blob_mime_type": None,
             "meta": {"meta_key": "meta_value"},
-            "embedding": "[0.1, 0.2, 0.3]",
+            "embedding": np.array([0.1, 0.2, 0.3]),
         },
         {
             "id": "2",
@@ -180,7 +190,7 @@ def test_from_pg_to_haystack_documents():
             "blob_meta": None,
             "blob_mime_type": None,
             "meta": {"meta_key": "meta_value"},
-            "embedding": "[0.4, 0.5, 0.6]",
+            "embedding": np.array([0.4, 0.5, 0.6]),
         },
         {
             "id": "3",
@@ -190,16 +200,11 @@ def test_from_pg_to_haystack_documents():
             "blob_meta": {"blob_meta_key": "blob_meta_value"},
             "blob_mime_type": "mime_type",
             "meta": {"meta_key": "meta_value"},
-            "embedding": "[0.7, 0.8, 0.9]",
+            "embedding": np.array([0.7, 0.8, 0.9]),
         },
     ]
 
-    with patch(
-        "haystack_integrations.document_stores.pgvector.document_store.PgvectorDocumentStore.__init__"
-    ) as mock_init:
-        mock_init.return_value = None
-        ds = PgvectorDocumentStore(connection_string="test")
-
+    ds = PgvectorDocumentStore(connection_string=Secret.from_token("test"))
     haystack_docs = ds._from_pg_to_haystack_documents(pg_docs)
 
     assert haystack_docs[0].id == "1"

{pgvector_haystack-0.2.0 → pgvector_haystack-0.4.0}/tests/test_retrievers.py

@@ -48,7 +48,9 @@ class TestEmbeddingRetriever:
                         "hnsw_recreate_index_if_exists": False,
                         "language": "english",
                         "hnsw_index_creation_kwargs": {},
+                        "hnsw_index_name": "haystack_hnsw_index",
                         "hnsw_ef_search": None,
+                        "keyword_index_name": "haystack_keyword_index",
                     },
                 },
                 "filters": {"field": "value"},
@@ -75,7 +77,9 @@ class TestEmbeddingRetriever:
                         "search_strategy": "exact_nearest_neighbor",
                         "hnsw_recreate_index_if_exists": False,
                         "hnsw_index_creation_kwargs": {},
+                        "hnsw_index_name": "haystack_hnsw_index",
                         "hnsw_ef_search": None,
+                        "keyword_index_name": "haystack_keyword_index",
                     },
                 },
                 "filters": {"field": "value"},
@@ -96,7 +100,9 @@ class TestEmbeddingRetriever:
         assert document_store.search_strategy == "exact_nearest_neighbor"
         assert not document_store.hnsw_recreate_index_if_exists
         assert document_store.hnsw_index_creation_kwargs == {}
+        assert document_store.hnsw_index_name == "haystack_hnsw_index"
         assert document_store.hnsw_ef_search is None
+        assert document_store.keyword_index_name == "haystack_keyword_index"
 
         assert retriever.filters == {"field": "value"}
         assert retriever.top_k == 5
@@ -149,7 +155,9 @@ class TestKeywordRetriever:
                         "hnsw_recreate_index_if_exists": False,
                         "language": "english",
                         "hnsw_index_creation_kwargs": {},
+                        "hnsw_index_name": "haystack_hnsw_index",
                         "hnsw_ef_search": None,
+                        "keyword_index_name": "haystack_keyword_index",
                     },
                 },
                 "filters": {"field": "value"},
@@ -175,7 +183,9 @@ class TestKeywordRetriever:
                         "search_strategy": "exact_nearest_neighbor",
                         "hnsw_recreate_index_if_exists": False,
                         "hnsw_index_creation_kwargs": {},
+                        "hnsw_index_name": "haystack_hnsw_index",
                         "hnsw_ef_search": None,
+                        "keyword_index_name": "haystack_keyword_index",
                     },
                 },
                 "filters": {"field": "value"},
@@ -195,7 +205,9 @@ class TestKeywordRetriever:
         assert document_store.search_strategy == "exact_nearest_neighbor"
         assert not document_store.hnsw_recreate_index_if_exists
         assert document_store.hnsw_index_creation_kwargs == {}
+        assert document_store.hnsw_index_name == "haystack_hnsw_index"
         assert document_store.hnsw_ef_search is None
+        assert document_store.keyword_index_name == "haystack_keyword_index"
 
         assert retriever.filters == {"field": "value"}
         assert retriever.top_k == 5