pgvector-haystack 0.0.1__tar.gz → 0.1.0__tar.gz

{pgvector_haystack-0.0.1 → pgvector_haystack-0.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pgvector-haystack
-Version: 0.0.1
+Version: 0.1.0
 Summary: An integration of pgvector (vector search extension for Postgres) with Haystack
 Project-URL: Source, https://github.com/deepset-ai/haystack-core-integrations
 Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/blob/main/integrations/pgvector/README.md
@@ -45,12 +45,23 @@ pip install pgvector-haystack
 
 ## Testing
 
-TODO
+Ensure that you have a PostgreSQL running with the `pgvector` extension. For a quick setup using Docker, run:
+```
+docker run -d -p 5432:5432 -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres -e POSTGRES_DB=postgres ankane/pgvector
+```
+
+then run the tests:
 
 ```console
 hatch run test
 ```
 
+To run the coverage report:
+
+```console
+hatch run cov
+```
+
 ## License
 
 `pgvector-haystack` is distributed under the terms of the [Apache-2.0](https://spdx.org/licenses/Apache-2.0.html) license.

{pgvector_haystack-0.0.1 → pgvector_haystack-0.1.0}/README.md

@@ -20,12 +20,23 @@ pip install pgvector-haystack
 
 ## Testing
 
-TODO
+Ensure that you have a PostgreSQL running with the `pgvector` extension. For a quick setup using Docker, run:
+```
+docker run -d -p 5432:5432 -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres -e POSTGRES_DB=postgres ankane/pgvector
+```
+
+then run the tests:
 
 ```console
 hatch run test
 ```
 
+To run the coverage report:
+
+```console
+hatch run cov
+```
+
 ## License
 
 `pgvector-haystack` is distributed under the terms of the [Apache-2.0](https://spdx.org/licenses/Apache-2.0.html) license.

pgvector_haystack-0.1.0/examples/example.py

@@ -0,0 +1,60 @@
+# Before running this example, ensure you have PostgreSQL installed with the pgvector extension.
+# For a quick setup using Docker:
+# docker run -d -p 5432:5432 -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres
+# -e POSTGRES_DB=postgres ankane/pgvector
+
+# Install required packages for this example, including pgvector-haystack and other libraries needed
+# for Markdown conversion and embeddings generation. Use the following command:
+# pip install pgvector-haystack markdown-it-py mdit_plain "sentence-transformers>=2.2.0"
+
+# Download some Markdown files to index.
+# git clone https://github.com/anakin87/neural-search-pills
+
+import glob
+import os
+
+from haystack import Pipeline
+from haystack.components.converters import MarkdownToDocument
+from haystack.components.embedders import SentenceTransformersDocumentEmbedder, SentenceTransformersTextEmbedder
+from haystack.components.preprocessors import DocumentSplitter
+from haystack.components.writers import DocumentWriter
+from haystack_integrations.components.retrievers.pgvector import PgvectorEmbeddingRetriever
+from haystack_integrations.document_stores.pgvector import PgvectorDocumentStore
+
+os.environ["PG_CONN_STR"] = "postgresql://postgres:postgres@localhost:5432/postgres"
+
+# Initialize PgvectorDocumentStore
+document_store = PgvectorDocumentStore(
+    table_name="haystack_test",
+    embedding_dimension=768,
+    vector_function="cosine_similarity",
+    recreate_table=True,
+    search_strategy="hnsw",
+)
+
+# Create the indexing Pipeline and index some documents
+file_paths = glob.glob("neural-search-pills/pills/*.md")
+
+
+indexing = Pipeline()
+indexing.add_component("converter", MarkdownToDocument())
+indexing.add_component("splitter", DocumentSplitter(split_by="sentence", split_length=2))
+indexing.add_component("embedder", SentenceTransformersDocumentEmbedder())
+indexing.add_component("writer", DocumentWriter(document_store))
+indexing.connect("converter", "splitter")
+indexing.connect("splitter", "embedder")
+indexing.connect("embedder", "writer")
+
+indexing.run({"converter": {"sources": file_paths}})
+
+# Create the querying Pipeline and try a query
+querying = Pipeline()
+querying.add_component("embedder", SentenceTransformersTextEmbedder())
+querying.add_component("retriever", PgvectorEmbeddingRetriever(document_store=document_store, top_k=3))
+querying.connect("embedder", "retriever")
+
+results = querying.run({"embedder": {"text": "What is a cross-encoder?"}})
+
+for doc in results["retriever"]["documents"]:
+    print(doc)
+    print("-" * 10)

pgvector_haystack-0.1.0/pydoc/config.yml

@@ -0,0 +1,30 @@
+loaders:
+  - type: haystack_pydoc_tools.loaders.CustomPythonLoader
+    search_path: [../src]
+    modules: [
+      "haystack_integrations.components.retrievers.pgvector.embedding_retriever",
+      "haystack_integrations.document_stores.pgvector.document_store",
+      "haystack_integrations.document_stores.pgvector.filters",
+    ]
+    ignore_when_discovered: ["__init__"]
+processors:
+  - type: filter
+    expression:
+    documented_only: true
+    do_not_filter_modules: false
+    skip_empty_modules: true
+  - type: smart
+  - type: crossref
+renderer:
+  type: haystack_pydoc_tools.renderers.ReadmePreviewRenderer
+  excerpt: Pgvector integration for Haystack
+  category_slug: haystack-integrations
+  title: Pgvector
+  slug: integrations-pgvector
+  order: 140
+  markdown:
+    descriptive_class_title: false
+    descriptive_module_title: true
+    add_method_class_prefix: true
+    add_member_class_prefix: false
+    filename: _readme_pgvector.md

{pgvector_haystack-0.0.1 → pgvector_haystack-0.1.0}/pyproject.toml

@@ -51,6 +51,7 @@ dependencies = [
   "coverage[toml]>=6.5",
   "pytest",
   "ipython",
+  "haystack-pydoc-tools",  
 ]
 [tool.hatch.envs.default.scripts]
 test = "pytest {args:tests}"
@@ -63,6 +64,9 @@ cov = [
   "test-cov",
   "cov-report",
 ]
+docs = [
+  "pydoc-markdown pydoc/config.yml"
+]
 
 [[tool.hatch.envs.all.matrix]]
 python = ["3.8", "3.9", "3.10", "3.11", "3.12"]
@@ -134,6 +138,8 @@ ignore = [
   "S105", "S106", "S107",
   # Ignore complexity
   "C901", "PLR0911", "PLR0912", "PLR0913", "PLR0915",
+  # ignore function-call-in-default-argument
+  "B008",
 ]
 unfixable = [
   # Don't touch unused imports
@@ -149,24 +155,25 @@ ban-relative-imports = "parents"
 [tool.ruff.per-file-ignores]
 # Tests can use magic values, assertions, and relative imports
 "tests/**/*" = ["PLR2004", "S101", "TID252"]
+# examples can contain "print" commands
+"examples/**/*" = ["T201"]
+
 
 [tool.coverage.run]
-source_pkgs = ["src", "tests"]
+source = ["haystack_integrations"]
 branch = true
 parallel = true
 
-
-[tool.coverage.paths]
-weaviate_haystack = ["src/haystack_integrations", "*/pgvector-haystack/src"]
-tests = ["tests", "*/pgvector-haystack/tests"]
-
 [tool.coverage.report]
+omit = ["*/tests/*", "*/__init__.py"]
+show_missing=true
 exclude_lines = [
   "no cov",
   "if __name__ == .__main__.:",
   "if TYPE_CHECKING:",
 ]
 
+
 [[tool.mypy.overrides]]
 module = [
   "haystack.*",

{pgvector_haystack-0.0.1 → pgvector_haystack-0.1.0}/src/haystack_integrations/components/retrievers/pgvector/embedding_retriever.py

@@ -68,9 +68,8 @@ class PgvectorEmbeddingRetriever:
 
     @classmethod
     def from_dict(cls, data: Dict[str, Any]) -> "PgvectorEmbeddingRetriever":
-        data["init_parameters"]["document_store"] = default_from_dict(
-            PgvectorDocumentStore, data["init_parameters"]["document_store"]
-        )
+        doc_store_params = data["init_parameters"]["document_store"]
+        data["init_parameters"]["document_store"] = PgvectorDocumentStore.from_dict(doc_store_params)
         return default_from_dict(cls, data)
 
     @component.output_types(documents=List[Document])

{pgvector_haystack-0.0.1 → pgvector_haystack-0.1.0}/src/haystack_integrations/document_stores/pgvector/document_store.py

@@ -4,10 +4,11 @@
 import logging
 from typing import Any, Dict, List, Literal, Optional
 
-from haystack import default_to_dict
+from haystack import default_from_dict, default_to_dict
 from haystack.dataclasses.document import ByteStream, Document
 from haystack.document_stores.errors import DocumentStoreError, DuplicateDocumentError
 from haystack.document_stores.types import DuplicatePolicy
+from haystack.utils.auth import Secret, deserialize_secrets_inplace
 from haystack.utils.filters import convert
 from psycopg import Error, IntegrityError, connect
 from psycopg.abc import Query
@@ -69,7 +70,7 @@ class PgvectorDocumentStore:
     def __init__(
         self,
         *,
-        connection_string: str,
+        connection_string: Secret = Secret.from_env_var("PG_CONN_STR"),
         table_name: str = "haystack_documents",
         embedding_dimension: int = 768,
         vector_function: Literal["cosine_similarity", "inner_product", "l2_distance"] = "cosine_similarity",
@@ -84,8 +85,8 @@ class PgvectorDocumentStore:
         It is meant to be connected to a PostgreSQL database with the pgvector extension installed.
         A specific table to store Haystack documents will be created if it doesn't exist yet.
 
-        :param connection_string: The connection string to use to connect to the PostgreSQL database.
-            e.g. "postgresql://USER:PASSWORD@HOST:PORT/DB_NAME"
+        :param connection_string: The connection string to use to connect to the PostgreSQL database, defined as an
+            environment variable, e.g.: PG_CONN_STR="postgresql://USER:PASSWORD@HOST:PORT/DB_NAME"
         :param table_name: The name of the table to use to store Haystack documents. Defaults to "haystack_documents".
         :param embedding_dimension: The dimension of the embedding. Defaults to 768.
         :param vector_function: The similarity function to use when searching for similar embeddings.
@@ -130,7 +131,7 @@ class PgvectorDocumentStore:
         self.hnsw_index_creation_kwargs = hnsw_index_creation_kwargs or {}
         self.hnsw_ef_search = hnsw_ef_search
 
-        connection = connect(connection_string)
+        connection = connect(self.connection_string.resolve_value())
         connection.autocommit = True
         self._connection = connection
 
@@ -151,7 +152,7 @@ class PgvectorDocumentStore:
     def to_dict(self) -> Dict[str, Any]:
         return default_to_dict(
             self,
-            connection_string=self.connection_string,
+            connection_string=self.connection_string.to_dict(),
             table_name=self.table_name,
             embedding_dimension=self.embedding_dimension,
             vector_function=self.vector_function,
@@ -162,6 +163,11 @@ class PgvectorDocumentStore:
             hnsw_ef_search=self.hnsw_ef_search,
         )
 
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "PgvectorDocumentStore":
+        deserialize_secrets_inplace(data["init_parameters"], ["connection_string"])
+        return default_from_dict(cls, data)
+
     def _execute_sql(
         self, sql_query: Query, params: Optional[tuple] = None, error_msg: str = "", cursor: Optional[Cursor] = None
     ):
@@ -221,7 +227,7 @@ class PgvectorDocumentStore:
             )
             self._execute_sql(sql_set_hnsw_ef_search, error_msg="Could not set hnsw.ef_search")
 
-        index_esists = bool(
+        index_exists = bool(
             self._execute_sql(
                 "SELECT 1 FROM pg_indexes WHERE tablename = %s AND indexname = %s",
                 (self.table_name, HNSW_INDEX_NAME),
@@ -229,7 +235,7 @@ class PgvectorDocumentStore:
             ).fetchone()
         )
 
-        if index_esists and not self.hnsw_recreate_index_if_exists:
+        if index_exists and not self.hnsw_recreate_index_if_exists:
             logger.warning(
                 "HNSW index already exists and won't be recreated. "
                 "If you want to recreate it, pass 'hnsw_recreate_index_if_exists=True' to the "
@@ -373,7 +379,8 @@ class PgvectorDocumentStore:
 
         return written_docs
 
-    def _from_haystack_to_pg_documents(self, documents: List[Document]) -> List[Dict[str, Any]]:
+    @staticmethod
+    def _from_haystack_to_pg_documents(documents: List[Document]) -> List[Dict[str, Any]]:
         """
         Internal method to convert a list of Haystack Documents to a list of dictionaries that can be used to insert
         documents into the PgvectorDocumentStore.
@@ -395,7 +402,8 @@ class PgvectorDocumentStore:
 
         return db_documents
 
-    def _from_pg_to_haystack_documents(self, documents: List[Dict[str, Any]]) -> List[Document]:
+    @staticmethod
+    def _from_pg_to_haystack_documents(documents: List[Dict[str, Any]]) -> List[Document]:
         """
         Internal method to convert a list of dictionaries from pgvector to a list of Haystack Documents.
         """

{pgvector_haystack-0.0.1 → pgvector_haystack-0.1.0}/tests/conftest.py

@@ -1,10 +1,12 @@
+import os
+
 import pytest
 from haystack_integrations.document_stores.pgvector import PgvectorDocumentStore
 
 
 @pytest.fixture
 def document_store(request):
-    connection_string = "postgresql://postgres:postgres@localhost:5432/postgres"
+    os.environ["PG_CONN_STR"] = "postgresql://postgres:postgres@localhost:5432/postgres"
     table_name = f"haystack_{request.node.name}"
     embedding_dimension = 768
     vector_function = "cosine_similarity"
@@ -12,13 +14,13 @@ def document_store(request):
     search_strategy = "exact_nearest_neighbor"
 
     store = PgvectorDocumentStore(
-        connection_string=connection_string,
         table_name=table_name,
         embedding_dimension=embedding_dimension,
         vector_function=vector_function,
         recreate_table=recreate_table,
         search_strategy=search_strategy,
     )
+
     yield store
 
     store.delete_table()

{pgvector_haystack-0.0.1 → pgvector_haystack-0.1.0}/tests/test_document_store.py

@@ -41,7 +41,6 @@ class TestDocumentStore(CountDocumentsTest, WriteDocumentsTest, DeleteDocumentsT
 
     def test_init(self):
         document_store = PgvectorDocumentStore(
-            connection_string="postgresql://postgres:postgres@localhost:5432/postgres",
             table_name="my_table",
             embedding_dimension=512,
             vector_function="l2_distance",
@@ -52,7 +51,6 @@ class TestDocumentStore(CountDocumentsTest, WriteDocumentsTest, DeleteDocumentsT
             hnsw_ef_search=50,
         )
 
-        assert document_store.connection_string == "postgresql://postgres:postgres@localhost:5432/postgres"
         assert document_store.table_name == "my_table"
         assert document_store.embedding_dimension == 512
         assert document_store.vector_function == "l2_distance"
@@ -64,7 +62,6 @@ class TestDocumentStore(CountDocumentsTest, WriteDocumentsTest, DeleteDocumentsT
 
     def test_to_dict(self):
         document_store = PgvectorDocumentStore(
-            connection_string="postgresql://postgres:postgres@localhost:5432/postgres",
             table_name="my_table",
             embedding_dimension=512,
             vector_function="l2_distance",
@@ -78,7 +75,7 @@ class TestDocumentStore(CountDocumentsTest, WriteDocumentsTest, DeleteDocumentsT
         assert document_store.to_dict() == {
             "type": "haystack_integrations.document_stores.pgvector.document_store.PgvectorDocumentStore",
             "init_parameters": {
-                "connection_string": "postgresql://postgres:postgres@localhost:5432/postgres",
+                "connection_string": {"env_vars": ["PG_CONN_STR"], "strict": True, "type": "env_var"},
                 "table_name": "my_table",
                 "embedding_dimension": 512,
                 "vector_function": "l2_distance",

{pgvector_haystack-0.0.1 → pgvector_haystack-0.1.0}/tests/test_retriever.py

@@ -4,6 +4,7 @@
 from unittest.mock import Mock
 
 from haystack.dataclasses import Document
+from haystack.utils.auth import EnvVarSecret
 from haystack_integrations.components.retrievers.pgvector import PgvectorEmbeddingRetriever
 from haystack_integrations.document_stores.pgvector import PgvectorDocumentStore
 
@@ -37,7 +38,7 @@ class TestRetriever:
                 "document_store": {
                     "type": "haystack_integrations.document_stores.pgvector.document_store.PgvectorDocumentStore",
                     "init_parameters": {
-                        "connection_string": "postgresql://postgres:postgres@localhost:5432/postgres",
+                        "connection_string": {"env_vars": ["PG_CONN_STR"], "strict": True, "type": "env_var"},
                         "table_name": "haystack_test_to_dict",
                         "embedding_dimension": 768,
                         "vector_function": "cosine_similarity",
@@ -62,7 +63,7 @@ class TestRetriever:
                 "document_store": {
                     "type": "haystack_integrations.document_stores.pgvector.document_store.PgvectorDocumentStore",
                     "init_parameters": {
-                        "connection_string": "postgresql://postgres:postgres@localhost:5432/postgres",
+                        "connection_string": {"env_vars": ["PG_CONN_STR"], "strict": True, "type": "env_var"},
                         "table_name": "haystack_test_to_dict",
                         "embedding_dimension": 768,
                         "vector_function": "cosine_similarity",
@@ -83,7 +84,7 @@ class TestRetriever:
         document_store = retriever.document_store
 
         assert isinstance(document_store, PgvectorDocumentStore)
-        assert document_store.connection_string == "postgresql://postgres:postgres@localhost:5432/postgres"
+        assert isinstance(document_store.connection_string, EnvVarSecret)
         assert document_store.table_name == "haystack_test_to_dict"
         assert document_store.embedding_dimension == 768
         assert document_store.vector_function == "cosine_similarity"