pgvector-haystack 0.4.0__tar.gz → 0.5.1__tar.gz

{pgvector_haystack-0.4.0 → pgvector_haystack-0.5.1}/.gitignore

@@ -135,3 +135,12 @@ dmypy.json
 # Docs generation artifacts
 _readme_*.md
 .idea
+
+# macOS
+.DS_Store
+
+# http cache (requests-cache)
+**/http_cache.sqlite
+
+# ruff
+.ruff_cache

{pgvector_haystack-0.4.0 → pgvector_haystack-0.5.1}/CHANGELOG.md

@@ -1,10 +1,11 @@
 # Changelog
 
-## [unreleased]
+## [integrations/pgvector-v0.4.0] - 2024-06-20
 
 ### 🚀 Features
 
 - Defer the database connection to when it's needed (#773)
+- Add customizable index names for pgvector (#818)
 
 ## [integrations/pgvector-v0.2.0] - 2024-05-08
 

{pgvector_haystack-0.4.0 → pgvector_haystack-0.5.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: pgvector-haystack
-Version: 0.4.0
+Version: 0.5.1
 Summary: An integration of pgvector (vector search extension for Postgres) with Haystack
 Project-URL: Source, https://github.com/deepset-ai/haystack-core-integrations
 Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/blob/main/integrations/pgvector/README.md

{pgvector_haystack-0.4.0 → pgvector_haystack-0.5.1}/pyproject.toml

@@ -10,9 +10,7 @@ readme = "README.md"
 requires-python = ">=3.8"
 license = "Apache-2.0"
 keywords = []
-authors = [
-  { name = "deepset GmbH", email = "info@deepset.ai" },
-]
+authors = [{ name = "deepset GmbH", email = "info@deepset.ai" }]
 classifiers = [
   "License :: OSI Approved :: Apache Software License",
   "Development Status :: 4 - Beta",
@@ -25,11 +23,7 @@ classifiers = [
   "Programming Language :: Python :: Implementation :: CPython",
   "Programming Language :: Python :: Implementation :: PyPy",
 ]
-dependencies = [
-  "haystack-ai",
-  "pgvector",
-  "psycopg[binary]"
-]
+dependencies = ["haystack-ai", "pgvector", "psycopg[binary]"]
 
 [project.urls]
 Source = "https://github.com/deepset-ai/haystack-core-integrations"
@@ -51,49 +45,28 @@ git_describe_command = 'git describe --tags --match="integrations/pgvector-v[0-9
 dependencies = [
   "coverage[toml]>=6.5",
   "pytest",
+  "pytest-rerunfailures",
   "ipython",
   "haystack-pydoc-tools",
 ]
 [tool.hatch.envs.default.scripts]
-test = "pytest {args:tests}"
-test-cov = "coverage run -m pytest {args:tests}"
-cov-report = [
-  "- coverage combine",
-  "coverage report",
-]
-cov = [
-  "test-cov",
-  "cov-report",
-]
-docs = [
-  "pydoc-markdown pydoc/config.yml"
-]
+test = "pytest --reruns 3 --reruns-delay 30 -x {args:tests}"
+test-cov = "coverage run -m pytest --reruns 3 --reruns-delay 30 -x {args:tests}"
+cov-report = ["- coverage combine", "coverage report"]
+cov = ["test-cov", "cov-report"]
+docs = ["pydoc-markdown pydoc/config.yml"]
 
 [[tool.hatch.envs.all.matrix]]
 python = ["3.8", "3.9", "3.10", "3.11", "3.12"]
 
 [tool.hatch.envs.lint]
 detached = true
-dependencies = [
-  "black>=23.1.0",
-  "mypy>=1.0.0",
-  "ruff>=0.0.243",
-]
+dependencies = ["black>=23.1.0", "mypy>=1.0.0", "ruff>=0.0.243"]
 [tool.hatch.envs.lint.scripts]
 typing = "mypy --install-types --non-interactive --explicit-package-bases {args:src/ tests}"
-style = [
-  "ruff {args:.}",
-  "black --check --diff {args:.}",
-]
-fmt = [
-  "black {args:.}",
-  "ruff --fix {args:.}",
-  "style",
-]
-all = [
-  "style",
-  "typing",
-]
+style = ["ruff check {args:.}", "black --check --diff {args:.}"]
+fmt = ["black {args:.}", "ruff --fix {args:.}", "style"]
+all = ["style", "typing"]
 
 [tool.black]
 target-version = ["py38"]
@@ -136,9 +109,15 @@ ignore = [
   # Allow boolean positional values in function calls, like `dict.get(... True)`
   "FBT003",
   # Ignore checks for possible passwords
-  "S105", "S106", "S107",
+  "S105",
+  "S106",
+  "S107",
   # Ignore complexity
-  "C901", "PLR0911", "PLR0912", "PLR0913", "PLR0915",
+  "C901",
+  "PLR0911",
+  "PLR0912",
+  "PLR0913",
+  "PLR0915",
   # ignore function-call-in-default-argument
   "B008",
 ]
@@ -167,17 +146,11 @@ parallel = false
 
 [tool.coverage.report]
 omit = ["*/tests/*", "*/__init__.py"]
-show_missing=true
-exclude_lines = [
-  "no cov",
-  "if __name__ == .__main__.:",
-  "if TYPE_CHECKING:",
-]
+show_missing = true
+exclude_lines = ["no cov", "if __name__ == .__main__.:", "if TYPE_CHECKING:"]
 
 [tool.pytest.ini_options]
-markers = [
-  "integration: integration tests"
-]
+markers = ["integration: integration tests"]
 
 
 [[tool.mypy.overrides]]
@@ -186,6 +159,6 @@ module = [
   "haystack_integrations.*",
   "pgvector.*",
   "psycopg.*",
-  "pytest.*"
+  "pytest.*",
 ]
 ignore_missing_imports = true

{pgvector_haystack-0.4.0 → pgvector_haystack-0.5.1}/src/haystack_integrations/components/retrievers/pgvector/embedding_retriever.py

@@ -1,10 +1,12 @@
 # SPDX-FileCopyrightText: 2023-present deepset GmbH <info@deepset.ai>
 #
 # SPDX-License-Identifier: Apache-2.0
-from typing import Any, Dict, List, Literal, Optional
+from typing import Any, Dict, List, Literal, Optional, Union
 
 from haystack import component, default_from_dict, default_to_dict
 from haystack.dataclasses import Document
+from haystack.document_stores.types import FilterPolicy
+from haystack.document_stores.types.filter_policy import apply_filter_policy
 from haystack_integrations.document_stores.pgvector import PgvectorDocumentStore
 from haystack_integrations.document_stores.pgvector.document_store import VALID_VECTOR_FUNCTIONS
 
@@ -62,6 +64,7 @@ class PgvectorEmbeddingRetriever:
         filters: Optional[Dict[str, Any]] = None,
         top_k: int = 10,
         vector_function: Optional[Literal["cosine_similarity", "inner_product", "l2_distance"]] = None,
+        filter_policy: Union[str, FilterPolicy] = FilterPolicy.REPLACE,
     ):
         """
         :param document_store: An instance of `PgvectorDocumentStore`.
@@ -75,7 +78,7 @@ class PgvectorEmbeddingRetriever:
             and the most similar documents are the ones with the smallest score.
             **Important**: if the document store is using the `"hnsw"` search strategy, the vector function
             should match the one utilized during index creation to take advantage of the index.
-
+        :param filter_policy: Policy to determine how filters are applied.
         :raises ValueError: If `document_store` is not an instance of `PgvectorDocumentStore` or if `vector_function`
             is not one of the valid options.
         """
@@ -91,6 +94,9 @@ class PgvectorEmbeddingRetriever:
         self.filters = filters or {}
         self.top_k = top_k
         self.vector_function = vector_function or document_store.vector_function
+        self.filter_policy = (
+            filter_policy if isinstance(filter_policy, FilterPolicy) else FilterPolicy.from_str(filter_policy)
+        )
 
     def to_dict(self) -> Dict[str, Any]:
         """
@@ -104,6 +110,7 @@ class PgvectorEmbeddingRetriever:
             filters=self.filters,
             top_k=self.top_k,
             vector_function=self.vector_function,
+            filter_policy=self.filter_policy.value,
             document_store=self.document_store.to_dict(),
         )
 
@@ -119,6 +126,10 @@ class PgvectorEmbeddingRetriever:
         """
         doc_store_params = data["init_parameters"]["document_store"]
         data["init_parameters"]["document_store"] = PgvectorDocumentStore.from_dict(doc_store_params)
+        # Pipelines serialized with old versions of the component might not
+        # have the filter_policy field.
+        if filter_policy := data["init_parameters"].get("filter_policy"):
+            data["init_parameters"]["filter_policy"] = FilterPolicy.from_str(filter_policy)
         return default_from_dict(cls, data)
 
     @component.output_types(documents=List[Document])
@@ -133,13 +144,15 @@ class PgvectorEmbeddingRetriever:
         Retrieve documents from the `PgvectorDocumentStore`, based on their embeddings.
 
         :param query_embedding: Embedding of the query.
-        :param filters: Filters applied to the retrieved Documents.
+        :param filters: Filters applied to the retrieved Documents. The way runtime filters are applied depends on
+                        the `filter_policy` chosen at retriever initialization. See init method docstring for more
+                        details.
         :param top_k: Maximum number of Documents to return.
         :param vector_function: The similarity function to use when searching for similar embeddings.
 
         :returns: List of Documents similar to `query_embedding`.
         """
-        filters = filters or self.filters
+        filters = apply_filter_policy(self.filter_policy, self.filters, filters)
         top_k = top_k or self.top_k
         vector_function = vector_function or self.vector_function
 

{pgvector_haystack-0.4.0 → pgvector_haystack-0.5.1}/src/haystack_integrations/components/retrievers/pgvector/keyword_retriever.py

@@ -1,10 +1,12 @@
 # SPDX-FileCopyrightText: 2023-present deepset GmbH <info@deepset.ai>
 #
 # SPDX-License-Identifier: Apache-2.0
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Optional, Union
 
 from haystack import component, default_from_dict, default_to_dict
 from haystack.dataclasses import Document
+from haystack.document_stores.types import FilterPolicy
+from haystack.document_stores.types.filter_policy import apply_filter_policy
 from haystack_integrations.document_stores.pgvector import PgvectorDocumentStore
 
 
@@ -51,12 +53,13 @@ class PgvectorKeywordRetriever:
         document_store: PgvectorDocumentStore,
         filters: Optional[Dict[str, Any]] = None,
         top_k: int = 10,
+        filter_policy: Union[str, FilterPolicy] = FilterPolicy.REPLACE,
     ):
         """
         :param document_store: An instance of `PgvectorDocumentStore`.
         :param filters: Filters applied to the retrieved Documents.
         :param top_k: Maximum number of Documents to return.
-
+        :param filter_policy: Policy to determine how filters are applied.
         :raises ValueError: If `document_store` is not an instance of `PgvectorDocumentStore`.
         """
         if not isinstance(document_store, PgvectorDocumentStore):
@@ -66,6 +69,9 @@ class PgvectorKeywordRetriever:
         self.document_store = document_store
         self.filters = filters or {}
         self.top_k = top_k
+        self.filter_policy = (
+            filter_policy if isinstance(filter_policy, FilterPolicy) else FilterPolicy.from_str(filter_policy)
+        )
 
     def to_dict(self) -> Dict[str, Any]:
         """
@@ -78,6 +84,7 @@ class PgvectorKeywordRetriever:
             self,
             filters=self.filters,
             top_k=self.top_k,
+            filter_policy=self.filter_policy.value,
             document_store=self.document_store.to_dict(),
         )
 
@@ -93,6 +100,10 @@ class PgvectorKeywordRetriever:
         """
         doc_store_params = data["init_parameters"]["document_store"]
         data["init_parameters"]["document_store"] = PgvectorDocumentStore.from_dict(doc_store_params)
+        # Pipelines serialized with old versions of the component might not
+        # have the filter_policy field.
+        if filter_policy := data["init_parameters"].get("filter_policy"):
+            data["init_parameters"]["filter_policy"] = FilterPolicy.from_str(filter_policy)
         return default_from_dict(cls, data)
 
     @component.output_types(documents=List[Document])
@@ -106,13 +117,16 @@ class PgvectorKeywordRetriever:
         Retrieve documents from the `PgvectorDocumentStore`, based on keywords.
 
         :param query: String to search in `Document`s' content.
-        :param filters: Filters applied to the retrieved Documents.
+        :param filters: Filters applied to the retrieved Documents. The way runtime filters are applied depends on
+                        the `filter_policy` chosen at retriever initialization. See init method docstring for more
+                        details.
         :param top_k: Maximum number of Documents to return.
 
         :returns: A dictionary with the following keys:
             - `documents`: List of `Document`s that match the query.
         """
-        filters = filters or self.filters
+        filters = apply_filter_policy(self.filter_policy, self.filters, filters)
+
         top_k = top_k or self.top_k
 
         docs = self.document_store._keyword_retrieval(

{pgvector_haystack-0.4.0 → pgvector_haystack-0.5.1}/tests/test_retrievers.py

@@ -5,6 +5,7 @@ from unittest.mock import Mock
 
 import pytest
 from haystack.dataclasses import Document
+from haystack.document_stores.types import FilterPolicy
 from haystack.utils.auth import EnvVarSecret
 from haystack_integrations.components.retrievers.pgvector import PgvectorEmbeddingRetriever, PgvectorKeywordRetriever
 from haystack_integrations.document_stores.pgvector import PgvectorDocumentStore
@@ -16,8 +17,15 @@ class TestEmbeddingRetriever:
         assert retriever.document_store == mock_store
         assert retriever.filters == {}
         assert retriever.top_k == 10
+        assert retriever.filter_policy == FilterPolicy.REPLACE
         assert retriever.vector_function == mock_store.vector_function
 
+        retriever = PgvectorEmbeddingRetriever(document_store=mock_store, filter_policy="merge")
+        assert retriever.filter_policy == FilterPolicy.MERGE
+
+        with pytest.raises(ValueError):
+            PgvectorEmbeddingRetriever(document_store=mock_store, filter_policy="invalid")
+
     def test_init(self, mock_store):
         retriever = PgvectorEmbeddingRetriever(
             document_store=mock_store, filters={"field": "value"}, top_k=5, vector_function="l2_distance"
@@ -25,6 +33,7 @@ class TestEmbeddingRetriever:
         assert retriever.document_store == mock_store
         assert retriever.filters == {"field": "value"}
         assert retriever.top_k == 5
+        assert retriever.filter_policy == FilterPolicy.REPLACE
         assert retriever.vector_function == "l2_distance"
 
     def test_to_dict(self, mock_store):
@@ -56,6 +65,7 @@ class TestEmbeddingRetriever:
                 "filters": {"field": "value"},
                 "top_k": 5,
                 "vector_function": "l2_distance",
+                "filter_policy": "replace",
             },
         }
 
@@ -85,6 +95,7 @@ class TestEmbeddingRetriever:
                 "filters": {"field": "value"},
                 "top_k": 5,
                 "vector_function": "l2_distance",
+                "filter_policy": "replace",
             },
         }
 
@@ -106,6 +117,7 @@ class TestEmbeddingRetriever:
 
         assert retriever.filters == {"field": "value"}
         assert retriever.top_k == 5
+        assert retriever.filter_policy == FilterPolicy.REPLACE
         assert retriever.vector_function == "l2_distance"
 
     def test_run(self):
@@ -130,12 +142,27 @@ class TestKeywordRetriever:
         assert retriever.filters == {}
         assert retriever.top_k == 10
 
+        retriever = PgvectorKeywordRetriever(document_store=mock_store, filter_policy="merge")
+        assert retriever.filter_policy == FilterPolicy.MERGE
+
+        with pytest.raises(ValueError):
+            PgvectorKeywordRetriever(document_store=mock_store, filter_policy="invalid")
+
     def test_init(self, mock_store):
         retriever = PgvectorKeywordRetriever(document_store=mock_store, filters={"field": "value"}, top_k=5)
         assert retriever.document_store == mock_store
         assert retriever.filters == {"field": "value"}
         assert retriever.top_k == 5
 
+    def test_init_with_filter_policy(self, mock_store):
+        retriever = PgvectorKeywordRetriever(
+            document_store=mock_store, filters={"field": "value"}, top_k=5, filter_policy=FilterPolicy.MERGE
+        )
+        assert retriever.document_store == mock_store
+        assert retriever.filters == {"field": "value"}
+        assert retriever.top_k == 5
+        assert retriever.filter_policy == FilterPolicy.MERGE
+
     def test_to_dict(self, mock_store):
         retriever = PgvectorKeywordRetriever(document_store=mock_store, filters={"field": "value"}, top_k=5)
         res = retriever.to_dict()
@@ -162,6 +189,7 @@ class TestKeywordRetriever:
                 },
                 "filters": {"field": "value"},
                 "top_k": 5,
+                "filter_policy": "replace",
             },
         }
 
@@ -190,6 +218,7 @@ class TestKeywordRetriever:
                 },
                 "filters": {"field": "value"},
                 "top_k": 5,
+                "filter_policy": "replace",
             },
         }
 
@@ -212,6 +241,54 @@ class TestKeywordRetriever:
         assert retriever.filters == {"field": "value"}
         assert retriever.top_k == 5
 
+    @pytest.mark.usefixtures("patches_for_unit_tests")
+    def test_from_dict_without_filter_policy(self, monkeypatch):
+        monkeypatch.setenv("PG_CONN_STR", "some-connection-string")
+        t = "haystack_integrations.components.retrievers.pgvector.keyword_retriever.PgvectorKeywordRetriever"
+        data = {
+            "type": t,
+            "init_parameters": {
+                "document_store": {
+                    "type": "haystack_integrations.document_stores.pgvector.document_store.PgvectorDocumentStore",
+                    "init_parameters": {
+                        "connection_string": {"env_vars": ["PG_CONN_STR"], "strict": True, "type": "env_var"},
+                        "table_name": "haystack_test_to_dict",
+                        "embedding_dimension": 768,
+                        "vector_function": "cosine_similarity",
+                        "recreate_table": True,
+                        "search_strategy": "exact_nearest_neighbor",
+                        "hnsw_recreate_index_if_exists": False,
+                        "hnsw_index_creation_kwargs": {},
+                        "hnsw_index_name": "haystack_hnsw_index",
+                        "hnsw_ef_search": None,
+                        "keyword_index_name": "haystack_keyword_index",
+                    },
+                },
+                "filters": {"field": "value"},
+                "top_k": 5,
+            },
+        }
+
+        retriever = PgvectorKeywordRetriever.from_dict(data)
+        document_store = retriever.document_store
+
+        assert isinstance(document_store, PgvectorDocumentStore)
+        assert isinstance(document_store.connection_string, EnvVarSecret)
+        assert document_store.table_name == "haystack_test_to_dict"
+        assert document_store.embedding_dimension == 768
+        assert document_store.vector_function == "cosine_similarity"
+        assert document_store.recreate_table
+        assert document_store.search_strategy == "exact_nearest_neighbor"
+        assert not document_store.hnsw_recreate_index_if_exists
+        assert document_store.hnsw_index_creation_kwargs == {}
+        assert document_store.hnsw_index_name == "haystack_hnsw_index"
+        assert document_store.hnsw_ef_search is None
+        assert document_store.keyword_index_name == "haystack_keyword_index"
+
+        assert retriever.filters == {"field": "value"}
+        assert retriever.filter_policy == FilterPolicy.REPLACE  # defaults to REPLACE
+        assert retriever.top_k == 5
+
     def test_run(self):
         mock_store = Mock(spec=PgvectorDocumentStore)
         doc = Document(content="Test doc", embedding=[0.1, 0.2])
@@ -223,3 +300,19 @@ class TestKeywordRetriever:
         mock_store._keyword_retrieval.assert_called_once_with(query="test query", filters={}, top_k=10)
 
         assert res == {"documents": [doc]}
+
+    def test_run_with_filters(self):
+        mock_store = Mock(spec=PgvectorDocumentStore)
+        doc = Document(content="Test doc", embedding=[0.1, 0.2])
+        mock_store._keyword_retrieval.return_value = [doc]
+
+        retriever = PgvectorKeywordRetriever(
+            document_store=mock_store, filter_policy=FilterPolicy.MERGE, filters={"field": "value"}
+        )
+        res = retriever.run(query="test query", filters={"field2": "value2"})
+
+        mock_store._keyword_retrieval.assert_called_once_with(
+            query="test query", filters={"field": "value", "field2": "value2"}, top_k=10
+        )
+
+        assert res == {"documents": [doc]}