pgvector-haystack 0.5.0__tar.gz → 1.0.0__tar.gz

{pgvector_haystack-0.5.0 → pgvector_haystack-1.0.0}/CHANGELOG.md

@@ -1,5 +1,20 @@
 # Changelog
 
+## [unreleased]
+
+### 🚀 Features
+
+- Add filter_policy to pgvector integration (#820)
+
+### 🐛 Bug Fixes
+
+- `PgVector` - Fallback to default filter policy when deserializing retrievers without the init parameter (#900)
+
+### ⚙️ Miscellaneous Tasks
+
+- Retry tests to reduce flakyness (#836)
+- Update ruff invocation to include check parameter (#853)
+
 ## [integrations/pgvector-v0.4.0] - 2024-06-20
 
 ### 🚀 Features

{pgvector_haystack-0.5.0 → pgvector_haystack-1.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: pgvector-haystack
-Version: 0.5.0
+Version: 1.0.0
 Summary: An integration of pgvector (vector search extension for Postgres) with Haystack
 Project-URL: Source, https://github.com/deepset-ai/haystack-core-integrations
 Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/blob/main/integrations/pgvector/README.md

{pgvector_haystack-0.5.0 → pgvector_haystack-1.0.0}/examples/embedding_retrieval.py

@@ -17,6 +17,7 @@ from haystack.components.converters import MarkdownToDocument
 from haystack.components.embedders import SentenceTransformersDocumentEmbedder, SentenceTransformersTextEmbedder
 from haystack.components.preprocessors import DocumentSplitter
 from haystack.components.writers import DocumentWriter
+
 from haystack_integrations.components.retrievers.pgvector import PgvectorEmbeddingRetriever
 from haystack_integrations.document_stores.pgvector import PgvectorDocumentStore
 

{pgvector_haystack-0.5.0 → pgvector_haystack-1.0.0}/examples/hybrid_retrieval.py

@@ -18,6 +18,7 @@ from haystack.components.embedders import SentenceTransformersDocumentEmbedder,
 from haystack.components.joiners import DocumentJoiner
 from haystack.components.preprocessors import DocumentSplitter
 from haystack.components.writers import DocumentWriter
+
 from haystack_integrations.components.retrievers.pgvector import PgvectorEmbeddingRetriever, PgvectorKeywordRetriever
 from haystack_integrations.document_stores.pgvector import PgvectorDocumentStore
 

{pgvector_haystack-0.5.0 → pgvector_haystack-1.0.0}/pyproject.toml

@@ -50,10 +50,12 @@ dependencies = [
   "haystack-pydoc-tools",
 ]
 [tool.hatch.envs.default.scripts]
-test = "pytest --reruns 3 --reruns-delay 30 -x {args:tests}"
-test-cov = "coverage run -m pytest --reruns 3 --reruns-delay 30 -x {args:tests}"
+test = "pytest {args:tests}"
+test-cov = "coverage run -m pytest {args:tests}"
+test-cov-retry = "test-cov --reruns 3 --reruns-delay 30 -x"
 cov-report = ["- coverage combine", "coverage report"]
 cov = ["test-cov", "cov-report"]
+cov-retry = ["test-cov-retry", "cov-report"]
 docs = ["pydoc-markdown pydoc/config.yml"]
 
 [[tool.hatch.envs.all.matrix]]
@@ -64,8 +66,8 @@ detached = true
 dependencies = ["black>=23.1.0", "mypy>=1.0.0", "ruff>=0.0.243"]
 [tool.hatch.envs.lint.scripts]
 typing = "mypy --install-types --non-interactive --explicit-package-bases {args:src/ tests}"
-style = ["ruff check {args:.}", "black --check --diff {args:.}"]
-fmt = ["black {args:.}", "ruff --fix {args:.}", "style"]
+style = ["ruff check {args:. --exclude tests/, examples/}", "black --check --diff {args:.}"]
+fmt = ["black {args:.}", "ruff --fix {args:. --exclude tests/, examples/}", "style"]
 all = ["style", "typing"]
 
 [tool.black]

{pgvector_haystack-0.5.0 → pgvector_haystack-1.0.0}/src/haystack_integrations/components/retrievers/pgvector/embedding_retriever.py

@@ -7,6 +7,7 @@ from haystack import component, default_from_dict, default_to_dict
 from haystack.dataclasses import Document
 from haystack.document_stores.types import FilterPolicy
 from haystack.document_stores.types.filter_policy import apply_filter_policy
+
 from haystack_integrations.document_stores.pgvector import PgvectorDocumentStore
 from haystack_integrations.document_stores.pgvector.document_store import VALID_VECTOR_FUNCTIONS
 
@@ -126,7 +127,10 @@ class PgvectorEmbeddingRetriever:
         """
         doc_store_params = data["init_parameters"]["document_store"]
         data["init_parameters"]["document_store"] = PgvectorDocumentStore.from_dict(doc_store_params)
-        data["init_parameters"]["filter_policy"] = FilterPolicy.from_str(data["init_parameters"]["filter_policy"])
+        # Pipelines serialized with old versions of the component might not
+        # have the filter_policy field.
+        if filter_policy := data["init_parameters"].get("filter_policy"):
+            data["init_parameters"]["filter_policy"] = FilterPolicy.from_str(filter_policy)
         return default_from_dict(cls, data)
 
     @component.output_types(documents=List[Document])

{pgvector_haystack-0.5.0 → pgvector_haystack-1.0.0}/src/haystack_integrations/components/retrievers/pgvector/keyword_retriever.py

@@ -7,6 +7,7 @@ from haystack import component, default_from_dict, default_to_dict
 from haystack.dataclasses import Document
 from haystack.document_stores.types import FilterPolicy
 from haystack.document_stores.types.filter_policy import apply_filter_policy
+
 from haystack_integrations.document_stores.pgvector import PgvectorDocumentStore
 
 
@@ -100,7 +101,10 @@ class PgvectorKeywordRetriever:
         """
         doc_store_params = data["init_parameters"]["document_store"]
         data["init_parameters"]["document_store"] = PgvectorDocumentStore.from_dict(doc_store_params)
-        data["init_parameters"]["filter_policy"] = FilterPolicy.from_str(data["init_parameters"]["filter_policy"])
+        # Pipelines serialized with old versions of the component might not
+        # have the filter_policy field.
+        if filter_policy := data["init_parameters"].get("filter_policy"):
+            data["init_parameters"]["filter_policy"] = FilterPolicy.from_str(filter_policy)
         return default_from_dict(cls, data)
 
     @component.output_types(documents=List[Document])

{pgvector_haystack-0.5.0 → pgvector_haystack-1.0.0}/src/haystack_integrations/document_stores/pgvector/document_store.py

@@ -9,7 +9,6 @@ from haystack.dataclasses.document import ByteStream, Document
 from haystack.document_stores.errors import DocumentStoreError, DuplicateDocumentError
 from haystack.document_stores.types import DuplicatePolicy
 from haystack.utils.auth import Secret, deserialize_secrets_inplace
-from haystack.utils.filters import convert
 from psycopg import Error, IntegrityError, connect
 from psycopg.abc import Query
 from psycopg.cursor import Cursor
@@ -389,7 +388,8 @@ class PgvectorDocumentStore:
                 msg = "Filters must be a dictionary"
                 raise TypeError(msg)
             if "operator" not in filters and "conditions" not in filters:
-                filters = convert(filters)
+                msg = "Invalid filter syntax. See https://docs.haystack.deepset.ai/docs/metadata-filtering for details."
+                raise ValueError(msg)
 
         sql_filter = SQL("SELECT * FROM {table_name}").format(table_name=Identifier(self.table_name))
 

{pgvector_haystack-0.5.0 → pgvector_haystack-1.0.0}/tests/conftest.py

@@ -2,6 +2,7 @@ import os
 from unittest.mock import patch
 
 import pytest
+
 from haystack_integrations.document_stores.pgvector import PgvectorDocumentStore
 
 

{pgvector_haystack-0.5.0 → pgvector_haystack-1.0.0}/tests/test_document_store.py

@@ -11,9 +11,10 @@ from haystack.document_stores.errors import DuplicateDocumentError
 from haystack.document_stores.types import DuplicatePolicy
 from haystack.testing.document_store import CountDocumentsTest, DeleteDocumentsTest, WriteDocumentsTest
 from haystack.utils import Secret
-from haystack_integrations.document_stores.pgvector import PgvectorDocumentStore
 from pandas import DataFrame
 
+from haystack_integrations.document_stores.pgvector import PgvectorDocumentStore
+
 
 @pytest.mark.integration
 class TestDocumentStore(CountDocumentsTest, WriteDocumentsTest, DeleteDocumentsTest):

{pgvector_haystack-0.5.0 → pgvector_haystack-1.0.0}/tests/test_embedding_retrieval.py

@@ -6,9 +6,10 @@ from typing import List
 
 import pytest
 from haystack.dataclasses.document import Document
-from haystack_integrations.document_stores.pgvector import PgvectorDocumentStore
 from numpy.random import rand
 
+from haystack_integrations.document_stores.pgvector import PgvectorDocumentStore
+
 
 @pytest.mark.integration
 class TestEmbeddingRetrieval:

{pgvector_haystack-0.5.0 → pgvector_haystack-1.0.0}/tests/test_filters.py

@@ -3,6 +3,10 @@ from typing import List
 import pytest
 from haystack.dataclasses.document import Document
 from haystack.testing.document_store import FilterDocumentsTest
+from pandas import DataFrame
+from psycopg.sql import SQL
+from psycopg.types.json import Jsonb
+
 from haystack_integrations.document_stores.pgvector.filters import (
     FilterError,
     _convert_filters_to_where_clause_and_params,
@@ -10,9 +14,6 @@ from haystack_integrations.document_stores.pgvector.filters import (
     _parse_logical_condition,
     _treat_meta_field,
 )
-from pandas import DataFrame
-from psycopg.sql import SQL
-from psycopg.types.json import Jsonb
 
 
 @pytest.mark.integration

{pgvector_haystack-0.5.0 → pgvector_haystack-1.0.0}/tests/test_keyword_retrieval.py

@@ -1,5 +1,6 @@
 import pytest
 from haystack.dataclasses.document import Document
+
 from haystack_integrations.document_stores.pgvector import PgvectorDocumentStore
 
 

{pgvector_haystack-0.5.0 → pgvector_haystack-1.0.0}/tests/test_retrievers.py

@@ -7,6 +7,7 @@ import pytest
 from haystack.dataclasses import Document
 from haystack.document_stores.types import FilterPolicy
 from haystack.utils.auth import EnvVarSecret
+
 from haystack_integrations.components.retrievers.pgvector import PgvectorEmbeddingRetriever, PgvectorKeywordRetriever
 from haystack_integrations.document_stores.pgvector import PgvectorDocumentStore
 
@@ -241,6 +242,54 @@ class TestKeywordRetriever:
         assert retriever.filters == {"field": "value"}
         assert retriever.top_k == 5
 
+    @pytest.mark.usefixtures("patches_for_unit_tests")
+    def test_from_dict_without_filter_policy(self, monkeypatch):
+        monkeypatch.setenv("PG_CONN_STR", "some-connection-string")
+        t = "haystack_integrations.components.retrievers.pgvector.keyword_retriever.PgvectorKeywordRetriever"
+        data = {
+            "type": t,
+            "init_parameters": {
+                "document_store": {
+                    "type": "haystack_integrations.document_stores.pgvector.document_store.PgvectorDocumentStore",
+                    "init_parameters": {
+                        "connection_string": {"env_vars": ["PG_CONN_STR"], "strict": True, "type": "env_var"},
+                        "table_name": "haystack_test_to_dict",
+                        "embedding_dimension": 768,
+                        "vector_function": "cosine_similarity",
+                        "recreate_table": True,
+                        "search_strategy": "exact_nearest_neighbor",
+                        "hnsw_recreate_index_if_exists": False,
+                        "hnsw_index_creation_kwargs": {},
+                        "hnsw_index_name": "haystack_hnsw_index",
+                        "hnsw_ef_search": None,
+                        "keyword_index_name": "haystack_keyword_index",
+                    },
+                },
+                "filters": {"field": "value"},
+                "top_k": 5,
+            },
+        }
+
+        retriever = PgvectorKeywordRetriever.from_dict(data)
+        document_store = retriever.document_store
+
+        assert isinstance(document_store, PgvectorDocumentStore)
+        assert isinstance(document_store.connection_string, EnvVarSecret)
+        assert document_store.table_name == "haystack_test_to_dict"
+        assert document_store.embedding_dimension == 768
+        assert document_store.vector_function == "cosine_similarity"
+        assert document_store.recreate_table
+        assert document_store.search_strategy == "exact_nearest_neighbor"
+        assert not document_store.hnsw_recreate_index_if_exists
+        assert document_store.hnsw_index_creation_kwargs == {}
+        assert document_store.hnsw_index_name == "haystack_hnsw_index"
+        assert document_store.hnsw_ef_search is None
+        assert document_store.keyword_index_name == "haystack_keyword_index"
+
+        assert retriever.filters == {"field": "value"}
+        assert retriever.filter_policy == FilterPolicy.REPLACE  # defaults to REPLACE
+        assert retriever.top_k == 5
+
     def test_run(self):
         mock_store = Mock(spec=PgvectorDocumentStore)
         doc = Document(content="Test doc", embedding=[0.1, 0.2])
@@ -263,7 +312,7 @@ class TestKeywordRetriever:
         )
         res = retriever.run(query="test query", filters={"field2": "value2"})
 
-        mock_store._keyword_retrieval.assert_called_once_with(
+        mock_store.mock_calls[0].assert_called_once_with(
             query="test query", filters={"field": "value", "field2": "value2"}, top_k=10
         )