fastembed-haystack 1.1.0__tar.gz → 1.3.0__tar.gz

{fastembed_haystack-1.1.0 → fastembed_haystack-1.3.0}/.gitignore

@@ -135,3 +135,12 @@ dmypy.json
 # Docs generation artifacts
 _readme_*.md
 .idea
+
+# macOS
+.DS_Store
+
+# http cache (requests-cache)
+**/http_cache.sqlite
+
+# ruff
+.ruff_cache

fastembed_haystack-1.3.0/CHANGELOG.md

@@ -0,0 +1,63 @@
+# Changelog
+
+## [unreleased]
+
+### ⚙️ Miscellaneous Tasks
+
+- Retry tests to reduce flakyness (#836)
+- Update ruff invocation to include check parameter (#853)
+
+### Fix
+
+- Typo on Sparse embedders. The parameter should be "progress_bar" … (#814)
+
+## [integrations/fastembed-v1.1.0] - 2024-05-15
+
+## [integrations/fastembed-v1.0.0] - 2024-05-06
+
+## [integrations/fastembed-v0.1.0] - 2024-04-10
+
+### 🚀 Features
+
+- *(FastEmbed)* Support for SPLADE Sparse Embedder (#579)
+
+### 📚 Documentation
+
+- Disable-class-def (#556)
+
+## [integrations/fastembed-v0.0.6] - 2024-03-07
+
+### 📚 Documentation
+
+- Review and normalize docstrings - `integrations.fastembed` (#519)
+- Small consistency improvements (#536)
+
+## [integrations/fastembed-v0.0.5] - 2024-02-20
+
+### 🐛 Bug Fixes
+
+- Fix order of API docs (#447)
+
+This PR will also push the docs to Readme
+
+### 📚 Documentation
+
+- Update category slug (#442)
+
+## [integrations/fastembed-v0.0.4] - 2024-02-16
+
+## [integrations/fastembed-v0.0.3] - 2024-02-12
+
+### 🐛 Bug Fixes
+
+- From numpy float to float (#391)
+
+### 📚 Documentation
+
+- Update paths and titles (#397)
+
+## [integrations/fastembed-v0.0.2] - 2024-02-11
+
+## [integrations/fastembed-v0.0.1] - 2024-02-10
+
+<!-- generated by git-cliff -->

{fastembed_haystack-1.1.0 → fastembed_haystack-1.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: fastembed-haystack
-Version: 1.1.0
+Version: 1.3.0
 Summary: Haystack 2.x component to embed strings and Documents using fastembed embedding model
 Project-URL: Source, https://github.com/deepset-ai/haystack-core-integrations
 Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/blob/main/integrations/fastembed/README.md

{fastembed_haystack-1.1.0 → fastembed_haystack-1.3.0}/examples/example.py

@@ -1,6 +1,7 @@
 from haystack import Document, Pipeline
 from haystack.components.retrievers.in_memory import InMemoryEmbeddingRetriever
 from haystack.document_stores.in_memory import InMemoryDocumentStore
+
 from haystack_integrations.components.embedders.fastembed import FastembedDocumentEmbedder, FastembedTextEmbedder
 
 document_store = InMemoryDocumentStore(embedding_similarity_function="cosine")

{fastembed_haystack-1.1.0 → fastembed_haystack-1.3.0}/examples/sparse_example.py

@@ -4,6 +4,7 @@
 # involving indexing and retrieval of documents.
 
 from haystack import Document
+
 from haystack_integrations.components.embedders.fastembed import FastembedSparseDocumentEmbedder
 
 document_list = [

{fastembed_haystack-1.1.0 → fastembed_haystack-1.3.0}/pyproject.toml

@@ -10,9 +10,7 @@ readme = "README.md"
 requires-python = ">=3.8"
 license = "Apache-2.0"
 keywords = []
-authors = [
-  { name = "deepset GmbH", email = "info@deepset.ai" },
-]
+authors = [{ name = "deepset GmbH", email = "info@deepset.ai" }]
 classifiers = [
   "License :: OSI Approved :: Apache Software License",
   "Development Status :: 4 - Beta",
@@ -25,10 +23,7 @@ classifiers = [
   "Programming Language :: Python :: Implementation :: CPython",
   "Programming Language :: Python :: Implementation :: PyPy",
 ]
-dependencies = [
-"haystack-ai>=2.0.1",
-"fastembed>=0.2.5",
-]
+dependencies = ["haystack-ai>=2.0.1", "fastembed>=0.2.5"]
 
 [project.urls]
 Source = "https://github.com/deepset-ai/haystack-core-integrations"
@@ -50,50 +45,30 @@ git_describe_command = 'git describe --tags --match="integrations/fastembed-v[0-
 dependencies = [
   "coverage[toml]>=6.5",
   "pytest",
+  "pytest-rerunfailures",
   "ipython",
   "haystack-pydoc-tools",
 ]
 [tool.hatch.envs.default.scripts]
 test = "pytest {args:tests}"
 test-cov = "coverage run -m pytest {args:tests}"
-cov-report = [
-  "- coverage combine",
-  "coverage report",
-]
-cov = [
-  "test-cov",
-  "cov-report",
-]
-docs = [
-  "pydoc-markdown pydoc/config.yml"
-]
+test-cov-retry = "test-cov --reruns 3 --reruns-delay 30 -x"
+cov-report = ["- coverage combine", "coverage report"]
+cov = ["test-cov", "cov-report"]
+cov-retry = ["test-cov-retry", "cov-report"]
+docs = ["pydoc-markdown pydoc/config.yml"]
 
 [[tool.hatch.envs.all.matrix]]
 python = ["3.8", "3.9", "3.10", "3.11", "3.12"]
 
 [tool.hatch.envs.lint]
 detached = true
-dependencies = [
-  "black>=23.1.0",
-  "mypy>=1.0.0",
-  "ruff>=0.0.243",
-  "numpy"
-]
+dependencies = ["black>=23.1.0", "mypy>=1.0.0", "ruff>=0.0.243", "numpy"]
 [tool.hatch.envs.lint.scripts]
 typing = "mypy --install-types --non-interactive --explicit-package-bases {args:src/ tests}"
-style = [
-  "ruff {args:.}",
-  "black --check --diff {args:.}",
-]
-fmt = [
-  "black {args:.}",
-  "ruff --fix {args:.}",
-  "style",
-]
-all = [
-  "style",
-  "typing",
-]
+style = ["ruff check {args:.}", "black --check --diff {args:.}"]
+fmt = ["black {args:.}", "ruff check --fix {args:.}", "style"]
+all = ["style", "typing"]
 
 [tool.black]
 target-version = ["py38"]
@@ -103,6 +78,8 @@ skip-string-normalization = true
 [tool.ruff]
 target-version = "py38"
 line-length = 120
+
+[tool.ruff.lint]
 select = [
   "A",
   "ARG",
@@ -135,24 +112,31 @@ ignore = [
   "B027",
   # Allow boolean positional values in function calls, like `dict.get(... True)`
   "FBT003",
-  "FBT001", "FBT002",
+  "FBT001",
+  "FBT002",
   # Ignore checks for possible passwords
-  "S105", "S106", "S107",
+  "S105",
+  "S106",
+  "S107",
   # Ignore complexity
-  "C901", "PLR0911", "PLR0912", "PLR0913", "PLR0915",
+  "C901",
+  "PLR0911",
+  "PLR0912",
+  "PLR0913",
+  "PLR0915",
 ]
 unfixable = [
   # Don't touch unused imports
   "F401",
 ]
 
-[tool.ruff.isort]
-known-first-party = ["src"]
+[tool.ruff.lint.isort]
+known-first-party = ["haystack_integrations"]
 
-[tool.ruff.flake8-tidy-imports]
+[tool.ruff.lint.flake8-tidy-imports]
 ban-relative-imports = "parents"
 
-[tool.ruff.per-file-ignores]
+[tool.ruff.lint.per-file-ignores]
 # Tests can use magic values, assertions, and relative imports
 "tests/**/*" = ["PLR2004", "S101", "TID252"]
 # examples can contain "print" commands
@@ -165,12 +149,8 @@ parallel = false
 
 [tool.coverage.report]
 omit = ["*/tests/*", "*/__init__.py"]
-show_missing=true
-exclude_lines = [
-  "no cov",
-  "if __name__ == .__main__.:",
-  "if TYPE_CHECKING:",
-]
+show_missing = true
+exclude_lines = ["no cov", "if __name__ == .__main__.:", "if TYPE_CHECKING:"]
 
 [[tool.mypy.overrides]]
 module = [
@@ -178,6 +158,6 @@ module = [
   "haystack_integrations.*",
   "fastembed.*",
   "pytest.*",
-  "numpy.*"
+  "numpy.*",
 ]
 ignore_missing_imports = true

{fastembed_haystack-1.1.0 → fastembed_haystack-1.3.0}/src/haystack_integrations/components/embedders/fastembed/embedding_backend/fastembed_backend.py

@@ -1,4 +1,4 @@
-from typing import ClassVar, Dict, List, Optional
+from typing import Any, ClassVar, Dict, List, Optional
 
 from haystack.dataclasses.sparse_embedding import SparseEmbedding
 from tqdm import tqdm
@@ -73,14 +73,19 @@ class _FastembedSparseEmbeddingBackendFactory:
         cache_dir: Optional[str] = None,
         threads: Optional[int] = None,
         local_files_only: bool = False,
+        model_kwargs: Optional[Dict[str, Any]] = None,
     ):
-        embedding_backend_id = f"{model_name}{cache_dir}{threads}"
+        embedding_backend_id = f"{model_name}{cache_dir}{threads}{local_files_only}{model_kwargs}"
 
         if embedding_backend_id in _FastembedSparseEmbeddingBackendFactory._instances:
             return _FastembedSparseEmbeddingBackendFactory._instances[embedding_backend_id]
 
         embedding_backend = _FastembedSparseEmbeddingBackend(
-            model_name=model_name, cache_dir=cache_dir, threads=threads, local_files_only=local_files_only
+            model_name=model_name,
+            cache_dir=cache_dir,
+            threads=threads,
+            local_files_only=local_files_only,
+            model_kwargs=model_kwargs,
         )
         _FastembedSparseEmbeddingBackendFactory._instances[embedding_backend_id] = embedding_backend
         return embedding_backend
@@ -97,9 +102,16 @@ class _FastembedSparseEmbeddingBackend:
         cache_dir: Optional[str] = None,
         threads: Optional[int] = None,
         local_files_only: bool = False,
+        model_kwargs: Optional[Dict[str, Any]] = None,
     ):
+        model_kwargs = model_kwargs or {}
+
         self.model = SparseTextEmbedding(
-            model_name=model_name, cache_dir=cache_dir, threads=threads, local_files_only=local_files_only
+            model_name=model_name,
+            cache_dir=cache_dir,
+            threads=threads,
+            local_files_only=local_files_only,
+            **model_kwargs,
         )
 
     def embed(self, data: List[List[str]], progress_bar=True, **kwargs) -> List[SparseEmbedding]:

{fastembed_haystack-1.1.0 → fastembed_haystack-1.3.0}/src/haystack_integrations/components/embedders/fastembed/fastembed_sparse_document_embedder.py

@@ -62,6 +62,7 @@ class FastembedSparseDocumentEmbedder:
         local_files_only: bool = False,
         meta_fields_to_embed: Optional[List[str]] = None,
         embedding_separator: str = "\n",
+        model_kwargs: Optional[Dict[str, Any]] = None,
     ):
         """
         Create an FastembedDocumentEmbedder component.
@@ -81,6 +82,7 @@ class FastembedSparseDocumentEmbedder:
         :param local_files_only: If `True`, only use the model files in the `cache_dir`.
         :param meta_fields_to_embed: List of meta fields that should be embedded along with the Document content.
         :param embedding_separator: Separator used to concatenate the meta fields to the Document content.
+        :param model_kwargs: Dictionary containing model parameters such as `k`, `b`, `avg_len`, `language`.
         """
 
         self.model_name = model
@@ -92,6 +94,7 @@ class FastembedSparseDocumentEmbedder:
         self.local_files_only = local_files_only
         self.meta_fields_to_embed = meta_fields_to_embed or []
         self.embedding_separator = embedding_separator
+        self.model_kwargs = model_kwargs
 
     def to_dict(self) -> Dict[str, Any]:
         """
@@ -110,6 +113,7 @@ class FastembedSparseDocumentEmbedder:
             local_files_only=self.local_files_only,
             meta_fields_to_embed=self.meta_fields_to_embed,
             embedding_separator=self.embedding_separator,
+            model_kwargs=self.model_kwargs,
         )
 
     def warm_up(self):
@@ -122,6 +126,7 @@ class FastembedSparseDocumentEmbedder:
                 cache_dir=self.cache_dir,
                 threads=self.threads,
                 local_files_only=self.local_files_only,
+                model_kwargs=self.model_kwargs,
             )
 
     def _prepare_texts_to_embed(self, documents: List[Document]) -> List[str]:
@@ -159,7 +164,7 @@ class FastembedSparseDocumentEmbedder:
         embeddings = self.embedding_backend.embed(
             texts_to_embed,
             batch_size=self.batch_size,
-            show_progress_bar=self.progress_bar,
+            progress_bar=self.progress_bar,
             parallel=self.parallel,
         )
 

{fastembed_haystack-1.1.0 → fastembed_haystack-1.3.0}/src/haystack_integrations/components/embedders/fastembed/fastembed_sparse_text_embedder.py

@@ -35,6 +35,7 @@ class FastembedSparseTextEmbedder:
         progress_bar: bool = True,
         parallel: Optional[int] = None,
         local_files_only: bool = False,
+        model_kwargs: Optional[Dict[str, Any]] = None,
     ):
         """
         Create a FastembedSparseTextEmbedder component.
@@ -50,6 +51,7 @@ class FastembedSparseTextEmbedder:
                 If 0, use all available cores.
                 If None, don't use data-parallel processing, use default onnxruntime threading instead.
         :param local_files_only: If `True`, only use the model files in the `cache_dir`.
+        :param model_kwargs: Dictionary containing model parameters such as `k`, `b`, `avg_len`, `language`.
         """
 
         self.model_name = model
@@ -58,6 +60,7 @@ class FastembedSparseTextEmbedder:
         self.progress_bar = progress_bar
         self.parallel = parallel
         self.local_files_only = local_files_only
+        self.model_kwargs = model_kwargs
 
     def to_dict(self) -> Dict[str, Any]:
         """
@@ -74,6 +77,7 @@ class FastembedSparseTextEmbedder:
             progress_bar=self.progress_bar,
             parallel=self.parallel,
             local_files_only=self.local_files_only,
+            model_kwargs=self.model_kwargs,
         )
 
     def warm_up(self):
@@ -86,6 +90,7 @@ class FastembedSparseTextEmbedder:
                 cache_dir=self.cache_dir,
                 threads=self.threads,
                 local_files_only=self.local_files_only,
+                model_kwargs=self.model_kwargs,
             )
 
     @component.output_types(sparse_embedding=SparseEmbedding)
@@ -111,7 +116,7 @@ class FastembedSparseTextEmbedder:
 
         embedding = self.embedding_backend.embed(
             [text],
-            show_progress_bar=self.progress_bar,
+            progress_bar=self.progress_bar,
             parallel=self.parallel,
         )[0]
         return {"sparse_embedding": embedding}

{fastembed_haystack-1.1.0 → fastembed_haystack-1.3.0}/tests/test_fastembed_backend.py

@@ -2,6 +2,7 @@ from unittest.mock import patch
 
 from haystack_integrations.components.embedders.fastembed.embedding_backend.fastembed_backend import (
     _FastembedEmbeddingBackendFactory,
+    _FastembedSparseEmbeddingBackendFactory,
 )
 
 
@@ -44,3 +45,28 @@ def test_embedding_function_with_kwargs(mock_instructor):  # noqa: ARG001
     embedding_backend.model.embed.assert_called_once_with(data)
     # restore the factory stateTrue
     _FastembedEmbeddingBackendFactory._instances = {}
+
+
+@patch("haystack_integrations.components.embedders.fastembed.embedding_backend.fastembed_backend.SparseTextEmbedding")
+def test_model_kwargs_initialization(mock_instructor):
+    bm25_config = {
+        "k": 1.2,
+        "b": 0.75,
+        "avg_len": 300.0,
+        "language": "english",
+        "token_max_length": 40,
+    }
+
+    # Invoke the backend factory with the BM25 configuration
+    _FastembedSparseEmbeddingBackendFactory.get_embedding_backend(
+        model_name="Qdrant/bm25",
+        model_kwargs=bm25_config,
+    )
+
+    # Check if SparseTextEmbedding was called with the correct arguments
+    mock_instructor.assert_called_once_with(
+        model_name="Qdrant/bm25", cache_dir=None, threads=None, local_files_only=False, **bm25_config
+    )
+
+    # Restore factory state after the test
+    _FastembedSparseEmbeddingBackendFactory._instances = {}

{fastembed_haystack-1.1.0 → fastembed_haystack-1.3.0}/tests/test_fastembed_document_embedder.py

@@ -3,6 +3,7 @@ from unittest.mock import MagicMock, patch
 import numpy as np
 import pytest
 from haystack import Document, default_from_dict
+
 from haystack_integrations.components.embedders.fastembed.fastembed_document_embedder import (
     FastembedDocumentEmbedder,
 )

{fastembed_haystack-1.1.0 → fastembed_haystack-1.3.0}/tests/test_fastembed_sparse_document_embedder.py

@@ -4,6 +4,7 @@ import numpy as np
 import pytest
 from haystack import Document, default_from_dict
 from haystack.dataclasses.sparse_embedding import SparseEmbedding
+
 from haystack_integrations.components.embedders.fastembed.fastembed_sparse_document_embedder import (
     FastembedSparseDocumentEmbedder,
 )
@@ -68,6 +69,7 @@ class TestFastembedSparseDocumentEmbedderDoc:
                 "local_files_only": False,
                 "embedding_separator": "\n",
                 "meta_fields_to_embed": [],
+                "model_kwargs": None,
             },
         }
 
@@ -99,6 +101,7 @@ class TestFastembedSparseDocumentEmbedderDoc:
                 "local_files_only": True,
                 "meta_fields_to_embed": ["test_field"],
                 "embedding_separator": " | ",
+                "model_kwargs": None,
             },
         }
 
@@ -173,7 +176,11 @@ class TestFastembedSparseDocumentEmbedderDoc:
         mocked_factory.get_embedding_backend.assert_not_called()
         embedder.warm_up()
         mocked_factory.get_embedding_backend.assert_called_once_with(
-            model_name="prithvida/Splade_PP_en_v1", cache_dir=None, threads=None, local_files_only=False
+            model_name="prithvida/Splade_PP_en_v1",
+            cache_dir=None,
+            threads=None,
+            local_files_only=False,
+            model_kwargs=None,
         )
 
     @patch(
@@ -270,10 +277,56 @@ class TestFastembedSparseDocumentEmbedderDoc:
                 "meta_value 4\ndocument-number 4",
             ],
             batch_size=32,
-            show_progress_bar=True,
+            progress_bar=True,
             parallel=None,
         )
 
+    def test_init_with_model_kwargs_parameters(self):
+        """
+        Test initialization of FastembedSparseDocumentEmbedder with model_kwargs parameters.
+        """
+        bm25_config = {
+            "k": 1.2,
+            "b": 0.75,
+            "avg_len": 300.0,
+            "language": "english",
+            "token_max_length": 50,
+        }
+
+        embedder = FastembedSparseDocumentEmbedder(
+            model="Qdrant/bm25",
+            model_kwargs=bm25_config,
+        )
+
+        assert embedder.model_kwargs == bm25_config
+
+    @pytest.mark.integration
+    def test_run_with_model_kwargs(self):
+        """
+        Integration test to check the embedding with model_kwargs parameters.
+        """
+        bm42_config = {
+            "alpha": 0.2,
+        }
+
+        embedder = FastembedSparseDocumentEmbedder(
+            model="Qdrant/bm42-all-minilm-l6-v2-attentions",
+            model_kwargs=bm42_config,
+        )
+        embedder.warm_up()
+
+        doc = Document(content="Example content using BM42")
+
+        result = embedder.run(documents=[doc])
+        embedding = result["documents"][0].sparse_embedding
+        embedding_dict = embedding.to_dict()
+
+        assert isinstance(embedding, SparseEmbedding)
+        assert isinstance(embedding_dict["indices"], list)
+        assert isinstance(embedding_dict["values"], list)
+        assert isinstance(embedding_dict["indices"][0], int)
+        assert isinstance(embedding_dict["values"][0], float)
+
     @pytest.mark.integration
     def test_run(self):
         embedder = FastembedSparseDocumentEmbedder(

{fastembed_haystack-1.1.0 → fastembed_haystack-1.3.0}/tests/test_fastembed_sparse_text_embedder.py

@@ -4,6 +4,7 @@ import numpy as np
 import pytest
 from haystack import default_from_dict
 from haystack.dataclasses.sparse_embedding import SparseEmbedding
+
 from haystack_integrations.components.embedders.fastembed.fastembed_sparse_text_embedder import (
     FastembedSparseTextEmbedder,
 )
@@ -53,6 +54,7 @@ class TestFastembedSparseTextEmbedder:
                 "progress_bar": True,
                 "parallel": None,
                 "local_files_only": False,
+                "model_kwargs": None,
             },
         }
 
@@ -78,6 +80,7 @@ class TestFastembedSparseTextEmbedder:
                 "progress_bar": False,
                 "parallel": 1,
                 "local_files_only": True,
+                "model_kwargs": None,
             },
         }
 
@@ -134,7 +137,11 @@ class TestFastembedSparseTextEmbedder:
         mocked_factory.get_embedding_backend.assert_not_called()
         embedder.warm_up()
         mocked_factory.get_embedding_backend.assert_called_once_with(
-            model_name="prithvida/Splade_PP_en_v1", cache_dir=None, threads=None, local_files_only=False
+            model_name="prithvida/Splade_PP_en_v1",
+            cache_dir=None,
+            threads=None,
+            local_files_only=False,
+            model_kwargs=None,
         )
 
     @patch(
@@ -194,6 +201,54 @@ class TestFastembedSparseTextEmbedder:
         with pytest.raises(TypeError, match="FastembedSparseTextEmbedder expects a string as input"):
             embedder.run(text=list_integers_input)
 
+    def test_init_with_model_kwargs_parameters(self):
+        """
+        Test initialization of FastembedSparseTextEmbedder with model_kwargs parameters.
+        """
+        bm25_config = {
+            "k": 1.2,
+            "b": 0.75,
+            "avg_len": 300.0,
+            "language": "english",
+            "token_max_length": 50,
+        }
+
+        embedder = FastembedSparseTextEmbedder(
+            model="Qdrant/bm25",
+            model_kwargs=bm25_config,
+        )
+
+        assert embedder.model_kwargs == bm25_config
+
+    @pytest.mark.integration
+    def test_run_with_model_kwargs(self):
+        """
+        Integration test to check the embedding with model_kwargs parameters.
+        """
+        bm25_config = {
+            "k": 1.2,
+            "b": 0.75,
+            "avg_len": 256.0,
+        }
+
+        embedder = FastembedSparseTextEmbedder(
+            model="Qdrant/bm25",
+            model_kwargs=bm25_config,
+        )
+        embedder.warm_up()
+
+        text = "Example content using BM25"
+
+        result = embedder.run(text=text)
+        embedding = result["sparse_embedding"]
+        embedding_dict = embedding.to_dict()
+
+        assert isinstance(embedding, SparseEmbedding)
+        assert isinstance(embedding_dict["indices"], list)
+        assert isinstance(embedding_dict["values"], list)
+        assert isinstance(embedding_dict["indices"][0], int)
+        assert isinstance(embedding_dict["values"][0], float)
+
     @pytest.mark.integration
     def test_run(self):
         embedder = FastembedSparseTextEmbedder(

{fastembed_haystack-1.1.0 → fastembed_haystack-1.3.0}/tests/test_fastembed_text_embedder.py

@@ -3,6 +3,7 @@ from unittest.mock import MagicMock, patch
 import numpy as np
 import pytest
 from haystack import default_from_dict
+
 from haystack_integrations.components.embedders.fastembed.fastembed_text_embedder import (
     FastembedTextEmbedder,
 )