fastembed-haystack 1.6.0__tar.gz → 2.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {fastembed_haystack-1.6.0 → fastembed_haystack-2.1.0}/CHANGELOG.md +33 -0
- {fastembed_haystack-1.6.0 → fastembed_haystack-2.1.0}/PKG-INFO +3 -4
- {fastembed_haystack-1.6.0 → fastembed_haystack-2.1.0}/examples/example.py +0 -1
- {fastembed_haystack-1.6.0 → fastembed_haystack-2.1.0}/examples/ranker_example.py +0 -1
- {fastembed_haystack-1.6.0 → fastembed_haystack-2.1.0}/examples/sparse_example.py +0 -1
- {fastembed_haystack-1.6.0 → fastembed_haystack-2.1.0}/pyproject.toml +2 -8
- {fastembed_haystack-1.6.0 → fastembed_haystack-2.1.0}/src/haystack_integrations/components/embedders/fastembed/embedding_backend/fastembed_backend.py +11 -11
- {fastembed_haystack-1.6.0 → fastembed_haystack-2.1.0}/src/haystack_integrations/components/embedders/fastembed/fastembed_document_embedder.py +7 -9
- {fastembed_haystack-1.6.0 → fastembed_haystack-2.1.0}/src/haystack_integrations/components/embedders/fastembed/fastembed_sparse_document_embedder.py +8 -10
- {fastembed_haystack-1.6.0 → fastembed_haystack-2.1.0}/src/haystack_integrations/components/embedders/fastembed/fastembed_sparse_text_embedder.py +6 -7
- {fastembed_haystack-1.6.0 → fastembed_haystack-2.1.0}/src/haystack_integrations/components/embedders/fastembed/fastembed_text_embedder.py +5 -6
- {fastembed_haystack-1.6.0 → fastembed_haystack-2.1.0}/src/haystack_integrations/components/rankers/fastembed/ranker.py +10 -11
- {fastembed_haystack-1.6.0 → fastembed_haystack-2.1.0}/tests/test_fastembed_document_embedder.py +15 -1
- {fastembed_haystack-1.6.0 → fastembed_haystack-2.1.0}/tests/test_fastembed_ranker.py +17 -2
- {fastembed_haystack-1.6.0 → fastembed_haystack-2.1.0}/tests/test_fastembed_sparse_document_embedder.py +14 -3
- {fastembed_haystack-1.6.0 → fastembed_haystack-2.1.0}/tests/test_fastembed_sparse_text_embedder.py +14 -1
- {fastembed_haystack-1.6.0 → fastembed_haystack-2.1.0}/tests/test_fastembed_text_embedder.py +14 -1
- {fastembed_haystack-1.6.0 → fastembed_haystack-2.1.0}/.gitignore +0 -0
- {fastembed_haystack-1.6.0 → fastembed_haystack-2.1.0}/LICENSE.txt +0 -0
- {fastembed_haystack-1.6.0 → fastembed_haystack-2.1.0}/README.md +0 -0
- {fastembed_haystack-1.6.0 → fastembed_haystack-2.1.0}/pydoc/config_docusaurus.yml +0 -0
- {fastembed_haystack-1.6.0 → fastembed_haystack-2.1.0}/src/haystack_integrations/components/embedders/fastembed/__init__.py +0 -0
- {fastembed_haystack-1.6.0 → fastembed_haystack-2.1.0}/src/haystack_integrations/components/embedders/fastembed/embedding_backend/__init__.py +0 -0
- {fastembed_haystack-1.6.0 → fastembed_haystack-2.1.0}/src/haystack_integrations/components/embedders/py.typed +0 -0
- {fastembed_haystack-1.6.0 → fastembed_haystack-2.1.0}/src/haystack_integrations/components/rankers/fastembed/__init__.py +0 -0
- {fastembed_haystack-1.6.0 → fastembed_haystack-2.1.0}/src/haystack_integrations/components/rankers/py.typed +0 -0
- {fastembed_haystack-1.6.0 → fastembed_haystack-2.1.0}/tests/__init__.py +0 -0
- {fastembed_haystack-1.6.0 → fastembed_haystack-2.1.0}/tests/test_fastembed_backend.py +0 -0
|
@@ -1,5 +1,38 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [integrations/fastembed-v2.0.0] - 2026-01-12
|
|
4
|
+
|
|
5
|
+
### 🧹 Chores
|
|
6
|
+
|
|
7
|
+
- [**breaking**] Fastembed - drop Python 3.9 and use X|Y typing (#2698)
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
## [integrations/fastembed-v1.6.0] - 2026-01-09
|
|
11
|
+
|
|
12
|
+
### 🚀 Features
|
|
13
|
+
|
|
14
|
+
- Update FastEmbed components to auto call run `warm_up` and don't modify Documents in place (#2678)
|
|
15
|
+
|
|
16
|
+
### 📚 Documentation
|
|
17
|
+
|
|
18
|
+
- Add pydoc configurations for Docusaurus (#2411)
|
|
19
|
+
|
|
20
|
+
### ⚙️ CI
|
|
21
|
+
|
|
22
|
+
- Change pytest command (#2475)
|
|
23
|
+
|
|
24
|
+
### 🧹 Chores
|
|
25
|
+
|
|
26
|
+
- Remove black (#1985)
|
|
27
|
+
- Standardize readmes - part 1 (#2202)
|
|
28
|
+
- Fix linting fastembed (#2254)
|
|
29
|
+
- Remove Readme API CI workflow and configs (#2573)
|
|
30
|
+
- Make fmt command more forgiving (#2671)
|
|
31
|
+
|
|
32
|
+
### 🌀 Miscellaneous
|
|
33
|
+
|
|
34
|
+
- Adopt PEP 585 type hinting (part 3) (#2510)
|
|
35
|
+
|
|
3
36
|
## [integrations/fastembed-v1.5.0] - 2025-06-17
|
|
4
37
|
|
|
5
38
|
### 🐛 Bug Fixes
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: fastembed-haystack
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 2.1.0
|
|
4
4
|
Summary: Haystack 2.x component to embed strings and Documents using fastembed embedding model
|
|
5
5
|
Project-URL: Source, https://github.com/deepset-ai/haystack-core-integrations
|
|
6
6
|
Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/blob/main/integrations/fastembed/README.md
|
|
@@ -11,16 +11,15 @@ License-File: LICENSE.txt
|
|
|
11
11
|
Classifier: Development Status :: 4 - Beta
|
|
12
12
|
Classifier: License :: OSI Approved :: Apache Software License
|
|
13
13
|
Classifier: Programming Language :: Python
|
|
14
|
-
Classifier: Programming Language :: Python :: 3.9
|
|
15
14
|
Classifier: Programming Language :: Python :: 3.10
|
|
16
15
|
Classifier: Programming Language :: Python :: 3.11
|
|
17
16
|
Classifier: Programming Language :: Python :: 3.12
|
|
18
17
|
Classifier: Programming Language :: Python :: 3.13
|
|
19
18
|
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
20
19
|
Classifier: Programming Language :: Python :: Implementation :: PyPy
|
|
21
|
-
Requires-Python: >=3.
|
|
20
|
+
Requires-Python: >=3.10
|
|
22
21
|
Requires-Dist: fastembed>=0.4.2
|
|
23
|
-
Requires-Dist: haystack-ai>=2.0
|
|
22
|
+
Requires-Dist: haystack-ai>=2.22.0
|
|
24
23
|
Description-Content-Type: text/markdown
|
|
25
24
|
|
|
26
25
|
# fastembed-haystack
|
|
@@ -7,7 +7,7 @@ name = "fastembed-haystack"
|
|
|
7
7
|
dynamic = ["version"]
|
|
8
8
|
description = "Haystack 2.x component to embed strings and Documents using fastembed embedding model"
|
|
9
9
|
readme = "README.md"
|
|
10
|
-
requires-python = ">=3.
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
11
|
license = "Apache-2.0"
|
|
12
12
|
keywords = []
|
|
13
13
|
authors = [{ name = "deepset GmbH", email = "info@deepset.ai" }]
|
|
@@ -15,7 +15,6 @@ classifiers = [
|
|
|
15
15
|
"License :: OSI Approved :: Apache Software License",
|
|
16
16
|
"Development Status :: 4 - Beta",
|
|
17
17
|
"Programming Language :: Python",
|
|
18
|
-
"Programming Language :: Python :: 3.9",
|
|
19
18
|
"Programming Language :: Python :: 3.10",
|
|
20
19
|
"Programming Language :: Python :: 3.11",
|
|
21
20
|
"Programming Language :: Python :: 3.12",
|
|
@@ -23,7 +22,7 @@ classifiers = [
|
|
|
23
22
|
"Programming Language :: Python :: Implementation :: CPython",
|
|
24
23
|
"Programming Language :: Python :: Implementation :: PyPy",
|
|
25
24
|
]
|
|
26
|
-
dependencies = ["haystack-ai>=2.0
|
|
25
|
+
dependencies = ["haystack-ai>=2.22.0", "fastembed>=0.4.2"]
|
|
27
26
|
|
|
28
27
|
[project.urls]
|
|
29
28
|
Source = "https://github.com/deepset-ai/haystack-core-integrations"
|
|
@@ -75,7 +74,6 @@ check_untyped_defs = true
|
|
|
75
74
|
disallow_incomplete_defs = true
|
|
76
75
|
|
|
77
76
|
[tool.ruff]
|
|
78
|
-
target-version = "py39"
|
|
79
77
|
line-length = 120
|
|
80
78
|
|
|
81
79
|
[tool.ruff.lint]
|
|
@@ -124,10 +122,6 @@ ignore = [
|
|
|
124
122
|
"PLR0913",
|
|
125
123
|
"PLR0915",
|
|
126
124
|
]
|
|
127
|
-
unfixable = [
|
|
128
|
-
# Don't touch unused imports
|
|
129
|
-
"F401",
|
|
130
|
-
]
|
|
131
125
|
|
|
132
126
|
[tool.ruff.lint.isort]
|
|
133
127
|
known-first-party = ["haystack_integrations"]
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
from typing import Any, ClassVar
|
|
1
|
+
from typing import Any, ClassVar
|
|
2
2
|
|
|
3
3
|
from haystack.dataclasses.sparse_embedding import SparseEmbedding
|
|
4
4
|
from tqdm import tqdm
|
|
@@ -17,8 +17,8 @@ class _FastembedEmbeddingBackendFactory:
|
|
|
17
17
|
@staticmethod
|
|
18
18
|
def get_embedding_backend(
|
|
19
19
|
model_name: str,
|
|
20
|
-
cache_dir:
|
|
21
|
-
threads:
|
|
20
|
+
cache_dir: str | None = None,
|
|
21
|
+
threads: int | None = None,
|
|
22
22
|
local_files_only: bool = False,
|
|
23
23
|
) -> "_FastembedEmbeddingBackend":
|
|
24
24
|
embedding_backend_id = f"{model_name}{cache_dir}{threads}"
|
|
@@ -41,8 +41,8 @@ class _FastembedEmbeddingBackend:
|
|
|
41
41
|
def __init__(
|
|
42
42
|
self,
|
|
43
43
|
model_name: str,
|
|
44
|
-
cache_dir:
|
|
45
|
-
threads:
|
|
44
|
+
cache_dir: str | None = None,
|
|
45
|
+
threads: int | None = None,
|
|
46
46
|
local_files_only: bool = False,
|
|
47
47
|
):
|
|
48
48
|
self.model = TextEmbedding(
|
|
@@ -70,10 +70,10 @@ class _FastembedSparseEmbeddingBackendFactory:
|
|
|
70
70
|
@staticmethod
|
|
71
71
|
def get_embedding_backend(
|
|
72
72
|
model_name: str,
|
|
73
|
-
cache_dir:
|
|
74
|
-
threads:
|
|
73
|
+
cache_dir: str | None = None,
|
|
74
|
+
threads: int | None = None,
|
|
75
75
|
local_files_only: bool = False,
|
|
76
|
-
model_kwargs:
|
|
76
|
+
model_kwargs: dict[str, Any] | None = None,
|
|
77
77
|
) -> "_FastembedSparseEmbeddingBackend":
|
|
78
78
|
embedding_backend_id = f"{model_name}{cache_dir}{threads}{local_files_only}{model_kwargs}"
|
|
79
79
|
|
|
@@ -99,10 +99,10 @@ class _FastembedSparseEmbeddingBackend:
|
|
|
99
99
|
def __init__(
|
|
100
100
|
self,
|
|
101
101
|
model_name: str,
|
|
102
|
-
cache_dir:
|
|
103
|
-
threads:
|
|
102
|
+
cache_dir: str | None = None,
|
|
103
|
+
threads: int | None = None,
|
|
104
104
|
local_files_only: bool = False,
|
|
105
|
-
model_kwargs:
|
|
105
|
+
model_kwargs: dict[str, Any] | None = None,
|
|
106
106
|
):
|
|
107
107
|
model_kwargs = model_kwargs or {}
|
|
108
108
|
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
# SPDX-License-Identifier: Apache-2.0
|
|
4
4
|
|
|
5
5
|
from dataclasses import replace
|
|
6
|
-
from typing import Any
|
|
6
|
+
from typing import Any
|
|
7
7
|
|
|
8
8
|
from haystack import Document, component, default_to_dict
|
|
9
9
|
|
|
@@ -29,8 +29,6 @@ class FastembedDocumentEmbedder:
|
|
|
29
29
|
batch_size=256,
|
|
30
30
|
)
|
|
31
31
|
|
|
32
|
-
doc_embedder.warm_up()
|
|
33
|
-
|
|
34
32
|
# Text taken from PubMed QA Dataset (https://huggingface.co/datasets/pubmed_qa)
|
|
35
33
|
document_list = [
|
|
36
34
|
Document(
|
|
@@ -63,15 +61,15 @@ class FastembedDocumentEmbedder:
|
|
|
63
61
|
def __init__(
|
|
64
62
|
self,
|
|
65
63
|
model: str = "BAAI/bge-small-en-v1.5",
|
|
66
|
-
cache_dir:
|
|
67
|
-
threads:
|
|
64
|
+
cache_dir: str | None = None,
|
|
65
|
+
threads: int | None = None,
|
|
68
66
|
prefix: str = "",
|
|
69
67
|
suffix: str = "",
|
|
70
68
|
batch_size: int = 256,
|
|
71
69
|
progress_bar: bool = True,
|
|
72
|
-
parallel:
|
|
70
|
+
parallel: int | None = None,
|
|
73
71
|
local_files_only: bool = False,
|
|
74
|
-
meta_fields_to_embed:
|
|
72
|
+
meta_fields_to_embed: list[str] | None = None,
|
|
75
73
|
embedding_separator: str = "\n",
|
|
76
74
|
) -> None:
|
|
77
75
|
"""
|
|
@@ -107,7 +105,7 @@ class FastembedDocumentEmbedder:
|
|
|
107
105
|
self.local_files_only = local_files_only
|
|
108
106
|
self.meta_fields_to_embed = meta_fields_to_embed or []
|
|
109
107
|
self.embedding_separator = embedding_separator
|
|
110
|
-
self.embedding_backend:
|
|
108
|
+
self.embedding_backend: _FastembedEmbeddingBackend | None = None
|
|
111
109
|
|
|
112
110
|
def to_dict(self) -> dict[str, Any]:
|
|
113
111
|
"""
|
|
@@ -184,7 +182,7 @@ class FastembedDocumentEmbedder:
|
|
|
184
182
|
)
|
|
185
183
|
|
|
186
184
|
new_documents = []
|
|
187
|
-
for doc, emb in zip(documents, embeddings):
|
|
185
|
+
for doc, emb in zip(documents, embeddings, strict=True):
|
|
188
186
|
new_documents.append(replace(doc, embedding=emb))
|
|
189
187
|
|
|
190
188
|
return {"documents": new_documents}
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
# SPDX-License-Identifier: Apache-2.0
|
|
4
4
|
|
|
5
5
|
from dataclasses import replace
|
|
6
|
-
from typing import Any
|
|
6
|
+
from typing import Any
|
|
7
7
|
|
|
8
8
|
from haystack import Document, component, default_to_dict
|
|
9
9
|
|
|
@@ -28,8 +28,6 @@ class FastembedSparseDocumentEmbedder:
|
|
|
28
28
|
batch_size=32,
|
|
29
29
|
)
|
|
30
30
|
|
|
31
|
-
sparse_doc_embedder.warm_up()
|
|
32
|
-
|
|
33
31
|
# Text taken from PubMed QA Dataset (https://huggingface.co/datasets/pubmed_qa)
|
|
34
32
|
document_list = [
|
|
35
33
|
Document(
|
|
@@ -62,15 +60,15 @@ class FastembedSparseDocumentEmbedder:
|
|
|
62
60
|
def __init__(
|
|
63
61
|
self,
|
|
64
62
|
model: str = "prithivida/Splade_PP_en_v1",
|
|
65
|
-
cache_dir:
|
|
66
|
-
threads:
|
|
63
|
+
cache_dir: str | None = None,
|
|
64
|
+
threads: int | None = None,
|
|
67
65
|
batch_size: int = 32,
|
|
68
66
|
progress_bar: bool = True,
|
|
69
|
-
parallel:
|
|
67
|
+
parallel: int | None = None,
|
|
70
68
|
local_files_only: bool = False,
|
|
71
|
-
meta_fields_to_embed:
|
|
69
|
+
meta_fields_to_embed: list[str] | None = None,
|
|
72
70
|
embedding_separator: str = "\n",
|
|
73
|
-
model_kwargs:
|
|
71
|
+
model_kwargs: dict[str, Any] | None = None,
|
|
74
72
|
) -> None:
|
|
75
73
|
"""
|
|
76
74
|
Create an FastembedDocumentEmbedder component.
|
|
@@ -103,7 +101,7 @@ class FastembedSparseDocumentEmbedder:
|
|
|
103
101
|
self.meta_fields_to_embed = meta_fields_to_embed or []
|
|
104
102
|
self.embedding_separator = embedding_separator
|
|
105
103
|
self.model_kwargs = model_kwargs
|
|
106
|
-
self.embedding_backend:
|
|
104
|
+
self.embedding_backend: _FastembedSparseEmbeddingBackend | None = None
|
|
107
105
|
|
|
108
106
|
def to_dict(self) -> dict[str, Any]:
|
|
109
107
|
"""
|
|
@@ -179,7 +177,7 @@ class FastembedSparseDocumentEmbedder:
|
|
|
179
177
|
)
|
|
180
178
|
|
|
181
179
|
new_documents = []
|
|
182
|
-
for doc, emb in zip(documents, embeddings):
|
|
180
|
+
for doc, emb in zip(documents, embeddings, strict=True):
|
|
183
181
|
new_documents.append(replace(doc, sparse_embedding=emb))
|
|
184
182
|
|
|
185
183
|
return {"documents": new_documents}
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
#
|
|
3
3
|
# SPDX-License-Identifier: Apache-2.0
|
|
4
4
|
|
|
5
|
-
from typing import Any
|
|
5
|
+
from typing import Any
|
|
6
6
|
|
|
7
7
|
from haystack import component, default_to_dict
|
|
8
8
|
from haystack.dataclasses.sparse_embedding import SparseEmbedding
|
|
@@ -28,7 +28,6 @@ class FastembedSparseTextEmbedder:
|
|
|
28
28
|
sparse_text_embedder = FastembedSparseTextEmbedder(
|
|
29
29
|
model="prithivida/Splade_PP_en_v1"
|
|
30
30
|
)
|
|
31
|
-
sparse_text_embedder.warm_up()
|
|
32
31
|
|
|
33
32
|
sparse_embedding = sparse_text_embedder.run(text)["sparse_embedding"]
|
|
34
33
|
```
|
|
@@ -37,12 +36,12 @@ class FastembedSparseTextEmbedder:
|
|
|
37
36
|
def __init__(
|
|
38
37
|
self,
|
|
39
38
|
model: str = "prithivida/Splade_PP_en_v1",
|
|
40
|
-
cache_dir:
|
|
41
|
-
threads:
|
|
39
|
+
cache_dir: str | None = None,
|
|
40
|
+
threads: int | None = None,
|
|
42
41
|
progress_bar: bool = True,
|
|
43
|
-
parallel:
|
|
42
|
+
parallel: int | None = None,
|
|
44
43
|
local_files_only: bool = False,
|
|
45
|
-
model_kwargs:
|
|
44
|
+
model_kwargs: dict[str, Any] | None = None,
|
|
46
45
|
) -> None:
|
|
47
46
|
"""
|
|
48
47
|
Create a FastembedSparseTextEmbedder component.
|
|
@@ -68,7 +67,7 @@ class FastembedSparseTextEmbedder:
|
|
|
68
67
|
self.parallel = parallel
|
|
69
68
|
self.local_files_only = local_files_only
|
|
70
69
|
self.model_kwargs = model_kwargs
|
|
71
|
-
self.embedding_backend:
|
|
70
|
+
self.embedding_backend: _FastembedSparseEmbeddingBackend | None = None
|
|
72
71
|
|
|
73
72
|
def to_dict(self) -> dict[str, Any]:
|
|
74
73
|
"""
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
#
|
|
3
3
|
# SPDX-License-Identifier: Apache-2.0
|
|
4
4
|
|
|
5
|
-
from typing import Any
|
|
5
|
+
from typing import Any
|
|
6
6
|
|
|
7
7
|
from haystack import component, default_to_dict
|
|
8
8
|
|
|
@@ -24,7 +24,6 @@ class FastembedTextEmbedder:
|
|
|
24
24
|
text_embedder = FastembedTextEmbedder(
|
|
25
25
|
model="BAAI/bge-small-en-v1.5"
|
|
26
26
|
)
|
|
27
|
-
text_embedder.warm_up()
|
|
28
27
|
|
|
29
28
|
embedding = text_embedder.run(text)["embedding"]
|
|
30
29
|
```
|
|
@@ -33,12 +32,12 @@ class FastembedTextEmbedder:
|
|
|
33
32
|
def __init__(
|
|
34
33
|
self,
|
|
35
34
|
model: str = "BAAI/bge-small-en-v1.5",
|
|
36
|
-
cache_dir:
|
|
37
|
-
threads:
|
|
35
|
+
cache_dir: str | None = None,
|
|
36
|
+
threads: int | None = None,
|
|
38
37
|
prefix: str = "",
|
|
39
38
|
suffix: str = "",
|
|
40
39
|
progress_bar: bool = True,
|
|
41
|
-
parallel:
|
|
40
|
+
parallel: int | None = None,
|
|
42
41
|
local_files_only: bool = False,
|
|
43
42
|
) -> None:
|
|
44
43
|
"""
|
|
@@ -67,7 +66,7 @@ class FastembedTextEmbedder:
|
|
|
67
66
|
self.progress_bar = progress_bar
|
|
68
67
|
self.parallel = parallel
|
|
69
68
|
self.local_files_only = local_files_only
|
|
70
|
-
self.embedding_backend:
|
|
69
|
+
self.embedding_backend: _FastembedEmbeddingBackend | None = None
|
|
71
70
|
|
|
72
71
|
def to_dict(self) -> dict[str, Any]:
|
|
73
72
|
"""
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
#
|
|
3
3
|
# SPDX-License-Identifier: Apache-2.0
|
|
4
4
|
|
|
5
|
-
from typing import Any
|
|
5
|
+
from typing import Any
|
|
6
6
|
|
|
7
7
|
from haystack import Document, component, default_from_dict, default_to_dict, logging
|
|
8
8
|
|
|
@@ -39,12 +39,12 @@ class FastembedRanker:
|
|
|
39
39
|
self,
|
|
40
40
|
model_name: str = "Xenova/ms-marco-MiniLM-L-6-v2",
|
|
41
41
|
top_k: int = 10,
|
|
42
|
-
cache_dir:
|
|
43
|
-
threads:
|
|
42
|
+
cache_dir: str | None = None,
|
|
43
|
+
threads: int | None = None,
|
|
44
44
|
batch_size: int = 64,
|
|
45
|
-
parallel:
|
|
45
|
+
parallel: int | None = None,
|
|
46
46
|
local_files_only: bool = False,
|
|
47
|
-
meta_fields_to_embed:
|
|
47
|
+
meta_fields_to_embed: list[str] | None = None,
|
|
48
48
|
meta_data_separator: str = "\n",
|
|
49
49
|
):
|
|
50
50
|
"""
|
|
@@ -80,7 +80,7 @@ class FastembedRanker:
|
|
|
80
80
|
self.local_files_only = local_files_only
|
|
81
81
|
self.meta_fields_to_embed = meta_fields_to_embed or []
|
|
82
82
|
self.meta_data_separator = meta_data_separator
|
|
83
|
-
self._model:
|
|
83
|
+
self._model: TextCrossEncoder | None = None
|
|
84
84
|
|
|
85
85
|
def to_dict(self) -> dict[str, Any]:
|
|
86
86
|
"""
|
|
@@ -144,7 +144,7 @@ class FastembedRanker:
|
|
|
144
144
|
return concatenated_input_list
|
|
145
145
|
|
|
146
146
|
@component.output_types(documents=list[Document])
|
|
147
|
-
def run(self, query: str, documents: list[Document], top_k:
|
|
147
|
+
def run(self, query: str, documents: list[Document], top_k: int | None = None) -> dict[str, list[Document]]:
|
|
148
148
|
"""
|
|
149
149
|
Returns a list of documents ranked by their similarity to the given query, using FastEmbed.
|
|
150
150
|
|
|
@@ -177,13 +177,12 @@ class FastembedRanker:
|
|
|
177
177
|
raise ValueError(msg)
|
|
178
178
|
|
|
179
179
|
if self._model is None:
|
|
180
|
-
|
|
181
|
-
raise RuntimeError(msg)
|
|
180
|
+
self.warm_up()
|
|
182
181
|
|
|
183
182
|
fastembed_input_docs = self._prepare_fastembed_input_docs(documents)
|
|
184
183
|
|
|
185
184
|
scores = list(
|
|
186
|
-
self._model.rerank(
|
|
185
|
+
self._model.rerank( # type: ignore[union-attr]
|
|
187
186
|
query=query,
|
|
188
187
|
documents=fastembed_input_docs,
|
|
189
188
|
batch_size=self.batch_size,
|
|
@@ -192,7 +191,7 @@ class FastembedRanker:
|
|
|
192
191
|
)
|
|
193
192
|
|
|
194
193
|
# Combine the two lists into a single list of tuples
|
|
195
|
-
doc_scores = list(zip(documents, scores))
|
|
194
|
+
doc_scores = list(zip(documents, scores, strict=True))
|
|
196
195
|
|
|
197
196
|
# Sort the list of tuples by the score in descending order
|
|
198
197
|
sorted_doc_scores = sorted(doc_scores, key=lambda x: x[1], reverse=True)
|
{fastembed_haystack-1.6.0 → fastembed_haystack-2.1.0}/tests/test_fastembed_document_embedder.py
RENAMED
|
@@ -262,6 +262,7 @@ class TestFastembedDocumentEmbedder:
|
|
|
262
262
|
embedding_separator="\n",
|
|
263
263
|
)
|
|
264
264
|
embedder.embedding_backend = MagicMock()
|
|
265
|
+
embedder.embedding_backend.embed.return_value = [np.random.rand(3, 16).tolist() for _ in range(5)]
|
|
265
266
|
|
|
266
267
|
documents = [Document(content=f"document-number {i}", meta={"meta_field": f"meta_value {i}"}) for i in range(5)]
|
|
267
268
|
|
|
@@ -280,12 +281,25 @@ class TestFastembedDocumentEmbedder:
|
|
|
280
281
|
parallel=None,
|
|
281
282
|
)
|
|
282
283
|
|
|
284
|
+
def test_run_calls_warm_up(self):
|
|
285
|
+
embedder = FastembedDocumentEmbedder()
|
|
286
|
+
assert embedder.embedding_backend is None
|
|
287
|
+
|
|
288
|
+
mock_backend = MagicMock()
|
|
289
|
+
mock_backend.embed.return_value = [[0.1, 0.2, 0.3]]
|
|
290
|
+
|
|
291
|
+
with patch.object(
|
|
292
|
+
embedder, "warm_up", side_effect=lambda: setattr(embedder, "embedding_backend", mock_backend)
|
|
293
|
+
) as mock_warm_up:
|
|
294
|
+
embedder.run(documents=[Document(content="test document")])
|
|
295
|
+
|
|
296
|
+
mock_warm_up.assert_called_once()
|
|
297
|
+
|
|
283
298
|
@pytest.mark.integration
|
|
284
299
|
def test_run(self):
|
|
285
300
|
embedder = FastembedDocumentEmbedder(
|
|
286
301
|
model="BAAI/bge-small-en-v1.5",
|
|
287
302
|
)
|
|
288
|
-
embedder.warm_up()
|
|
289
303
|
|
|
290
304
|
doc = Document(content="Parton energy loss in QCD matter")
|
|
291
305
|
|
|
@@ -2,8 +2,9 @@
|
|
|
2
2
|
#
|
|
3
3
|
# SPDX-License-Identifier: Apache-2.0
|
|
4
4
|
|
|
5
|
-
from unittest.mock import MagicMock
|
|
5
|
+
from unittest.mock import MagicMock, patch
|
|
6
6
|
|
|
7
|
+
import numpy as np
|
|
7
8
|
import pytest
|
|
8
9
|
from haystack import Document, default_from_dict
|
|
9
10
|
|
|
@@ -237,6 +238,7 @@ class TestFastembedRanker:
|
|
|
237
238
|
meta_fields_to_embed=["meta_field"],
|
|
238
239
|
)
|
|
239
240
|
ranker._model = MagicMock()
|
|
241
|
+
ranker._model.rerank.return_value = [np.random.rand(3, 16).tolist() for _ in range(5)]
|
|
240
242
|
|
|
241
243
|
documents = [Document(content=f"document-number {i}", meta={"meta_field": f"meta_value {i}"}) for i in range(5)]
|
|
242
244
|
query = "test"
|
|
@@ -255,10 +257,23 @@ class TestFastembedRanker:
|
|
|
255
257
|
parallel=None,
|
|
256
258
|
)
|
|
257
259
|
|
|
260
|
+
def test_run_calls_warm_up(self):
|
|
261
|
+
"""
|
|
262
|
+
Unit test to check that warm_up is called when run is called for the first time.
|
|
263
|
+
"""
|
|
264
|
+
ranker = FastembedRanker()
|
|
265
|
+
|
|
266
|
+
mock_model = MagicMock()
|
|
267
|
+
mock_model.rerank.return_value = [0.5]
|
|
268
|
+
|
|
269
|
+
with patch.object(ranker, "warm_up", side_effect=lambda: setattr(ranker, "_model", mock_model)) as mock_warm_up:
|
|
270
|
+
ranker.run(query="test query", documents=[Document(content="test document")])
|
|
271
|
+
|
|
272
|
+
mock_warm_up.assert_called_once()
|
|
273
|
+
|
|
258
274
|
@pytest.mark.integration
|
|
259
275
|
def test_run(self):
|
|
260
276
|
ranker = FastembedRanker(model_name="Xenova/ms-marco-MiniLM-L-6-v2", top_k=2)
|
|
261
|
-
ranker.warm_up()
|
|
262
277
|
|
|
263
278
|
query = "Who is maintaining Qdrant?"
|
|
264
279
|
documents = [
|
|
@@ -267,7 +267,7 @@ class TestFastembedSparseDocumentEmbedderDoc:
|
|
|
267
267
|
embedding_separator="\n",
|
|
268
268
|
)
|
|
269
269
|
embedder.embedding_backend = MagicMock()
|
|
270
|
-
|
|
270
|
+
embedder.embedding_backend.embed.return_value = [self._generate_mocked_sparse_embedding(3) for _ in range(5)]
|
|
271
271
|
documents = [Document(content=f"document-number {i}", meta={"meta_field": f"meta_value {i}"}) for i in range(5)]
|
|
272
272
|
|
|
273
273
|
embedder.run(documents=documents)
|
|
@@ -304,6 +304,19 @@ class TestFastembedSparseDocumentEmbedderDoc:
|
|
|
304
304
|
|
|
305
305
|
assert embedder.model_kwargs == bm25_config
|
|
306
306
|
|
|
307
|
+
def test_run_calls_warm_up(self):
|
|
308
|
+
embedder = FastembedSparseDocumentEmbedder()
|
|
309
|
+
|
|
310
|
+
mock_backend = MagicMock()
|
|
311
|
+
mock_backend.embed.return_value = [{"indices": [0], "values": [0.5]}]
|
|
312
|
+
|
|
313
|
+
with patch.object(
|
|
314
|
+
embedder, "warm_up", side_effect=lambda: setattr(embedder, "embedding_backend", mock_backend)
|
|
315
|
+
) as mock_warm_up:
|
|
316
|
+
embedder.run(documents=[Document(content="test document")])
|
|
317
|
+
|
|
318
|
+
mock_warm_up.assert_called_once()
|
|
319
|
+
|
|
307
320
|
@pytest.mark.integration
|
|
308
321
|
def test_run_with_model_kwargs(self):
|
|
309
322
|
"""
|
|
@@ -317,7 +330,6 @@ class TestFastembedSparseDocumentEmbedderDoc:
|
|
|
317
330
|
model="Qdrant/bm42-all-minilm-l6-v2-attentions",
|
|
318
331
|
model_kwargs=bm42_config,
|
|
319
332
|
)
|
|
320
|
-
embedder.warm_up()
|
|
321
333
|
|
|
322
334
|
doc = Document(content="Example content using BM42")
|
|
323
335
|
|
|
@@ -336,7 +348,6 @@ class TestFastembedSparseDocumentEmbedderDoc:
|
|
|
336
348
|
embedder = FastembedSparseDocumentEmbedder(
|
|
337
349
|
model="prithivida/Splade_PP_en_v1",
|
|
338
350
|
)
|
|
339
|
-
embedder.warm_up()
|
|
340
351
|
|
|
341
352
|
doc = Document(content="Parton energy loss in QCD matter")
|
|
342
353
|
|
{fastembed_haystack-1.6.0 → fastembed_haystack-2.1.0}/tests/test_fastembed_sparse_text_embedder.py
RENAMED
|
@@ -224,6 +224,20 @@ class TestFastembedSparseTextEmbedder:
|
|
|
224
224
|
|
|
225
225
|
assert embedder.model_kwargs == bm25_config
|
|
226
226
|
|
|
227
|
+
def test_run_calls_warm_up(self):
|
|
228
|
+
embedder = FastembedSparseTextEmbedder()
|
|
229
|
+
assert embedder.embedding_backend is None
|
|
230
|
+
|
|
231
|
+
mock_backend = MagicMock()
|
|
232
|
+
mock_backend.embed.return_value = [[0.1, 0.2, 0.3]]
|
|
233
|
+
|
|
234
|
+
with patch.object(
|
|
235
|
+
embedder, "warm_up", side_effect=lambda: setattr(embedder, "embedding_backend", mock_backend)
|
|
236
|
+
) as mock_warm_up:
|
|
237
|
+
embedder.run(text="test text")
|
|
238
|
+
|
|
239
|
+
mock_warm_up.assert_called_once()
|
|
240
|
+
|
|
227
241
|
@pytest.mark.integration
|
|
228
242
|
def test_run_with_model_kwargs(self):
|
|
229
243
|
"""
|
|
@@ -258,7 +272,6 @@ class TestFastembedSparseTextEmbedder:
|
|
|
258
272
|
embedder = FastembedSparseTextEmbedder(
|
|
259
273
|
model="prithivida/Splade_PP_en_v1",
|
|
260
274
|
)
|
|
261
|
-
embedder.warm_up()
|
|
262
275
|
|
|
263
276
|
text = "Parton energy loss in QCD matter"
|
|
264
277
|
|
|
@@ -202,12 +202,25 @@ class TestFastembedTextEmbedder:
|
|
|
202
202
|
with pytest.raises(TypeError, match="FastembedTextEmbedder expects a string as input"):
|
|
203
203
|
embedder.run(text=list_integers_input)
|
|
204
204
|
|
|
205
|
+
def test_run_calls_warm_up(self):
|
|
206
|
+
embedder = FastembedTextEmbedder()
|
|
207
|
+
assert embedder.embedding_backend is None
|
|
208
|
+
|
|
209
|
+
mock_backend = MagicMock()
|
|
210
|
+
mock_backend.embed.return_value = [[0.1, 0.2, 0.3]]
|
|
211
|
+
|
|
212
|
+
with patch.object(
|
|
213
|
+
embedder, "warm_up", side_effect=lambda: setattr(embedder, "embedding_backend", mock_backend)
|
|
214
|
+
) as mock_warm_up:
|
|
215
|
+
embedder.run(text="test text")
|
|
216
|
+
|
|
217
|
+
mock_warm_up.assert_called_once()
|
|
218
|
+
|
|
205
219
|
@pytest.mark.integration
|
|
206
220
|
def test_run(self):
|
|
207
221
|
embedder = FastembedTextEmbedder(
|
|
208
222
|
model="BAAI/bge-small-en-v1.5",
|
|
209
223
|
)
|
|
210
|
-
embedder.warm_up()
|
|
211
224
|
|
|
212
225
|
text = "Parton energy loss in QCD matter"
|
|
213
226
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|