fastembed-haystack 1.5.0__tar.gz → 1.6.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {fastembed_haystack-1.5.0 → fastembed_haystack-1.6.0}/CHANGELOG.md +17 -0
- {fastembed_haystack-1.5.0 → fastembed_haystack-1.6.0}/LICENSE.txt +1 -1
- fastembed_haystack-1.6.0/PKG-INFO +38 -0
- fastembed_haystack-1.6.0/README.md +13 -0
- fastembed_haystack-1.6.0/pydoc/config_docusaurus.yml +32 -0
- {fastembed_haystack-1.5.0 → fastembed_haystack-1.6.0}/pyproject.toml +4 -9
- {fastembed_haystack-1.5.0 → fastembed_haystack-1.6.0}/src/haystack_integrations/components/embedders/fastembed/__init__.py +1 -0
- {fastembed_haystack-1.5.0 → fastembed_haystack-1.6.0}/src/haystack_integrations/components/embedders/fastembed/embedding_backend/fastembed_backend.py +7 -7
- {fastembed_haystack-1.5.0 → fastembed_haystack-1.6.0}/src/haystack_integrations/components/embedders/fastembed/fastembed_document_embedder.py +24 -16
- {fastembed_haystack-1.5.0 → fastembed_haystack-1.6.0}/src/haystack_integrations/components/embedders/fastembed/fastembed_sparse_document_embedder.py +29 -17
- {fastembed_haystack-1.5.0 → fastembed_haystack-1.6.0}/src/haystack_integrations/components/embedders/fastembed/fastembed_sparse_text_embedder.py +20 -13
- {fastembed_haystack-1.5.0 → fastembed_haystack-1.6.0}/src/haystack_integrations/components/embedders/fastembed/fastembed_text_embedder.py +17 -13
- fastembed_haystack-1.6.0/src/haystack_integrations/components/rankers/fastembed/__init__.py +7 -0
- {fastembed_haystack-1.5.0 → fastembed_haystack-1.6.0}/src/haystack_integrations/components/rankers/fastembed/ranker.py +11 -7
- {fastembed_haystack-1.5.0 → fastembed_haystack-1.6.0}/tests/test_fastembed_backend.py +4 -0
- {fastembed_haystack-1.5.0 → fastembed_haystack-1.6.0}/tests/test_fastembed_document_embedder.py +6 -2
- {fastembed_haystack-1.5.0 → fastembed_haystack-1.6.0}/tests/test_fastembed_ranker.py +6 -16
- {fastembed_haystack-1.5.0 → fastembed_haystack-1.6.0}/tests/test_fastembed_sparse_document_embedder.py +6 -2
- {fastembed_haystack-1.5.0 → fastembed_haystack-1.6.0}/tests/test_fastembed_sparse_text_embedder.py +4 -0
- {fastembed_haystack-1.5.0 → fastembed_haystack-1.6.0}/tests/test_fastembed_text_embedder.py +4 -0
- fastembed_haystack-1.5.0/PKG-INFO +0 -125
- fastembed_haystack-1.5.0/README.md +0 -100
- fastembed_haystack-1.5.0/pydoc/config.yml +0 -34
- fastembed_haystack-1.5.0/src/haystack_integrations/components/rankers/fastembed/__init__.py +0 -3
- {fastembed_haystack-1.5.0 → fastembed_haystack-1.6.0}/.gitignore +0 -0
- {fastembed_haystack-1.5.0 → fastembed_haystack-1.6.0}/examples/example.py +0 -0
- {fastembed_haystack-1.5.0 → fastembed_haystack-1.6.0}/examples/ranker_example.py +0 -0
- {fastembed_haystack-1.5.0 → fastembed_haystack-1.6.0}/examples/sparse_example.py +0 -0
- {fastembed_haystack-1.5.0 → fastembed_haystack-1.6.0}/src/haystack_integrations/components/embedders/fastembed/embedding_backend/__init__.py +0 -0
- {fastembed_haystack-1.5.0 → fastembed_haystack-1.6.0}/src/haystack_integrations/components/embedders/py.typed +0 -0
- {fastembed_haystack-1.5.0 → fastembed_haystack-1.6.0}/src/haystack_integrations/components/rankers/py.typed +0 -0
- {fastembed_haystack-1.5.0 → fastembed_haystack-1.6.0}/tests/__init__.py +0 -0
|
@@ -1,5 +1,22 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [integrations/fastembed-v1.5.0] - 2025-06-17
|
|
4
|
+
|
|
5
|
+
### 🐛 Bug Fixes
|
|
6
|
+
|
|
7
|
+
- Fix Fastembed types + add py.typed (#1954)
|
|
8
|
+
|
|
9
|
+
### ⚙️ CI
|
|
10
|
+
|
|
11
|
+
- Review testing workflows (#1541)
|
|
12
|
+
|
|
13
|
+
### 🧹 Chores
|
|
14
|
+
|
|
15
|
+
- Fix linting/isort (#1215)
|
|
16
|
+
- Remove Python 3.8 support (#1421)
|
|
17
|
+
- Align core-integrations Hatch scripts (#1898)
|
|
18
|
+
|
|
19
|
+
|
|
3
20
|
## [integrations/fastembed-v1.4.1] - 2024-11-19
|
|
4
21
|
|
|
5
22
|
### 🌀 Miscellaneous
|
|
@@ -58,7 +58,7 @@ APPENDIX: How to apply the Apache License to your work.
|
|
|
58
58
|
|
|
59
59
|
To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives.
|
|
60
60
|
|
|
61
|
-
Copyright
|
|
61
|
+
Copyright 2024 deepset GmbH
|
|
62
62
|
|
|
63
63
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
64
64
|
you may not use this file except in compliance with the License.
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: fastembed-haystack
|
|
3
|
+
Version: 1.6.0
|
|
4
|
+
Summary: Haystack 2.x component to embed strings and Documents using fastembed embedding model
|
|
5
|
+
Project-URL: Source, https://github.com/deepset-ai/haystack-core-integrations
|
|
6
|
+
Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/blob/main/integrations/fastembed/README.md
|
|
7
|
+
Project-URL: Issues, https://github.com/deepset-ai/haystack-core-integrations/issues
|
|
8
|
+
Author-email: deepset GmbH <info@deepset.ai>
|
|
9
|
+
License-Expression: Apache-2.0
|
|
10
|
+
License-File: LICENSE.txt
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
13
|
+
Classifier: Programming Language :: Python
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
+
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
20
|
+
Classifier: Programming Language :: Python :: Implementation :: PyPy
|
|
21
|
+
Requires-Python: >=3.9
|
|
22
|
+
Requires-Dist: fastembed>=0.4.2
|
|
23
|
+
Requires-Dist: haystack-ai>=2.0.1
|
|
24
|
+
Description-Content-Type: text/markdown
|
|
25
|
+
|
|
26
|
+
# fastembed-haystack
|
|
27
|
+
|
|
28
|
+
[](https://pypi.org/project/fastembed-haystack)
|
|
29
|
+
[](https://pypi.org/project/fastembed-haystack)
|
|
30
|
+
|
|
31
|
+
- [Integration page](https://haystack.deepset.ai/integrations/fastembed)
|
|
32
|
+
- [Changelog](https://github.com/deepset-ai/haystack-core-integrations/blob/main/integrations/fastembed/CHANGELOG.md)
|
|
33
|
+
|
|
34
|
+
---
|
|
35
|
+
|
|
36
|
+
## Contributing
|
|
37
|
+
|
|
38
|
+
Refer to the general [Contribution Guidelines](https://github.com/deepset-ai/haystack-core-integrations/blob/main/CONTRIBUTING.md).
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
# fastembed-haystack
|
|
2
|
+
|
|
3
|
+
[](https://pypi.org/project/fastembed-haystack)
|
|
4
|
+
[](https://pypi.org/project/fastembed-haystack)
|
|
5
|
+
|
|
6
|
+
- [Integration page](https://haystack.deepset.ai/integrations/fastembed)
|
|
7
|
+
- [Changelog](https://github.com/deepset-ai/haystack-core-integrations/blob/main/integrations/fastembed/CHANGELOG.md)
|
|
8
|
+
|
|
9
|
+
---
|
|
10
|
+
|
|
11
|
+
## Contributing
|
|
12
|
+
|
|
13
|
+
Refer to the general [Contribution Guidelines](https://github.com/deepset-ai/haystack-core-integrations/blob/main/CONTRIBUTING.md).
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
loaders:
|
|
2
|
+
- ignore_when_discovered:
|
|
3
|
+
- __init__
|
|
4
|
+
modules:
|
|
5
|
+
- haystack_integrations.components.embedders.fastembed.fastembed_document_embedder
|
|
6
|
+
- haystack_integrations.components.embedders.fastembed.fastembed_text_embedder
|
|
7
|
+
- haystack_integrations.components.embedders.fastembed.fastembed_sparse_document_embedder
|
|
8
|
+
- haystack_integrations.components.embedders.fastembed.fastembed_sparse_text_embedder
|
|
9
|
+
- haystack_integrations.components.rankers.fastembed.ranker
|
|
10
|
+
search_path:
|
|
11
|
+
- ../src
|
|
12
|
+
type: haystack_pydoc_tools.loaders.CustomPythonLoader
|
|
13
|
+
processors:
|
|
14
|
+
- do_not_filter_modules: false
|
|
15
|
+
documented_only: true
|
|
16
|
+
expression: null
|
|
17
|
+
skip_empty_modules: true
|
|
18
|
+
type: filter
|
|
19
|
+
- type: smart
|
|
20
|
+
- type: crossref
|
|
21
|
+
renderer:
|
|
22
|
+
description: FastEmbed integration for Haystack
|
|
23
|
+
id: fastembed-embedders
|
|
24
|
+
markdown:
|
|
25
|
+
add_member_class_prefix: false
|
|
26
|
+
add_method_class_prefix: true
|
|
27
|
+
classdef_code_block: false
|
|
28
|
+
descriptive_class_title: false
|
|
29
|
+
descriptive_module_title: true
|
|
30
|
+
filename: fastembed.md
|
|
31
|
+
title: FastEmbed
|
|
32
|
+
type: haystack_pydoc_tools.renderers.DocusaurusRenderer
|
|
@@ -45,8 +45,8 @@ git_describe_command = 'git describe --tags --match="integrations/fastembed-v[0-
|
|
|
45
45
|
installer = "uv"
|
|
46
46
|
dependencies = ["haystack-pydoc-tools", "ruff"]
|
|
47
47
|
[tool.hatch.envs.default.scripts]
|
|
48
|
-
docs = ["pydoc-markdown pydoc/
|
|
49
|
-
fmt = "ruff check --fix {args}
|
|
48
|
+
docs = ["pydoc-markdown pydoc/config_docusaurus.yml"]
|
|
49
|
+
fmt = "ruff check --fix {args}; ruff format {args}"
|
|
50
50
|
fmt-check = "ruff check {args} && ruff format --check {args}"
|
|
51
51
|
|
|
52
52
|
[tool.hatch.envs.test]
|
|
@@ -64,7 +64,7 @@ dependencies = [
|
|
|
64
64
|
unit = 'pytest -m "not integration" {args:tests}'
|
|
65
65
|
integration = 'pytest -m "integration" {args:tests}'
|
|
66
66
|
all = 'pytest {args:tests}'
|
|
67
|
-
cov-retry = '
|
|
67
|
+
cov-retry = 'pytest --cov=haystack_integrations --reruns 3 --reruns-delay 30 -x {args:tests}'
|
|
68
68
|
types = """mypy -p haystack_integrations.components.embedders.fastembed \
|
|
69
69
|
-p haystack_integrations.components.rankers.fastembed {args}"""
|
|
70
70
|
|
|
@@ -74,13 +74,8 @@ non_interactive = true
|
|
|
74
74
|
check_untyped_defs = true
|
|
75
75
|
disallow_incomplete_defs = true
|
|
76
76
|
|
|
77
|
-
[tool.black]
|
|
78
|
-
target-version = ["py38"]
|
|
79
|
-
line-length = 120
|
|
80
|
-
skip-string-normalization = true
|
|
81
|
-
|
|
82
77
|
[tool.ruff]
|
|
83
|
-
target-version = "
|
|
78
|
+
target-version = "py39"
|
|
84
79
|
line-length = 120
|
|
85
80
|
|
|
86
81
|
[tool.ruff.lint]
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# SPDX-FileCopyrightText: 2024-present deepset GmbH <info@deepset.ai>
|
|
2
2
|
#
|
|
3
3
|
# SPDX-License-Identifier: Apache-2.0
|
|
4
|
+
|
|
4
5
|
from .fastembed_document_embedder import FastembedDocumentEmbedder
|
|
5
6
|
from .fastembed_sparse_document_embedder import FastembedSparseDocumentEmbedder
|
|
6
7
|
from .fastembed_sparse_text_embedder import FastembedSparseTextEmbedder
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
from typing import Any, ClassVar,
|
|
1
|
+
from typing import Any, ClassVar, Optional
|
|
2
2
|
|
|
3
3
|
from haystack.dataclasses.sparse_embedding import SparseEmbedding
|
|
4
4
|
from tqdm import tqdm
|
|
@@ -12,7 +12,7 @@ class _FastembedEmbeddingBackendFactory:
|
|
|
12
12
|
Factory class to create instances of fastembed embedding backends.
|
|
13
13
|
"""
|
|
14
14
|
|
|
15
|
-
_instances: ClassVar[
|
|
15
|
+
_instances: ClassVar[dict[str, "_FastembedEmbeddingBackend"]] = {}
|
|
16
16
|
|
|
17
17
|
@staticmethod
|
|
18
18
|
def get_embedding_backend(
|
|
@@ -49,7 +49,7 @@ class _FastembedEmbeddingBackend:
|
|
|
49
49
|
model_name=model_name, cache_dir=cache_dir, threads=threads, local_files_only=local_files_only
|
|
50
50
|
)
|
|
51
51
|
|
|
52
|
-
def embed(self, data:
|
|
52
|
+
def embed(self, data: list[str], progress_bar: bool = True, **kwargs: Any) -> list[list[float]]:
|
|
53
53
|
# the embed method returns a Iterable[np.ndarray], so we convert it to a list of lists
|
|
54
54
|
embeddings = []
|
|
55
55
|
embeddings_iterable = self.model.embed(data, **kwargs)
|
|
@@ -65,7 +65,7 @@ class _FastembedSparseEmbeddingBackendFactory:
|
|
|
65
65
|
Factory class to create instances of fastembed sparse embedding backends.
|
|
66
66
|
"""
|
|
67
67
|
|
|
68
|
-
_instances: ClassVar[
|
|
68
|
+
_instances: ClassVar[dict[str, "_FastembedSparseEmbeddingBackend"]] = {}
|
|
69
69
|
|
|
70
70
|
@staticmethod
|
|
71
71
|
def get_embedding_backend(
|
|
@@ -73,7 +73,7 @@ class _FastembedSparseEmbeddingBackendFactory:
|
|
|
73
73
|
cache_dir: Optional[str] = None,
|
|
74
74
|
threads: Optional[int] = None,
|
|
75
75
|
local_files_only: bool = False,
|
|
76
|
-
model_kwargs: Optional[
|
|
76
|
+
model_kwargs: Optional[dict[str, Any]] = None,
|
|
77
77
|
) -> "_FastembedSparseEmbeddingBackend":
|
|
78
78
|
embedding_backend_id = f"{model_name}{cache_dir}{threads}{local_files_only}{model_kwargs}"
|
|
79
79
|
|
|
@@ -102,7 +102,7 @@ class _FastembedSparseEmbeddingBackend:
|
|
|
102
102
|
cache_dir: Optional[str] = None,
|
|
103
103
|
threads: Optional[int] = None,
|
|
104
104
|
local_files_only: bool = False,
|
|
105
|
-
model_kwargs: Optional[
|
|
105
|
+
model_kwargs: Optional[dict[str, Any]] = None,
|
|
106
106
|
):
|
|
107
107
|
model_kwargs = model_kwargs or {}
|
|
108
108
|
|
|
@@ -114,7 +114,7 @@ class _FastembedSparseEmbeddingBackend:
|
|
|
114
114
|
**model_kwargs,
|
|
115
115
|
)
|
|
116
116
|
|
|
117
|
-
def embed(self, data:
|
|
117
|
+
def embed(self, data: list[str], progress_bar: bool = True, **kwargs: Any) -> list[SparseEmbedding]:
|
|
118
118
|
# The embed method returns a Iterable[SparseEmbedding], so we convert to Haystack SparseEmbedding type.
|
|
119
119
|
# Each SparseEmbedding contains an `indices` key containing a list of int and
|
|
120
120
|
# an `values` key containing a list of floats.
|
|
@@ -1,8 +1,13 @@
|
|
|
1
|
-
|
|
1
|
+
# SPDX-FileCopyrightText: 2024-present deepset GmbH <info@deepset.ai>
|
|
2
|
+
#
|
|
3
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
4
|
+
|
|
5
|
+
from dataclasses import replace
|
|
6
|
+
from typing import Any, Optional
|
|
2
7
|
|
|
3
8
|
from haystack import Document, component, default_to_dict
|
|
4
9
|
|
|
5
|
-
from .embedding_backend.fastembed_backend import _FastembedEmbeddingBackendFactory
|
|
10
|
+
from .embedding_backend.fastembed_backend import _FastembedEmbeddingBackend, _FastembedEmbeddingBackendFactory
|
|
6
11
|
|
|
7
12
|
|
|
8
13
|
@component
|
|
@@ -66,9 +71,9 @@ class FastembedDocumentEmbedder:
|
|
|
66
71
|
progress_bar: bool = True,
|
|
67
72
|
parallel: Optional[int] = None,
|
|
68
73
|
local_files_only: bool = False,
|
|
69
|
-
meta_fields_to_embed: Optional[
|
|
74
|
+
meta_fields_to_embed: Optional[list[str]] = None,
|
|
70
75
|
embedding_separator: str = "\n",
|
|
71
|
-
):
|
|
76
|
+
) -> None:
|
|
72
77
|
"""
|
|
73
78
|
Create an FastembedDocumentEmbedder component.
|
|
74
79
|
|
|
@@ -102,8 +107,9 @@ class FastembedDocumentEmbedder:
|
|
|
102
107
|
self.local_files_only = local_files_only
|
|
103
108
|
self.meta_fields_to_embed = meta_fields_to_embed or []
|
|
104
109
|
self.embedding_separator = embedding_separator
|
|
110
|
+
self.embedding_backend: Optional[_FastembedEmbeddingBackend] = None
|
|
105
111
|
|
|
106
|
-
def to_dict(self) ->
|
|
112
|
+
def to_dict(self) -> dict[str, Any]:
|
|
107
113
|
"""
|
|
108
114
|
Serializes the component to a dictionary.
|
|
109
115
|
:returns:
|
|
@@ -124,11 +130,11 @@ class FastembedDocumentEmbedder:
|
|
|
124
130
|
embedding_separator=self.embedding_separator,
|
|
125
131
|
)
|
|
126
132
|
|
|
127
|
-
def warm_up(self):
|
|
133
|
+
def warm_up(self) -> None:
|
|
128
134
|
"""
|
|
129
135
|
Initializes the component.
|
|
130
136
|
"""
|
|
131
|
-
if
|
|
137
|
+
if self.embedding_backend is None:
|
|
132
138
|
self.embedding_backend = _FastembedEmbeddingBackendFactory.get_embedding_backend(
|
|
133
139
|
model_name=self.model_name,
|
|
134
140
|
cache_dir=self.cache_dir,
|
|
@@ -136,7 +142,7 @@ class FastembedDocumentEmbedder:
|
|
|
136
142
|
local_files_only=self.local_files_only,
|
|
137
143
|
)
|
|
138
144
|
|
|
139
|
-
def _prepare_texts_to_embed(self, documents:
|
|
145
|
+
def _prepare_texts_to_embed(self, documents: list[Document]) -> list[str]:
|
|
140
146
|
texts_to_embed = []
|
|
141
147
|
for doc in documents:
|
|
142
148
|
meta_values_to_embed = [
|
|
@@ -149,14 +155,15 @@ class FastembedDocumentEmbedder:
|
|
|
149
155
|
texts_to_embed.append(text_to_embed)
|
|
150
156
|
return texts_to_embed
|
|
151
157
|
|
|
152
|
-
@component.output_types(documents=
|
|
153
|
-
def run(self, documents:
|
|
158
|
+
@component.output_types(documents=list[Document])
|
|
159
|
+
def run(self, documents: list[Document]) -> dict[str, list[Document]]:
|
|
154
160
|
"""
|
|
155
161
|
Embeds a list of Documents.
|
|
156
162
|
|
|
157
163
|
:param documents: List of Documents to embed.
|
|
158
164
|
:returns: A dictionary with the following keys:
|
|
159
165
|
- `documents`: List of Documents with each Document's `embedding` field set to the computed embeddings.
|
|
166
|
+
:raises TypeError: If the input is not a list of Documents.
|
|
160
167
|
"""
|
|
161
168
|
if not isinstance(documents, list) or (documents and not isinstance(documents[0], Document)):
|
|
162
169
|
msg = (
|
|
@@ -164,19 +171,20 @@ class FastembedDocumentEmbedder:
|
|
|
164
171
|
"In case you want to embed a list of strings, please use the FastembedTextEmbedder."
|
|
165
172
|
)
|
|
166
173
|
raise TypeError(msg)
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
174
|
+
|
|
175
|
+
if self.embedding_backend is None:
|
|
176
|
+
self.warm_up()
|
|
170
177
|
|
|
171
178
|
texts_to_embed = self._prepare_texts_to_embed(documents=documents)
|
|
172
|
-
embeddings = self.embedding_backend.embed(
|
|
179
|
+
embeddings = self.embedding_backend.embed( # type: ignore[union-attr]
|
|
173
180
|
texts_to_embed,
|
|
174
181
|
batch_size=self.batch_size,
|
|
175
182
|
progress_bar=self.progress_bar,
|
|
176
183
|
parallel=self.parallel,
|
|
177
184
|
)
|
|
178
185
|
|
|
186
|
+
new_documents = []
|
|
179
187
|
for doc, emb in zip(documents, embeddings):
|
|
180
|
-
doc
|
|
188
|
+
new_documents.append(replace(doc, embedding=emb))
|
|
181
189
|
|
|
182
|
-
return {"documents":
|
|
190
|
+
return {"documents": new_documents}
|
|
@@ -1,8 +1,16 @@
|
|
|
1
|
-
|
|
1
|
+
# SPDX-FileCopyrightText: 2024-present deepset GmbH <info@deepset.ai>
|
|
2
|
+
#
|
|
3
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
4
|
+
|
|
5
|
+
from dataclasses import replace
|
|
6
|
+
from typing import Any, Optional
|
|
2
7
|
|
|
3
8
|
from haystack import Document, component, default_to_dict
|
|
4
9
|
|
|
5
|
-
from .embedding_backend.fastembed_backend import
|
|
10
|
+
from .embedding_backend.fastembed_backend import (
|
|
11
|
+
_FastembedSparseEmbeddingBackend,
|
|
12
|
+
_FastembedSparseEmbeddingBackendFactory,
|
|
13
|
+
)
|
|
6
14
|
|
|
7
15
|
|
|
8
16
|
@component
|
|
@@ -60,10 +68,10 @@ class FastembedSparseDocumentEmbedder:
|
|
|
60
68
|
progress_bar: bool = True,
|
|
61
69
|
parallel: Optional[int] = None,
|
|
62
70
|
local_files_only: bool = False,
|
|
63
|
-
meta_fields_to_embed: Optional[
|
|
71
|
+
meta_fields_to_embed: Optional[list[str]] = None,
|
|
64
72
|
embedding_separator: str = "\n",
|
|
65
|
-
model_kwargs: Optional[
|
|
66
|
-
):
|
|
73
|
+
model_kwargs: Optional[dict[str, Any]] = None,
|
|
74
|
+
) -> None:
|
|
67
75
|
"""
|
|
68
76
|
Create an FastembedDocumentEmbedder component.
|
|
69
77
|
|
|
@@ -95,8 +103,9 @@ class FastembedSparseDocumentEmbedder:
|
|
|
95
103
|
self.meta_fields_to_embed = meta_fields_to_embed or []
|
|
96
104
|
self.embedding_separator = embedding_separator
|
|
97
105
|
self.model_kwargs = model_kwargs
|
|
106
|
+
self.embedding_backend: Optional[_FastembedSparseEmbeddingBackend] = None
|
|
98
107
|
|
|
99
|
-
def to_dict(self) ->
|
|
108
|
+
def to_dict(self) -> dict[str, Any]:
|
|
100
109
|
"""
|
|
101
110
|
Serializes the component to a dictionary.
|
|
102
111
|
:returns:
|
|
@@ -116,11 +125,11 @@ class FastembedSparseDocumentEmbedder:
|
|
|
116
125
|
model_kwargs=self.model_kwargs,
|
|
117
126
|
)
|
|
118
127
|
|
|
119
|
-
def warm_up(self):
|
|
128
|
+
def warm_up(self) -> None:
|
|
120
129
|
"""
|
|
121
130
|
Initializes the component.
|
|
122
131
|
"""
|
|
123
|
-
if
|
|
132
|
+
if self.embedding_backend is None:
|
|
124
133
|
self.embedding_backend = _FastembedSparseEmbeddingBackendFactory.get_embedding_backend(
|
|
125
134
|
model_name=self.model_name,
|
|
126
135
|
cache_dir=self.cache_dir,
|
|
@@ -129,7 +138,7 @@ class FastembedSparseDocumentEmbedder:
|
|
|
129
138
|
model_kwargs=self.model_kwargs,
|
|
130
139
|
)
|
|
131
140
|
|
|
132
|
-
def _prepare_texts_to_embed(self, documents:
|
|
141
|
+
def _prepare_texts_to_embed(self, documents: list[Document]) -> list[str]:
|
|
133
142
|
texts_to_embed = []
|
|
134
143
|
for doc in documents:
|
|
135
144
|
meta_values_to_embed = [
|
|
@@ -140,8 +149,8 @@ class FastembedSparseDocumentEmbedder:
|
|
|
140
149
|
texts_to_embed.append(text_to_embed)
|
|
141
150
|
return texts_to_embed
|
|
142
151
|
|
|
143
|
-
@component.output_types(documents=
|
|
144
|
-
def run(self, documents:
|
|
152
|
+
@component.output_types(documents=list[Document])
|
|
153
|
+
def run(self, documents: list[Document]) -> dict[str, list[Document]]:
|
|
145
154
|
"""
|
|
146
155
|
Embeds a list of Documents.
|
|
147
156
|
|
|
@@ -149,6 +158,7 @@ class FastembedSparseDocumentEmbedder:
|
|
|
149
158
|
:returns: A dictionary with the following keys:
|
|
150
159
|
- `documents`: List of Documents with each Document's `sparse_embedding`
|
|
151
160
|
field set to the computed embeddings.
|
|
161
|
+
:raises TypeError: If the input is not a list of Documents.
|
|
152
162
|
"""
|
|
153
163
|
if not isinstance(documents, list) or (documents and not isinstance(documents[0], Document)):
|
|
154
164
|
msg = (
|
|
@@ -156,18 +166,20 @@ class FastembedSparseDocumentEmbedder:
|
|
|
156
166
|
"In case you want to embed a list of strings, please use the FastembedTextEmbedder."
|
|
157
167
|
)
|
|
158
168
|
raise TypeError(msg)
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
169
|
+
|
|
170
|
+
if self.embedding_backend is None:
|
|
171
|
+
self.warm_up()
|
|
162
172
|
|
|
163
173
|
texts_to_embed = self._prepare_texts_to_embed(documents=documents)
|
|
164
|
-
embeddings = self.embedding_backend.embed(
|
|
174
|
+
embeddings = self.embedding_backend.embed( # type: ignore[union-attr]
|
|
165
175
|
texts_to_embed,
|
|
166
176
|
batch_size=self.batch_size,
|
|
167
177
|
progress_bar=self.progress_bar,
|
|
168
178
|
parallel=self.parallel,
|
|
169
179
|
)
|
|
170
180
|
|
|
181
|
+
new_documents = []
|
|
171
182
|
for doc, emb in zip(documents, embeddings):
|
|
172
|
-
doc
|
|
173
|
-
|
|
183
|
+
new_documents.append(replace(doc, sparse_embedding=emb))
|
|
184
|
+
|
|
185
|
+
return {"documents": new_documents}
|
|
@@ -1,9 +1,16 @@
|
|
|
1
|
-
|
|
1
|
+
# SPDX-FileCopyrightText: 2024-present deepset GmbH <info@deepset.ai>
|
|
2
|
+
#
|
|
3
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
4
|
+
|
|
5
|
+
from typing import Any, Optional
|
|
2
6
|
|
|
3
7
|
from haystack import component, default_to_dict
|
|
4
8
|
from haystack.dataclasses.sparse_embedding import SparseEmbedding
|
|
5
9
|
|
|
6
|
-
from .embedding_backend.fastembed_backend import
|
|
10
|
+
from .embedding_backend.fastembed_backend import (
|
|
11
|
+
_FastembedSparseEmbeddingBackend,
|
|
12
|
+
_FastembedSparseEmbeddingBackendFactory,
|
|
13
|
+
)
|
|
7
14
|
|
|
8
15
|
|
|
9
16
|
@component
|
|
@@ -35,8 +42,8 @@ class FastembedSparseTextEmbedder:
|
|
|
35
42
|
progress_bar: bool = True,
|
|
36
43
|
parallel: Optional[int] = None,
|
|
37
44
|
local_files_only: bool = False,
|
|
38
|
-
model_kwargs: Optional[
|
|
39
|
-
):
|
|
45
|
+
model_kwargs: Optional[dict[str, Any]] = None,
|
|
46
|
+
) -> None:
|
|
40
47
|
"""
|
|
41
48
|
Create a FastembedSparseTextEmbedder component.
|
|
42
49
|
|
|
@@ -61,8 +68,9 @@ class FastembedSparseTextEmbedder:
|
|
|
61
68
|
self.parallel = parallel
|
|
62
69
|
self.local_files_only = local_files_only
|
|
63
70
|
self.model_kwargs = model_kwargs
|
|
71
|
+
self.embedding_backend: Optional[_FastembedSparseEmbeddingBackend] = None
|
|
64
72
|
|
|
65
|
-
def to_dict(self) ->
|
|
73
|
+
def to_dict(self) -> dict[str, Any]:
|
|
66
74
|
"""
|
|
67
75
|
Serializes the component to a dictionary.
|
|
68
76
|
|
|
@@ -80,11 +88,11 @@ class FastembedSparseTextEmbedder:
|
|
|
80
88
|
model_kwargs=self.model_kwargs,
|
|
81
89
|
)
|
|
82
90
|
|
|
83
|
-
def warm_up(self):
|
|
91
|
+
def warm_up(self) -> None:
|
|
84
92
|
"""
|
|
85
93
|
Initializes the component.
|
|
86
94
|
"""
|
|
87
|
-
if
|
|
95
|
+
if self.embedding_backend is None:
|
|
88
96
|
self.embedding_backend = _FastembedSparseEmbeddingBackendFactory.get_embedding_backend(
|
|
89
97
|
model_name=self.model_name,
|
|
90
98
|
cache_dir=self.cache_dir,
|
|
@@ -94,7 +102,7 @@ class FastembedSparseTextEmbedder:
|
|
|
94
102
|
)
|
|
95
103
|
|
|
96
104
|
@component.output_types(sparse_embedding=SparseEmbedding)
|
|
97
|
-
def run(self, text: str) ->
|
|
105
|
+
def run(self, text: str) -> dict[str, SparseEmbedding]:
|
|
98
106
|
"""
|
|
99
107
|
Embeds text using the Fastembed model.
|
|
100
108
|
|
|
@@ -102,7 +110,6 @@ class FastembedSparseTextEmbedder:
|
|
|
102
110
|
:returns: A dictionary with the following keys:
|
|
103
111
|
- `embedding`: A list of floats representing the embedding of the input text.
|
|
104
112
|
:raises TypeError: If the input is not a string.
|
|
105
|
-
:raises RuntimeError: If the embedding model has not been loaded.
|
|
106
113
|
"""
|
|
107
114
|
if not isinstance(text, str):
|
|
108
115
|
msg = (
|
|
@@ -110,11 +117,11 @@ class FastembedSparseTextEmbedder:
|
|
|
110
117
|
"In case you want to embed a list of Documents, please use the FastembedDocumentEmbedder."
|
|
111
118
|
)
|
|
112
119
|
raise TypeError(msg)
|
|
113
|
-
if not hasattr(self, "embedding_backend"):
|
|
114
|
-
msg = "The embedding model has not been loaded. Please call warm_up() before running."
|
|
115
|
-
raise RuntimeError(msg)
|
|
116
120
|
|
|
117
|
-
|
|
121
|
+
if self.embedding_backend is None:
|
|
122
|
+
self.warm_up()
|
|
123
|
+
|
|
124
|
+
embedding = self.embedding_backend.embed( # type: ignore[union-attr]
|
|
118
125
|
[text],
|
|
119
126
|
progress_bar=self.progress_bar,
|
|
120
127
|
parallel=self.parallel,
|
|
@@ -1,8 +1,12 @@
|
|
|
1
|
-
|
|
1
|
+
# SPDX-FileCopyrightText: 2024-present deepset GmbH <info@deepset.ai>
|
|
2
|
+
#
|
|
3
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
4
|
+
|
|
5
|
+
from typing import Any, Optional
|
|
2
6
|
|
|
3
7
|
from haystack import component, default_to_dict
|
|
4
8
|
|
|
5
|
-
from .embedding_backend.fastembed_backend import _FastembedEmbeddingBackendFactory
|
|
9
|
+
from .embedding_backend.fastembed_backend import _FastembedEmbeddingBackend, _FastembedEmbeddingBackendFactory
|
|
6
10
|
|
|
7
11
|
|
|
8
12
|
@component
|
|
@@ -36,7 +40,7 @@ class FastembedTextEmbedder:
|
|
|
36
40
|
progress_bar: bool = True,
|
|
37
41
|
parallel: Optional[int] = None,
|
|
38
42
|
local_files_only: bool = False,
|
|
39
|
-
):
|
|
43
|
+
) -> None:
|
|
40
44
|
"""
|
|
41
45
|
Create a FastembedTextEmbedder component.
|
|
42
46
|
|
|
@@ -63,8 +67,9 @@ class FastembedTextEmbedder:
|
|
|
63
67
|
self.progress_bar = progress_bar
|
|
64
68
|
self.parallel = parallel
|
|
65
69
|
self.local_files_only = local_files_only
|
|
70
|
+
self.embedding_backend: Optional[_FastembedEmbeddingBackend] = None
|
|
66
71
|
|
|
67
|
-
def to_dict(self) ->
|
|
72
|
+
def to_dict(self) -> dict[str, Any]:
|
|
68
73
|
"""
|
|
69
74
|
Serializes the component to a dictionary.
|
|
70
75
|
|
|
@@ -83,11 +88,11 @@ class FastembedTextEmbedder:
|
|
|
83
88
|
local_files_only=self.local_files_only,
|
|
84
89
|
)
|
|
85
90
|
|
|
86
|
-
def warm_up(self):
|
|
91
|
+
def warm_up(self) -> None:
|
|
87
92
|
"""
|
|
88
93
|
Initializes the component.
|
|
89
94
|
"""
|
|
90
|
-
if
|
|
95
|
+
if self.embedding_backend is None:
|
|
91
96
|
self.embedding_backend = _FastembedEmbeddingBackendFactory.get_embedding_backend(
|
|
92
97
|
model_name=self.model_name,
|
|
93
98
|
cache_dir=self.cache_dir,
|
|
@@ -95,8 +100,8 @@ class FastembedTextEmbedder:
|
|
|
95
100
|
local_files_only=self.local_files_only,
|
|
96
101
|
)
|
|
97
102
|
|
|
98
|
-
@component.output_types(embedding=
|
|
99
|
-
def run(self, text: str) ->
|
|
103
|
+
@component.output_types(embedding=list[float])
|
|
104
|
+
def run(self, text: str) -> dict[str, list[float]]:
|
|
100
105
|
"""
|
|
101
106
|
Embeds text using the Fastembed model.
|
|
102
107
|
|
|
@@ -104,7 +109,6 @@ class FastembedTextEmbedder:
|
|
|
104
109
|
:returns: A dictionary with the following keys:
|
|
105
110
|
- `embedding`: A list of floats representing the embedding of the input text.
|
|
106
111
|
:raises TypeError: If the input is not a string.
|
|
107
|
-
:raises RuntimeError: If the embedding model has not been loaded.
|
|
108
112
|
"""
|
|
109
113
|
if not isinstance(text, str):
|
|
110
114
|
msg = (
|
|
@@ -112,13 +116,13 @@ class FastembedTextEmbedder:
|
|
|
112
116
|
"In case you want to embed a list of Documents, please use the FastembedDocumentEmbedder."
|
|
113
117
|
)
|
|
114
118
|
raise TypeError(msg)
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
119
|
+
|
|
120
|
+
if self.embedding_backend is None:
|
|
121
|
+
self.warm_up()
|
|
118
122
|
|
|
119
123
|
text_to_embed = [self.prefix + text + self.suffix]
|
|
120
124
|
embedding = list(
|
|
121
|
-
self.embedding_backend.embed(
|
|
125
|
+
self.embedding_backend.embed( # type: ignore[union-attr]
|
|
122
126
|
text_to_embed,
|
|
123
127
|
progress_bar=self.progress_bar,
|
|
124
128
|
parallel=self.parallel,
|
|
@@ -1,4 +1,8 @@
|
|
|
1
|
-
|
|
1
|
+
# SPDX-FileCopyrightText: 2024-present deepset GmbH <info@deepset.ai>
|
|
2
|
+
#
|
|
3
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
4
|
+
|
|
5
|
+
from typing import Any, Optional
|
|
2
6
|
|
|
3
7
|
from haystack import Document, component, default_from_dict, default_to_dict, logging
|
|
4
8
|
|
|
@@ -40,7 +44,7 @@ class FastembedRanker:
|
|
|
40
44
|
batch_size: int = 64,
|
|
41
45
|
parallel: Optional[int] = None,
|
|
42
46
|
local_files_only: bool = False,
|
|
43
|
-
meta_fields_to_embed: Optional[
|
|
47
|
+
meta_fields_to_embed: Optional[list[str]] = None,
|
|
44
48
|
meta_data_separator: str = "\n",
|
|
45
49
|
):
|
|
46
50
|
"""
|
|
@@ -78,7 +82,7 @@ class FastembedRanker:
|
|
|
78
82
|
self.meta_data_separator = meta_data_separator
|
|
79
83
|
self._model: Optional[TextCrossEncoder] = None
|
|
80
84
|
|
|
81
|
-
def to_dict(self) ->
|
|
85
|
+
def to_dict(self) -> dict[str, Any]:
|
|
82
86
|
"""
|
|
83
87
|
Serializes the component to a dictionary.
|
|
84
88
|
|
|
@@ -99,7 +103,7 @@ class FastembedRanker:
|
|
|
99
103
|
)
|
|
100
104
|
|
|
101
105
|
@classmethod
|
|
102
|
-
def from_dict(cls, data:
|
|
106
|
+
def from_dict(cls, data: dict[str, Any]) -> "FastembedRanker":
|
|
103
107
|
"""
|
|
104
108
|
Deserializes the component from a dictionary.
|
|
105
109
|
|
|
@@ -122,7 +126,7 @@ class FastembedRanker:
|
|
|
122
126
|
local_files_only=self.local_files_only,
|
|
123
127
|
)
|
|
124
128
|
|
|
125
|
-
def _prepare_fastembed_input_docs(self, documents:
|
|
129
|
+
def _prepare_fastembed_input_docs(self, documents: list[Document]) -> list[str]:
|
|
126
130
|
"""
|
|
127
131
|
Prepare the input by concatenating the document text with the metadata fields specified.
|
|
128
132
|
:param documents: The list of Document objects.
|
|
@@ -139,8 +143,8 @@ class FastembedRanker:
|
|
|
139
143
|
|
|
140
144
|
return concatenated_input_list
|
|
141
145
|
|
|
142
|
-
@component.output_types(documents=
|
|
143
|
-
def run(self, query: str, documents:
|
|
146
|
+
@component.output_types(documents=list[Document])
|
|
147
|
+
def run(self, query: str, documents: list[Document], top_k: Optional[int] = None) -> dict[str, list[Document]]:
|
|
144
148
|
"""
|
|
145
149
|
Returns a list of documents ranked by their similarity to the given query, using FastEmbed.
|
|
146
150
|
|
{fastembed_haystack-1.5.0 → fastembed_haystack-1.6.0}/tests/test_fastembed_document_embedder.py
RENAMED
|
@@ -1,3 +1,7 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: 2024-present deepset GmbH <info@deepset.ai>
|
|
2
|
+
#
|
|
3
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
4
|
+
|
|
1
5
|
from unittest.mock import MagicMock, patch
|
|
2
6
|
|
|
3
7
|
import numpy as np
|
|
@@ -237,13 +241,13 @@ class TestFastembedDocumentEmbedder:
|
|
|
237
241
|
|
|
238
242
|
with pytest.raises(
|
|
239
243
|
TypeError,
|
|
240
|
-
match="FastembedDocumentEmbedder expects a list of Documents as input
|
|
244
|
+
match=r"FastembedDocumentEmbedder expects a list of Documents as input\.",
|
|
241
245
|
):
|
|
242
246
|
embedder.run(documents=string_input)
|
|
243
247
|
|
|
244
248
|
with pytest.raises(
|
|
245
249
|
TypeError,
|
|
246
|
-
match="FastembedDocumentEmbedder expects a list of Documents as input
|
|
250
|
+
match=r"FastembedDocumentEmbedder expects a list of Documents as input\.",
|
|
247
251
|
):
|
|
248
252
|
embedder.run(documents=list_integers_input)
|
|
249
253
|
|
|
@@ -1,3 +1,7 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: 2024-present deepset GmbH <info@deepset.ai>
|
|
2
|
+
#
|
|
3
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
4
|
+
|
|
1
5
|
from unittest.mock import MagicMock
|
|
2
6
|
|
|
3
7
|
import pytest
|
|
@@ -189,13 +193,13 @@ class TestFastembedRanker:
|
|
|
189
193
|
|
|
190
194
|
with pytest.raises(
|
|
191
195
|
TypeError,
|
|
192
|
-
match="FastembedRanker expects a list of Documents as input
|
|
196
|
+
match=r"FastembedRanker expects a list of Documents as input\.",
|
|
193
197
|
):
|
|
194
198
|
ranker.run(query=query, documents=string_input)
|
|
195
199
|
|
|
196
200
|
with pytest.raises(
|
|
197
201
|
TypeError,
|
|
198
|
-
match="FastembedRanker expects a list of Documents as input
|
|
202
|
+
match=r"FastembedRanker expects a list of Documents as input\.",
|
|
199
203
|
):
|
|
200
204
|
ranker.run(query=query, documents=list_integers_input)
|
|
201
205
|
|
|
@@ -211,20 +215,6 @@ class TestFastembedRanker:
|
|
|
211
215
|
):
|
|
212
216
|
ranker.run(query=query, documents=list_document, top_k=-3)
|
|
213
217
|
|
|
214
|
-
def test_run_no_warmup(self):
|
|
215
|
-
"""
|
|
216
|
-
Test for checking error when calling without a warmup.
|
|
217
|
-
"""
|
|
218
|
-
ranker = FastembedRanker(model_name="Xenova/ms-marco-MiniLM-L-12-v2")
|
|
219
|
-
|
|
220
|
-
query = "query"
|
|
221
|
-
list_document = [Document("Document 1")]
|
|
222
|
-
|
|
223
|
-
with pytest.raises(
|
|
224
|
-
RuntimeError,
|
|
225
|
-
):
|
|
226
|
-
ranker.run(query=query, documents=list_document)
|
|
227
|
-
|
|
228
218
|
def test_run_empty_document_list(self):
|
|
229
219
|
"""
|
|
230
220
|
Test for no error when sending no documents.
|
|
@@ -1,3 +1,7 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: 2024-present deepset GmbH <info@deepset.ai>
|
|
2
|
+
#
|
|
3
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
4
|
+
|
|
1
5
|
from unittest.mock import MagicMock, patch
|
|
2
6
|
|
|
3
7
|
import numpy as np
|
|
@@ -242,13 +246,13 @@ class TestFastembedSparseDocumentEmbedderDoc:
|
|
|
242
246
|
|
|
243
247
|
with pytest.raises(
|
|
244
248
|
TypeError,
|
|
245
|
-
match="FastembedSparseDocumentEmbedder expects a list of Documents as input
|
|
249
|
+
match=r"FastembedSparseDocumentEmbedder expects a list of Documents as input\.",
|
|
246
250
|
):
|
|
247
251
|
embedder.run(documents=string_input)
|
|
248
252
|
|
|
249
253
|
with pytest.raises(
|
|
250
254
|
TypeError,
|
|
251
|
-
match="FastembedSparseDocumentEmbedder expects a list of Documents as input
|
|
255
|
+
match=r"FastembedSparseDocumentEmbedder expects a list of Documents as input\.",
|
|
252
256
|
):
|
|
253
257
|
embedder.run(documents=list_integers_input)
|
|
254
258
|
|
|
@@ -1,125 +0,0 @@
|
|
|
1
|
-
Metadata-Version: 2.4
|
|
2
|
-
Name: fastembed-haystack
|
|
3
|
-
Version: 1.5.0
|
|
4
|
-
Summary: Haystack 2.x component to embed strings and Documents using fastembed embedding model
|
|
5
|
-
Project-URL: Source, https://github.com/deepset-ai/haystack-core-integrations
|
|
6
|
-
Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/blob/main/integrations/fastembed/README.md
|
|
7
|
-
Project-URL: Issues, https://github.com/deepset-ai/haystack-core-integrations/issues
|
|
8
|
-
Author-email: deepset GmbH <info@deepset.ai>
|
|
9
|
-
License-Expression: Apache-2.0
|
|
10
|
-
License-File: LICENSE.txt
|
|
11
|
-
Classifier: Development Status :: 4 - Beta
|
|
12
|
-
Classifier: License :: OSI Approved :: Apache Software License
|
|
13
|
-
Classifier: Programming Language :: Python
|
|
14
|
-
Classifier: Programming Language :: Python :: 3.9
|
|
15
|
-
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
-
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
-
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
-
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
-
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
20
|
-
Classifier: Programming Language :: Python :: Implementation :: PyPy
|
|
21
|
-
Requires-Python: >=3.9
|
|
22
|
-
Requires-Dist: fastembed>=0.4.2
|
|
23
|
-
Requires-Dist: haystack-ai>=2.0.1
|
|
24
|
-
Description-Content-Type: text/markdown
|
|
25
|
-
|
|
26
|
-
# fastembed-haystack
|
|
27
|
-
|
|
28
|
-
[](https://pypi.org/project/fastembed-haystack)
|
|
29
|
-
[](https://pypi.org/project/fastembed-haystack)
|
|
30
|
-
|
|
31
|
-
-----
|
|
32
|
-
|
|
33
|
-
**Table of Contents**
|
|
34
|
-
|
|
35
|
-
- [Installation](#installation)
|
|
36
|
-
- [Usage](#Usage)
|
|
37
|
-
- [License](#license)
|
|
38
|
-
|
|
39
|
-
## Installation
|
|
40
|
-
|
|
41
|
-
```console
|
|
42
|
-
pip install fastembed-haystack
|
|
43
|
-
```
|
|
44
|
-
|
|
45
|
-
## Usage
|
|
46
|
-
|
|
47
|
-
You can use `FastembedTextEmbedder` and `FastembedDocumentEmbedder` by importing as:
|
|
48
|
-
|
|
49
|
-
```python
|
|
50
|
-
from haystack_integrations.components.embedders.fastembed import FastembedTextEmbedder
|
|
51
|
-
|
|
52
|
-
text = "fastembed is supported by and maintained by Qdrant."
|
|
53
|
-
text_embedder = FastembedTextEmbedder(
|
|
54
|
-
model="BAAI/bge-small-en-v1.5"
|
|
55
|
-
)
|
|
56
|
-
text_embedder.warm_up()
|
|
57
|
-
embedding = text_embedder.run(text)["embedding"]
|
|
58
|
-
```
|
|
59
|
-
|
|
60
|
-
```python
|
|
61
|
-
from haystack_integrations.components.embedders.fastembed import FastembedDocumentEmbedder
|
|
62
|
-
from haystack import Document
|
|
63
|
-
|
|
64
|
-
embedder = FastembedDocumentEmbedder(
|
|
65
|
-
model="BAAI/bge-small-en-v1.5",
|
|
66
|
-
)
|
|
67
|
-
embedder.warm_up()
|
|
68
|
-
doc = Document(content="fastembed is supported by and maintained by Qdrant.", meta={"long_answer": "no",})
|
|
69
|
-
result = embedder.run(documents=[doc])
|
|
70
|
-
```
|
|
71
|
-
|
|
72
|
-
You can use `FastembedSparseTextEmbedder` and `FastembedSparseDocumentEmbedder` by importing as:
|
|
73
|
-
|
|
74
|
-
```python
|
|
75
|
-
from haystack_integrations.components.embedders.fastembed import FastembedSparseTextEmbedder
|
|
76
|
-
|
|
77
|
-
text = "fastembed is supported by and maintained by Qdrant."
|
|
78
|
-
text_embedder = FastembedSparseTextEmbedder(
|
|
79
|
-
model="prithivida/Splade_PP_en_v1"
|
|
80
|
-
)
|
|
81
|
-
text_embedder.warm_up()
|
|
82
|
-
embedding = text_embedder.run(text)["sparse_embedding"]
|
|
83
|
-
```
|
|
84
|
-
|
|
85
|
-
```python
|
|
86
|
-
from haystack_integrations.components.embedders.fastembed import FastembedSparseDocumentEmbedder
|
|
87
|
-
from haystack import Document
|
|
88
|
-
|
|
89
|
-
embedder = FastembedSparseDocumentEmbedder(
|
|
90
|
-
model="prithivida/Splade_PP_en_v1",
|
|
91
|
-
)
|
|
92
|
-
embedder.warm_up()
|
|
93
|
-
doc = Document(content="fastembed is supported by and maintained by Qdrant.", meta={"long_answer": "no",})
|
|
94
|
-
result = embedder.run(documents=[doc])
|
|
95
|
-
```
|
|
96
|
-
|
|
97
|
-
You can use `FastembedRanker` by importing as:
|
|
98
|
-
|
|
99
|
-
```python
|
|
100
|
-
from haystack import Document
|
|
101
|
-
|
|
102
|
-
from haystack_integrations.components.rankers.fastembed import FastembedRanker
|
|
103
|
-
|
|
104
|
-
query = "Who is maintaining Qdrant?"
|
|
105
|
-
documents = [
|
|
106
|
-
Document(
|
|
107
|
-
content="This is built to be faster and lighter than other embedding libraries e.g. Transformers, Sentence-Transformers, etc."
|
|
108
|
-
),
|
|
109
|
-
Document(content="fastembed is supported by and maintained by Qdrant."),
|
|
110
|
-
]
|
|
111
|
-
|
|
112
|
-
ranker = FastembedRanker(model_name="Xenova/ms-marco-MiniLM-L-6-v2")
|
|
113
|
-
ranker.warm_up()
|
|
114
|
-
reranked_documents = ranker.run(query=query, documents=documents)["documents"]
|
|
115
|
-
|
|
116
|
-
print(reranked_documents[0])
|
|
117
|
-
|
|
118
|
-
# Document(id=...,
|
|
119
|
-
# content: 'fastembed is supported by and maintained by Qdrant.',
|
|
120
|
-
# score: 5.472434997558594..)
|
|
121
|
-
```
|
|
122
|
-
|
|
123
|
-
## License
|
|
124
|
-
|
|
125
|
-
`fastembed-haystack` is distributed under the terms of the [Apache-2.0](https://spdx.org/licenses/Apache-2.0.html) license.
|
|
@@ -1,100 +0,0 @@
|
|
|
1
|
-
# fastembed-haystack
|
|
2
|
-
|
|
3
|
-
[](https://pypi.org/project/fastembed-haystack)
|
|
4
|
-
[](https://pypi.org/project/fastembed-haystack)
|
|
5
|
-
|
|
6
|
-
-----
|
|
7
|
-
|
|
8
|
-
**Table of Contents**
|
|
9
|
-
|
|
10
|
-
- [Installation](#installation)
|
|
11
|
-
- [Usage](#Usage)
|
|
12
|
-
- [License](#license)
|
|
13
|
-
|
|
14
|
-
## Installation
|
|
15
|
-
|
|
16
|
-
```console
|
|
17
|
-
pip install fastembed-haystack
|
|
18
|
-
```
|
|
19
|
-
|
|
20
|
-
## Usage
|
|
21
|
-
|
|
22
|
-
You can use `FastembedTextEmbedder` and `FastembedDocumentEmbedder` by importing as:
|
|
23
|
-
|
|
24
|
-
```python
|
|
25
|
-
from haystack_integrations.components.embedders.fastembed import FastembedTextEmbedder
|
|
26
|
-
|
|
27
|
-
text = "fastembed is supported by and maintained by Qdrant."
|
|
28
|
-
text_embedder = FastembedTextEmbedder(
|
|
29
|
-
model="BAAI/bge-small-en-v1.5"
|
|
30
|
-
)
|
|
31
|
-
text_embedder.warm_up()
|
|
32
|
-
embedding = text_embedder.run(text)["embedding"]
|
|
33
|
-
```
|
|
34
|
-
|
|
35
|
-
```python
|
|
36
|
-
from haystack_integrations.components.embedders.fastembed import FastembedDocumentEmbedder
|
|
37
|
-
from haystack import Document
|
|
38
|
-
|
|
39
|
-
embedder = FastembedDocumentEmbedder(
|
|
40
|
-
model="BAAI/bge-small-en-v1.5",
|
|
41
|
-
)
|
|
42
|
-
embedder.warm_up()
|
|
43
|
-
doc = Document(content="fastembed is supported by and maintained by Qdrant.", meta={"long_answer": "no",})
|
|
44
|
-
result = embedder.run(documents=[doc])
|
|
45
|
-
```
|
|
46
|
-
|
|
47
|
-
You can use `FastembedSparseTextEmbedder` and `FastembedSparseDocumentEmbedder` by importing as:
|
|
48
|
-
|
|
49
|
-
```python
|
|
50
|
-
from haystack_integrations.components.embedders.fastembed import FastembedSparseTextEmbedder
|
|
51
|
-
|
|
52
|
-
text = "fastembed is supported by and maintained by Qdrant."
|
|
53
|
-
text_embedder = FastembedSparseTextEmbedder(
|
|
54
|
-
model="prithivida/Splade_PP_en_v1"
|
|
55
|
-
)
|
|
56
|
-
text_embedder.warm_up()
|
|
57
|
-
embedding = text_embedder.run(text)["sparse_embedding"]
|
|
58
|
-
```
|
|
59
|
-
|
|
60
|
-
```python
|
|
61
|
-
from haystack_integrations.components.embedders.fastembed import FastembedSparseDocumentEmbedder
|
|
62
|
-
from haystack import Document
|
|
63
|
-
|
|
64
|
-
embedder = FastembedSparseDocumentEmbedder(
|
|
65
|
-
model="prithivida/Splade_PP_en_v1",
|
|
66
|
-
)
|
|
67
|
-
embedder.warm_up()
|
|
68
|
-
doc = Document(content="fastembed is supported by and maintained by Qdrant.", meta={"long_answer": "no",})
|
|
69
|
-
result = embedder.run(documents=[doc])
|
|
70
|
-
```
|
|
71
|
-
|
|
72
|
-
You can use `FastembedRanker` by importing as:
|
|
73
|
-
|
|
74
|
-
```python
|
|
75
|
-
from haystack import Document
|
|
76
|
-
|
|
77
|
-
from haystack_integrations.components.rankers.fastembed import FastembedRanker
|
|
78
|
-
|
|
79
|
-
query = "Who is maintaining Qdrant?"
|
|
80
|
-
documents = [
|
|
81
|
-
Document(
|
|
82
|
-
content="This is built to be faster and lighter than other embedding libraries e.g. Transformers, Sentence-Transformers, etc."
|
|
83
|
-
),
|
|
84
|
-
Document(content="fastembed is supported by and maintained by Qdrant."),
|
|
85
|
-
]
|
|
86
|
-
|
|
87
|
-
ranker = FastembedRanker(model_name="Xenova/ms-marco-MiniLM-L-6-v2")
|
|
88
|
-
ranker.warm_up()
|
|
89
|
-
reranked_documents = ranker.run(query=query, documents=documents)["documents"]
|
|
90
|
-
|
|
91
|
-
print(reranked_documents[0])
|
|
92
|
-
|
|
93
|
-
# Document(id=...,
|
|
94
|
-
# content: 'fastembed is supported by and maintained by Qdrant.',
|
|
95
|
-
# score: 5.472434997558594..)
|
|
96
|
-
```
|
|
97
|
-
|
|
98
|
-
## License
|
|
99
|
-
|
|
100
|
-
`fastembed-haystack` is distributed under the terms of the [Apache-2.0](https://spdx.org/licenses/Apache-2.0.html) license.
|
|
@@ -1,34 +0,0 @@
|
|
|
1
|
-
loaders:
|
|
2
|
-
- type: haystack_pydoc_tools.loaders.CustomPythonLoader
|
|
3
|
-
search_path: [../src]
|
|
4
|
-
modules:
|
|
5
|
-
[
|
|
6
|
-
"haystack_integrations.components.embedders.fastembed.fastembed_document_embedder",
|
|
7
|
-
"haystack_integrations.components.embedders.fastembed.fastembed_text_embedder",
|
|
8
|
-
"haystack_integrations.components.embedders.fastembed.fastembed_sparse_document_embedder",
|
|
9
|
-
"haystack_integrations.components.embedders.fastembed.fastembed_sparse_text_embedder",
|
|
10
|
-
"haystack_integrations.components.rankers.fastembed.ranker"
|
|
11
|
-
]
|
|
12
|
-
ignore_when_discovered: ["__init__"]
|
|
13
|
-
processors:
|
|
14
|
-
- type: filter
|
|
15
|
-
expression:
|
|
16
|
-
documented_only: true
|
|
17
|
-
do_not_filter_modules: false
|
|
18
|
-
skip_empty_modules: true
|
|
19
|
-
- type: smart
|
|
20
|
-
- type: crossref
|
|
21
|
-
renderer:
|
|
22
|
-
type: haystack_pydoc_tools.renderers.ReadmeIntegrationRenderer
|
|
23
|
-
excerpt: FastEmbed integration for Haystack
|
|
24
|
-
category_slug: integrations-api
|
|
25
|
-
title: FastEmbed
|
|
26
|
-
slug: fastembed-embedders
|
|
27
|
-
order: 80
|
|
28
|
-
markdown:
|
|
29
|
-
descriptive_class_title: false
|
|
30
|
-
classdef_code_block: false
|
|
31
|
-
descriptive_module_title: true
|
|
32
|
-
add_method_class_prefix: true
|
|
33
|
-
add_member_class_prefix: false
|
|
34
|
-
filename: _readme_fastembed.md
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|