fastembed-haystack 1.4.1__tar.gz → 1.6.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fastembed_haystack-1.6.0/CHANGELOG.md +147 -0
- {fastembed_haystack-1.4.1 → fastembed_haystack-1.6.0}/LICENSE.txt +1 -1
- fastembed_haystack-1.6.0/PKG-INFO +38 -0
- fastembed_haystack-1.6.0/README.md +13 -0
- fastembed_haystack-1.6.0/pydoc/config_docusaurus.yml +32 -0
- {fastembed_haystack-1.4.1 → fastembed_haystack-1.6.0}/pyproject.toml +33 -45
- {fastembed_haystack-1.4.1 → fastembed_haystack-1.6.0}/src/haystack_integrations/components/embedders/fastembed/__init__.py +2 -1
- {fastembed_haystack-1.4.1 → fastembed_haystack-1.6.0}/src/haystack_integrations/components/embedders/fastembed/embedding_backend/fastembed_backend.py +9 -9
- {fastembed_haystack-1.4.1 → fastembed_haystack-1.6.0}/src/haystack_integrations/components/embedders/fastembed/fastembed_document_embedder.py +25 -17
- {fastembed_haystack-1.4.1 → fastembed_haystack-1.6.0}/src/haystack_integrations/components/embedders/fastembed/fastembed_sparse_document_embedder.py +30 -18
- {fastembed_haystack-1.4.1 → fastembed_haystack-1.6.0}/src/haystack_integrations/components/embedders/fastembed/fastembed_sparse_text_embedder.py +20 -13
- {fastembed_haystack-1.4.1 → fastembed_haystack-1.6.0}/src/haystack_integrations/components/embedders/fastembed/fastembed_text_embedder.py +17 -13
- fastembed_haystack-1.6.0/src/haystack_integrations/components/embedders/py.typed +0 -0
- fastembed_haystack-1.6.0/src/haystack_integrations/components/rankers/fastembed/__init__.py +7 -0
- {fastembed_haystack-1.4.1 → fastembed_haystack-1.6.0}/src/haystack_integrations/components/rankers/fastembed/ranker.py +13 -9
- fastembed_haystack-1.6.0/src/haystack_integrations/components/rankers/py.typed +0 -0
- {fastembed_haystack-1.4.1 → fastembed_haystack-1.6.0}/tests/test_fastembed_backend.py +4 -0
- {fastembed_haystack-1.4.1 → fastembed_haystack-1.6.0}/tests/test_fastembed_document_embedder.py +6 -2
- {fastembed_haystack-1.4.1 → fastembed_haystack-1.6.0}/tests/test_fastembed_ranker.py +6 -16
- {fastembed_haystack-1.4.1 → fastembed_haystack-1.6.0}/tests/test_fastembed_sparse_document_embedder.py +6 -2
- {fastembed_haystack-1.4.1 → fastembed_haystack-1.6.0}/tests/test_fastembed_sparse_text_embedder.py +4 -0
- {fastembed_haystack-1.4.1 → fastembed_haystack-1.6.0}/tests/test_fastembed_text_embedder.py +4 -0
- fastembed_haystack-1.4.1/CHANGELOG.md +0 -78
- fastembed_haystack-1.4.1/PKG-INFO +0 -124
- fastembed_haystack-1.4.1/README.md +0 -100
- fastembed_haystack-1.4.1/pydoc/config.yml +0 -34
- fastembed_haystack-1.4.1/src/haystack_integrations/components/rankers/fastembed/__init__.py +0 -3
- {fastembed_haystack-1.4.1 → fastembed_haystack-1.6.0}/.gitignore +0 -0
- {fastembed_haystack-1.4.1 → fastembed_haystack-1.6.0}/examples/example.py +0 -0
- {fastembed_haystack-1.4.1 → fastembed_haystack-1.6.0}/examples/ranker_example.py +0 -0
- {fastembed_haystack-1.4.1 → fastembed_haystack-1.6.0}/examples/sparse_example.py +0 -0
- {fastembed_haystack-1.4.1 → fastembed_haystack-1.6.0}/src/haystack_integrations/components/embedders/fastembed/embedding_backend/__init__.py +0 -0
- {fastembed_haystack-1.4.1 → fastembed_haystack-1.6.0}/tests/__init__.py +0 -0
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## [integrations/fastembed-v1.5.0] - 2025-06-17
|
|
4
|
+
|
|
5
|
+
### 🐛 Bug Fixes
|
|
6
|
+
|
|
7
|
+
- Fix Fastembed types + add py.typed (#1954)
|
|
8
|
+
|
|
9
|
+
### ⚙️ CI
|
|
10
|
+
|
|
11
|
+
- Review testing workflows (#1541)
|
|
12
|
+
|
|
13
|
+
### 🧹 Chores
|
|
14
|
+
|
|
15
|
+
- Fix linting/isort (#1215)
|
|
16
|
+
- Remove Python 3.8 support (#1421)
|
|
17
|
+
- Align core-integrations Hatch scripts (#1898)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
## [integrations/fastembed-v1.4.1] - 2024-11-19
|
|
21
|
+
|
|
22
|
+
### 🌀 Miscellaneous
|
|
23
|
+
|
|
24
|
+
- Add new example with the ranker and fix the old ones (#1198)
|
|
25
|
+
- Fix: Fastembed - Change default Sparse model as the used one is deprecated due to a typo (#1201)
|
|
26
|
+
|
|
27
|
+
## [integrations/fastembed-v1.4.0] - 2024-11-13
|
|
28
|
+
|
|
29
|
+
### ⚙️ CI
|
|
30
|
+
|
|
31
|
+
- Adopt uv as installer (#1142)
|
|
32
|
+
|
|
33
|
+
### 🌀 Miscellaneous
|
|
34
|
+
|
|
35
|
+
- Chore: fastembed - pin `onnxruntime<1.20.0` (#1164)
|
|
36
|
+
- Feat: Fastembed - add FastembedRanker (#1178)
|
|
37
|
+
|
|
38
|
+
## [integrations/fastembed-v1.3.0] - 2024-10-07
|
|
39
|
+
|
|
40
|
+
### 🚀 Features
|
|
41
|
+
|
|
42
|
+
- Introduce `model_kwargs` in Sparse Embedders (can be used for BM25 parameters) (#1126)
|
|
43
|
+
|
|
44
|
+
### 🧪 Testing
|
|
45
|
+
|
|
46
|
+
- Do not retry tests in `hatch run test` command (#954)
|
|
47
|
+
|
|
48
|
+
### ⚙️ CI
|
|
49
|
+
|
|
50
|
+
- Retry tests to reduce flakyness (#836)
|
|
51
|
+
|
|
52
|
+
### 🧹 Chores
|
|
53
|
+
|
|
54
|
+
- Update ruff invocation to include check parameter (#853)
|
|
55
|
+
- Update ruff linting scripts and settings (#1105)
|
|
56
|
+
|
|
57
|
+
### 🌀 Miscellaneous
|
|
58
|
+
|
|
59
|
+
- Ci: install `pytest-rerunfailures` where needed; add retry config to `test-cov` script (#845)
|
|
60
|
+
- Typo on Sparse embedders. The parameter should be "progress_bar" … (#814)
|
|
61
|
+
- Chore: fastembed - ruff update, don't ruff tests (#997)
|
|
62
|
+
|
|
63
|
+
## [integrations/fastembed-v1.1.0] - 2024-05-15
|
|
64
|
+
|
|
65
|
+
### 🌀 Miscellaneous
|
|
66
|
+
|
|
67
|
+
- Chore: change the pydoc renderer class (#718)
|
|
68
|
+
- Use the local_files_only option available as of fastembed==0.2.7. It … (#736)
|
|
69
|
+
|
|
70
|
+
## [integrations/fastembed-v1.0.0] - 2024-05-06
|
|
71
|
+
|
|
72
|
+
### 🌀 Miscellaneous
|
|
73
|
+
|
|
74
|
+
- Chore: add license classifiers (#680)
|
|
75
|
+
- `FastembedSparseTextEmbedder` - remove `batch_size` (#688)
|
|
76
|
+
|
|
77
|
+
## [integrations/fastembed-v0.1.0] - 2024-04-10
|
|
78
|
+
|
|
79
|
+
### 🚀 Features
|
|
80
|
+
|
|
81
|
+
- *(FastEmbed)* Support for SPLADE Sparse Embedder (#579)
|
|
82
|
+
|
|
83
|
+
### 📚 Documentation
|
|
84
|
+
|
|
85
|
+
- Disable-class-def (#556)
|
|
86
|
+
|
|
87
|
+
### 🌀 Miscellaneous
|
|
88
|
+
|
|
89
|
+
- Remove references to Python 3.7 (#601)
|
|
90
|
+
|
|
91
|
+
## [integrations/fastembed-v0.0.6] - 2024-03-07
|
|
92
|
+
|
|
93
|
+
### 📚 Documentation
|
|
94
|
+
|
|
95
|
+
- Review and normalize docstrings - `integrations.fastembed` (#519)
|
|
96
|
+
- Small consistency improvements (#536)
|
|
97
|
+
|
|
98
|
+
### 🌀 Miscellaneous
|
|
99
|
+
|
|
100
|
+
- Docs: Fix `integrations.fastembed` API docs (#540)
|
|
101
|
+
- Improvements to FastEmbed integration (#558)
|
|
102
|
+
|
|
103
|
+
## [integrations/fastembed-v0.0.5] - 2024-02-20
|
|
104
|
+
|
|
105
|
+
### 🐛 Bug Fixes
|
|
106
|
+
|
|
107
|
+
- Fix order of API docs (#447)
|
|
108
|
+
|
|
109
|
+
### 📚 Documentation
|
|
110
|
+
|
|
111
|
+
- Update category slug (#442)
|
|
112
|
+
|
|
113
|
+
### 🌀 Miscellaneous
|
|
114
|
+
|
|
115
|
+
- Fastembed integration new parameters (#446)
|
|
116
|
+
|
|
117
|
+
## [integrations/fastembed-v0.0.4] - 2024-02-16
|
|
118
|
+
|
|
119
|
+
### 🌀 Miscellaneous
|
|
120
|
+
|
|
121
|
+
- Fastembed integration: add example (#401)
|
|
122
|
+
- Fastembed fix: add parallel (#403)
|
|
123
|
+
|
|
124
|
+
## [integrations/fastembed-v0.0.3] - 2024-02-12
|
|
125
|
+
|
|
126
|
+
### 🐛 Bug Fixes
|
|
127
|
+
|
|
128
|
+
- From numpy float to float (#391)
|
|
129
|
+
|
|
130
|
+
### 📚 Documentation
|
|
131
|
+
|
|
132
|
+
- Update paths and titles (#397)
|
|
133
|
+
|
|
134
|
+
## [integrations/fastembed-v0.0.2] - 2024-02-11
|
|
135
|
+
|
|
136
|
+
### 🌀 Miscellaneous
|
|
137
|
+
|
|
138
|
+
- Updated labeler and readme (#389)
|
|
139
|
+
- Fastembed fix: added prefix and suffix (#390)
|
|
140
|
+
|
|
141
|
+
## [integrations/fastembed-v0.0.1] - 2024-02-10
|
|
142
|
+
|
|
143
|
+
### 🌀 Miscellaneous
|
|
144
|
+
|
|
145
|
+
- Add Fastembed Embeddings integration (#383)
|
|
146
|
+
|
|
147
|
+
<!-- generated by git-cliff -->
|
|
@@ -58,7 +58,7 @@ APPENDIX: How to apply the Apache License to your work.
|
|
|
58
58
|
|
|
59
59
|
To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives.
|
|
60
60
|
|
|
61
|
-
Copyright
|
|
61
|
+
Copyright 2024 deepset GmbH
|
|
62
62
|
|
|
63
63
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
64
64
|
you may not use this file except in compliance with the License.
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: fastembed-haystack
|
|
3
|
+
Version: 1.6.0
|
|
4
|
+
Summary: Haystack 2.x component to embed strings and Documents using fastembed embedding model
|
|
5
|
+
Project-URL: Source, https://github.com/deepset-ai/haystack-core-integrations
|
|
6
|
+
Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/blob/main/integrations/fastembed/README.md
|
|
7
|
+
Project-URL: Issues, https://github.com/deepset-ai/haystack-core-integrations/issues
|
|
8
|
+
Author-email: deepset GmbH <info@deepset.ai>
|
|
9
|
+
License-Expression: Apache-2.0
|
|
10
|
+
License-File: LICENSE.txt
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
13
|
+
Classifier: Programming Language :: Python
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
+
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
20
|
+
Classifier: Programming Language :: Python :: Implementation :: PyPy
|
|
21
|
+
Requires-Python: >=3.9
|
|
22
|
+
Requires-Dist: fastembed>=0.4.2
|
|
23
|
+
Requires-Dist: haystack-ai>=2.0.1
|
|
24
|
+
Description-Content-Type: text/markdown
|
|
25
|
+
|
|
26
|
+
# fastembed-haystack
|
|
27
|
+
|
|
28
|
+
[](https://pypi.org/project/fastembed-haystack)
|
|
29
|
+
[](https://pypi.org/project/fastembed-haystack)
|
|
30
|
+
|
|
31
|
+
- [Integration page](https://haystack.deepset.ai/integrations/fastembed)
|
|
32
|
+
- [Changelog](https://github.com/deepset-ai/haystack-core-integrations/blob/main/integrations/fastembed/CHANGELOG.md)
|
|
33
|
+
|
|
34
|
+
---
|
|
35
|
+
|
|
36
|
+
## Contributing
|
|
37
|
+
|
|
38
|
+
Refer to the general [Contribution Guidelines](https://github.com/deepset-ai/haystack-core-integrations/blob/main/CONTRIBUTING.md).
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
# fastembed-haystack
|
|
2
|
+
|
|
3
|
+
[](https://pypi.org/project/fastembed-haystack)
|
|
4
|
+
[](https://pypi.org/project/fastembed-haystack)
|
|
5
|
+
|
|
6
|
+
- [Integration page](https://haystack.deepset.ai/integrations/fastembed)
|
|
7
|
+
- [Changelog](https://github.com/deepset-ai/haystack-core-integrations/blob/main/integrations/fastembed/CHANGELOG.md)
|
|
8
|
+
|
|
9
|
+
---
|
|
10
|
+
|
|
11
|
+
## Contributing
|
|
12
|
+
|
|
13
|
+
Refer to the general [Contribution Guidelines](https://github.com/deepset-ai/haystack-core-integrations/blob/main/CONTRIBUTING.md).
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
loaders:
|
|
2
|
+
- ignore_when_discovered:
|
|
3
|
+
- __init__
|
|
4
|
+
modules:
|
|
5
|
+
- haystack_integrations.components.embedders.fastembed.fastembed_document_embedder
|
|
6
|
+
- haystack_integrations.components.embedders.fastembed.fastembed_text_embedder
|
|
7
|
+
- haystack_integrations.components.embedders.fastembed.fastembed_sparse_document_embedder
|
|
8
|
+
- haystack_integrations.components.embedders.fastembed.fastembed_sparse_text_embedder
|
|
9
|
+
- haystack_integrations.components.rankers.fastembed.ranker
|
|
10
|
+
search_path:
|
|
11
|
+
- ../src
|
|
12
|
+
type: haystack_pydoc_tools.loaders.CustomPythonLoader
|
|
13
|
+
processors:
|
|
14
|
+
- do_not_filter_modules: false
|
|
15
|
+
documented_only: true
|
|
16
|
+
expression: null
|
|
17
|
+
skip_empty_modules: true
|
|
18
|
+
type: filter
|
|
19
|
+
- type: smart
|
|
20
|
+
- type: crossref
|
|
21
|
+
renderer:
|
|
22
|
+
description: FastEmbed integration for Haystack
|
|
23
|
+
id: fastembed-embedders
|
|
24
|
+
markdown:
|
|
25
|
+
add_member_class_prefix: false
|
|
26
|
+
add_method_class_prefix: true
|
|
27
|
+
classdef_code_block: false
|
|
28
|
+
descriptive_class_title: false
|
|
29
|
+
descriptive_module_title: true
|
|
30
|
+
filename: fastembed.md
|
|
31
|
+
title: FastEmbed
|
|
32
|
+
type: haystack_pydoc_tools.renderers.DocusaurusRenderer
|
|
@@ -7,7 +7,7 @@ name = "fastembed-haystack"
|
|
|
7
7
|
dynamic = ["version"]
|
|
8
8
|
description = "Haystack 2.x component to embed strings and Documents using fastembed embedding model"
|
|
9
9
|
readme = "README.md"
|
|
10
|
-
requires-python = ">=3.
|
|
10
|
+
requires-python = ">=3.9"
|
|
11
11
|
license = "Apache-2.0"
|
|
12
12
|
keywords = []
|
|
13
13
|
authors = [{ name = "deepset GmbH", email = "info@deepset.ai" }]
|
|
@@ -15,11 +15,11 @@ classifiers = [
|
|
|
15
15
|
"License :: OSI Approved :: Apache Software License",
|
|
16
16
|
"Development Status :: 4 - Beta",
|
|
17
17
|
"Programming Language :: Python",
|
|
18
|
-
"Programming Language :: Python :: 3.8",
|
|
19
18
|
"Programming Language :: Python :: 3.9",
|
|
20
19
|
"Programming Language :: Python :: 3.10",
|
|
21
20
|
"Programming Language :: Python :: 3.11",
|
|
22
21
|
"Programming Language :: Python :: 3.12",
|
|
22
|
+
"Programming Language :: Python :: 3.13",
|
|
23
23
|
"Programming Language :: Python :: Implementation :: CPython",
|
|
24
24
|
"Programming Language :: Python :: Implementation :: PyPy",
|
|
25
25
|
]
|
|
@@ -43,42 +43,39 @@ git_describe_command = 'git describe --tags --match="integrations/fastembed-v[0-
|
|
|
43
43
|
|
|
44
44
|
[tool.hatch.envs.default]
|
|
45
45
|
installer = "uv"
|
|
46
|
+
dependencies = ["haystack-pydoc-tools", "ruff"]
|
|
47
|
+
[tool.hatch.envs.default.scripts]
|
|
48
|
+
docs = ["pydoc-markdown pydoc/config_docusaurus.yml"]
|
|
49
|
+
fmt = "ruff check --fix {args}; ruff format {args}"
|
|
50
|
+
fmt-check = "ruff check {args} && ruff format --check {args}"
|
|
51
|
+
|
|
52
|
+
[tool.hatch.envs.test]
|
|
46
53
|
dependencies = [
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
54
|
+
"pytest",
|
|
55
|
+
"pytest-asyncio",
|
|
56
|
+
"pytest-cov",
|
|
57
|
+
"pytest-rerunfailures",
|
|
58
|
+
"mypy",
|
|
59
|
+
"pip",
|
|
60
|
+
"ipython"
|
|
52
61
|
]
|
|
53
|
-
|
|
54
|
-
test
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
cov =
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
[
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
detached = true
|
|
68
|
-
dependencies = ["pip", "black>=23.1.0", "mypy>=1.0.0", "ruff>=0.0.243", "numpy"]
|
|
69
|
-
[tool.hatch.envs.lint.scripts]
|
|
70
|
-
typing = "mypy --install-types --non-interactive --explicit-package-bases {args:src/ tests}"
|
|
71
|
-
style = ["ruff check {args:.}", "black --check --diff {args:.}"]
|
|
72
|
-
fmt = ["black {args:.}", "ruff check --fix {args:.}", "style"]
|
|
73
|
-
all = ["style", "typing"]
|
|
74
|
-
|
|
75
|
-
[tool.black]
|
|
76
|
-
target-version = ["py38"]
|
|
77
|
-
line-length = 120
|
|
78
|
-
skip-string-normalization = true
|
|
62
|
+
|
|
63
|
+
[tool.hatch.envs.test.scripts]
|
|
64
|
+
unit = 'pytest -m "not integration" {args:tests}'
|
|
65
|
+
integration = 'pytest -m "integration" {args:tests}'
|
|
66
|
+
all = 'pytest {args:tests}'
|
|
67
|
+
cov-retry = 'pytest --cov=haystack_integrations --reruns 3 --reruns-delay 30 -x {args:tests}'
|
|
68
|
+
types = """mypy -p haystack_integrations.components.embedders.fastembed \
|
|
69
|
+
-p haystack_integrations.components.rankers.fastembed {args}"""
|
|
70
|
+
|
|
71
|
+
[tool.mypy]
|
|
72
|
+
install_types = true
|
|
73
|
+
non_interactive = true
|
|
74
|
+
check_untyped_defs = true
|
|
75
|
+
disallow_incomplete_defs = true
|
|
79
76
|
|
|
80
77
|
[tool.ruff]
|
|
81
|
-
target-version = "
|
|
78
|
+
target-version = "py39"
|
|
82
79
|
line-length = 120
|
|
83
80
|
|
|
84
81
|
[tool.ruff.lint]
|
|
@@ -156,14 +153,5 @@ exclude_lines = ["no cov", "if __name__ == .__main__.:", "if TYPE_CHECKING:"]
|
|
|
156
153
|
|
|
157
154
|
[tool.pytest.ini_options]
|
|
158
155
|
minversion = "6.0"
|
|
159
|
-
markers = ["
|
|
160
|
-
|
|
161
|
-
[[tool.mypy.overrides]]
|
|
162
|
-
module = [
|
|
163
|
-
"haystack.*",
|
|
164
|
-
"haystack_integrations.*",
|
|
165
|
-
"fastembed.*",
|
|
166
|
-
"pytest.*",
|
|
167
|
-
"numpy.*",
|
|
168
|
-
]
|
|
169
|
-
ignore_missing_imports = true
|
|
156
|
+
markers = ["integration: integration tests"]
|
|
157
|
+
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# SPDX-FileCopyrightText: 2024-present deepset GmbH <info@deepset.ai>
|
|
2
2
|
#
|
|
3
3
|
# SPDX-License-Identifier: Apache-2.0
|
|
4
|
+
|
|
4
5
|
from .fastembed_document_embedder import FastembedDocumentEmbedder
|
|
5
6
|
from .fastembed_sparse_document_embedder import FastembedSparseDocumentEmbedder
|
|
6
7
|
from .fastembed_sparse_text_embedder import FastembedSparseTextEmbedder
|
|
@@ -8,7 +9,7 @@ from .fastembed_text_embedder import FastembedTextEmbedder
|
|
|
8
9
|
|
|
9
10
|
__all__ = [
|
|
10
11
|
"FastembedDocumentEmbedder",
|
|
11
|
-
"FastembedTextEmbedder",
|
|
12
12
|
"FastembedSparseDocumentEmbedder",
|
|
13
13
|
"FastembedSparseTextEmbedder",
|
|
14
|
+
"FastembedTextEmbedder",
|
|
14
15
|
]
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
from typing import Any, ClassVar,
|
|
1
|
+
from typing import Any, ClassVar, Optional
|
|
2
2
|
|
|
3
3
|
from haystack.dataclasses.sparse_embedding import SparseEmbedding
|
|
4
4
|
from tqdm import tqdm
|
|
@@ -12,7 +12,7 @@ class _FastembedEmbeddingBackendFactory:
|
|
|
12
12
|
Factory class to create instances of fastembed embedding backends.
|
|
13
13
|
"""
|
|
14
14
|
|
|
15
|
-
_instances: ClassVar[
|
|
15
|
+
_instances: ClassVar[dict[str, "_FastembedEmbeddingBackend"]] = {}
|
|
16
16
|
|
|
17
17
|
@staticmethod
|
|
18
18
|
def get_embedding_backend(
|
|
@@ -20,7 +20,7 @@ class _FastembedEmbeddingBackendFactory:
|
|
|
20
20
|
cache_dir: Optional[str] = None,
|
|
21
21
|
threads: Optional[int] = None,
|
|
22
22
|
local_files_only: bool = False,
|
|
23
|
-
):
|
|
23
|
+
) -> "_FastembedEmbeddingBackend":
|
|
24
24
|
embedding_backend_id = f"{model_name}{cache_dir}{threads}"
|
|
25
25
|
|
|
26
26
|
if embedding_backend_id in _FastembedEmbeddingBackendFactory._instances:
|
|
@@ -49,7 +49,7 @@ class _FastembedEmbeddingBackend:
|
|
|
49
49
|
model_name=model_name, cache_dir=cache_dir, threads=threads, local_files_only=local_files_only
|
|
50
50
|
)
|
|
51
51
|
|
|
52
|
-
def embed(self, data:
|
|
52
|
+
def embed(self, data: list[str], progress_bar: bool = True, **kwargs: Any) -> list[list[float]]:
|
|
53
53
|
# the embed method returns a Iterable[np.ndarray], so we convert it to a list of lists
|
|
54
54
|
embeddings = []
|
|
55
55
|
embeddings_iterable = self.model.embed(data, **kwargs)
|
|
@@ -65,7 +65,7 @@ class _FastembedSparseEmbeddingBackendFactory:
|
|
|
65
65
|
Factory class to create instances of fastembed sparse embedding backends.
|
|
66
66
|
"""
|
|
67
67
|
|
|
68
|
-
_instances: ClassVar[
|
|
68
|
+
_instances: ClassVar[dict[str, "_FastembedSparseEmbeddingBackend"]] = {}
|
|
69
69
|
|
|
70
70
|
@staticmethod
|
|
71
71
|
def get_embedding_backend(
|
|
@@ -73,8 +73,8 @@ class _FastembedSparseEmbeddingBackendFactory:
|
|
|
73
73
|
cache_dir: Optional[str] = None,
|
|
74
74
|
threads: Optional[int] = None,
|
|
75
75
|
local_files_only: bool = False,
|
|
76
|
-
model_kwargs: Optional[
|
|
77
|
-
):
|
|
76
|
+
model_kwargs: Optional[dict[str, Any]] = None,
|
|
77
|
+
) -> "_FastembedSparseEmbeddingBackend":
|
|
78
78
|
embedding_backend_id = f"{model_name}{cache_dir}{threads}{local_files_only}{model_kwargs}"
|
|
79
79
|
|
|
80
80
|
if embedding_backend_id in _FastembedSparseEmbeddingBackendFactory._instances:
|
|
@@ -102,7 +102,7 @@ class _FastembedSparseEmbeddingBackend:
|
|
|
102
102
|
cache_dir: Optional[str] = None,
|
|
103
103
|
threads: Optional[int] = None,
|
|
104
104
|
local_files_only: bool = False,
|
|
105
|
-
model_kwargs: Optional[
|
|
105
|
+
model_kwargs: Optional[dict[str, Any]] = None,
|
|
106
106
|
):
|
|
107
107
|
model_kwargs = model_kwargs or {}
|
|
108
108
|
|
|
@@ -114,7 +114,7 @@ class _FastembedSparseEmbeddingBackend:
|
|
|
114
114
|
**model_kwargs,
|
|
115
115
|
)
|
|
116
116
|
|
|
117
|
-
def embed(self, data:
|
|
117
|
+
def embed(self, data: list[str], progress_bar: bool = True, **kwargs: Any) -> list[SparseEmbedding]:
|
|
118
118
|
# The embed method returns a Iterable[SparseEmbedding], so we convert to Haystack SparseEmbedding type.
|
|
119
119
|
# Each SparseEmbedding contains an `indices` key containing a list of int and
|
|
120
120
|
# an `values` key containing a list of floats.
|
|
@@ -1,8 +1,13 @@
|
|
|
1
|
-
|
|
1
|
+
# SPDX-FileCopyrightText: 2024-present deepset GmbH <info@deepset.ai>
|
|
2
|
+
#
|
|
3
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
4
|
+
|
|
5
|
+
from dataclasses import replace
|
|
6
|
+
from typing import Any, Optional
|
|
2
7
|
|
|
3
8
|
from haystack import Document, component, default_to_dict
|
|
4
9
|
|
|
5
|
-
from .embedding_backend.fastembed_backend import _FastembedEmbeddingBackendFactory
|
|
10
|
+
from .embedding_backend.fastembed_backend import _FastembedEmbeddingBackend, _FastembedEmbeddingBackendFactory
|
|
6
11
|
|
|
7
12
|
|
|
8
13
|
@component
|
|
@@ -66,9 +71,9 @@ class FastembedDocumentEmbedder:
|
|
|
66
71
|
progress_bar: bool = True,
|
|
67
72
|
parallel: Optional[int] = None,
|
|
68
73
|
local_files_only: bool = False,
|
|
69
|
-
meta_fields_to_embed: Optional[
|
|
74
|
+
meta_fields_to_embed: Optional[list[str]] = None,
|
|
70
75
|
embedding_separator: str = "\n",
|
|
71
|
-
):
|
|
76
|
+
) -> None:
|
|
72
77
|
"""
|
|
73
78
|
Create an FastembedDocumentEmbedder component.
|
|
74
79
|
|
|
@@ -102,8 +107,9 @@ class FastembedDocumentEmbedder:
|
|
|
102
107
|
self.local_files_only = local_files_only
|
|
103
108
|
self.meta_fields_to_embed = meta_fields_to_embed or []
|
|
104
109
|
self.embedding_separator = embedding_separator
|
|
110
|
+
self.embedding_backend: Optional[_FastembedEmbeddingBackend] = None
|
|
105
111
|
|
|
106
|
-
def to_dict(self) ->
|
|
112
|
+
def to_dict(self) -> dict[str, Any]:
|
|
107
113
|
"""
|
|
108
114
|
Serializes the component to a dictionary.
|
|
109
115
|
:returns:
|
|
@@ -124,11 +130,11 @@ class FastembedDocumentEmbedder:
|
|
|
124
130
|
embedding_separator=self.embedding_separator,
|
|
125
131
|
)
|
|
126
132
|
|
|
127
|
-
def warm_up(self):
|
|
133
|
+
def warm_up(self) -> None:
|
|
128
134
|
"""
|
|
129
135
|
Initializes the component.
|
|
130
136
|
"""
|
|
131
|
-
if
|
|
137
|
+
if self.embedding_backend is None:
|
|
132
138
|
self.embedding_backend = _FastembedEmbeddingBackendFactory.get_embedding_backend(
|
|
133
139
|
model_name=self.model_name,
|
|
134
140
|
cache_dir=self.cache_dir,
|
|
@@ -136,7 +142,7 @@ class FastembedDocumentEmbedder:
|
|
|
136
142
|
local_files_only=self.local_files_only,
|
|
137
143
|
)
|
|
138
144
|
|
|
139
|
-
def _prepare_texts_to_embed(self, documents:
|
|
145
|
+
def _prepare_texts_to_embed(self, documents: list[Document]) -> list[str]:
|
|
140
146
|
texts_to_embed = []
|
|
141
147
|
for doc in documents:
|
|
142
148
|
meta_values_to_embed = [
|
|
@@ -149,34 +155,36 @@ class FastembedDocumentEmbedder:
|
|
|
149
155
|
texts_to_embed.append(text_to_embed)
|
|
150
156
|
return texts_to_embed
|
|
151
157
|
|
|
152
|
-
@component.output_types(documents=
|
|
153
|
-
def run(self, documents:
|
|
158
|
+
@component.output_types(documents=list[Document])
|
|
159
|
+
def run(self, documents: list[Document]) -> dict[str, list[Document]]:
|
|
154
160
|
"""
|
|
155
161
|
Embeds a list of Documents.
|
|
156
162
|
|
|
157
163
|
:param documents: List of Documents to embed.
|
|
158
164
|
:returns: A dictionary with the following keys:
|
|
159
165
|
- `documents`: List of Documents with each Document's `embedding` field set to the computed embeddings.
|
|
166
|
+
:raises TypeError: If the input is not a list of Documents.
|
|
160
167
|
"""
|
|
161
|
-
if not isinstance(documents, list) or documents and not isinstance(documents[0], Document):
|
|
168
|
+
if not isinstance(documents, list) or (documents and not isinstance(documents[0], Document)):
|
|
162
169
|
msg = (
|
|
163
170
|
"FastembedDocumentEmbedder expects a list of Documents as input. "
|
|
164
171
|
"In case you want to embed a list of strings, please use the FastembedTextEmbedder."
|
|
165
172
|
)
|
|
166
173
|
raise TypeError(msg)
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
174
|
+
|
|
175
|
+
if self.embedding_backend is None:
|
|
176
|
+
self.warm_up()
|
|
170
177
|
|
|
171
178
|
texts_to_embed = self._prepare_texts_to_embed(documents=documents)
|
|
172
|
-
embeddings = self.embedding_backend.embed(
|
|
179
|
+
embeddings = self.embedding_backend.embed( # type: ignore[union-attr]
|
|
173
180
|
texts_to_embed,
|
|
174
181
|
batch_size=self.batch_size,
|
|
175
182
|
progress_bar=self.progress_bar,
|
|
176
183
|
parallel=self.parallel,
|
|
177
184
|
)
|
|
178
185
|
|
|
186
|
+
new_documents = []
|
|
179
187
|
for doc, emb in zip(documents, embeddings):
|
|
180
|
-
doc
|
|
188
|
+
new_documents.append(replace(doc, embedding=emb))
|
|
181
189
|
|
|
182
|
-
return {"documents":
|
|
190
|
+
return {"documents": new_documents}
|
|
@@ -1,8 +1,16 @@
|
|
|
1
|
-
|
|
1
|
+
# SPDX-FileCopyrightText: 2024-present deepset GmbH <info@deepset.ai>
|
|
2
|
+
#
|
|
3
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
4
|
+
|
|
5
|
+
from dataclasses import replace
|
|
6
|
+
from typing import Any, Optional
|
|
2
7
|
|
|
3
8
|
from haystack import Document, component, default_to_dict
|
|
4
9
|
|
|
5
|
-
from .embedding_backend.fastembed_backend import
|
|
10
|
+
from .embedding_backend.fastembed_backend import (
|
|
11
|
+
_FastembedSparseEmbeddingBackend,
|
|
12
|
+
_FastembedSparseEmbeddingBackendFactory,
|
|
13
|
+
)
|
|
6
14
|
|
|
7
15
|
|
|
8
16
|
@component
|
|
@@ -60,10 +68,10 @@ class FastembedSparseDocumentEmbedder:
|
|
|
60
68
|
progress_bar: bool = True,
|
|
61
69
|
parallel: Optional[int] = None,
|
|
62
70
|
local_files_only: bool = False,
|
|
63
|
-
meta_fields_to_embed: Optional[
|
|
71
|
+
meta_fields_to_embed: Optional[list[str]] = None,
|
|
64
72
|
embedding_separator: str = "\n",
|
|
65
|
-
model_kwargs: Optional[
|
|
66
|
-
):
|
|
73
|
+
model_kwargs: Optional[dict[str, Any]] = None,
|
|
74
|
+
) -> None:
|
|
67
75
|
"""
|
|
68
76
|
Create an FastembedDocumentEmbedder component.
|
|
69
77
|
|
|
@@ -95,8 +103,9 @@ class FastembedSparseDocumentEmbedder:
|
|
|
95
103
|
self.meta_fields_to_embed = meta_fields_to_embed or []
|
|
96
104
|
self.embedding_separator = embedding_separator
|
|
97
105
|
self.model_kwargs = model_kwargs
|
|
106
|
+
self.embedding_backend: Optional[_FastembedSparseEmbeddingBackend] = None
|
|
98
107
|
|
|
99
|
-
def to_dict(self) ->
|
|
108
|
+
def to_dict(self) -> dict[str, Any]:
|
|
100
109
|
"""
|
|
101
110
|
Serializes the component to a dictionary.
|
|
102
111
|
:returns:
|
|
@@ -116,11 +125,11 @@ class FastembedSparseDocumentEmbedder:
|
|
|
116
125
|
model_kwargs=self.model_kwargs,
|
|
117
126
|
)
|
|
118
127
|
|
|
119
|
-
def warm_up(self):
|
|
128
|
+
def warm_up(self) -> None:
|
|
120
129
|
"""
|
|
121
130
|
Initializes the component.
|
|
122
131
|
"""
|
|
123
|
-
if
|
|
132
|
+
if self.embedding_backend is None:
|
|
124
133
|
self.embedding_backend = _FastembedSparseEmbeddingBackendFactory.get_embedding_backend(
|
|
125
134
|
model_name=self.model_name,
|
|
126
135
|
cache_dir=self.cache_dir,
|
|
@@ -129,7 +138,7 @@ class FastembedSparseDocumentEmbedder:
|
|
|
129
138
|
model_kwargs=self.model_kwargs,
|
|
130
139
|
)
|
|
131
140
|
|
|
132
|
-
def _prepare_texts_to_embed(self, documents:
|
|
141
|
+
def _prepare_texts_to_embed(self, documents: list[Document]) -> list[str]:
|
|
133
142
|
texts_to_embed = []
|
|
134
143
|
for doc in documents:
|
|
135
144
|
meta_values_to_embed = [
|
|
@@ -140,8 +149,8 @@ class FastembedSparseDocumentEmbedder:
|
|
|
140
149
|
texts_to_embed.append(text_to_embed)
|
|
141
150
|
return texts_to_embed
|
|
142
151
|
|
|
143
|
-
@component.output_types(documents=
|
|
144
|
-
def run(self, documents:
|
|
152
|
+
@component.output_types(documents=list[Document])
|
|
153
|
+
def run(self, documents: list[Document]) -> dict[str, list[Document]]:
|
|
145
154
|
"""
|
|
146
155
|
Embeds a list of Documents.
|
|
147
156
|
|
|
@@ -149,25 +158,28 @@ class FastembedSparseDocumentEmbedder:
|
|
|
149
158
|
:returns: A dictionary with the following keys:
|
|
150
159
|
- `documents`: List of Documents with each Document's `sparse_embedding`
|
|
151
160
|
field set to the computed embeddings.
|
|
161
|
+
:raises TypeError: If the input is not a list of Documents.
|
|
152
162
|
"""
|
|
153
|
-
if not isinstance(documents, list) or documents and not isinstance(documents[0], Document):
|
|
163
|
+
if not isinstance(documents, list) or (documents and not isinstance(documents[0], Document)):
|
|
154
164
|
msg = (
|
|
155
165
|
"FastembedSparseDocumentEmbedder expects a list of Documents as input. "
|
|
156
166
|
"In case you want to embed a list of strings, please use the FastembedTextEmbedder."
|
|
157
167
|
)
|
|
158
168
|
raise TypeError(msg)
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
169
|
+
|
|
170
|
+
if self.embedding_backend is None:
|
|
171
|
+
self.warm_up()
|
|
162
172
|
|
|
163
173
|
texts_to_embed = self._prepare_texts_to_embed(documents=documents)
|
|
164
|
-
embeddings = self.embedding_backend.embed(
|
|
174
|
+
embeddings = self.embedding_backend.embed( # type: ignore[union-attr]
|
|
165
175
|
texts_to_embed,
|
|
166
176
|
batch_size=self.batch_size,
|
|
167
177
|
progress_bar=self.progress_bar,
|
|
168
178
|
parallel=self.parallel,
|
|
169
179
|
)
|
|
170
180
|
|
|
181
|
+
new_documents = []
|
|
171
182
|
for doc, emb in zip(documents, embeddings):
|
|
172
|
-
doc
|
|
173
|
-
|
|
183
|
+
new_documents.append(replace(doc, sparse_embedding=emb))
|
|
184
|
+
|
|
185
|
+
return {"documents": new_documents}
|