jina-haystack 0.0.1__tar.gz

jina_haystack-0.0.1/.gitignore

@@ -0,0 +1,133 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+volumes/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# IDEs
+.vscode

jina_haystack-0.0.1/LICENSE.txt

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2023-present deepset GmbH
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

jina_haystack-0.0.1/PKG-INFO

@@ -0,0 +1,55 @@
+Metadata-Version: 2.1
+Name: jina-haystack
+Version: 0.0.1
+Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/jina/jina-haystack#readme
+Project-URL: Issues, https://github.com/deepset-ai/haystack-core-integrations/issues
+Project-URL: Source, https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/jina/jina-haystack
+Author-email: Joan Fontanals Martinez <joan.fontanals.martinez@jina.ai>
+License-Expression: Apache-2.0
+License-File: LICENSE.txt
+Classifier: Development Status :: 4 - Beta
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Programming Language :: Python :: Implementation :: PyPy
+Requires-Python: >=3.7
+Requires-Dist: haystack-ai
+Requires-Dist: requests
+Description-Content-Type: text/markdown
+
+# jina-haystack
+
+[![PyPI - Version](https://img.shields.io/pypi/v/jina-haystack.svg)](https://pypi.org/project/jina-haystack)
+[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/jina-haystack.svg)](https://pypi.org/project/jina-haystack)
+
+-----
+
+**Table of Contents**
+
+- [jina-haystack](#jina-haystack)
+  - [Installation](#installation)
+  - [Usage](#usage)
+  - [License](#license)
+
+## Installation
+
+```console
+pip install jina-haystack
+```
+
+## Usage
+
+You can use `JinaTextEmbedder` and `JinaDocumentEmbedder` by importing as:
+
+```python
+from jina_haystack.document_embedder import JinaDocumentEmbedder
+from jina_haystack.text_embedder import JinaTextEmbedder
+```
+
+## License
+
+`jina-haystack` is distributed under the terms of the [Apache-2.0](https://spdx.org/licenses/Apache-2.0.html) license.

jina_haystack-0.0.1/README.md

@@ -0,0 +1,32 @@
+# jina-haystack
+
+[![PyPI - Version](https://img.shields.io/pypi/v/jina-haystack.svg)](https://pypi.org/project/jina-haystack)
+[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/jina-haystack.svg)](https://pypi.org/project/jina-haystack)
+
+-----
+
+**Table of Contents**
+
+- [jina-haystack](#jina-haystack)
+  - [Installation](#installation)
+  - [Usage](#usage)
+  - [License](#license)
+
+## Installation
+
+```console
+pip install jina-haystack
+```
+
+## Usage
+
+You can use `JinaTextEmbedder` and `JinaDocumentEmbedder` by importing as:
+
+```python
+from jina_haystack.document_embedder import JinaDocumentEmbedder
+from jina_haystack.text_embedder import JinaTextEmbedder
+```
+
+## License
+
+`jina-haystack` is distributed under the terms of the [Apache-2.0](https://spdx.org/licenses/Apache-2.0.html) license.

jina_haystack-0.0.1/pyproject.toml

@@ -0,0 +1,161 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "jina-haystack"
+dynamic = ["version"]
+description = ''
+readme = "README.md"
+requires-python = ">=3.7"
+license = "Apache-2.0"
+keywords = []
+authors = [
+  { name = "Joan Fontanals Martinez", email = "joan.fontanals.martinez@jina.ai" },
+]
+classifiers = [
+  "Development Status :: 4 - Beta",
+  "Programming Language :: Python",
+  "Programming Language :: Python :: 3.7",
+  "Programming Language :: Python :: 3.8",
+  "Programming Language :: Python :: 3.9",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: Implementation :: CPython",
+  "Programming Language :: Python :: Implementation :: PyPy",
+]
+dependencies = ["requests", "haystack-ai"]
+
+[project.urls]
+Documentation = "https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/jina/jina-haystack#readme"
+Issues = "https://github.com/deepset-ai/haystack-core-integrations/issues"
+Source = "https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/jina/jina-haystack"
+
+[tool.hatch.version]
+path = "src/jina_haystack/__about__.py"
+
+[tool.hatch.envs.default]
+dependencies = [
+  "coverage[toml]>=6.5",
+  "pytest",
+]
+[tool.hatch.envs.default.scripts]
+test = "pytest {args:tests}"
+test-cov = "coverage run -m pytest {args:tests}"
+cov-report = [
+  "- coverage combine",
+  "coverage report",
+]
+cov = [
+  "test-cov",
+  "cov-report",
+]
+
+[[tool.hatch.envs.all.matrix]]
+python = ["3.7", "3.8", "3.9", "3.10", "3.11"]
+
+[tool.hatch.envs.lint]
+detached = true
+dependencies = [
+  "black>=23.1.0",
+  "mypy>=1.0.0",
+  "ruff>=0.0.243",
+]
+[tool.hatch.envs.lint.scripts]
+typing = "mypy --install-types --non-interactive {args:src/jina_haystack tests}"
+style = [
+  "ruff {args:.}",
+  "black --check --diff {args:.}",
+]
+fmt = [
+  "black {args:.}",
+  "ruff --fix {args:.}",
+  "style",
+]
+all = [
+  "style",
+  "typing",
+]
+
+[tool.black]
+target-version = ["py37"]
+line-length = 120
+skip-string-normalization = true
+
+[tool.ruff]
+target-version = "py37"
+line-length = 120
+select = [
+  "A",
+  "ARG",
+  "B",
+  "C",
+  "DTZ",
+  "E",
+  "EM",
+  "F",
+  "I",
+  "ICN",
+  "ISC",
+  "N",
+  "PLC",
+  "PLE",
+  "PLR",
+  "PLW",
+  "Q",
+  "RUF",
+  "S",
+  "T",
+  "TID",
+  "UP",
+  "W",
+  "YTT",
+]
+ignore = [
+  # Allow non-abstract empty methods in abstract base classes
+  "B027",
+  # Ignore checks for possible passwords
+  "S105", "S106", "S107",
+  # Ignore complexity
+  "C901", "PLR0911", "PLR0912", "PLR0913", "PLR0915",
+]
+unfixable = [
+  # Don't touch unused imports
+  "F401",
+]
+
+[tool.ruff.isort]
+known-first-party = ["jina_haystack"]
+
+[tool.ruff.flake8-tidy-imports]
+ban-relative-imports = "all"
+
+[tool.ruff.per-file-ignores]
+# Tests can use magic values, assertions, and relative imports
+"tests/**/*" = ["PLR2004", "S101", "TID252"]
+
+[tool.coverage.run]
+source_pkgs = ["jina_haystack", "tests"]
+branch = true
+parallel = true
+omit = [
+  "src/jina_haystack/__about__.py",
+]
+
+[tool.coverage.paths]
+jina_haystack = ["src/jina_haystack", "*/jina-haystack/src/jina_haystack"]
+tests = ["tests", "*/jina-haystack/tests"]
+
+[tool.coverage.report]
+exclude_lines = [
+  "no cov",
+  "if __name__ == .__main__.:",
+  "if TYPE_CHECKING:",
+]
+
+[[tool.mypy.overrides]]
+module = [
+  "haystack.*",
+  "pytest.*"
+]
+ignore_missing_imports = true

jina_haystack-0.0.1/src/jina_haystack/__about__.py

@@ -0,0 +1,4 @@
+# SPDX-FileCopyrightText: 2023-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+__version__ = "0.0.1"

jina_haystack-0.0.1/src/jina_haystack/__init__.py

@@ -0,0 +1,8 @@
+# SPDX-FileCopyrightText: 2023-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from jina_haystack.document_embedder import JinaDocumentEmbedder
+from jina_haystack.text_embedder import JinaTextEmbedder
+
+__all__ = ["JinaDocumentEmbedder", "JinaTextEmbedder"]

jina_haystack-0.0.1/src/jina_haystack/document_embedder.py

@@ -0,0 +1,180 @@
+# SPDX-FileCopyrightText: 2023-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+import os
+from typing import Any, Dict, List, Optional, Tuple
+
+import requests
+from haystack import Document, component, default_to_dict
+from tqdm import tqdm
+
+JINA_API_URL: str = "https://api.jina.ai/v1/embeddings"
+
+
+@component
+class JinaDocumentEmbedder:
+    """
+    A component for computing Document embeddings using Jina AI models.
+    The embedding of each Document is stored in the `embedding` field of the Document.
+
+    Usage example:
+    ```python
+    from haystack import Document
+    from jina_haystack import JinaDocumentEmbedder
+
+    doc = Document(content="I love pizza!")
+
+    document_embedder = JinaDocumentEmbedder()
+
+    result = document_embedder.run([doc])
+    print(result['documents'][0].embedding)
+
+    # [0.017020374536514282, -0.023255806416273117, ...]
+    ```
+    """
+
+    def __init__(
+        self,
+        api_key: Optional[str] = None,
+        model_name: str = "jina-embeddings-v2-base-en",
+        prefix: str = "",
+        suffix: str = "",
+        batch_size: int = 32,
+        progress_bar: bool = True,
+        metadata_fields_to_embed: Optional[List[str]] = None,
+        embedding_separator: str = "\n",
+    ):
+        """
+        Create a JinaDocumentEmbedder component.
+        :param api_key: The Jina API key. It can be explicitly provided or automatically read from the
+            environment variable JINA_API_KEY (recommended).
+        :param model_name: The name of the Jina model to use. Check the list of available models on `https://jina.ai/embeddings/`
+        :param prefix: A string to add to the beginning of each text.
+        :param suffix: A string to add to the end of each text.
+        :param batch_size: Number of Documents to encode at once.
+        :param progress_bar: Whether to show a progress bar or not. Can be helpful to disable in production deployments
+                             to keep the logs clean.
+        :param metadata_fields_to_embed: List of meta fields that should be embedded along with the Document text.
+        :param embedding_separator: Separator used to concatenate the meta fields to the Document text.
+        """
+        # if the user does not provide the API key, check if it is set in the module client
+        if api_key is None:
+            try:
+                api_key = os.environ["JINA_API_KEY"]
+            except KeyError as e:
+                msg = (
+                    "JinaDocumentEmbedder expects a Jina API key. "
+                    "Set the JINA_API_KEY environment variable (recommended) or pass it explicitly."
+                )
+                raise ValueError(msg) from e
+
+        self.model_name = model_name
+        self.prefix = prefix
+        self.suffix = suffix
+        self.prefix = prefix
+        self.suffix = suffix
+        self.batch_size = batch_size
+        self.progress_bar = progress_bar
+        self.metadata_fields_to_embed = metadata_fields_to_embed or []
+        self.embedding_separator = embedding_separator
+        self._session = requests.Session()
+        self._session.headers.update(
+            {
+                "Authorization": f"Bearer {api_key}",
+                "Accept-Encoding": "identity",
+                "Content-type": "application/json",
+            }
+        )
+
+    def _get_telemetry_data(self) -> Dict[str, Any]:
+        """
+        Data that is sent to Posthog for usage analytics.
+        """
+        return {"model": self.model_name}
+
+    def to_dict(self) -> Dict[str, Any]:
+        """
+        This method overrides the default serializer in order to avoid leaking the `api_key` value passed
+        to the constructor.
+        """
+        return default_to_dict(
+            self,
+            model_name=self.model_name,
+            prefix=self.prefix,
+            suffix=self.suffix,
+            batch_size=self.batch_size,
+            progress_bar=self.progress_bar,
+            metadata_fields_to_embed=self.metadata_fields_to_embed,
+            embedding_separator=self.embedding_separator,
+        )
+
+    def _prepare_texts_to_embed(self, documents: List[Document]) -> List[str]:
+        """
+        Prepare the texts to embed by concatenating the Document text with the metadata fields to embed.
+        """
+        texts_to_embed = []
+        for doc in documents:
+            meta_values_to_embed = [
+                str(doc.meta[key])
+                for key in self.metadata_fields_to_embed
+                if key in doc.meta and doc.meta[key] is not None
+            ]
+            text_to_embed = (
+                self.prefix + self.embedding_separator.join([*meta_values_to_embed, doc.content or ""]) + self.suffix
+            )
+
+            texts_to_embed.append(text_to_embed)
+        return texts_to_embed
+
+    def _embed_batch(self, texts_to_embed: List[str], batch_size: int) -> Tuple[List[List[float]], Dict[str, Any]]:
+        """
+        Embed a list of texts in batches.
+        """
+
+        all_embeddings = []
+        metadata = {}
+        for i in tqdm(
+            range(0, len(texts_to_embed), batch_size), disable=not self.progress_bar, desc="Calculating embeddings"
+        ):
+            batch = texts_to_embed[i : i + batch_size]
+            response = self._session.post(JINA_API_URL, json={"input": batch, "model": self.model_name}).json()
+            if "data" not in response:
+                raise RuntimeError(response["detail"])
+
+            # Sort resulting embeddings by index
+            sorted_embeddings = sorted(response["data"], key=lambda e: e["index"])
+            embeddings = [result["embedding"] for result in sorted_embeddings]
+            all_embeddings.extend(embeddings)
+            if "model" not in metadata:
+                metadata["model"] = response["model"]
+            if "usage" not in metadata:
+                metadata["usage"] = dict(response["usage"].items())
+            else:
+                metadata["usage"]["prompt_tokens"] += response["usage"]["prompt_tokens"]
+                metadata["usage"]["total_tokens"] += response["usage"]["total_tokens"]
+
+        return all_embeddings, metadata
+
+    @component.output_types(documents=List[Document], metadata=Dict[str, Any])
+    def run(self, documents: List[Document]):
+        """
+        Embed a list of Documents.
+        The embedding of each Document is stored in the `embedding` field of the Document.
+
+        :param documents: A list of Documents to embed.
+        """
+        if not isinstance(documents, list) or documents and not isinstance(documents[0], Document):
+            msg = (
+                "JinaDocumentEmbedder expects a list of Documents as input."
+                "In case you want to embed a string, please use the JinaTextEmbedder."
+            )
+            raise TypeError(msg)
+
+        texts_to_embed = self._prepare_texts_to_embed(documents=documents)
+
+        embeddings, metadata = self._embed_batch(texts_to_embed=texts_to_embed, batch_size=self.batch_size)
+
+        for doc, emb in zip(documents, embeddings):
+            doc.embedding = emb
+
+        return {"documents": documents, "metadata": metadata}

jina_haystack-0.0.1/src/jina_haystack/text_embedder.py

@@ -0,0 +1,106 @@
+# SPDX-FileCopyrightText: 2023-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+import os
+from typing import Any, Dict, List, Optional
+
+import requests
+from haystack import component, default_to_dict
+
+JINA_API_URL: str = "https://api.jina.ai/v1/embeddings"
+
+
+@component
+class JinaTextEmbedder:
+    """
+    A component for embedding strings using Jina models.
+
+    Usage example:
+    ```python
+    from jina_haystack import JinaTextEmbedder
+
+    text_to_embed = "I love pizza!"
+
+    text_embedder = JinaTextEmbedder()
+
+    print(text_embedder.run(text_to_embed))
+
+    # {'embedding': [0.017020374536514282, -0.023255806416273117, ...],
+    # 'metadata': {'model': 'jina-embeddings-v2-base-en',
+    #              'usage': {'prompt_tokens': 4, 'total_tokens': 4}}}
+    ```
+    """
+
+    def __init__(
+        self,
+        api_key: Optional[str] = None,
+        model_name: str = "jina-embeddings-v2-base-en",
+        prefix: str = "",
+        suffix: str = "",
+    ):
+        """
+        Create an JinaTextEmbedder component.
+
+        :param api_key: The Jina API key. It can be explicitly provided or automatically read from the
+            environment variable JINA_API_KEY (recommended).
+        :param model_name: The name of the Jina model to use. Check the list of available models on `https://jina.ai/embeddings/`
+        :param prefix: A string to add to the beginning of each text.
+        :param suffix: A string to add to the end of each text.
+        """
+        # if the user does not provide the API key, check if it is set in the module client
+        if api_key is None:
+            try:
+                api_key = os.environ["JINA_API_KEY"]
+            except KeyError as e:
+                msg = (
+                    "JinaTextEmbedder expects a Jina API key. "
+                    "Set the JINA_API_KEY environment variable (recommended) or pass it explicitly."
+                )
+                raise ValueError(msg) from e
+
+        self.model_name = model_name
+        self.prefix = prefix
+        self.suffix = suffix
+        self._session = requests.Session()
+        self._session.headers.update(
+            {
+                "Authorization": f"Bearer {api_key}",
+                "Accept-Encoding": "identity",
+                "Content-type": "application/json",
+            }
+        )
+
+    def _get_telemetry_data(self) -> Dict[str, Any]:
+        """
+        Data that is sent to Posthog for usage analytics.
+        """
+        return {"model": self.model_name}
+
+    def to_dict(self) -> Dict[str, Any]:
+        """
+        This method overrides the default serializer in order to avoid leaking the `api_key` value passed
+        to the constructor.
+        """
+
+        return default_to_dict(self, model_name=self.model_name, prefix=self.prefix, suffix=self.suffix)
+
+    @component.output_types(embedding=List[float], metadata=Dict[str, Any])
+    def run(self, text: str):
+        """Embed a string."""
+        if not isinstance(text, str):
+            msg = (
+                "JinaTextEmbedder expects a string as an input."
+                "In case you want to embed a list of Documents, please use the JinaDocumentEmbedder."
+            )
+            raise TypeError(msg)
+
+        text_to_embed = self.prefix + text + self.suffix
+
+        resp = self._session.post(JINA_API_URL, json={"input": [text_to_embed], "model": self.model_name}).json()
+        if "data" not in resp:
+            raise RuntimeError(resp["detail"])
+
+        metadata = {"model": resp["model"], "usage": dict(resp["usage"].items())}
+        embedding = resp["data"][0]["embedding"]
+
+        return {"embedding": embedding, "metadata": metadata}

jina_haystack-0.0.1/tests/__init__.py

@@ -0,0 +1,3 @@
+# SPDX-FileCopyrightText: 2023-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0

jina_haystack-0.0.1/tests/test_document_embedder.py

@@ -0,0 +1,239 @@
+# SPDX-FileCopyrightText: 2023-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+import json
+from unittest.mock import patch
+
+import pytest
+import requests
+from haystack import Document
+
+from jina_haystack import JinaDocumentEmbedder
+
+
+def mock_session_post_response(*args, **kwargs):  # noqa: ARG001
+    inputs = kwargs["json"]["input"]
+    model = kwargs["json"]["model"]
+    mock_response = requests.Response()
+    mock_response.status_code = 200
+    data = [{"object": "embedding", "index": i, "embedding": [0.1, 0.2, 0.3]} for i in range(len(inputs))]
+    mock_response._content = json.dumps(
+        {"model": model, "object": "list", "usage": {"total_tokens": 4, "prompt_tokens": 4}, "data": data}
+    ).encode()
+
+    return mock_response
+
+
+class TestJinaDocumentEmbedder:
+    def test_init_default(self, monkeypatch):
+        monkeypatch.setenv("JINA_API_KEY", "fake-api-key")
+        embedder = JinaDocumentEmbedder()
+
+        assert embedder.model_name == "jina-embeddings-v2-base-en"
+        assert embedder.prefix == ""
+        assert embedder.suffix == ""
+        assert embedder.batch_size == 32
+        assert embedder.progress_bar is True
+        assert embedder.metadata_fields_to_embed == []
+        assert embedder.embedding_separator == "\n"
+
+    def test_init_with_parameters(self):
+        embedder = JinaDocumentEmbedder(
+            api_key="fake-api-key",
+            model_name="model",
+            prefix="prefix",
+            suffix="suffix",
+            batch_size=64,
+            progress_bar=False,
+            metadata_fields_to_embed=["test_field"],
+            embedding_separator=" | ",
+        )
+        assert embedder.model_name == "model"
+        assert embedder.prefix == "prefix"
+        assert embedder.suffix == "suffix"
+        assert embedder.batch_size == 64
+        assert embedder.progress_bar is False
+        assert embedder.metadata_fields_to_embed == ["test_field"]
+        assert embedder.embedding_separator == " | "
+
+    def test_init_fail_wo_api_key(self, monkeypatch):
+        monkeypatch.delenv("JINA_API_KEY", raising=False)
+        with pytest.raises(ValueError, match="JinaDocumentEmbedder expects a Jina API key"):
+            JinaDocumentEmbedder()
+
+    def test_to_dict(self):
+        component = JinaDocumentEmbedder(api_key="fake-api-key")
+        data = component.to_dict()
+        assert data == {
+            "type": "jina_haystack.document_embedder.JinaDocumentEmbedder",
+            "init_parameters": {
+                "model_name": "jina-embeddings-v2-base-en",
+                "prefix": "",
+                "suffix": "",
+                "batch_size": 32,
+                "progress_bar": True,
+                "metadata_fields_to_embed": [],
+                "embedding_separator": "\n",
+            },
+        }
+
+    def test_to_dict_with_custom_init_parameters(self):
+        component = JinaDocumentEmbedder(
+            api_key="fake-api-key",
+            model_name="model",
+            prefix="prefix",
+            suffix="suffix",
+            batch_size=64,
+            progress_bar=False,
+            metadata_fields_to_embed=["test_field"],
+            embedding_separator=" | ",
+        )
+        data = component.to_dict()
+        assert data == {
+            "type": "jina_haystack.document_embedder.JinaDocumentEmbedder",
+            "init_parameters": {
+                "model_name": "model",
+                "prefix": "prefix",
+                "suffix": "suffix",
+                "batch_size": 64,
+                "progress_bar": False,
+                "metadata_fields_to_embed": ["test_field"],
+                "embedding_separator": " | ",
+            },
+        }
+
+    def test_prepare_texts_to_embed_w_metadata(self):
+        documents = [
+            Document(content=f"document number {i}:\ncontent", meta={"meta_field": f"meta_value {i}"}) for i in range(5)
+        ]
+
+        embedder = JinaDocumentEmbedder(
+            api_key="fake-api-key", metadata_fields_to_embed=["meta_field"], embedding_separator=" | "
+        )
+
+        prepared_texts = embedder._prepare_texts_to_embed(documents)
+
+        # note that newline is replaced by space
+        assert prepared_texts == [
+            "meta_value 0 | document number 0:\ncontent",
+            "meta_value 1 | document number 1:\ncontent",
+            "meta_value 2 | document number 2:\ncontent",
+            "meta_value 3 | document number 3:\ncontent",
+            "meta_value 4 | document number 4:\ncontent",
+        ]
+
+    def test_prepare_texts_to_embed_w_suffix(self):
+        documents = [Document(content=f"document number {i}") for i in range(5)]
+
+        embedder = JinaDocumentEmbedder(api_key="fake-api-key", prefix="my_prefix ", suffix=" my_suffix")
+
+        prepared_texts = embedder._prepare_texts_to_embed(documents)
+
+        assert prepared_texts == [
+            "my_prefix document number 0 my_suffix",
+            "my_prefix document number 1 my_suffix",
+            "my_prefix document number 2 my_suffix",
+            "my_prefix document number 3 my_suffix",
+            "my_prefix document number 4 my_suffix",
+        ]
+
+    def test_embed_batch(self):
+        texts = ["text 1", "text 2", "text 3", "text 4", "text 5"]
+
+        with patch("requests.sessions.Session.post", side_effect=mock_session_post_response):
+            embedder = JinaDocumentEmbedder(api_key="fake-api-key", model_name="model")
+
+            embeddings, metadata = embedder._embed_batch(texts_to_embed=texts, batch_size=2)
+
+        assert isinstance(embeddings, list)
+        assert len(embeddings) == len(texts)
+        for embedding in embeddings:
+            assert isinstance(embedding, list)
+            assert len(embedding) == 3
+            assert all(isinstance(x, float) for x in embedding)
+
+        assert metadata == {"model": "model", "usage": {"prompt_tokens": 3 * 4, "total_tokens": 3 * 4}}
+
+    def test_run(self):
+        docs = [
+            Document(content="I love cheese", meta={"topic": "Cuisine"}),
+            Document(content="A transformer is a deep learning architecture", meta={"topic": "ML"}),
+        ]
+
+        model = "jina-embeddings-v2-base-en"
+        with patch("requests.sessions.Session.post", side_effect=mock_session_post_response):
+            embedder = JinaDocumentEmbedder(
+                api_key="fake-api-key",
+                model_name=model,
+                prefix="prefix ",
+                suffix=" suffix",
+                metadata_fields_to_embed=["topic"],
+                embedding_separator=" | ",
+            )
+
+            result = embedder.run(documents=docs)
+
+        documents_with_embeddings = result["documents"]
+        metadata = result["metadata"]
+
+        assert isinstance(documents_with_embeddings, list)
+        assert len(documents_with_embeddings) == len(docs)
+        for doc in documents_with_embeddings:
+            assert isinstance(doc, Document)
+            assert isinstance(doc.embedding, list)
+            assert len(doc.embedding) == 3
+            assert all(isinstance(x, float) for x in doc.embedding)
+        assert metadata == {"model": model, "usage": {"prompt_tokens": 4, "total_tokens": 4}}
+
+    def test_run_custom_batch_size(self):
+        docs = [
+            Document(content="I love cheese", meta={"topic": "Cuisine"}),
+            Document(content="A transformer is a deep learning architecture", meta={"topic": "ML"}),
+        ]
+        model = "jina-embeddings-v2-base-en"
+        with patch("requests.sessions.Session.post", side_effect=mock_session_post_response):
+            embedder = JinaDocumentEmbedder(
+                api_key="fake-api-key",
+                model_name=model,
+                prefix="prefix ",
+                suffix=" suffix",
+                metadata_fields_to_embed=["topic"],
+                embedding_separator=" | ",
+                batch_size=1,
+            )
+
+            result = embedder.run(documents=docs)
+
+        documents_with_embeddings = result["documents"]
+        metadata = result["metadata"]
+
+        assert isinstance(documents_with_embeddings, list)
+        assert len(documents_with_embeddings) == len(docs)
+        for doc in documents_with_embeddings:
+            assert isinstance(doc, Document)
+            assert isinstance(doc.embedding, list)
+            assert len(doc.embedding) == 3
+            assert all(isinstance(x, float) for x in doc.embedding)
+
+        assert metadata == {"model": model, "usage": {"prompt_tokens": 2 * 4, "total_tokens": 2 * 4}}
+
+    def test_run_wrong_input_format(self):
+        embedder = JinaDocumentEmbedder(api_key="fake-api-key")
+
+        string_input = "text"
+        list_integers_input = [1, 2, 3]
+
+        with pytest.raises(TypeError, match="JinaDocumentEmbedder expects a list of Documents as input"):
+            embedder.run(documents=string_input)
+
+        with pytest.raises(TypeError, match="JinaDocumentEmbedder expects a list of Documents as input"):
+            embedder.run(documents=list_integers_input)
+
+    def test_run_on_empty_list(self):
+        embedder = JinaDocumentEmbedder(api_key="fake-api-key")
+
+        empty_list_input = []
+        result = embedder.run(documents=empty_list_input)
+
+        assert result["documents"] is not None
+        assert not result["documents"]  # empty list

jina_haystack-0.0.1/tests/test_text_embedder.py

@@ -0,0 +1,100 @@
+# SPDX-FileCopyrightText: 2023-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+import json
+from unittest.mock import patch
+
+import pytest
+import requests
+
+from jina_haystack import JinaTextEmbedder
+
+
+class TestJinaTextEmbedder:
+    def test_init_default(self, monkeypatch):
+        monkeypatch.setenv("JINA_API_KEY", "fake-api-key")
+        embedder = JinaTextEmbedder()
+
+        assert embedder.model_name == "jina-embeddings-v2-base-en"
+        assert embedder.prefix == ""
+        assert embedder.suffix == ""
+
+    def test_init_with_parameters(self):
+        embedder = JinaTextEmbedder(
+            api_key="fake-api-key",
+            model_name="model",
+            prefix="prefix",
+            suffix="suffix",
+        )
+        assert embedder.model_name == "model"
+        assert embedder.prefix == "prefix"
+        assert embedder.suffix == "suffix"
+
+    def test_init_fail_wo_api_key(self, monkeypatch):
+        monkeypatch.delenv("JINA_API_KEY", raising=False)
+        with pytest.raises(ValueError, match="JinaTextEmbedder expects a Jina API key"):
+            JinaTextEmbedder()
+
+    def test_to_dict(self):
+        component = JinaTextEmbedder(api_key="fake-api-key")
+        data = component.to_dict()
+        assert data == {
+            "type": "jina_haystack.text_embedder.JinaTextEmbedder",
+            "init_parameters": {
+                "model_name": "jina-embeddings-v2-base-en",
+                "prefix": "",
+                "suffix": "",
+            },
+        }
+
+    def test_to_dict_with_custom_init_parameters(self):
+        component = JinaTextEmbedder(
+            api_key="fake-api-key",
+            model_name="model",
+            prefix="prefix",
+            suffix="suffix",
+        )
+        data = component.to_dict()
+        assert data == {
+            "type": "jina_haystack.text_embedder.JinaTextEmbedder",
+            "init_parameters": {
+                "model_name": "model",
+                "prefix": "prefix",
+                "suffix": "suffix",
+            },
+        }
+
+    def test_run(self):
+        model = "jina-embeddings-v2-base-en"
+        with patch("requests.sessions.Session.post") as mock_post:
+            # Configure the mock to return a specific response
+            mock_response = requests.Response()
+            mock_response.status_code = 200
+            mock_response._content = json.dumps(
+                {
+                    "model": "jina-embeddings-v2-base-en",
+                    "object": "list",
+                    "usage": {"total_tokens": 6, "prompt_tokens": 6},
+                    "data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2, 0.3]}],
+                }
+            ).encode()
+
+            mock_post.return_value = mock_response
+
+            embedder = JinaTextEmbedder(api_key="fake-api-key", model_name=model, prefix="prefix ", suffix=" suffix")
+            result = embedder.run(text="The food was delicious")
+
+        assert len(result["embedding"]) == 3
+        assert all(isinstance(x, float) for x in result["embedding"])
+        assert result["metadata"] == {
+            "model": "jina-embeddings-v2-base-en",
+            "usage": {"prompt_tokens": 6, "total_tokens": 6},
+        }
+
+    def test_run_wrong_input_format(self):
+        embedder = JinaTextEmbedder(api_key="fake-api-key")
+
+        list_integers_input = [1, 2, 3]
+
+        with pytest.raises(TypeError, match="JinaTextEmbedder expects a string as an input"):
+            embedder.run(text=list_integers_input)