haystack-experimental 0.14.3__py3-none-any.whl → 0.15.1__py3-none-any.whl

haystack_experimental/components/retrievers/multi_query_text_retriever.py

@@ -1,150 +0,0 @@
-# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
-#
-# SPDX-License-Identifier: Apache-2.0
-
-from concurrent.futures import ThreadPoolExecutor
-from typing import Any, Optional
-
-from haystack import Document, component, default_from_dict, default_to_dict
-from haystack.core.serialization import component_to_dict
-from haystack.utils.deserialization import deserialize_component_inplace
-
-from haystack_experimental.components.retrievers.types import TextRetriever
-
-
-@component
-class MultiQueryTextRetriever:
-    """
-    A component that retrieves documents using multiple queries in parallel with a text-based retriever.
-
-    This component takes a list of text queries and uses a text-based retriever to find relevant documents for each
-    query in parallel, using a thread pool to manage concurrent execution. The results are combined and sorted by
-    relevance score.
-
-    You can use this component in combination with QueryExpander component to enhance the retrieval process.
-
-    ### Usage example
-    ```python
-    from haystack import Document
-    from haystack.components.writers import DocumentWriter
-    from haystack.document_stores.in_memory import InMemoryDocumentStore
-    from haystack.document_stores.types import DuplicatePolicy
-    from haystack.components.retrievers import InMemoryBM25Retriever
-    from haystack_experimental.components.query import QueryExpander
-    from haystack_experimental.components.retrievers.multi_query_text_retriever import MultiQueryTextRetriever
-
-    documents = [
-        Document(content="Renewable energy is energy that is collected from renewable resources."),
-        Document(content="Solar energy is a type of green energy that is harnessed from the sun."),
-        Document(content="Wind energy is another type of green energy that is generated by wind turbines."),
-        Document(content="Hydropower is a form of renewable energy using the flow of water to generate electricity."),
-        Document(content="Geothermal energy is heat that comes from the sub-surface of the earth.")
-    ]
-
-    document_store = InMemoryDocumentStore()
-    doc_writer = DocumentWriter(document_store=document_store, policy=DuplicatePolicy.SKIP)
-    doc_writer.run(documents=documents)
-
-    in_memory_retriever = InMemoryBM25Retriever(document_store=document_store, top_k=1)
-    multiquery_retriever = MultiQueryTextRetriever(retriever=in_memory_retriever)
-    results = multiquery_retriever.run(queries=["renewable energy?", "Geothermal", "Hydropower"])
-    for doc in results["documents"]:
-        print(f"Content: {doc.content}, Score: {doc.score}")
-    >>
-    >> Content: Geothermal energy is heat that comes from the sub-surface of the earth., Score: 1.6474448833731097
-    >> Content: Hydropower is a form of renewable energy using the flow of water to generate electricity., Score: 1.6157822790079805
-    >> Content: Renewable energy is energy that is collected from renewable resources., Score: 1.5255309812344944
-    ```
-    """  # noqa E501
-
-    def __init__(self, *, retriever: TextRetriever, max_workers: int = 3) -> None:
-        """
-        Initialize MultiQueryTextRetriever.
-
-        :param retriever: The text-based retriever to use for document retrieval.
-        :param max_workers: Maximum number of worker threads for parallel processing. Default is 3.
-        """
-        self.retriever = retriever
-        self.max_workers = max_workers
-        self._is_warmed_up = False
-
-    def warm_up(self) -> None:
-        """
-        Warm up the retriever if it has a warm_up method.
-        """
-        if not self._is_warmed_up:
-            if hasattr(self.retriever, "warm_up") and callable(getattr(self.retriever, "warm_up")):
-                self.retriever.warm_up()
-            self._is_warmed_up = True
-
-    @component.output_types(documents=list[Document])
-    def run(self, queries: list[str], retriever_kwargs: Optional[dict[str, Any]] = None) -> dict[str, list[Document]]:
-        """
-        Retrieve documents using multiple queries in parallel.
-
-        :param queries: List of text queries to process.
-        :param retriever_kwargs: Optional dictionary of arguments to pass to the retriever's run method.
-        :returns:
-            A dictionary containing:
-                `documents`: List of retrieved documents sorted by relevance score.
-        """
-        docs: list[Document] = []
-        seen_contents = set()
-        retriever_kwargs = retriever_kwargs or {}
-
-        with ThreadPoolExecutor(max_workers=self.max_workers) as executor:
-            queries_results = executor.map(lambda query: self._run_on_thread(query, retriever_kwargs), queries)
-            for result in queries_results:
-                if not result:
-                    continue
-                # deduplicate based on content
-                for doc in result:
-                    if doc.content not in seen_contents:
-                        docs.append(doc)
-                        seen_contents.add(doc.content)
-
-        docs.sort(key=lambda x: x.score or 0.0, reverse=True)
-        return {"documents": docs}
-
-    def _run_on_thread(
-        self,
-        query: str,
-        retriever_kwargs: Optional[dict[str, Any]] = None,
-    ) -> Optional[list[Document]]:
-        """
-        Process a single query on a separate thread.
-
-        :param query: The text query to process.
-        :param retriever_kwargs: Optional dictionary of arguments to pass to the retriever's run method.
-        :returns:
-            List of retrieved documents or None if no results.
-        """
-        result = self.retriever.run(query=query, **(retriever_kwargs or {}))
-        if result and "documents" in result:
-            return result["documents"]
-        return None
-
-    def to_dict(self) -> dict[str, Any]:
-        """
-        Serializes the component to a dictionary.
-
-        :returns:
-            The serialized component as a dictionary.
-        """
-        return default_to_dict(
-            self,
-            retriever=component_to_dict(obj=self.retriever, name="retriever"),
-            max_workers=self.max_workers,
-        )
-
-    @classmethod
-    def from_dict(cls, data: dict[str, Any]) -> "MultiQueryTextRetriever":
-        """
-        Deserializes the component from a dictionary.
-
-        :param data: The dictionary to deserialize from.
-        :returns:
-            The deserialized component.
-        """
-        deserialize_component_inplace(data["init_parameters"], key="retriever")
-        return default_from_dict(cls, data)

haystack_experimental/super_components/__init__.py

@@ -1,3 +0,0 @@
-# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
-#
-# SPDX-License-Identifier: Apache-2.0

haystack_experimental/super_components/indexers/__init__.py

@@ -1,11 +0,0 @@
-# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
-#
-# SPDX-License-Identifier: Apache-2.0
-
-from haystack_experimental.super_components.indexers.sentence_transformers_document_indexer import (
-    SentenceTransformersDocumentIndexer,
-)
-
-__all__ = [
-    "SentenceTransformersDocumentIndexer",
-]

haystack_experimental/super_components/indexers/sentence_transformers_document_indexer.py

@@ -1,199 +0,0 @@
-# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
-#
-# SPDX-License-Identifier: Apache-2.0
-
-from typing import Any, Dict, List, Literal, Optional
-
-from haystack import Pipeline, component, default_from_dict, default_to_dict
-from haystack.components.embedders import SentenceTransformersDocumentEmbedder
-from haystack.components.writers import DocumentWriter
-from haystack.core.super_component import SuperComponent
-from haystack.document_stores.types import DocumentStore, DuplicatePolicy
-from haystack.utils import (
-    ComponentDevice,
-    Secret,
-    deserialize_document_store_in_init_params_inplace,
-    deserialize_secrets_inplace,
-)
-from haystack.utils.hf import deserialize_hf_model_kwargs, serialize_hf_model_kwargs
-
-
-@component
-class SentenceTransformersDocumentIndexer(SuperComponent):
-    """
-    A document indexer that takes a list of documents, embeds them using SentenceTransformers, and stores them.
-
-    Usage:
-
-    ```python
-    >>> from haystack import Document
-    >>> from haystack.document_stores.in_memory import InMemoryDocumentStore
-    >>> document_store = InMemoryDocumentStore()
-    >>> doc = Document(content="I love pizza!")
-    >>> indexer = SentenceTransformersDocumentIndexer(document_store=document_store)
-    >>> indexer.warm_up()
-    >>> result = indexer.run(documents=[doc])
-    >>> print(result)
-    {'documents_written': 1}
-    >>> document_store.count_documents()
-    1
-    ```
-    """
-
-    def __init__(  # pylint: disable=R0917
-        self,
-        document_store: DocumentStore,
-        model: str = "sentence-transformers/all-mpnet-base-v2",
-        device: Optional[ComponentDevice] = None,
-        token: Optional[Secret] = Secret.from_env_var(["HF_API_TOKEN", "HF_TOKEN"], strict=False),
-        prefix: str = "",
-        suffix: str = "",
-        batch_size: int = 32,
-        progress_bar: bool = True,
-        normalize_embeddings: bool = False,
-        meta_fields_to_embed: Optional[List[str]] = None,
-        embedding_separator: str = "\n",
-        trust_remote_code: bool = False,
-        truncate_dim: Optional[int] = None,
-        model_kwargs: Optional[Dict[str, Any]] = None,
-        tokenizer_kwargs: Optional[Dict[str, Any]] = None,
-        config_kwargs: Optional[Dict[str, Any]] = None,
-        precision: Literal["float32", "int8", "uint8", "binary", "ubinary"] = "float32",
-        duplicate_policy: DuplicatePolicy = DuplicatePolicy.OVERWRITE,
-    ) -> None:
-        """
-        Initialize the SentenceTransformersDocumentIndexer component.
-
-        :param document_store: The document store where the documents should be stored.
-        :param model: The embedding model to use (local path or Hugging Face model ID).
-        :param device: The device to use for loading the model.
-        :param token: The API token to download private models from Hugging Face.
-        :param prefix: String to add at the beginning of each document text.
-        :param suffix: String to add at the end of each document text.
-        :param batch_size: Number of documents to embed at once.
-        :param progress_bar: If True, shows a progress bar when embedding documents.
-        :param normalize_embeddings: If True, embeddings are L2 normalized.
-        :param meta_fields_to_embed: List of metadata fields to embed along with the document text.
-        :param embedding_separator: Separator used to concatenate metadata fields to document text.
-        :param trust_remote_code: If True, allows custom models and scripts.
-        :param truncate_dim: Dimension to truncate sentence embeddings to.
-        :param model_kwargs: Additional keyword arguments for model initialization.
-        :param tokenizer_kwargs: Additional keyword arguments for tokenizer initialization.
-        :param config_kwargs: Additional keyword arguments for model configuration.
-        :param precision: The precision to use for the embeddings.
-        :param duplicate_policy: The duplicate policy to use when writing documents.
-        """
-        self.document_store = document_store
-        self.model = model
-        self.device = device
-        self.token = token
-        self.prefix = prefix
-        self.suffix = suffix
-        self.batch_size = batch_size
-        self.progress_bar = progress_bar
-        self.normalize_embeddings = normalize_embeddings
-        self.meta_fields_to_embed = meta_fields_to_embed
-        self.embedding_separator = embedding_separator
-        self.trust_remote_code = trust_remote_code
-        self.truncate_dim = truncate_dim
-        self.model_kwargs = model_kwargs
-        self.tokenizer_kwargs = tokenizer_kwargs
-        self.config_kwargs = config_kwargs
-        self.precision = precision
-        self.duplicate_policy = duplicate_policy
-
-        pipeline = Pipeline()
-
-        pipeline.add_component(
-            "embedder",
-            SentenceTransformersDocumentEmbedder(
-                model=self.model,
-                device=self.device,
-                token=self.token,
-                prefix=self.prefix,
-                suffix=self.suffix,
-                batch_size=self.batch_size,
-                progress_bar=self.progress_bar,
-                normalize_embeddings=self.normalize_embeddings,
-                meta_fields_to_embed=self.meta_fields_to_embed,
-                embedding_separator=self.embedding_separator,
-                trust_remote_code=self.trust_remote_code,
-                truncate_dim=self.truncate_dim,
-                model_kwargs=self.model_kwargs,
-                tokenizer_kwargs=self.tokenizer_kwargs,
-                config_kwargs=self.config_kwargs,
-                precision=self.precision,
-            ),
-        )
-        pipeline.add_component(
-            "writer",
-            DocumentWriter(
-                document_store=self.document_store,
-                policy=self.duplicate_policy,
-            ),
-        )
-
-        pipeline.connect("embedder.documents", "writer.documents")
-
-        super(SentenceTransformersDocumentIndexer, self).__init__(
-            pipeline=pipeline,
-            input_mapping={"documents": ["embedder.documents"]},
-            output_mapping={"writer.documents_written": "documents_written"},
-        )
-
-    def to_dict(self) -> Dict[str, Any]:
-        """
-        Serialize this instance to a dictionary.
-        """
-        serialization_dict = default_to_dict(
-            self,
-            document_store=self.document_store.to_dict(),
-            model=self.model,
-            device=self.device.to_dict() if self.device else None,
-            token=self.token.to_dict() if self.token else None,
-            prefix=self.prefix,
-            suffix=self.suffix,
-            batch_size=self.batch_size,
-            progress_bar=self.progress_bar,
-            normalize_embeddings=self.normalize_embeddings,
-            meta_fields_to_embed=self.meta_fields_to_embed,
-            embedding_separator=self.embedding_separator,
-            trust_remote_code=self.trust_remote_code,
-            truncate_dim=self.truncate_dim,
-            model_kwargs=self.model_kwargs,
-            tokenizer_kwargs=self.tokenizer_kwargs,
-            config_kwargs=self.config_kwargs,
-            precision=self.precision,
-            duplicate_policy=self.duplicate_policy.value,
-        )
-
-        if serialization_dict["init_parameters"].get("model_kwargs") is not None:
-            serialize_hf_model_kwargs(serialization_dict["init_parameters"]["model_kwargs"])
-
-        return serialization_dict
-
-    @classmethod
-    def from_dict(cls, data: Dict[str, Any]) -> "SentenceTransformersDocumentIndexer":
-        """
-        Load an instance of this component from a dictionary.
-        """
-        deserialize_document_store_in_init_params_inplace(data)
-        init_params = data.get("init_parameters", {})
-
-        # Handle device deserialization
-        if init_params.get("device") is not None:
-            init_params["device"] = ComponentDevice.from_dict(init_params["device"])
-
-        # Handle secrets deserialization
-        deserialize_secrets_inplace(init_params, keys=["token"])
-
-        # Handle model kwargs deserialization
-        if init_params.get("model_kwargs") is not None:
-            deserialize_hf_model_kwargs(init_params["model_kwargs"])
-
-        # Handle duplicate policy deserialization
-        if policy_value := init_params.get("duplicate_policy"):
-            init_params["duplicate_policy"] = DuplicatePolicy(policy_value)
-
-        data["init_parameters"] = init_params
-        return default_from_dict(cls, data)