haystack-experimental 0.0.1__tar.gz → 0.0.2.dev0__tar.gz

{haystack_experimental-0.0.1 → haystack_experimental-0.0.2.dev0}/LICENSE

@@ -186,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright [yyyy] [name of copyright owner]
+   Copyright 2024-present deepset GmbH <info@deepset.ai>
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

haystack_experimental-0.0.2.dev0/PKG-INFO

@@ -0,0 +1,145 @@
+Metadata-Version: 2.3
+Name: haystack-experimental
+Version: 0.0.2.dev0
+Summary: Experimental components and features for the Haystack LLM framework.
+Project-URL: CI: GitHub, https://github.com/deepset-ai/haystack-experimental/actions
+Project-URL: GitHub: issues, https://github.com/deepset-ai/haystack-experimental/issues
+Project-URL: GitHub: repo, https://github.com/deepset-ai/haystack-experimental
+Project-URL: Homepage, https://github.com/deepset-ai/haystack-experimental
+Author-email: "deepset.ai" <info@deepset.ai>
+License: Apache-2.0
+License-File: LICENSE
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: Freely Distributable
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.8
+Requires-Dist: haystack-ai
+Description-Content-Type: text/markdown
+
+[![PyPI - Version](https://img.shields.io/pypi/v/haystack-experimental.svg)](https://pypi.org/project/haystack-experimental)
+[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/haystack-experimental.svg)](https://pypi.org/project/haystack-experimental)
+[![Tests](https://github.com/deepset-ai/haystack-experimental/actions/workflows/tests.yml/badge.svg)](https://github.com/deepset-ai/haystack-experimental/actions/workflows/tests.yml)
+[![Project release on PyPi](https://github.com/deepset-ai/haystack-experimental/actions/workflows/pypi_release.yml/badge.svg)](https://github.com/deepset-ai/haystack-experimental/actions/workflows/pypi_release.yml)
+[![Hatch project](https://img.shields.io/badge/%F0%9F%A5%9A-Hatch-4051b5.svg)](https://github.com/pypa/hatch)
+[![Checked with mypy](https://www.mypy-lang.org/static/mypy_badge.svg)](https://mypy-lang.org/)
+
+# Haystack experimental package
+
+The `haystack-experimental` package provides Haystack users with access to experimental features without immediately
+committing to their official release. The main goal is to gather user feedback and iterate on new features quickly.
+
+## Installation
+
+For simplicity, every release of `haystack-experimental` will ship all the available experiments at that time. To
+install the latest experimental features, run:
+
+```sh
+$ pip install -U haystack-experimental
+```
+
+> [!IMPORTANT]
+> The latest version of the experimental package is only tested against the latest version of Haystack. Compatibility
+> with older versions of Haystack is not guaranteed.
+
+
+## Experiments lifecycle
+
+Any experimental feature will be removed from `haystack-experimental` after a period of 3 months. After this time,
+the experiment will be either:
+- Merged into Haystack core and published in the next minor release
+- Released as a Core Integration,
+- Dropped.
+
+## Experiments catalog
+
+The latest version of the package contains the following experiments:
+
+| Name                     | Type                    | Experiment end date |
+| ------------------------ | ----------------------- | ------------------- |
+| [`EvaluationHarness`][1] | Evaluation orchestrator | August 2024         |
+| [`OpenAIFunctionCaller`][2] | Function Calling Component | August 2024         |
+
+[1]: https://github.com/deepset-ai/haystack-experimental/tree/main/haystack_experimental/evaluation/harness
+[2]: https://github.com/deepset-ai/haystack-experimental/tree/main/haystack_experimental/components/tools/openai
+
+## Usage
+
+Experimental new features can be imported like any other Haystack integration package:
+
+```python
+from haystack.dataclasses import ChatMessage
+from haystack_experimental.components.generators import FoobarGenerator
+
+c = FoobarGenerator()
+c.run([ChatMessage.from_user("What's an experiment? Be brief.")])
+```
+
+Experiments can also override existing Haystack features. For example, users can opt into an experimental type of
+`Pipeline` by just changing the usual import:
+
+```python
+# from haystack import Pipeline
+from haystack_experimental import Pipeline
+
+pipe = Pipeline()
+# ...
+pipe.run(...)
+```
+
+## Documentation
+
+Documentation for `haystack-experimental` can be found [here](https://docs.haystack.deepset.ai/reference/haystack-experimental-api).
+
+## Implementation
+
+Experiments should replicate the namespace of the core package. For example, a new generator:
+
+```python
+# in haystack_experimental/components/generators/foobar.py
+
+from haystack import component
+
+
+@component
+class FoobarGenerator:
+    ...
+
+```
+
+When the experiment overrides an existing feature, the new symbol should be created at the same path in the experimental
+package. This new symbol will override the original in `haystack-ai`: for classes, with a subclass and for bare
+functions, with a wrapper. For example:
+
+```python
+# in haystack_experiment/src/haystack_experiment/core/pipeline/pipeline.py
+
+from haystack.core.pipeline import Pipeline as HaystackPipeline
+
+
+class Pipeline(HaystackPipeline):
+	# Any new experimental method that doesn't exist in the original class
+	def run_async(self, inputs) -> Dict[str, Dict[str, Any]]:
+		...
+
+	# Existing methods with breaking changes to their signature, like adding a new mandatory param
+    def to_dict(new_param: str) -> Dict[str, Any]:
+        # do something with the new parameter
+        print(new_param)
+        # call the original method
+        return super().to_dict()
+
+```
+
+## Contributing
+
+Direct contributions to `haystack-experimental` are not expected, but Haystack maintainers might ask contributors to move pull requests that target the [core repository](https://github.com/deepset-ai/haystack) to this repository.

haystack_experimental-0.0.2.dev0/README.md

@@ -0,0 +1,117 @@
+[![PyPI - Version](https://img.shields.io/pypi/v/haystack-experimental.svg)](https://pypi.org/project/haystack-experimental)
+[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/haystack-experimental.svg)](https://pypi.org/project/haystack-experimental)
+[![Tests](https://github.com/deepset-ai/haystack-experimental/actions/workflows/tests.yml/badge.svg)](https://github.com/deepset-ai/haystack-experimental/actions/workflows/tests.yml)
+[![Project release on PyPi](https://github.com/deepset-ai/haystack-experimental/actions/workflows/pypi_release.yml/badge.svg)](https://github.com/deepset-ai/haystack-experimental/actions/workflows/pypi_release.yml)
+[![Hatch project](https://img.shields.io/badge/%F0%9F%A5%9A-Hatch-4051b5.svg)](https://github.com/pypa/hatch)
+[![Checked with mypy](https://www.mypy-lang.org/static/mypy_badge.svg)](https://mypy-lang.org/)
+
+# Haystack experimental package
+
+The `haystack-experimental` package provides Haystack users with access to experimental features without immediately
+committing to their official release. The main goal is to gather user feedback and iterate on new features quickly.
+
+## Installation
+
+For simplicity, every release of `haystack-experimental` will ship all the available experiments at that time. To
+install the latest experimental features, run:
+
+```sh
+$ pip install -U haystack-experimental
+```
+
+> [!IMPORTANT]
+> The latest version of the experimental package is only tested against the latest version of Haystack. Compatibility
+> with older versions of Haystack is not guaranteed.
+
+
+## Experiments lifecycle
+
+Any experimental feature will be removed from `haystack-experimental` after a period of 3 months. After this time,
+the experiment will be either:
+- Merged into Haystack core and published in the next minor release
+- Released as a Core Integration,
+- Dropped.
+
+## Experiments catalog
+
+The latest version of the package contains the following experiments:
+
+| Name                     | Type                    | Experiment end date |
+| ------------------------ | ----------------------- | ------------------- |
+| [`EvaluationHarness`][1] | Evaluation orchestrator | August 2024         |
+| [`OpenAIFunctionCaller`][2] | Function Calling Component | August 2024         |
+
+[1]: https://github.com/deepset-ai/haystack-experimental/tree/main/haystack_experimental/evaluation/harness
+[2]: https://github.com/deepset-ai/haystack-experimental/tree/main/haystack_experimental/components/tools/openai
+
+## Usage
+
+Experimental new features can be imported like any other Haystack integration package:
+
+```python
+from haystack.dataclasses import ChatMessage
+from haystack_experimental.components.generators import FoobarGenerator
+
+c = FoobarGenerator()
+c.run([ChatMessage.from_user("What's an experiment? Be brief.")])
+```
+
+Experiments can also override existing Haystack features. For example, users can opt into an experimental type of
+`Pipeline` by just changing the usual import:
+
+```python
+# from haystack import Pipeline
+from haystack_experimental import Pipeline
+
+pipe = Pipeline()
+# ...
+pipe.run(...)
+```
+
+## Documentation
+
+Documentation for `haystack-experimental` can be found [here](https://docs.haystack.deepset.ai/reference/haystack-experimental-api).
+
+## Implementation
+
+Experiments should replicate the namespace of the core package. For example, a new generator:
+
+```python
+# in haystack_experimental/components/generators/foobar.py
+
+from haystack import component
+
+
+@component
+class FoobarGenerator:
+    ...
+
+```
+
+When the experiment overrides an existing feature, the new symbol should be created at the same path in the experimental
+package. This new symbol will override the original in `haystack-ai`: for classes, with a subclass and for bare
+functions, with a wrapper. For example:
+
+```python
+# in haystack_experiment/src/haystack_experiment/core/pipeline/pipeline.py
+
+from haystack.core.pipeline import Pipeline as HaystackPipeline
+
+
+class Pipeline(HaystackPipeline):
+	# Any new experimental method that doesn't exist in the original class
+	def run_async(self, inputs) -> Dict[str, Dict[str, Any]]:
+		...
+
+	# Existing methods with breaking changes to their signature, like adding a new mandatory param
+    def to_dict(new_param: str) -> Dict[str, Any]:
+        # do something with the new parameter
+        print(new_param)
+        # call the original method
+        return super().to_dict()
+
+```
+
+## Contributing
+
+Direct contributions to `haystack-experimental` are not expected, but Haystack maintainers might ask contributors to move pull requests that target the [core repository](https://github.com/deepset-ai/haystack) to this repository.

haystack_experimental-0.0.2.dev0/haystack_experimental/__init__.py

@@ -0,0 +1,3 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0

haystack_experimental-0.0.2.dev0/haystack_experimental/components/__init__.py

@@ -0,0 +1,7 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from .tools import OpenAIFunctionCaller
+
+_all_ = [ "OpenAIFunctionCaller"]

haystack_experimental-0.0.2.dev0/haystack_experimental/components/tools/__init__.py

@@ -0,0 +1,7 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from .openai.function_caller import OpenAIFunctionCaller
+
+_all_ = ["OpenAIFunctionCaller"]

haystack_experimental-0.0.2.dev0/haystack_experimental/components/tools/openai/__init__.py

@@ -0,0 +1,7 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from .function_caller import OpenAIFunctionCaller
+
+_all_ = [ "OpenAIFunctionCaller"]

haystack_experimental-0.0.2.dev0/haystack_experimental/components/tools/openai/function_caller.py

@@ -0,0 +1,101 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import json
+from typing import Any, Callable, Dict, List
+
+from haystack import component, default_from_dict, default_to_dict
+from haystack.dataclasses import ChatMessage
+from haystack.utils import deserialize_callable, serialize_callable
+
+_FUNCTION_NAME_FAILURE = (
+    "I'm sorry, I tried to run a function that did not exist. Would you like me to correct it and try again?"
+)
+_FUNCTION_RUN_FAILURE = "Seems there was an error while running the function: {error}"
+
+
+@component
+class OpenAIFunctionCaller:
+    """
+    OpenAIFunctionCaller processes a list of chat messages and call Python functions when needed.
+
+    The OpenAIFunctionCaller expects a list of ChatMessages and if there is a tool call with a function name and
+    arguments, it runs the function and returns the result as a ChatMessage from role = 'function'
+    """
+
+    def __init__(self, available_functions: Dict[str, Callable]):
+        """
+        Initialize the OpenAIFunctionCaller component.
+
+        :param available_functions:
+            A dictionary of available functions. This dictionary expects key value pairs of function name,
+            and the function itself. For example, `{"weather_function": weather_function}`
+        """
+        self.available_functions = available_functions
+
+    def to_dict(self) -> Dict[str, Any]:
+        """
+        Serializes the component to a dictionary.
+
+        :returns:
+            Dictionary with serialized data.
+        """
+        available_function_paths = {}
+        for name, function in self.available_functions.items():
+            available_function_paths[name] = serialize_callable(function)
+        serialization_dict = default_to_dict(self, available_functions=available_function_paths)
+        return serialization_dict
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "OpenAIFunctionCaller":
+        """
+        Deserializes the component from a dictionary.
+
+        :param data:
+            The dictionary to deserialize from.
+        :returns:
+            The deserialized component.
+        """
+        available_function_paths = data.get("init_parameters", {}).get("available_functions")
+        available_functions = {}
+        for name, path in available_function_paths.items():
+            available_functions[name] = deserialize_callable(path)
+        data["init_parameters"]["available_functions"] = available_functions
+        return default_from_dict(cls, data)
+
+    @component.output_types(function_replies=List[ChatMessage], assistant_replies=List[ChatMessage])
+    def run(self, messages: List[ChatMessage]):
+        """
+        Evaluates `messages` and invokes available functions if the messages contain tool_calls.
+
+        :param messages: A list of messages generated from the `OpenAIChatGenerator`
+        :returns: This component returns a list of messages in one of two outputs
+            - `function_replies`: List of ChatMessages containing the result of a function invocation.
+                This message comes from role = 'function'. If the function name was hallucinated or wrong,
+                an assistant message explaining as such is returned
+            - `assistant_replies`: List of ChatMessages containing a regular assistant reply. In this case,
+                there were no tool_calls in the received messages
+        """
+        if messages[0].meta["finish_reason"] == "tool_calls":
+            function_calls = json.loads(messages[0].content)
+            for function_call in function_calls:
+                function_name = function_call["function"]["name"]
+                function_args = json.loads(function_call["function"]["arguments"])
+                if function_name in self.available_functions:
+                    function_to_call = self.available_functions[function_name]
+                    try:
+                        function_response = function_to_call(**function_args)
+                        messages.append(
+                            ChatMessage.from_function(
+                                content=json.dumps(function_response),
+                                name=function_name,
+                            )
+                        )
+                    # pylint: disable=broad-exception-caught
+                    except Exception as e:
+                        messages.append(ChatMessage.from_assistant(_FUNCTION_RUN_FAILURE.format(error=e)))
+                else:
+                    messages.append(ChatMessage.from_assistant(_FUNCTION_NAME_FAILURE))
+            return {"function_replies": messages}
+        return {"assistant_replies": messages}

haystack_experimental-0.0.2.dev0/haystack_experimental/evaluation/__init__.py

@@ -0,0 +1,7 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from .harness import EvaluationHarness, EvaluationRunOverrides
+
+_all_ = ["EvaluationHarness", "EvaluationRunOverrides"]

haystack_experimental-0.0.2.dev0/haystack_experimental/evaluation/harness/__init__.py

@@ -0,0 +1,7 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from .evalution_harness import EvaluationHarness, EvaluationRunOverrides
+
+_all_ = ["EvaluationHarness", "EvaluationRunOverrides"]

haystack_experimental-0.0.2.dev0/haystack_experimental/evaluation/harness/evalution_harness.py

@@ -0,0 +1,87 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import Any, Dict, Generic, Optional, Type, TypeVar
+
+from haystack import Pipeline
+from haystack.core.serialization import DeserializationCallbacks
+
+
+@dataclass
+class EvaluationRunOverrides:
+    """
+    Overrides for an evaluation run.
+
+    Used to override the init parameters of components in either
+    (or both) the evaluated and evaluation pipelines. Each key is
+    a component name and its value a dictionary with init parameters
+    to override.
+
+    :param evaluated_pipeline_overrides:
+        Overrides for the evaluated pipeline.
+    :param evaluation_pipeline_overrides:
+        Overrides for the evaluation pipeline.
+    """
+
+    evaluated_pipeline_overrides: Optional[Dict[str, Dict[str, Any]]] = None
+    evaluation_pipeline_overrides: Optional[Dict[str, Dict[str, Any]]] = None
+
+
+EvalRunInputT = TypeVar("EvalRunInputT")
+EvalRunOutputT = TypeVar("EvalRunOutputT")
+EvalRunOverridesT = TypeVar("EvalRunOverridesT")
+
+
+class EvaluationHarness(ABC, Generic[EvalRunInputT, EvalRunOverridesT, EvalRunOutputT]):
+    """
+    Executes a pipeline with a given set of parameters, inputs and evaluates its outputs with an evaluation pipeline.
+    """
+
+    @staticmethod
+    def _override_pipeline(pipeline: Pipeline, parameter_overrides: Optional[Dict[str, Any]]) -> Pipeline:
+        def component_pre_init_callback(name: str, cls: Type, init_params: Dict[str, Any]):  # pylint: disable=unused-argument
+            assert parameter_overrides is not None
+            overrides = parameter_overrides.get(name)
+            if overrides:
+                init_params.update(overrides)
+
+        def validate_overrides():
+            if parameter_overrides is None:
+                return
+
+            pipeline_components = pipeline.inputs(include_components_with_connected_inputs=True).keys()
+            for component_name in parameter_overrides.keys():
+                if component_name not in pipeline_components:
+                    raise ValueError(f"Cannot override non-existent component '{component_name}'")
+
+        callbacks = DeserializationCallbacks(component_pre_init_callback)
+        if parameter_overrides:
+            validate_overrides()
+            serialized_pipeline = pipeline.dumps()
+            pipeline = Pipeline.loads(serialized_pipeline, callbacks=callbacks)
+
+        return pipeline
+
+    @abstractmethod
+    def run(
+        self,
+        inputs: EvalRunInputT,
+        *,
+        overrides: Optional[EvalRunOverridesT] = None,
+        run_name: Optional[str] = None,
+    ) -> EvalRunOutputT:
+        """
+        Launch a evaluation run.
+
+        :param inputs:
+            Inputs to the evaluated and evaluation pipelines.
+        :param overrides:
+            Overrides for the harness.
+        :param run_name:
+            A name for the evaluation run.
+        :returns:
+            The output of the evaluation pipeline.
+        """

haystack_experimental-0.0.2.dev0/haystack_experimental/evaluation/harness/rag/__init__.py

@@ -0,0 +1,23 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from .harness import RAGEvaluationHarness
+from .parameters import (
+    RAGEvaluationInput,
+    RAGEvaluationMetric,
+    RAGEvaluationOutput,
+    RAGEvaluationOverrides,
+    RAGExpectedComponent,
+    RAGExpectedComponentMetadata,
+)
+
+_all_ = [
+    "RAGEvaluationHarness",
+    "RAGExpectedComponent",
+    "RAGExpectedComponentMetadata",
+    "RAGEvaluationMetric",
+    "RAGEvaluationOutput",
+    "RAGEvaluationOverrides",
+    "RAGEvaluationInput",
+]

haystack_experimental-0.0.2.dev0/haystack_experimental/evaluation/harness/rag/evaluation_pipeline.py

@@ -0,0 +1,55 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from functools import partial
+from typing import Set
+
+from haystack import Pipeline
+from haystack.components.evaluators import (
+    ContextRelevanceEvaluator,
+    DocumentMAPEvaluator,
+    DocumentMRREvaluator,
+    DocumentRecallEvaluator,
+    FaithfulnessEvaluator,
+    SASEvaluator,
+)
+from haystack.components.evaluators.document_recall import RecallMode
+
+from .parameters import RAGEvaluationMetric
+
+
+def default_rag_evaluation_pipeline(
+    metrics: Set[RAGEvaluationMetric],
+) -> Pipeline:
+    """
+    Builds the default evaluation pipeline for RAG.
+
+    :param metrics:
+        The set of metrics to include in the pipeline.
+    :returns:
+        The evaluation pipeline.
+    """
+    pipeline = Pipeline()
+
+    metric_ctors = {
+        RAGEvaluationMetric.DOCUMENT_MAP: DocumentMAPEvaluator,
+        RAGEvaluationMetric.DOCUMENT_MRR: DocumentMRREvaluator,
+        RAGEvaluationMetric.DOCUMENT_RECALL_SINGLE_HIT: partial(
+            DocumentRecallEvaluator, mode=RecallMode.SINGLE_HIT
+        ),
+        RAGEvaluationMetric.DOCUMENT_RECALL_MULTI_HIT: partial(
+            DocumentRecallEvaluator, mode=RecallMode.MULTI_HIT
+        ),
+        RAGEvaluationMetric.SEMANTIC_ANSWER_SIMILARITY: partial(
+            SASEvaluator, model="sentence-transformers/all-MiniLM-L6-v2"
+        ),
+        RAGEvaluationMetric.ANSWER_FAITHFULNESS: partial(FaithfulnessEvaluator, raise_on_failure=False),
+        RAGEvaluationMetric.CONTEXT_RELEVANCE: partial(ContextRelevanceEvaluator, raise_on_failure=False),
+    }
+
+    for metric in metrics:
+        ctor = metric_ctors[metric]
+        pipeline.add_component(metric.value, ctor())
+
+    return pipeline