azure-ai-evaluation 1.0.0b5__py3-none-any.whl → 1.0.1__py3-none-any.whl

azure/ai/evaluation/_evaluators/_multimodal/_content_safety_multimodal.py

@@ -28,12 +28,10 @@ class ContentSafetyMultimodalEvaluator:
     :param azure_ai_project: The scope of the Azure AI project, containing the subscription ID,
         resource group, and project name.
     :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
-    :param parallel: Specifies whether to use parallel execution for evaluators.
-        If True, evaluators execute in parallel; otherwise, they execute sequentially. Defaults to True.
-    :type parallel: bool
+    :param kwargs: Additional arguments to pass to the evaluator.
+    :type kwargs: Any
 
     :return: A function that evaluates multimodal chat messages and generates content safety metrics.
-    :rtype: Callable
 
     **Usage Example**
 
@@ -45,7 +43,7 @@ class ContentSafetyMultimodalEvaluator:
             "project_name": "<project_name>",
         }
         eval_fn = ContentSafetyMultimodalEvaluator(azure_ai_project)
-        result = eval_fn(
+        result = eval_fn(conversation=
             {
                 "messages": [
                     {
@@ -92,8 +90,11 @@ class ContentSafetyMultimodalEvaluator:
 
     """
 
-    def __init__(self, credential, azure_ai_project, parallel: bool = False):
-        self._parallel = parallel
+    id = "content_safety_multimodal"
+    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+
+    def __init__(self, credential, azure_ai_project, **kwargs):
+        self._parallel = kwargs.pop("_parallel", False)
         self._evaluators: List[Callable[..., Dict[str, Union[str, float]]]] = [
             ViolenceMultimodalEvaluator(credential=credential, azure_ai_project=azure_ai_project),
             SexualMultimodalEvaluator(credential=credential, azure_ai_project=azure_ai_project),
@@ -104,8 +105,9 @@ class ContentSafetyMultimodalEvaluator:
     def __call__(self, *, conversation, **kwargs):
         """
         Evaluates content-safety metrics for list of messages.
+
         :keyword conversation: The conversation contains list of messages to be evaluated.
-            Each message should have "role" and "content" keys.
+            Each message should have "role" and "content" keys. It supports single turn only.
         :paramtype conversation: ~azure.ai.evaluation.Conversation
         :return: The evaluation score based on the Content Safety Metrics.
         :rtype: Dict[str, Union[float, str]]

azure/ai/evaluation/_evaluators/_multimodal/_content_safety_multimodal_base.py

@@ -7,10 +7,8 @@ from azure.ai.evaluation._common.rai_service import evaluate_with_rai_service_mu
 from azure.ai.evaluation._common.constants import EvaluationMetrics, _InternalEvaluationMetrics
 from azure.ai.evaluation._common.utils import validate_conversation
 from azure.core.credentials import TokenCredential
-from azure.ai.evaluation._common._experimental import experimental
 
 
-@experimental
 class ContentSafetyMultimodalEvaluatorBase(ABC):
     """
     Initialize a evaluator for a specified Evaluation Metric. Base class that is not

azure/ai/evaluation/_evaluators/_multimodal/_hate_unfairness.py

@@ -28,7 +28,7 @@ class HateUnfairnessMultimodalEvaluator:
             "project_name": "<project_name>",
         }
         eval_fn = HateUnfairnessMultimodalEvaluator(azure_ai_project)
-        result = eval_fn(
+        result = eval_fn(conversation=
             {
                 "messages": [
                     {
@@ -66,14 +66,18 @@ class HateUnfairnessMultimodalEvaluator:
 
     """
 
+    id = "hate_unfairness_multimodal"
+    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+
     def __init__(self, credential, azure_ai_project):
         self._async_evaluator = _AsyncHateUnfairnessMultimodalEvaluator(credential, azure_ai_project)
 
     def __call__(self, *, conversation, **kwargs):
         """
         Evaluates hate unfairness content.
+
         :keyword conversation: The conversation contains list of messages to be evaluated.
-            Each message should have "role" and "content" keys.
+            Each message should have "role" and "content" keys. It supports single turn only.
         :paramtype conversation: ~azure.ai.evaluation.Conversation
         :return: The hate unfairness score.
         :rtype: Dict[str, Union[float, str]]

azure/ai/evaluation/_evaluators/_multimodal/_protected_material.py

@@ -2,10 +2,10 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
 from promptflow._utils.async_utils import async_run_allowing_running_loop
-from azure.ai.evaluation._common._experimental import experimental
 from azure.ai.evaluation._common.constants import EvaluationMetrics
 from azure.ai.evaluation._common.utils import validate_conversation
 from azure.ai.evaluation._common.rai_service import evaluate_with_rai_service_multimodal
+from azure.ai.evaluation._common._experimental import experimental
 
 
 @experimental
@@ -22,7 +22,6 @@ class ProtectedMaterialMultimodalEvaluator:
     :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
 
     :return: A dictionary containing the evaluation result label and reasoning.
-    :rtype: Dict[str, str]
 
     **Usage Example**
 
@@ -34,7 +33,7 @@ class ProtectedMaterialMultimodalEvaluator:
             "project_name": "<project_name>",
         }
         eval_fn = ProtectedMaterialMultimodalEvaluator(azure_ai_project)
-        result = eval_fn(
+        result = eval_fn(conversation=
             {
                 "messages": [
                     {
@@ -71,6 +70,9 @@ class ProtectedMaterialMultimodalEvaluator:
 
     """
 
+    id = "protected_material_multimodal"
+    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+
     def __init__(
         self,
         credential,
@@ -82,8 +84,9 @@ class ProtectedMaterialMultimodalEvaluator:
         """
         Evaluates protected materials content.
 
-        :keyword messages: The messages to be evaluated. Each message should have "role" and "content" keys.
-        :paramtype messages: ~azure.ai.evaluation.Conversation
+        :keyword conversation: The conversation contains list of messages to be evaluated.
+            Each message should have "role" and "content" keys. It supports single turn only.
+        :paramtype conversation: ~azure.ai.evaluation.Conversation
         :return: A dictionary containing a boolean label and reasoning.
         :rtype: Dict[str, str]
         """
@@ -101,8 +104,9 @@ class _AsyncProtectedMaterialMultimodalEvaluator:
     async def __call__(self, *, conversation, **kwargs):
         """
         Evaluates content according to this evaluator's metric.
+
         :keyword conversation: The conversation contains list of messages to be evaluated.
-        Each message should have "role" and "content" keys.
+            Each message should have "role" and "content" keys. It supports single turn only.
         :paramtype conversation: ~azure.ai.evaluation.Conversation
         :return: The evaluation score computation based on the Content Safety metric (self.metric).
         :rtype: Any

azure/ai/evaluation/_evaluators/_multimodal/_self_harm.py

@@ -28,7 +28,7 @@ class SelfHarmMultimodalEvaluator:
             "project_name": "<project_name>",
         }
         eval_fn = SelfHarmMultimodalEvaluator(azure_ai_project)
-        result = eval_fn(
+        result = eval_fn(conversation=
             {
                 "messages": [
                     {
@@ -66,14 +66,18 @@ class SelfHarmMultimodalEvaluator:
 
     """
 
+    id = "self_harm_multimodal"
+    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+
     def __init__(self, credential, azure_ai_project):
         self._async_evaluator = _AsyncSelfHarmMultimodalEvaluator(credential, azure_ai_project)
 
     def __call__(self, *, conversation, **kwargs):
         """
         Evaluates self harm content.
+
         :keyword conversation: The conversation contains list of messages to be evaluated.
-            Each message should have "role" and "content" keys.
+            Each message should have "role" and "content" keys. It supports single turn only.
         :paramtype conversation: ~azure.ai.evaluation.Conversation
         :return: The self harm score.
         :rtype: Dict[str, Union[float, str]]

azure/ai/evaluation/_evaluators/_multimodal/_sexual.py

@@ -28,7 +28,7 @@ class SexualMultimodalEvaluator:
             "project_name": "<project_name>",
         }
         eval_fn = SexualMultimodalEvaluator(azure_ai_project)
-        result = eval_fn(
+        result = eval_fn(conversation=
             {
                 "messages": [
                     {
@@ -66,14 +66,18 @@ class SexualMultimodalEvaluator:
 
     """
 
+    id = "sexual_multimodal"
+    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+
     def __init__(self, credential, azure_ai_project):
         self._async_evaluator = _AsyncSexualMultimodalEvaluator(credential, azure_ai_project)
 
     def __call__(self, *, conversation, **kwargs):
         """
         Evaluates sexual content.
+
         :keyword conversation: The conversation contains list of messages to be evaluated.
-            Each message should have "role" and "content" keys.
+            Each message should have "role" and "content" keys. It supports single turn only.
         :paramtype conversation: ~azure.ai.evaluation.Conversation
         :return: The sexual score.
         :rtype: Dict[str, Union[float, str]]

azure/ai/evaluation/_evaluators/_multimodal/_violence.py

@@ -28,7 +28,7 @@ class ViolenceMultimodalEvaluator:
             "project_name": "<project_name>",
         }
         eval_fn = ViolenceMultimodalEvaluator(azure_ai_project)
-        result = eval_fn(
+        result = eval_fn(conversation=
             {
                 "messages": [
                     {
@@ -66,14 +66,18 @@ class ViolenceMultimodalEvaluator:
 
     """
 
+    id = "violence_multimodal"
+    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+
     def __init__(self, credential, azure_ai_project):
         self._async_evaluator = _AsyncViolenceMultimodalEvaluator(credential, azure_ai_project)
 
     def __call__(self, *, conversation, **kwargs):
         """
         Evaluates violence content.
+
         :keyword conversation: The conversation contains list of messages to be evaluated.
-            Each message should have "role" and "content" keys.
+            Each message should have "role" and "content" keys. It supports single turn only.
         :paramtype conversation: ~azure.ai.evaluation.Conversation
         :return: The violence score.
         :rtype: Dict[str, Union[float, str]]

azure/ai/evaluation/_evaluators/_protected_material/_protected_material.py

@@ -2,21 +2,26 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
 
-from typing import Optional
+from typing import Dict, List, Optional, Union
 
-from typing_extensions import override
+from typing_extensions import overload, override
 
 from azure.ai.evaluation._common._experimental import experimental
 from azure.ai.evaluation._common.constants import EvaluationMetrics
 from azure.ai.evaluation._evaluators._common import RaiServiceEvaluatorBase
+from azure.ai.evaluation._model_configurations import Conversation
 
 
 @experimental
-class ProtectedMaterialEvaluator(RaiServiceEvaluatorBase):
+class ProtectedMaterialEvaluator(RaiServiceEvaluatorBase[Union[str, bool]]):
     """
-    Initialize a protected material evaluator to detect whether protected material
-    is present in the AI system's response. The evaluator outputs a Boolean label (`True` or `False`)
-    indicating the presence of protected material, along with AI-generated reasoning.
+    Evaluates the protected material score for a given query and response or a multi-turn conversation, with reasoning.
+
+    Protected material is any text that is under copyright, including song lyrics, recipes, and articles. Protected
+    material evaluation leverages the Azure AI Content Safety Protected Material for Text service to perform the
+    classification.
+
+    The protected material score is a boolean value, where True indicates that protected material was detected.
 
     :param credential: The credential required for connecting to the Azure AI project.
     :type credential: ~azure.core.credentials.TokenCredential
@@ -24,46 +29,64 @@ class ProtectedMaterialEvaluator(RaiServiceEvaluatorBase):
         resource group, and project name.
     :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
 
-    :return: A dictionary with a label indicating the presence of protected material and the reasoning.
-    :rtype: Dict[str, Union[bool, str]]
-
-    **Usage Example**
-
-    .. code-block:: python
-
-        azure_ai_project = {
-            "subscription_id": "<subscription_id>",
-            "resource_group_name": "<resource_group_name>",
-            "project_name": "<project_name>",
-        }
-        eval_fn = ProtectedMaterialEvaluator(azure_ai_project)
-        result = eval_fn(query="What is the capital of France?", response="Paris.")
-
-    **Output Format**
-
-    .. code-block:: json
-
-        {
-            "protected_material_label": false,
-            "protected_material_reason": "This query does not contain any protected material."
-        }
+    .. admonition:: Example:
 
+        .. literalinclude:: ../samples/evaluation_samples_evaluate.py
+            :start-after: [START protected_material_evaluator]
+            :end-before: [END protected_material_evaluator]
+            :language: python
+            :dedent: 8
+            :caption: Initialize and call a ProtectedMaterialEvaluator.
     """
 
+    id = "azureml://registries/azureml/models/Protected-Material-Evaluator/versions/3"
+    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+
     @override
     def __init__(
         self,
         credential,
         azure_ai_project,
-        eval_last_turn: bool = False,
     ):
         super().__init__(
             eval_metric=EvaluationMetrics.PROTECTED_MATERIAL,
             azure_ai_project=azure_ai_project,
             credential=credential,
-            eval_last_turn=eval_last_turn,
         )
 
+    @overload
+    def __call__(
+        self,
+        *,
+        query: str,
+        response: str,
+    ) -> Dict[str, Union[str, bool]]:
+        """Evaluate a given query/response pair for protected material
+
+        :keyword query: The query to be evaluated.
+        :paramtype query: str
+        :keyword response: The response to be evaluated.
+        :paramtype response: str
+        :return: The protected material score.
+        :rtype: Dict[str, Union[str, bool]]
+        """
+
+    @overload
+    def __call__(
+        self,
+        *,
+        conversation: Conversation,
+    ) -> Dict[str, Union[float, Dict[str, List[Union[str, bool]]]]]:
+        """Evaluate a conversation for protected material
+
+        :keyword conversation: The conversation to evaluate. Expected to contain a list of conversation turns under the
+            key "messages", and potentially a global context under the key "context". Conversation turns are expected
+            to be dictionaries with keys "content", "role", and possibly "context".
+        :paramtype conversation: Optional[~azure.ai.evaluation.Conversation]
+        :return: The protected material score.
+        :rtype: Dict[str, Union[str, bool, Dict[str, List[Union[str, bool]]]]]
+        """
+
     @override
     def __call__(
         self,
@@ -77,14 +100,14 @@ class ProtectedMaterialEvaluator(RaiServiceEvaluatorBase):
         Evaluate if protected material is present in your AI system's response.
 
         :keyword query: The query to be evaluated.
-        :paramtype query: str
+        :paramtype query: Optional[str]
         :keyword response: The response to be evaluated.
-        :paramtype response: str
+        :paramtype response: Optional[str]
         :keyword conversation: The conversation to evaluate. Expected to contain a list of conversation turns under the
             key "messages". Conversation turns are expected
             to be dictionaries with keys "content" and "role".
         :paramtype conversation: Optional[~azure.ai.evaluation.Conversation]
         :return: The fluency score.
-        :rtype: Union[Dict[str, Union[str, bool]], Dict[str, Union[str, bool, Dict[str, List[Union[str, bool]]]]]]
+        :rtype: Union[Dict[str, Union[str, bool]], Dict[str, Union[float, Dict[str, List[Union[str, bool]]]]]]
         """
         return super().__call__(query=query, response=response, conversation=conversation, **kwargs)

azure/ai/evaluation/_evaluators/_qa/_qa.py

@@ -3,7 +3,7 @@
 # ---------------------------------------------------------
 
 from concurrent.futures import as_completed
-from typing import Callable, Dict, List
+from typing import Callable, Dict, List, Union
 
 from promptflow.tracing import ThreadPoolExecutorWithContext as ThreadPoolExecutor
 
@@ -23,42 +23,32 @@ class QAEvaluator:
     :type model_config: Union[~azure.ai.evaluation.AzureOpenAIModelConfiguration,
         ~azure.ai.evaluation.OpenAIModelConfiguration]
     :return: A callable class that evaluates and generates metrics for "question-answering" scenario.
+    :param kwargs: Additional arguments to pass to the evaluator.
+    :type kwargs: Any
 
-    **Usage**
-
-    .. code-block:: python
-
-        eval_fn = QAEvaluator(model_config)
-        result = qa_eval(
-            query="Tokyo is the capital of which country?",
-            response="Japan",
-            context="Tokyo is the capital of Japan.",
-            ground_truth="Japan"
-        )
-
-    **Output format**
-
-    .. code-block:: python
-
-        {
-            "groundedness": 3.5,
-            "relevance": 4.0,
-            "coherence": 1.5,
-            "fluency": 4.0,
-            "similarity": 3.0,
-            "gpt_groundedness": 3.5,
-            "gpt_relevance": 4.0,
-            "gpt_coherence": 1.5,
-            "gpt_fluency": 4.0,
-            "gpt_similarity": 3.0,
-            "f1_score": 0.42
-        }
+    .. admonition:: Example:
+
+        .. literalinclude:: ../samples/evaluation_samples_evaluate.py
+            :start-after: [START qa_evaluator]
+            :end-before: [END qa_evaluator]
+            :language: python
+            :dedent: 8
+            :caption: Initialize and call a QAEvaluator.
+
+    .. note::
+
+        To align with our support of a diverse set of models, keys without the `gpt_` prefix has been added.
+        To maintain backwards compatibility, the old keys with the `gpt_` prefix are still be present in the output;
+        however, it is recommended to use the new keys moving forward as the old keys will be deprecated in the future.
     """
 
-    def __init__(self, model_config, parallel: bool = True):
-        self._parallel = parallel
+    id = "qa"
+    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+
+    def __init__(self, model_config, **kwargs):
+        self._parallel = kwargs.pop("_parallel", False)
 
-        self._evaluators: List[Callable[..., Dict[str, float]]] = [
+        self._evaluators: List[Union[Callable[..., Dict[str, Union[str, float]]], Callable[..., Dict[str, float]]]] = [
             GroundednessEvaluator(model_config),
             RelevanceEvaluator(model_config),
             CoherenceEvaluator(model_config),
@@ -79,12 +69,10 @@ class QAEvaluator:
         :paramtype context: str
         :keyword ground_truth: The ground truth to be evaluated.
         :paramtype ground_truth: str
-        :keyword parallel: Whether to evaluate in parallel. Defaults to True.
-        :paramtype parallel: bool
         :return: The scores for QA scenario.
-        :rtype: Dict[str, float]
+        :rtype: Dict[str, Union[str, float]]
         """
-        results: Dict[str, float] = {}
+        results: Dict[str, Union[str, float]] = {}
         if self._parallel:
             with ThreadPoolExecutor() as executor:
                 futures = {

azure/ai/evaluation/_evaluators/_relevance/_relevance.py

@@ -3,62 +3,97 @@
 # ---------------------------------------------------------
 
 import os
-from typing import Optional
+from typing import Dict, Union, List
 
-from typing_extensions import override
+from typing_extensions import overload, override
 
+from azure.ai.evaluation._model_configurations import Conversation
 from azure.ai.evaluation._evaluators._common import PromptyEvaluatorBase
 
 
 class RelevanceEvaluator(PromptyEvaluatorBase):
     """
-    Initialize a relevance evaluator configured for a specific Azure OpenAI model.
+    Evaluates relevance score for a given query and response or a multi-turn conversation, including reasoning.
+
+    The relevance measure assesses the ability of answers to capture the key points of the context.
+    High relevance scores signify the AI system's understanding of the input and its capability to produce coherent
+    and contextually appropriate outputs. Conversely, low relevance scores indicate that generated responses might
+    be off-topic, lacking in context, or insufficient in addressing the user's intended queries. Use the relevance
+    metric when evaluating the AI system's performance in understanding the input and generating contextually
+    appropriate responses.
+
+    Relevance scores range from 1 to 5, with 1 being the worst and 5 being the best.
 
     :param model_config: Configuration for the Azure OpenAI model.
     :type model_config: Union[~azure.ai.evaluation.AzureOpenAIModelConfiguration,
         ~azure.ai.evaluation.OpenAIModelConfiguration]
 
-    **Usage**
-
-    .. code-block:: python
+    .. admonition:: Example:
 
-        eval_fn = RelevanceEvaluator(model_config)
-        result = eval_fn(
-            query="What is the capital of Japan?",
-            response="The capital of Japan is Tokyo.")
+        .. literalinclude:: ../samples/evaluation_samples_evaluate.py
+            :start-after: [START relevance_evaluator]
+            :end-before: [END relevance_evaluator]
+            :language: python
+            :dedent: 8
+            :caption: Initialize and call a RelevanceEvaluator with a query, response, and context.
 
-    **Output format**
+    .. note::
 
-    .. code-block:: python
-
-        {
-            "relevance": 3.0,
-            "gpt_relevance": 3.0,
-            "relevance_reason": "The response is relevant to the query because it provides the correct answer.",
-        }
-
-    Note: To align with our support of a diverse set of models, a key without the `gpt_` prefix has been added.
-    To maintain backwards compatibility, the old key with the `gpt_` prefix is still be present in the output;
-    however, it is recommended to use the new key moving forward as the old key will be deprecated in the future.
+        To align with our support of a diverse set of models, an output key without the `gpt_` prefix has been added.
+        To maintain backwards compatibility, the old key with the `gpt_` prefix is still be present in the output;
+        however, it is recommended to use the new key moving forward as the old key will be deprecated in the future.
     """
 
     # Constants must be defined within eval's directory to be save/loadable
     _PROMPTY_FILE = "relevance.prompty"
     _RESULT_KEY = "relevance"
 
+    id = "azureml://registries/azureml/models/Relevance-Evaluator/versions/4"
+    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+
     @override
     def __init__(self, model_config):
         current_dir = os.path.dirname(__file__)
         prompty_path = os.path.join(current_dir, self._PROMPTY_FILE)
         super().__init__(model_config=model_config, prompty_file=prompty_path, result_key=self._RESULT_KEY)
 
-    @override
+    @overload
+    def __call__(
+        self,
+        *,
+        query: str,
+        response: str,
+    ) -> Dict[str, Union[str, float]]:
+        """Evaluate groundedness for given input of query, response, context
+
+        :keyword query: The query to be evaluated.
+        :paramtype query: str
+        :keyword response: The response to be evaluated.
+        :paramtype response: str
+        :return: The relevance score.
+        :rtype: Dict[str, float]
+        """
+
+    @overload
     def __call__(
         self,
         *,
-        query: Optional[str] = None,
-        response: Optional[str] = None,
-        conversation=None,
+        conversation: Conversation,
+    ) -> Dict[str, Union[float, Dict[str, List[Union[str, float]]]]]:
+        """Evaluate relevance for a conversation
+
+        :keyword conversation: The conversation to evaluate. Expected to contain a list of conversation turns under the
+            key "messages", and potentially a global context under the key "context". Conversation turns are expected
+            to be dictionaries with keys "content", "role", and possibly "context".
+        :paramtype conversation: Optional[~azure.ai.evaluation.Conversation]
+        :return: The relevance score.
+        :rtype: Dict[str, Union[float, Dict[str, List[float]]]]
+        """
+
+    @override
+    def __call__(  # pylint: disable=docstring-missing-param
+        self,
+        *args,
         **kwargs,
     ):
         """Evaluate relevance. Accepts either a query and response for a single evaluation,
@@ -74,7 +109,6 @@ class RelevanceEvaluator(PromptyEvaluatorBase):
             to be dictionaries with keys "content", "role", and possibly "context".
         :paramtype conversation: Optional[~azure.ai.evaluation.Conversation]
         :return: The relevance score.
-        :rtype: Union[Dict[str, float], Dict[str, Union[float, Dict[str, List[float]]]]]
+        :rtype: Union[Dict[str, Union[str, float]], Dict[str, Union[float, Dict[str, List[Union[str, float]]]]]]
         """
-
-        return super().__call__(query=query, response=response, conversation=conversation, **kwargs)
+        return super().__call__(*args, **kwargs)