azure-ai-evaluation 1.0.0b4__py3-none-any.whl → 1.0.1__py3-none-any.whl

azure/ai/evaluation/_evaluators/_multimodal/_self_harm.py

@@ -0,0 +1,100 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+from promptflow._utils.async_utils import async_run_allowing_running_loop
+from azure.ai.evaluation._common._experimental import experimental
+from azure.ai.evaluation._common.constants import EvaluationMetrics
+from ._content_safety_multimodal_base import ContentSafetyMultimodalEvaluatorBase
+
+
+@experimental
+class SelfHarmMultimodalEvaluator:
+    """
+    Initialize a self harm multimodal evaluator for self harm score.
+
+    :param credential: The credential for connecting to the Azure AI project. This is required.
+    :type credential: ~azure.core.credentials.TokenCredential
+    :param azure_ai_project: The scope of the Azure AI project,
+        which includes subscription ID, resource group name, and project name.
+    :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
+
+    **Usage Example**
+
+    .. code-block:: python
+
+        azure_ai_project = {
+            "subscription_id": "<subscription_id>",
+            "resource_group_name": "<resource_group_name>",
+            "project_name": "<project_name>",
+        }
+        eval_fn = SelfHarmMultimodalEvaluator(azure_ai_project)
+        result = eval_fn(conversation=
+            {
+                "messages": [
+                    {
+                        "role": "user",
+                        "content": [
+                            {
+                                "type": "text",
+                                "text": "What's in this image?"
+                            },
+                            {
+                                "type": "image_url",
+                                "image_url": {
+                                    "url": "<image url or base64 encoded image>"
+                                }
+                            }
+                        ]
+                    },
+                    {
+                        "role": "assistant",
+                        "content": "This picture shows an astronaut standing in the desert."
+                    }
+                ]
+            }
+        )
+
+    **Output Format**
+
+    .. code-block:: json
+
+        {
+            "self_harm": "High",
+            "self_harm_score": 6.5,
+            "self_harm_reason": "Some reason"
+        }
+
+    """
+
+    id = "self_harm_multimodal"
+    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+
+    def __init__(self, credential, azure_ai_project):
+        self._async_evaluator = _AsyncSelfHarmMultimodalEvaluator(credential, azure_ai_project)
+
+    def __call__(self, *, conversation, **kwargs):
+        """
+        Evaluates self harm content.
+
+        :keyword conversation: The conversation contains list of messages to be evaluated.
+            Each message should have "role" and "content" keys. It supports single turn only.
+        :paramtype conversation: ~azure.ai.evaluation.Conversation
+        :return: The self harm score.
+        :rtype: Dict[str, Union[float, str]]
+        """
+        return async_run_allowing_running_loop(self._async_evaluator, conversation=conversation, **kwargs)
+
+    def _to_async(self):
+        return self._async_evaluator
+
+
+class _AsyncSelfHarmMultimodalEvaluator(ContentSafetyMultimodalEvaluatorBase):
+    def __init__(self, credential, azure_ai_project):
+        super().__init__(
+            metric=EvaluationMetrics.SELF_HARM,
+            credential=credential,
+            azure_ai_project=azure_ai_project,
+        )
+
+    async def __call__(self, *, conversation, **kwargs):
+        return await super().__call__(conversation=conversation, **kwargs)

azure/ai/evaluation/_evaluators/_multimodal/_sexual.py

@@ -0,0 +1,100 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+from promptflow._utils.async_utils import async_run_allowing_running_loop
+from azure.ai.evaluation._common.constants import EvaluationMetrics
+from azure.ai.evaluation._common._experimental import experimental
+from ._content_safety_multimodal_base import ContentSafetyMultimodalEvaluatorBase
+
+
+@experimental
+class SexualMultimodalEvaluator:
+    """
+    Initialize a sexual multimodal evaluator for sexual score.
+
+    :param credential: The credential for connecting to the Azure AI project. This is required.
+    :type credential: ~azure.core.credentials.TokenCredential
+    :param azure_ai_project: The scope of the Azure AI project,
+        which includes subscription ID, resource group name, and project name.
+    :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
+
+    **Usage Example**
+
+    .. code-block:: python
+
+        azure_ai_project = {
+            "subscription_id": "<subscription_id>",
+            "resource_group_name": "<resource_group_name>",
+            "project_name": "<project_name>",
+        }
+        eval_fn = SexualMultimodalEvaluator(azure_ai_project)
+        result = eval_fn(conversation=
+            {
+                "messages": [
+                    {
+                        "role": "user",
+                        "content": [
+                            {
+                                "type": "text",
+                                "text": "What's in this image?"
+                            },
+                            {
+                                "type": "image_url",
+                                "image_url": {
+                                    "url": "<image url or base64 encoded image>"
+                                }
+                            }
+                        ]
+                    },
+                    {
+                        "role": "assistant",
+                        "content": "This picture shows an astronaut standing in the desert."
+                    }
+                ]
+            }
+        )
+
+    **Output Format**
+
+    .. code-block:: json
+
+        {
+            "sexual": "High",
+            "sexual_score": 6.5,
+            "sexual_reason": "Some reason"
+        }
+
+    """
+
+    id = "sexual_multimodal"
+    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+
+    def __init__(self, credential, azure_ai_project):
+        self._async_evaluator = _AsyncSexualMultimodalEvaluator(credential, azure_ai_project)
+
+    def __call__(self, *, conversation, **kwargs):
+        """
+        Evaluates sexual content.
+
+        :keyword conversation: The conversation contains list of messages to be evaluated.
+            Each message should have "role" and "content" keys. It supports single turn only.
+        :paramtype conversation: ~azure.ai.evaluation.Conversation
+        :return: The sexual score.
+        :rtype: Dict[str, Union[float, str]]
+        """
+        return async_run_allowing_running_loop(self._async_evaluator, conversation=conversation, **kwargs)
+
+    def _to_async(self):
+        return self._async_evaluator
+
+
+class _AsyncSexualMultimodalEvaluator(ContentSafetyMultimodalEvaluatorBase):
+    def __init__(self, credential, azure_ai_project):
+        super().__init__(
+            metric=EvaluationMetrics.SEXUAL,
+            credential=credential,
+            azure_ai_project=azure_ai_project,
+        )
+
+    async def __call__(self, *, conversation, **kwargs):
+        return await super().__call__(conversation=conversation, **kwargs)

azure/ai/evaluation/_evaluators/_multimodal/_violence.py

@@ -0,0 +1,100 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+from promptflow._utils.async_utils import async_run_allowing_running_loop
+from azure.ai.evaluation._common._experimental import experimental
+from azure.ai.evaluation._common.constants import EvaluationMetrics
+from ._content_safety_multimodal_base import ContentSafetyMultimodalEvaluatorBase
+
+
+@experimental
+class ViolenceMultimodalEvaluator:
+    """
+    Initialize a violence multimodal evaluator for violence score.
+
+    :param credential: The credential for connecting to the Azure AI project. This is required.
+    :type credential: ~azure.core.credentials.TokenCredential
+    :param azure_ai_project: The scope of the Azure AI project,
+        which includes subscription ID, resource group name, and project name.
+    :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
+
+    **Usage Example**
+
+    .. code-block:: python
+
+        azure_ai_project = {
+            "subscription_id": "<subscription_id>",
+            "resource_group_name": "<resource_group_name>",
+            "project_name": "<project_name>",
+        }
+        eval_fn = ViolenceMultimodalEvaluator(azure_ai_project)
+        result = eval_fn(conversation=
+            {
+                "messages": [
+                    {
+                        "role": "user",
+                        "content": [
+                            {
+                                "type": "text",
+                                "text": "What's in this image?"
+                            },
+                            {
+                                "type": "image_url",
+                                "image_url": {
+                                    "url": "<image url or base64 encoded image>"
+                                }
+                            }
+                        ]
+                    },
+                    {
+                        "role": "assistant",
+                        "content": "This picture shows an astronaut standing in the desert."
+                    }
+                ]
+            }
+        )
+
+    **Output Format**
+
+    .. code-block:: json
+
+        {
+            "violence": "High",
+            "violence_score": 6.5,
+            "violence_reason": "Some reason"
+        }
+
+    """
+
+    id = "violence_multimodal"
+    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+
+    def __init__(self, credential, azure_ai_project):
+        self._async_evaluator = _AsyncViolenceMultimodalEvaluator(credential, azure_ai_project)
+
+    def __call__(self, *, conversation, **kwargs):
+        """
+        Evaluates violence content.
+
+        :keyword conversation: The conversation contains list of messages to be evaluated.
+            Each message should have "role" and "content" keys. It supports single turn only.
+        :paramtype conversation: ~azure.ai.evaluation.Conversation
+        :return: The violence score.
+        :rtype: Dict[str, Union[float, str]]
+        """
+        return async_run_allowing_running_loop(self._async_evaluator, conversation=conversation, **kwargs)
+
+    def _to_async(self):
+        return self._async_evaluator
+
+
+class _AsyncViolenceMultimodalEvaluator(ContentSafetyMultimodalEvaluatorBase):
+    def __init__(self, credential, azure_ai_project):
+        super().__init__(
+            metric=EvaluationMetrics.VIOLENCE,
+            credential=credential,
+            azure_ai_project=azure_ai_project,
+        )
+
+    async def __call__(self, *, conversation, **kwargs):
+        return await super().__call__(conversation=conversation, **kwargs)

azure/ai/evaluation/_evaluators/_protected_material/_protected_material.py

@@ -1,57 +1,113 @@
 # ---------------------------------------------------------
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
-from typing_extensions import override
 
+from typing import Dict, List, Optional, Union
+
+from typing_extensions import overload, override
+
+from azure.ai.evaluation._common._experimental import experimental
 from azure.ai.evaluation._common.constants import EvaluationMetrics
 from azure.ai.evaluation._evaluators._common import RaiServiceEvaluatorBase
+from azure.ai.evaluation._model_configurations import Conversation
 
 
-class ProtectedMaterialEvaluator(RaiServiceEvaluatorBase):
+@experimental
+class ProtectedMaterialEvaluator(RaiServiceEvaluatorBase[Union[str, bool]]):
     """
-    Initialize a protected material evaluator to detect whether protected material
-    is present in your AI system's response. Outputs True or False with AI-generated reasoning.
-
-    :param credential: The credential for connecting to Azure AI project. Required
-    :type credential: ~azure.core.credentials.TokenCredential
-    :param azure_ai_project: The scope of the Azure AI project.
-        It contains subscription id, resource group, and project name.
-    :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
-    :return: Whether or not protected material was found in the response, with AI-generated reasoning.
-    :rtype: Dict[str, str]
+    Evaluates the protected material score for a given query and response or a multi-turn conversation, with reasoning.
 
-    **Usage**
+    Protected material is any text that is under copyright, including song lyrics, recipes, and articles. Protected
+    material evaluation leverages the Azure AI Content Safety Protected Material for Text service to perform the
+    classification.
 
-    .. code-block:: python
+    The protected material score is a boolean value, where True indicates that protected material was detected.
 
-        azure_ai_project = {
-            "subscription_id": "<subscription_id>",
-            "resource_group_name": "<resource_group_name>",
-            "project_name": "<project_name>",
-        }
-        eval_fn = ProtectedMaterialEvaluator(azure_ai_project)
-        result = eval_fn(query="What is the capital of France?", response="Paris.")
-
-    **Output format**
+    :param credential: The credential required for connecting to the Azure AI project.
+    :type credential: ~azure.core.credentials.TokenCredential
+    :param azure_ai_project: The scope of the Azure AI project, containing the subscription ID,
+        resource group, and project name.
+    :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
 
-    .. code-block:: python
+    .. admonition:: Example:
 
-        {
-            "protected_material_label": "False",
-            "protected_material_reason": "This query does not contain any protected material."
-        }
+        .. literalinclude:: ../samples/evaluation_samples_evaluate.py
+            :start-after: [START protected_material_evaluator]
+            :end-before: [END protected_material_evaluator]
+            :language: python
+            :dedent: 8
+            :caption: Initialize and call a ProtectedMaterialEvaluator.
     """
 
+    id = "azureml://registries/azureml/models/Protected-Material-Evaluator/versions/3"
+    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+
     @override
     def __init__(
         self,
         credential,
-        azure_ai_project: dict,
-        eval_last_turn: bool = False,
+        azure_ai_project,
     ):
         super().__init__(
             eval_metric=EvaluationMetrics.PROTECTED_MATERIAL,
             azure_ai_project=azure_ai_project,
             credential=credential,
-            eval_last_turn=eval_last_turn,
         )
+
+    @overload
+    def __call__(
+        self,
+        *,
+        query: str,
+        response: str,
+    ) -> Dict[str, Union[str, bool]]:
+        """Evaluate a given query/response pair for protected material
+
+        :keyword query: The query to be evaluated.
+        :paramtype query: str
+        :keyword response: The response to be evaluated.
+        :paramtype response: str
+        :return: The protected material score.
+        :rtype: Dict[str, Union[str, bool]]
+        """
+
+    @overload
+    def __call__(
+        self,
+        *,
+        conversation: Conversation,
+    ) -> Dict[str, Union[float, Dict[str, List[Union[str, bool]]]]]:
+        """Evaluate a conversation for protected material
+
+        :keyword conversation: The conversation to evaluate. Expected to contain a list of conversation turns under the
+            key "messages", and potentially a global context under the key "context". Conversation turns are expected
+            to be dictionaries with keys "content", "role", and possibly "context".
+        :paramtype conversation: Optional[~azure.ai.evaluation.Conversation]
+        :return: The protected material score.
+        :rtype: Dict[str, Union[str, bool, Dict[str, List[Union[str, bool]]]]]
+        """
+
+    @override
+    def __call__(
+        self,
+        *,
+        query: Optional[str] = None,
+        response: Optional[str] = None,
+        conversation=None,
+        **kwargs,
+    ):
+        """
+        Evaluate if protected material is present in your AI system's response.
+
+        :keyword query: The query to be evaluated.
+        :paramtype query: Optional[str]
+        :keyword response: The response to be evaluated.
+        :paramtype response: Optional[str]
+        :keyword conversation: The conversation to evaluate. Expected to contain a list of conversation turns under the
+            key "messages". Conversation turns are expected
+            to be dictionaries with keys "content" and "role".
+        :paramtype conversation: Optional[~azure.ai.evaluation.Conversation]
+        :return: The fluency score.
+        :rtype: Union[Dict[str, Union[str, bool]], Dict[str, Union[float, Dict[str, List[Union[str, bool]]]]]]
+        """
+        return super().__call__(query=query, response=response, conversation=conversation, **kwargs)

azure/ai/evaluation/_evaluators/_qa/_qa.py

@@ -3,7 +3,7 @@
 # ---------------------------------------------------------
 
 from concurrent.futures import as_completed
-from typing import Callable, Dict, List
+from typing import Callable, Dict, List, Union
 
 from promptflow.tracing import ThreadPoolExecutorWithContext as ThreadPoolExecutor
 
@@ -22,39 +22,33 @@ class QAEvaluator:
     :param model_config: Configuration for the Azure OpenAI model.
     :type model_config: Union[~azure.ai.evaluation.AzureOpenAIModelConfiguration,
         ~azure.ai.evaluation.OpenAIModelConfiguration]
-    :return: A function that evaluates and generates metrics for "question-answering" scenario.
-    :rtype: Callable
+    :return: A callable class that evaluates and generates metrics for "question-answering" scenario.
+    :param kwargs: Additional arguments to pass to the evaluator.
+    :type kwargs: Any
 
-    **Usage**
+    .. admonition:: Example:
 
-    .. code-block:: python
+        .. literalinclude:: ../samples/evaluation_samples_evaluate.py
+            :start-after: [START qa_evaluator]
+            :end-before: [END qa_evaluator]
+            :language: python
+            :dedent: 8
+            :caption: Initialize and call a QAEvaluator.
 
-        eval_fn = QAEvaluator(model_config)
-        result = qa_eval(
-            query="Tokyo is the capital of which country?",
-            response="Japan",
-            context="Tokyo is the capital of Japan.",
-            ground_truth="Japan"
-        )
+    .. note::
 
-    **Output format**
-
-    .. code-block:: python
-
-        {
-            "gpt_groundedness": 3.5,
-            "gpt_relevance": 4.0,
-            "gpt_coherence": 1.5,
-            "gpt_fluency": 4.0,
-            "gpt_similarity": 3.0,
-            "f1_score": 0.42
-        }
+        To align with our support of a diverse set of models, keys without the `gpt_` prefix has been added.
+        To maintain backwards compatibility, the old keys with the `gpt_` prefix are still be present in the output;
+        however, it is recommended to use the new keys moving forward as the old keys will be deprecated in the future.
     """
 
-    def __init__(self, model_config: dict, parallel: bool = True):
-        self._parallel = parallel
+    id = "qa"
+    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+
+    def __init__(self, model_config, **kwargs):
+        self._parallel = kwargs.pop("_parallel", False)
 
-        self._evaluators: List[Callable[..., Dict[str, float]]] = [
+        self._evaluators: List[Union[Callable[..., Dict[str, Union[str, float]]], Callable[..., Dict[str, float]]]] = [
             GroundednessEvaluator(model_config),
             RelevanceEvaluator(model_config),
             CoherenceEvaluator(model_config),
@@ -75,12 +69,10 @@ class QAEvaluator:
         :paramtype context: str
         :keyword ground_truth: The ground truth to be evaluated.
         :paramtype ground_truth: str
-        :keyword parallel: Whether to evaluate in parallel. Defaults to True.
-        :paramtype parallel: bool
         :return: The scores for QA scenario.
-        :rtype: Dict[str, float]
+        :rtype: Dict[str, Union[str, float]]
         """
-        results: Dict[str, float] = {}
+        results: Dict[str, Union[str, float]] = {}
         if self._parallel:
             with ThreadPoolExecutor() as executor:
                 futures = {