azure-ai-evaluation 1.0.0b3__py3-none-any.whl → 1.0.0b5__py3-none-any.whl

azure/ai/evaluation/_evaluators/_multimodal/_content_safety_multimodal_base.py

@@ -0,0 +1,57 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+from abc import ABC
+from typing import Union
+from azure.ai.evaluation._common.rai_service import evaluate_with_rai_service_multimodal
+from azure.ai.evaluation._common.constants import EvaluationMetrics, _InternalEvaluationMetrics
+from azure.ai.evaluation._common.utils import validate_conversation
+from azure.core.credentials import TokenCredential
+from azure.ai.evaluation._common._experimental import experimental
+
+
+@experimental
+class ContentSafetyMultimodalEvaluatorBase(ABC):
+    """
+    Initialize a evaluator for a specified Evaluation Metric. Base class that is not
+    meant to be instantiated by users.
+
+    :param metric: The metric to be evaluated.
+    :type metric: ~azure.ai.evaluation._evaluators._content_safety.flow.constants.EvaluationMetrics
+    :param credential: The credential for connecting to Azure AI project. Required
+    :type credential: ~azure.core.credentials.TokenCredential
+    :param azure_ai_project: The scope of the Azure AI project.
+        It contains subscription id, resource group, and project name.
+    :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
+    """
+
+    def __init__(
+        self,
+        metric: Union[EvaluationMetrics, _InternalEvaluationMetrics],
+        credential: TokenCredential,
+        azure_ai_project,
+    ):
+        self._metric = metric
+        self._azure_ai_project = azure_ai_project
+        self._credential = credential
+
+    async def __call__(self, *, conversation, **kwargs):
+        """
+        Evaluates content according to this evaluator's metric.
+        :keyword conversation: The conversation contains list of messages to be evaluated.
+            Each message should have "role" and "content" keys.
+        :paramtype conversation: ~azure.ai.evaluation.Conversation
+        :return: The evaluation score computation based on the Content Safety metric (self.metric).
+        :rtype: Dict[str, Union[float, str]]
+        """
+        # validate inputs
+        validate_conversation(conversation)
+        messages = conversation["messages"]
+        # Run score computation based on supplied metric.
+        result = await evaluate_with_rai_service_multimodal(
+            messages=messages,
+            metric_name=self._metric,
+            project_scope=self._azure_ai_project,
+            credential=self._credential,
+        )
+        return result

azure/ai/evaluation/_evaluators/_multimodal/_hate_unfairness.py

@@ -0,0 +1,96 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+from promptflow._utils.async_utils import async_run_allowing_running_loop
+from azure.ai.evaluation._common._experimental import experimental
+from azure.ai.evaluation._common.constants import EvaluationMetrics
+from ._content_safety_multimodal_base import ContentSafetyMultimodalEvaluatorBase
+
+
+@experimental
+class HateUnfairnessMultimodalEvaluator:
+    """
+    Initialize a hate-unfairness multimodal evaluator for hate unfairness score.
+
+    :param credential: The credential for connecting to the Azure AI project. This is required.
+    :type credential: ~azure.core.credentials.TokenCredential
+    :param azure_ai_project: The scope of the Azure AI project,
+        which includes subscription ID, resource group name, and project name.
+    :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
+
+    **Usage Example**
+
+    .. code-block:: python
+
+        azure_ai_project = {
+            "subscription_id": "<subscription_id>",
+            "resource_group_name": "<resource_group_name>",
+            "project_name": "<project_name>",
+        }
+        eval_fn = HateUnfairnessMultimodalEvaluator(azure_ai_project)
+        result = eval_fn(
+            {
+                "messages": [
+                    {
+                        "role": "user",
+                        "content": [
+                            {
+                                "type": "text",
+                                "text": "What's in this image?"
+                            },
+                            {
+                                "type": "image_url",
+                                "image_url": {
+                                    "url": "<image url or base64 encoded image>"
+                                }
+                            }
+                        ]
+                    },
+                    {
+                        "role": "assistant",
+                        "content": "This picture shows an astronaut standing in the desert."
+                    }
+                ]
+            }
+        )
+
+    **Output Format**
+
+    .. code-block:: json
+
+        {
+            "hate_unfairness": "High",
+            "hate_unfairness_score": 6.5,
+            "hate_unfairness_reason": "Some reason"
+        }
+
+    """
+
+    def __init__(self, credential, azure_ai_project):
+        self._async_evaluator = _AsyncHateUnfairnessMultimodalEvaluator(credential, azure_ai_project)
+
+    def __call__(self, *, conversation, **kwargs):
+        """
+        Evaluates hate unfairness content.
+        :keyword conversation: The conversation contains list of messages to be evaluated.
+            Each message should have "role" and "content" keys.
+        :paramtype conversation: ~azure.ai.evaluation.Conversation
+        :return: The hate unfairness score.
+        :rtype: Dict[str, Union[float, str]]
+        """
+        return async_run_allowing_running_loop(self._async_evaluator, conversation=conversation, **kwargs)
+
+    def _to_async(self):
+        return self._async_evaluator
+
+
+class _AsyncHateUnfairnessMultimodalEvaluator(ContentSafetyMultimodalEvaluatorBase):
+    def __init__(self, credential, azure_ai_project):
+        super().__init__(
+            metric=EvaluationMetrics.HATE_FAIRNESS,
+            credential=credential,
+            azure_ai_project=azure_ai_project,
+        )
+
+    async def __call__(self, *, conversation, **kwargs):
+        return await super().__call__(conversation=conversation, **kwargs)

azure/ai/evaluation/_evaluators/_multimodal/_protected_material.py

@@ -0,0 +1,120 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+from promptflow._utils.async_utils import async_run_allowing_running_loop
+from azure.ai.evaluation._common._experimental import experimental
+from azure.ai.evaluation._common.constants import EvaluationMetrics
+from azure.ai.evaluation._common.utils import validate_conversation
+from azure.ai.evaluation._common.rai_service import evaluate_with_rai_service_multimodal
+
+
+@experimental
+class ProtectedMaterialMultimodalEvaluator:
+    """
+    Initialize a protected materials evaluator to detect whether protected material
+    is present in multimodal messages. The evaluator outputs a Boolean label (`True` or `False`)
+    indicating the presence of protected material, along with AI-generated reasoning.
+
+    :param credential: The credential for connecting to the Azure AI project. This is required.
+    :type credential: ~azure.core.credentials.TokenCredential
+    :param azure_ai_project: The scope of the Azure AI project, containing the subscription ID,
+        resource group, and project name.
+    :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
+
+    :return: A dictionary containing the evaluation result label and reasoning.
+    :rtype: Dict[str, str]
+
+    **Usage Example**
+
+    .. code-block:: python
+
+        azure_ai_project = {
+            "subscription_id": "<subscription_id>",
+            "resource_group_name": "<resource_group_name>",
+            "project_name": "<project_name>",
+        }
+        eval_fn = ProtectedMaterialMultimodalEvaluator(azure_ai_project)
+        result = eval_fn(
+            {
+                "messages": [
+                    {
+                        "role": "user",
+                        "content": [
+                            {
+                                "type": "text",
+                                "text": "What's in this image?"
+                            },
+                            {
+                                "type": "image_url",
+                                "image_url": {
+                                    "url": "<image url or base64 encoded image>"
+                                }
+                            }
+                        ]
+                    },
+                    {
+                        "role": "assistant",
+                        "content": "This picture shows an astronaut standing in the desert."
+                    }
+                ]
+            }
+        )
+
+    **Output Format**
+
+    .. code-block:: json
+
+        {
+            "protected_material_label": "False",
+            "protected_material_reason": "This query does not contain any protected material."
+        }
+
+    """
+
+    def __init__(
+        self,
+        credential,
+        azure_ai_project,
+    ):
+        self._async_evaluator = _AsyncProtectedMaterialMultimodalEvaluator(credential, azure_ai_project)
+
+    def __call__(self, *, conversation, **kwargs):
+        """
+        Evaluates protected materials content.
+
+        :keyword messages: The messages to be evaluated. Each message should have "role" and "content" keys.
+        :paramtype messages: ~azure.ai.evaluation.Conversation
+        :return: A dictionary containing a boolean label and reasoning.
+        :rtype: Dict[str, str]
+        """
+        return async_run_allowing_running_loop(self._async_evaluator, conversation=conversation, **kwargs)
+
+    def _to_async(self):
+        return self._async_evaluator
+
+
+class _AsyncProtectedMaterialMultimodalEvaluator:
+    def __init__(self, credential, azure_ai_project):
+        self._credential = credential
+        self._azure_ai_project = azure_ai_project
+
+    async def __call__(self, *, conversation, **kwargs):
+        """
+        Evaluates content according to this evaluator's metric.
+        :keyword conversation: The conversation contains list of messages to be evaluated.
+        Each message should have "role" and "content" keys.
+        :paramtype conversation: ~azure.ai.evaluation.Conversation
+        :return: The evaluation score computation based on the Content Safety metric (self.metric).
+        :rtype: Any
+        """
+        # Validate inputs
+        validate_conversation(conversation)
+        messages = conversation["messages"]
+        # Run score computation based on supplied metric.
+        result = await evaluate_with_rai_service_multimodal(
+            messages=messages,
+            metric_name=EvaluationMetrics.PROTECTED_MATERIAL,
+            credential=self._credential,
+            project_scope=self._azure_ai_project,
+        )
+        return result

azure/ai/evaluation/_evaluators/_multimodal/_self_harm.py

@@ -0,0 +1,96 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+from promptflow._utils.async_utils import async_run_allowing_running_loop
+from azure.ai.evaluation._common._experimental import experimental
+from azure.ai.evaluation._common.constants import EvaluationMetrics
+from ._content_safety_multimodal_base import ContentSafetyMultimodalEvaluatorBase
+
+
+@experimental
+class SelfHarmMultimodalEvaluator:
+    """
+    Initialize a self harm multimodal evaluator for self harm score.
+
+    :param credential: The credential for connecting to the Azure AI project. This is required.
+    :type credential: ~azure.core.credentials.TokenCredential
+    :param azure_ai_project: The scope of the Azure AI project,
+        which includes subscription ID, resource group name, and project name.
+    :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
+
+    **Usage Example**
+
+    .. code-block:: python
+
+        azure_ai_project = {
+            "subscription_id": "<subscription_id>",
+            "resource_group_name": "<resource_group_name>",
+            "project_name": "<project_name>",
+        }
+        eval_fn = SelfHarmMultimodalEvaluator(azure_ai_project)
+        result = eval_fn(
+            {
+                "messages": [
+                    {
+                        "role": "user",
+                        "content": [
+                            {
+                                "type": "text",
+                                "text": "What's in this image?"
+                            },
+                            {
+                                "type": "image_url",
+                                "image_url": {
+                                    "url": "<image url or base64 encoded image>"
+                                }
+                            }
+                        ]
+                    },
+                    {
+                        "role": "assistant",
+                        "content": "This picture shows an astronaut standing in the desert."
+                    }
+                ]
+            }
+        )
+
+    **Output Format**
+
+    .. code-block:: json
+
+        {
+            "self_harm": "High",
+            "self_harm_score": 6.5,
+            "self_harm_reason": "Some reason"
+        }
+
+    """
+
+    def __init__(self, credential, azure_ai_project):
+        self._async_evaluator = _AsyncSelfHarmMultimodalEvaluator(credential, azure_ai_project)
+
+    def __call__(self, *, conversation, **kwargs):
+        """
+        Evaluates self harm content.
+        :keyword conversation: The conversation contains list of messages to be evaluated.
+            Each message should have "role" and "content" keys.
+        :paramtype conversation: ~azure.ai.evaluation.Conversation
+        :return: The self harm score.
+        :rtype: Dict[str, Union[float, str]]
+        """
+        return async_run_allowing_running_loop(self._async_evaluator, conversation=conversation, **kwargs)
+
+    def _to_async(self):
+        return self._async_evaluator
+
+
+class _AsyncSelfHarmMultimodalEvaluator(ContentSafetyMultimodalEvaluatorBase):
+    def __init__(self, credential, azure_ai_project):
+        super().__init__(
+            metric=EvaluationMetrics.SELF_HARM,
+            credential=credential,
+            azure_ai_project=azure_ai_project,
+        )
+
+    async def __call__(self, *, conversation, **kwargs):
+        return await super().__call__(conversation=conversation, **kwargs)

azure/ai/evaluation/_evaluators/_multimodal/_sexual.py

@@ -0,0 +1,96 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+from promptflow._utils.async_utils import async_run_allowing_running_loop
+from azure.ai.evaluation._common.constants import EvaluationMetrics
+from azure.ai.evaluation._common._experimental import experimental
+from ._content_safety_multimodal_base import ContentSafetyMultimodalEvaluatorBase
+
+
+@experimental
+class SexualMultimodalEvaluator:
+    """
+    Initialize a sexual multimodal evaluator for sexual score.
+
+    :param credential: The credential for connecting to the Azure AI project. This is required.
+    :type credential: ~azure.core.credentials.TokenCredential
+    :param azure_ai_project: The scope of the Azure AI project,
+        which includes subscription ID, resource group name, and project name.
+    :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
+
+    **Usage Example**
+
+    .. code-block:: python
+
+        azure_ai_project = {
+            "subscription_id": "<subscription_id>",
+            "resource_group_name": "<resource_group_name>",
+            "project_name": "<project_name>",
+        }
+        eval_fn = SexualMultimodalEvaluator(azure_ai_project)
+        result = eval_fn(
+            {
+                "messages": [
+                    {
+                        "role": "user",
+                        "content": [
+                            {
+                                "type": "text",
+                                "text": "What's in this image?"
+                            },
+                            {
+                                "type": "image_url",
+                                "image_url": {
+                                    "url": "<image url or base64 encoded image>"
+                                }
+                            }
+                        ]
+                    },
+                    {
+                        "role": "assistant",
+                        "content": "This picture shows an astronaut standing in the desert."
+                    }
+                ]
+            }
+        )
+
+    **Output Format**
+
+    .. code-block:: json
+
+        {
+            "sexual": "High",
+            "sexual_score": 6.5,
+            "sexual_reason": "Some reason"
+        }
+
+    """
+
+    def __init__(self, credential, azure_ai_project):
+        self._async_evaluator = _AsyncSexualMultimodalEvaluator(credential, azure_ai_project)
+
+    def __call__(self, *, conversation, **kwargs):
+        """
+        Evaluates sexual content.
+        :keyword conversation: The conversation contains list of messages to be evaluated.
+            Each message should have "role" and "content" keys.
+        :paramtype conversation: ~azure.ai.evaluation.Conversation
+        :return: The sexual score.
+        :rtype: Dict[str, Union[float, str]]
+        """
+        return async_run_allowing_running_loop(self._async_evaluator, conversation=conversation, **kwargs)
+
+    def _to_async(self):
+        return self._async_evaluator
+
+
+class _AsyncSexualMultimodalEvaluator(ContentSafetyMultimodalEvaluatorBase):
+    def __init__(self, credential, azure_ai_project):
+        super().__init__(
+            metric=EvaluationMetrics.SEXUAL,
+            credential=credential,
+            azure_ai_project=azure_ai_project,
+        )
+
+    async def __call__(self, *, conversation, **kwargs):
+        return await super().__call__(conversation=conversation, **kwargs)

azure/ai/evaluation/_evaluators/_multimodal/_violence.py

@@ -0,0 +1,96 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+from promptflow._utils.async_utils import async_run_allowing_running_loop
+from azure.ai.evaluation._common._experimental import experimental
+from azure.ai.evaluation._common.constants import EvaluationMetrics
+from ._content_safety_multimodal_base import ContentSafetyMultimodalEvaluatorBase
+
+
+@experimental
+class ViolenceMultimodalEvaluator:
+    """
+    Initialize a violence multimodal evaluator for violence score.
+
+    :param credential: The credential for connecting to the Azure AI project. This is required.
+    :type credential: ~azure.core.credentials.TokenCredential
+    :param azure_ai_project: The scope of the Azure AI project,
+        which includes subscription ID, resource group name, and project name.
+    :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
+
+    **Usage Example**
+
+    .. code-block:: python
+
+        azure_ai_project = {
+            "subscription_id": "<subscription_id>",
+            "resource_group_name": "<resource_group_name>",
+            "project_name": "<project_name>",
+        }
+        eval_fn = ViolenceMultimodalEvaluator(azure_ai_project)
+        result = eval_fn(
+            {
+                "messages": [
+                    {
+                        "role": "user",
+                        "content": [
+                            {
+                                "type": "text",
+                                "text": "What's in this image?"
+                            },
+                            {
+                                "type": "image_url",
+                                "image_url": {
+                                    "url": "<image url or base64 encoded image>"
+                                }
+                            }
+                        ]
+                    },
+                    {
+                        "role": "assistant",
+                        "content": "This picture shows an astronaut standing in the desert."
+                    }
+                ]
+            }
+        )
+
+    **Output Format**
+
+    .. code-block:: json
+
+        {
+            "violence": "High",
+            "violence_score": 6.5,
+            "violence_reason": "Some reason"
+        }
+
+    """
+
+    def __init__(self, credential, azure_ai_project):
+        self._async_evaluator = _AsyncViolenceMultimodalEvaluator(credential, azure_ai_project)
+
+    def __call__(self, *, conversation, **kwargs):
+        """
+        Evaluates violence content.
+        :keyword conversation: The conversation contains list of messages to be evaluated.
+            Each message should have "role" and "content" keys.
+        :paramtype conversation: ~azure.ai.evaluation.Conversation
+        :return: The violence score.
+        :rtype: Dict[str, Union[float, str]]
+        """
+        return async_run_allowing_running_loop(self._async_evaluator, conversation=conversation, **kwargs)
+
+    def _to_async(self):
+        return self._async_evaluator
+
+
+class _AsyncViolenceMultimodalEvaluator(ContentSafetyMultimodalEvaluatorBase):
+    def __init__(self, credential, azure_ai_project):
+        super().__init__(
+            metric=EvaluationMetrics.VIOLENCE,
+            credential=credential,
+            azure_ai_project=azure_ai_project,
+        )
+
+    async def __call__(self, *, conversation, **kwargs):
+        return await super().__call__(conversation=conversation, **kwargs)

azure/ai/evaluation/_evaluators/_protected_material/_protected_material.py

@@ -1,26 +1,33 @@
 # ---------------------------------------------------------
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
+
 from typing import Optional
+
 from typing_extensions import override
+
+from azure.ai.evaluation._common._experimental import experimental
 from azure.ai.evaluation._common.constants import EvaluationMetrics
 from azure.ai.evaluation._evaluators._common import RaiServiceEvaluatorBase
 
 
+@experimental
 class ProtectedMaterialEvaluator(RaiServiceEvaluatorBase):
     """
     Initialize a protected material evaluator to detect whether protected material
-    is present in your AI system's response. Outputs True or False with AI-generated reasoning.
+    is present in the AI system's response. The evaluator outputs a Boolean label (`True` or `False`)
+    indicating the presence of protected material, along with AI-generated reasoning.
 
-    :param azure_ai_project: The scope of the Azure AI project.
-        It contains subscription id, resource group, and project name.
+    :param credential: The credential required for connecting to the Azure AI project.
+    :type credential: ~azure.core.credentials.TokenCredential
+    :param azure_ai_project: The scope of the Azure AI project, containing the subscription ID,
+        resource group, and project name.
     :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
-    :param credential: The credential for connecting to Azure AI project.
-    :type credential: Optional[~azure.core.credentials.TokenCredential]
-    :return: Whether or not protected material was found in the response, with AI-generated reasoning.
-    :rtype: Dict[str, str]
 
-    **Usage**
+    :return: A dictionary with a label indicating the presence of protected material and the reasoning.
+    :rtype: Dict[str, Union[bool, str]]
+
+    **Usage Example**
 
     .. code-block:: python
 
@@ -32,21 +39,22 @@ class ProtectedMaterialEvaluator(RaiServiceEvaluatorBase):
         eval_fn = ProtectedMaterialEvaluator(azure_ai_project)
         result = eval_fn(query="What is the capital of France?", response="Paris.")
 
-    **Output format**
+    **Output Format**
 
-    .. code-block:: python
+    .. code-block:: json
 
         {
-            "protected_material_label": "False",
+            "protected_material_label": false,
             "protected_material_reason": "This query does not contain any protected material."
         }
+
     """
 
     @override
     def __init__(
         self,
-        azure_ai_project: dict,
-        credential: Optional[dict] = None,
+        credential,
+        azure_ai_project,
         eval_last_turn: bool = False,
     ):
         super().__init__(
@@ -55,3 +63,28 @@ class ProtectedMaterialEvaluator(RaiServiceEvaluatorBase):
             credential=credential,
             eval_last_turn=eval_last_turn,
         )
+
+    @override
+    def __call__(
+        self,
+        *,
+        query: Optional[str] = None,
+        response: Optional[str] = None,
+        conversation=None,
+        **kwargs,
+    ):
+        """
+        Evaluate if protected material is present in your AI system's response.
+
+        :keyword query: The query to be evaluated.
+        :paramtype query: str
+        :keyword response: The response to be evaluated.
+        :paramtype response: str
+        :keyword conversation: The conversation to evaluate. Expected to contain a list of conversation turns under the
+            key "messages". Conversation turns are expected
+            to be dictionaries with keys "content" and "role".
+        :paramtype conversation: Optional[~azure.ai.evaluation.Conversation]
+        :return: The fluency score.
+        :rtype: Union[Dict[str, Union[str, bool]], Dict[str, Union[str, bool, Dict[str, List[Union[str, bool]]]]]]
+        """
+        return super().__call__(query=query, response=response, conversation=conversation, **kwargs)