azure-ai-evaluation 0.0.0b0__py3-none-any.whl → 1.0.0__py3-none-any.whl

azure/ai/evaluation/_evaluators/_multimodal/_hate_unfairness.py

@@ -0,0 +1,100 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+from promptflow._utils.async_utils import async_run_allowing_running_loop
+from azure.ai.evaluation._common._experimental import experimental
+from azure.ai.evaluation._common.constants import EvaluationMetrics
+from ._content_safety_multimodal_base import ContentSafetyMultimodalEvaluatorBase
+
+
+@experimental
+class HateUnfairnessMultimodalEvaluator:
+    """
+    Initialize a hate-unfairness multimodal evaluator for hate unfairness score.
+
+    :param credential: The credential for connecting to the Azure AI project. This is required.
+    :type credential: ~azure.core.credentials.TokenCredential
+    :param azure_ai_project: The scope of the Azure AI project,
+        which includes subscription ID, resource group name, and project name.
+    :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
+
+    **Usage Example**
+
+    .. code-block:: python
+
+        azure_ai_project = {
+            "subscription_id": "<subscription_id>",
+            "resource_group_name": "<resource_group_name>",
+            "project_name": "<project_name>",
+        }
+        eval_fn = HateUnfairnessMultimodalEvaluator(azure_ai_project)
+        result = eval_fn(conversation=
+            {
+                "messages": [
+                    {
+                        "role": "user",
+                        "content": [
+                            {
+                                "type": "text",
+                                "text": "What's in this image?"
+                            },
+                            {
+                                "type": "image_url",
+                                "image_url": {
+                                    "url": "<image url or base64 encoded image>"
+                                }
+                            }
+                        ]
+                    },
+                    {
+                        "role": "assistant",
+                        "content": "This picture shows an astronaut standing in the desert."
+                    }
+                ]
+            }
+        )
+
+    **Output Format**
+
+    .. code-block:: json
+
+        {
+            "hate_unfairness": "High",
+            "hate_unfairness_score": 6.5,
+            "hate_unfairness_reason": "Some reason"
+        }
+
+    """
+
+    id = "hate_unfairness_multimodal"
+    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+
+    def __init__(self, credential, azure_ai_project):
+        self._async_evaluator = _AsyncHateUnfairnessMultimodalEvaluator(credential, azure_ai_project)
+
+    def __call__(self, *, conversation, **kwargs):
+        """
+        Evaluates hate unfairness content.
+
+        :keyword conversation: The conversation contains list of messages to be evaluated.
+            Each message should have "role" and "content" keys. It supports single turn only.
+        :paramtype conversation: ~azure.ai.evaluation.Conversation
+        :return: The hate unfairness score.
+        :rtype: Dict[str, Union[float, str]]
+        """
+        return async_run_allowing_running_loop(self._async_evaluator, conversation=conversation, **kwargs)
+
+    def _to_async(self):
+        return self._async_evaluator
+
+
+class _AsyncHateUnfairnessMultimodalEvaluator(ContentSafetyMultimodalEvaluatorBase):
+    def __init__(self, credential, azure_ai_project):
+        super().__init__(
+            metric=EvaluationMetrics.HATE_FAIRNESS,
+            credential=credential,
+            azure_ai_project=azure_ai_project,
+        )
+
+    async def __call__(self, *, conversation, **kwargs):
+        return await super().__call__(conversation=conversation, **kwargs)

azure/ai/evaluation/_evaluators/_multimodal/_protected_material.py

@@ -0,0 +1,124 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+from promptflow._utils.async_utils import async_run_allowing_running_loop
+from azure.ai.evaluation._common.constants import EvaluationMetrics
+from azure.ai.evaluation._common.utils import validate_conversation
+from azure.ai.evaluation._common.rai_service import evaluate_with_rai_service_multimodal
+from azure.ai.evaluation._common._experimental import experimental
+
+
+@experimental
+class ProtectedMaterialMultimodalEvaluator:
+    """
+    Initialize a protected materials evaluator to detect whether protected material
+    is present in multimodal messages. The evaluator outputs a Boolean label (`True` or `False`)
+    indicating the presence of protected material, along with AI-generated reasoning.
+
+    :param credential: The credential for connecting to the Azure AI project. This is required.
+    :type credential: ~azure.core.credentials.TokenCredential
+    :param azure_ai_project: The scope of the Azure AI project, containing the subscription ID,
+        resource group, and project name.
+    :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
+
+    :return: A dictionary containing the evaluation result label and reasoning.
+
+    **Usage Example**
+
+    .. code-block:: python
+
+        azure_ai_project = {
+            "subscription_id": "<subscription_id>",
+            "resource_group_name": "<resource_group_name>",
+            "project_name": "<project_name>",
+        }
+        eval_fn = ProtectedMaterialMultimodalEvaluator(azure_ai_project)
+        result = eval_fn(conversation=
+            {
+                "messages": [
+                    {
+                        "role": "user",
+                        "content": [
+                            {
+                                "type": "text",
+                                "text": "What's in this image?"
+                            },
+                            {
+                                "type": "image_url",
+                                "image_url": {
+                                    "url": "<image url or base64 encoded image>"
+                                }
+                            }
+                        ]
+                    },
+                    {
+                        "role": "assistant",
+                        "content": "This picture shows an astronaut standing in the desert."
+                    }
+                ]
+            }
+        )
+
+    **Output Format**
+
+    .. code-block:: json
+
+        {
+            "protected_material_label": "False",
+            "protected_material_reason": "This query does not contain any protected material."
+        }
+
+    """
+
+    id = "protected_material_multimodal"
+    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+
+    def __init__(
+        self,
+        credential,
+        azure_ai_project,
+    ):
+        self._async_evaluator = _AsyncProtectedMaterialMultimodalEvaluator(credential, azure_ai_project)
+
+    def __call__(self, *, conversation, **kwargs):
+        """
+        Evaluates protected materials content.
+
+        :keyword conversation: The conversation contains list of messages to be evaluated.
+            Each message should have "role" and "content" keys. It supports single turn only.
+        :paramtype conversation: ~azure.ai.evaluation.Conversation
+        :return: A dictionary containing a boolean label and reasoning.
+        :rtype: Dict[str, str]
+        """
+        return async_run_allowing_running_loop(self._async_evaluator, conversation=conversation, **kwargs)
+
+    def _to_async(self):
+        return self._async_evaluator
+
+
+class _AsyncProtectedMaterialMultimodalEvaluator:
+    def __init__(self, credential, azure_ai_project):
+        self._credential = credential
+        self._azure_ai_project = azure_ai_project
+
+    async def __call__(self, *, conversation, **kwargs):
+        """
+        Evaluates content according to this evaluator's metric.
+
+        :keyword conversation: The conversation contains list of messages to be evaluated.
+            Each message should have "role" and "content" keys. It supports single turn only.
+        :paramtype conversation: ~azure.ai.evaluation.Conversation
+        :return: The evaluation score computation based on the Content Safety metric (self.metric).
+        :rtype: Any
+        """
+        # Validate inputs
+        validate_conversation(conversation)
+        messages = conversation["messages"]
+        # Run score computation based on supplied metric.
+        result = await evaluate_with_rai_service_multimodal(
+            messages=messages,
+            metric_name=EvaluationMetrics.PROTECTED_MATERIAL,
+            credential=self._credential,
+            project_scope=self._azure_ai_project,
+        )
+        return result

azure/ai/evaluation/_evaluators/_multimodal/_self_harm.py

@@ -0,0 +1,100 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+from promptflow._utils.async_utils import async_run_allowing_running_loop
+from azure.ai.evaluation._common._experimental import experimental
+from azure.ai.evaluation._common.constants import EvaluationMetrics
+from ._content_safety_multimodal_base import ContentSafetyMultimodalEvaluatorBase
+
+
+@experimental
+class SelfHarmMultimodalEvaluator:
+    """
+    Initialize a self harm multimodal evaluator for self harm score.
+
+    :param credential: The credential for connecting to the Azure AI project. This is required.
+    :type credential: ~azure.core.credentials.TokenCredential
+    :param azure_ai_project: The scope of the Azure AI project,
+        which includes subscription ID, resource group name, and project name.
+    :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
+
+    **Usage Example**
+
+    .. code-block:: python
+
+        azure_ai_project = {
+            "subscription_id": "<subscription_id>",
+            "resource_group_name": "<resource_group_name>",
+            "project_name": "<project_name>",
+        }
+        eval_fn = SelfHarmMultimodalEvaluator(azure_ai_project)
+        result = eval_fn(conversation=
+            {
+                "messages": [
+                    {
+                        "role": "user",
+                        "content": [
+                            {
+                                "type": "text",
+                                "text": "What's in this image?"
+                            },
+                            {
+                                "type": "image_url",
+                                "image_url": {
+                                    "url": "<image url or base64 encoded image>"
+                                }
+                            }
+                        ]
+                    },
+                    {
+                        "role": "assistant",
+                        "content": "This picture shows an astronaut standing in the desert."
+                    }
+                ]
+            }
+        )
+
+    **Output Format**
+
+    .. code-block:: json
+
+        {
+            "self_harm": "High",
+            "self_harm_score": 6.5,
+            "self_harm_reason": "Some reason"
+        }
+
+    """
+
+    id = "self_harm_multimodal"
+    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+
+    def __init__(self, credential, azure_ai_project):
+        self._async_evaluator = _AsyncSelfHarmMultimodalEvaluator(credential, azure_ai_project)
+
+    def __call__(self, *, conversation, **kwargs):
+        """
+        Evaluates self harm content.
+
+        :keyword conversation: The conversation contains list of messages to be evaluated.
+            Each message should have "role" and "content" keys. It supports single turn only.
+        :paramtype conversation: ~azure.ai.evaluation.Conversation
+        :return: The self harm score.
+        :rtype: Dict[str, Union[float, str]]
+        """
+        return async_run_allowing_running_loop(self._async_evaluator, conversation=conversation, **kwargs)
+
+    def _to_async(self):
+        return self._async_evaluator
+
+
+class _AsyncSelfHarmMultimodalEvaluator(ContentSafetyMultimodalEvaluatorBase):
+    def __init__(self, credential, azure_ai_project):
+        super().__init__(
+            metric=EvaluationMetrics.SELF_HARM,
+            credential=credential,
+            azure_ai_project=azure_ai_project,
+        )
+
+    async def __call__(self, *, conversation, **kwargs):
+        return await super().__call__(conversation=conversation, **kwargs)

azure/ai/evaluation/_evaluators/_multimodal/_sexual.py

@@ -0,0 +1,100 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+from promptflow._utils.async_utils import async_run_allowing_running_loop
+from azure.ai.evaluation._common.constants import EvaluationMetrics
+from azure.ai.evaluation._common._experimental import experimental
+from ._content_safety_multimodal_base import ContentSafetyMultimodalEvaluatorBase
+
+
+@experimental
+class SexualMultimodalEvaluator:
+    """
+    Initialize a sexual multimodal evaluator for sexual score.
+
+    :param credential: The credential for connecting to the Azure AI project. This is required.
+    :type credential: ~azure.core.credentials.TokenCredential
+    :param azure_ai_project: The scope of the Azure AI project,
+        which includes subscription ID, resource group name, and project name.
+    :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
+
+    **Usage Example**
+
+    .. code-block:: python
+
+        azure_ai_project = {
+            "subscription_id": "<subscription_id>",
+            "resource_group_name": "<resource_group_name>",
+            "project_name": "<project_name>",
+        }
+        eval_fn = SexualMultimodalEvaluator(azure_ai_project)
+        result = eval_fn(conversation=
+            {
+                "messages": [
+                    {
+                        "role": "user",
+                        "content": [
+                            {
+                                "type": "text",
+                                "text": "What's in this image?"
+                            },
+                            {
+                                "type": "image_url",
+                                "image_url": {
+                                    "url": "<image url or base64 encoded image>"
+                                }
+                            }
+                        ]
+                    },
+                    {
+                        "role": "assistant",
+                        "content": "This picture shows an astronaut standing in the desert."
+                    }
+                ]
+            }
+        )
+
+    **Output Format**
+
+    .. code-block:: json
+
+        {
+            "sexual": "High",
+            "sexual_score": 6.5,
+            "sexual_reason": "Some reason"
+        }
+
+    """
+
+    id = "sexual_multimodal"
+    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+
+    def __init__(self, credential, azure_ai_project):
+        self._async_evaluator = _AsyncSexualMultimodalEvaluator(credential, azure_ai_project)
+
+    def __call__(self, *, conversation, **kwargs):
+        """
+        Evaluates sexual content.
+
+        :keyword conversation: The conversation contains list of messages to be evaluated.
+            Each message should have "role" and "content" keys. It supports single turn only.
+        :paramtype conversation: ~azure.ai.evaluation.Conversation
+        :return: The sexual score.
+        :rtype: Dict[str, Union[float, str]]
+        """
+        return async_run_allowing_running_loop(self._async_evaluator, conversation=conversation, **kwargs)
+
+    def _to_async(self):
+        return self._async_evaluator
+
+
+class _AsyncSexualMultimodalEvaluator(ContentSafetyMultimodalEvaluatorBase):
+    def __init__(self, credential, azure_ai_project):
+        super().__init__(
+            metric=EvaluationMetrics.SEXUAL,
+            credential=credential,
+            azure_ai_project=azure_ai_project,
+        )
+
+    async def __call__(self, *, conversation, **kwargs):
+        return await super().__call__(conversation=conversation, **kwargs)

azure/ai/evaluation/_evaluators/_multimodal/_violence.py

@@ -0,0 +1,100 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+from promptflow._utils.async_utils import async_run_allowing_running_loop
+from azure.ai.evaluation._common._experimental import experimental
+from azure.ai.evaluation._common.constants import EvaluationMetrics
+from ._content_safety_multimodal_base import ContentSafetyMultimodalEvaluatorBase
+
+
+@experimental
+class ViolenceMultimodalEvaluator:
+    """
+    Initialize a violence multimodal evaluator for violence score.
+
+    :param credential: The credential for connecting to the Azure AI project. This is required.
+    :type credential: ~azure.core.credentials.TokenCredential
+    :param azure_ai_project: The scope of the Azure AI project,
+        which includes subscription ID, resource group name, and project name.
+    :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
+
+    **Usage Example**
+
+    .. code-block:: python
+
+        azure_ai_project = {
+            "subscription_id": "<subscription_id>",
+            "resource_group_name": "<resource_group_name>",
+            "project_name": "<project_name>",
+        }
+        eval_fn = ViolenceMultimodalEvaluator(azure_ai_project)
+        result = eval_fn(conversation=
+            {
+                "messages": [
+                    {
+                        "role": "user",
+                        "content": [
+                            {
+                                "type": "text",
+                                "text": "What's in this image?"
+                            },
+                            {
+                                "type": "image_url",
+                                "image_url": {
+                                    "url": "<image url or base64 encoded image>"
+                                }
+                            }
+                        ]
+                    },
+                    {
+                        "role": "assistant",
+                        "content": "This picture shows an astronaut standing in the desert."
+                    }
+                ]
+            }
+        )
+
+    **Output Format**
+
+    .. code-block:: json
+
+        {
+            "violence": "High",
+            "violence_score": 6.5,
+            "violence_reason": "Some reason"
+        }
+
+    """
+
+    id = "violence_multimodal"
+    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+
+    def __init__(self, credential, azure_ai_project):
+        self._async_evaluator = _AsyncViolenceMultimodalEvaluator(credential, azure_ai_project)
+
+    def __call__(self, *, conversation, **kwargs):
+        """
+        Evaluates violence content.
+
+        :keyword conversation: The conversation contains list of messages to be evaluated.
+            Each message should have "role" and "content" keys. It supports single turn only.
+        :paramtype conversation: ~azure.ai.evaluation.Conversation
+        :return: The violence score.
+        :rtype: Dict[str, Union[float, str]]
+        """
+        return async_run_allowing_running_loop(self._async_evaluator, conversation=conversation, **kwargs)
+
+    def _to_async(self):
+        return self._async_evaluator
+
+
+class _AsyncViolenceMultimodalEvaluator(ContentSafetyMultimodalEvaluatorBase):
+    def __init__(self, credential, azure_ai_project):
+        super().__init__(
+            metric=EvaluationMetrics.VIOLENCE,
+            credential=credential,
+            azure_ai_project=azure_ai_project,
+        )
+
+    async def __call__(self, *, conversation, **kwargs):
+        return await super().__call__(conversation=conversation, **kwargs)

azure/ai/evaluation/_evaluators/_protected_material/__init__.py

@@ -0,0 +1,5 @@
+from ._protected_material import ProtectedMaterialEvaluator
+
+__all__ = [
+    "ProtectedMaterialEvaluator",
+]

azure/ai/evaluation/_evaluators/_protected_material/_protected_material.py

@@ -0,0 +1,113 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from typing import Dict, List, Optional, Union
+
+from typing_extensions import overload, override
+
+from azure.ai.evaluation._common._experimental import experimental
+from azure.ai.evaluation._common.constants import EvaluationMetrics
+from azure.ai.evaluation._evaluators._common import RaiServiceEvaluatorBase
+from azure.ai.evaluation._model_configurations import Conversation
+
+
+@experimental
+class ProtectedMaterialEvaluator(RaiServiceEvaluatorBase[Union[str, bool]]):
+    """
+    Evaluates the protected material score for a given query and response or a multi-turn conversation, with reasoning.
+
+    Protected material is any text that is under copyright, including song lyrics, recipes, and articles. Protected
+    material evaluation leverages the Azure AI Content Safety Protected Material for Text service to perform the
+    classification.
+
+    The protected material score is a boolean value, where True indicates that protected material was detected.
+
+    :param credential: The credential required for connecting to the Azure AI project.
+    :type credential: ~azure.core.credentials.TokenCredential
+    :param azure_ai_project: The scope of the Azure AI project, containing the subscription ID,
+        resource group, and project name.
+    :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
+
+    .. admonition:: Example:
+
+        .. literalinclude:: ../samples/evaluation_samples_evaluate.py
+            :start-after: [START protected_material_evaluator]
+            :end-before: [END protected_material_evaluator]
+            :language: python
+            :dedent: 8
+            :caption: Initialize and call a ProtectedMaterialEvaluator.
+    """
+
+    id = "azureml://registries/azureml/models/Protected-Material-Evaluator/versions/3"
+    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+
+    @override
+    def __init__(
+        self,
+        credential,
+        azure_ai_project,
+    ):
+        super().__init__(
+            eval_metric=EvaluationMetrics.PROTECTED_MATERIAL,
+            azure_ai_project=azure_ai_project,
+            credential=credential,
+        )
+
+    @overload
+    def __call__(
+        self,
+        *,
+        query: str,
+        response: str,
+    ) -> Dict[str, Union[str, bool]]:
+        """Evaluate a given query/response pair for protected material
+
+        :keyword query: The query to be evaluated.
+        :paramtype query: str
+        :keyword response: The response to be evaluated.
+        :paramtype response: str
+        :return: The protected material score.
+        :rtype: Dict[str, Union[str, bool]]
+        """
+
+    @overload
+    def __call__(
+        self,
+        *,
+        conversation: Conversation,
+    ) -> Dict[str, Union[float, Dict[str, List[Union[str, bool]]]]]:
+        """Evaluate a conversation for protected material
+
+        :keyword conversation: The conversation to evaluate. Expected to contain a list of conversation turns under the
+            key "messages", and potentially a global context under the key "context". Conversation turns are expected
+            to be dictionaries with keys "content", "role", and possibly "context".
+        :paramtype conversation: Optional[~azure.ai.evaluation.Conversation]
+        :return: The protected material score.
+        :rtype: Dict[str, Union[str, bool, Dict[str, List[Union[str, bool]]]]]
+        """
+
+    @override
+    def __call__(
+        self,
+        *,
+        query: Optional[str] = None,
+        response: Optional[str] = None,
+        conversation=None,
+        **kwargs,
+    ):
+        """
+        Evaluate if protected material is present in your AI system's response.
+
+        :keyword query: The query to be evaluated.
+        :paramtype query: Optional[str]
+        :keyword response: The response to be evaluated.
+        :paramtype response: Optional[str]
+        :keyword conversation: The conversation to evaluate. Expected to contain a list of conversation turns under the
+            key "messages". Conversation turns are expected
+            to be dictionaries with keys "content" and "role".
+        :paramtype conversation: Optional[~azure.ai.evaluation.Conversation]
+        :return: The fluency score.
+        :rtype: Union[Dict[str, Union[str, bool]], Dict[str, Union[float, Dict[str, List[Union[str, bool]]]]]]
+        """
+        return super().__call__(query=query, response=response, conversation=conversation, **kwargs)

azure/ai/evaluation/_evaluators/_qa/__init__.py

@@ -0,0 +1,9 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from ._qa import QAEvaluator
+
+__all__ = [
+    "QAEvaluator",
+]