azure-ai-evaluation 1.1.0__py3-none-any.whl → 1.3.0__py3-none-any.whl

azure/ai/evaluation/_evaluators/_multimodal/_content_safety_multimodal.py

@@ -1,132 +0,0 @@
-# ---------------------------------------------------------
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# ---------------------------------------------------------
-import logging
-from concurrent.futures import as_completed
-from typing import Callable, Dict, List, Union
-
-from promptflow.tracing import ThreadPoolExecutorWithContext as ThreadPoolExecutor
-from azure.ai.evaluation._common._experimental import experimental
-from azure.ai.evaluation._common.utils import validate_conversation
-from ._hate_unfairness import HateUnfairnessMultimodalEvaluator
-from ._self_harm import SelfHarmMultimodalEvaluator
-from ._sexual import SexualMultimodalEvaluator
-from ._violence import ViolenceMultimodalEvaluator
-
-
-logger = logging.getLogger(__name__)
-
-
-@experimental
-class ContentSafetyMultimodalEvaluator:
-    """
-    Initialize a content safety multimodal evaluator configured to evaluate content safety metrics
-    in a multimodal scenario.
-
-    :param credential: The credential required for connecting to the Azure AI project.
-    :type credential: ~azure.core.credentials.TokenCredential
-    :param azure_ai_project: The scope of the Azure AI project, containing the subscription ID,
-        resource group, and project name.
-    :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
-    :param kwargs: Additional arguments to pass to the evaluator.
-    :type kwargs: Any
-
-    :return: A function that evaluates multimodal chat messages and generates content safety metrics.
-
-    **Usage Example**
-
-    .. code-block:: python
-
-        azure_ai_project = {
-            "subscription_id": "<subscription_id>",
-            "resource_group_name": "<resource_group_name>",
-            "project_name": "<project_name>",
-        }
-        eval_fn = ContentSafetyMultimodalEvaluator(azure_ai_project)
-        result = eval_fn(conversation=
-            {
-                "messages": [
-                    {
-                        "role": "user",
-                        "content": [
-                            {
-                                "type": "text",
-                                "text": "What's in this image?"
-                            },
-                            {
-                                "type": "image_url",
-                                "image_url": {
-                                    "url": "<image url or base64 encoded image>"
-                                }
-                            }
-                        ]
-                    },
-                    {
-                        "role": "assistant",
-                        "content": "This picture shows an astronaut standing in the desert."
-                    }
-                ]
-            }
-        )
-
-    **Output Format**
-
-    .. code-block:: json
-
-        {
-            "violence": "Medium",
-            "violence_score": 5.0,
-            "violence_reason": "Some reason",
-            "sexual": "Medium",
-            "sexual_score": 5.0,
-            "sexual_reason": "Some reason",
-            "self_harm": "Medium",
-            "self_harm_score": 5.0,
-            "self_harm_reason": "Some reason",
-            "hate_unfairness": "Medium",
-            "hate_unfairness_score": 5.0,
-            "hate_unfairness_reason": "Some reason"
-        }
-
-    """
-
-    id = "content_safety_multimodal"
-    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
-
-    def __init__(self, credential, azure_ai_project, **kwargs):
-        self._parallel = kwargs.pop("_parallel", False)
-        self._evaluators: List[Callable[..., Dict[str, Union[str, float]]]] = [
-            ViolenceMultimodalEvaluator(credential=credential, azure_ai_project=azure_ai_project),
-            SexualMultimodalEvaluator(credential=credential, azure_ai_project=azure_ai_project),
-            SelfHarmMultimodalEvaluator(credential=credential, azure_ai_project=azure_ai_project),
-            HateUnfairnessMultimodalEvaluator(credential=credential, azure_ai_project=azure_ai_project),
-        ]
-
-    def __call__(self, *, conversation, **kwargs):
-        """
-        Evaluates content-safety metrics for list of messages.
-
-        :keyword conversation: The conversation contains list of messages to be evaluated.
-            Each message should have "role" and "content" keys. It supports single turn only.
-        :paramtype conversation: ~azure.ai.evaluation.Conversation
-        :return: The evaluation score based on the Content Safety Metrics.
-        :rtype: Dict[str, Union[float, str]]
-        """
-        # validate inputs
-        validate_conversation(conversation)
-        results: Dict[str, Union[str, float]] = {}
-        if self._parallel:
-            with ThreadPoolExecutor() as executor:
-                futures = {
-                    executor.submit(evaluator, conversation=conversation, **kwargs): evaluator
-                    for evaluator in self._evaluators
-                }
-
-                for future in as_completed(futures):
-                    results.update(future.result())
-        else:
-            for evaluator in self._evaluators:
-                result = evaluator(conversation=conversation, **kwargs)
-                results.update(result)
-
-        return results

azure/ai/evaluation/_evaluators/_multimodal/_content_safety_multimodal_base.py

@@ -1,55 +0,0 @@
-# ---------------------------------------------------------
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# ---------------------------------------------------------
-from abc import ABC
-from typing import Union
-from azure.ai.evaluation._common.rai_service import evaluate_with_rai_service_multimodal
-from azure.ai.evaluation._common.constants import EvaluationMetrics, _InternalEvaluationMetrics
-from azure.ai.evaluation._common.utils import validate_conversation
-from azure.core.credentials import TokenCredential
-
-
-class ContentSafetyMultimodalEvaluatorBase(ABC):
-    """
-    Initialize a evaluator for a specified Evaluation Metric. Base class that is not
-    meant to be instantiated by users.
-
-    :param metric: The metric to be evaluated.
-    :type metric: ~azure.ai.evaluation._evaluators._content_safety.flow.constants.EvaluationMetrics
-    :param credential: The credential for connecting to Azure AI project. Required
-    :type credential: ~azure.core.credentials.TokenCredential
-    :param azure_ai_project: The scope of the Azure AI project.
-        It contains subscription id, resource group, and project name.
-    :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
-    """
-
-    def __init__(
-        self,
-        metric: Union[EvaluationMetrics, _InternalEvaluationMetrics],
-        credential: TokenCredential,
-        azure_ai_project,
-    ):
-        self._metric = metric
-        self._azure_ai_project = azure_ai_project
-        self._credential = credential
-
-    async def __call__(self, *, conversation, **kwargs):
-        """
-        Evaluates content according to this evaluator's metric.
-        :keyword conversation: The conversation contains list of messages to be evaluated.
-            Each message should have "role" and "content" keys.
-        :paramtype conversation: ~azure.ai.evaluation.Conversation
-        :return: The evaluation score computation based on the Content Safety metric (self.metric).
-        :rtype: Dict[str, Union[float, str]]
-        """
-        # validate inputs
-        validate_conversation(conversation)
-        messages = conversation["messages"]
-        # Run score computation based on supplied metric.
-        result = await evaluate_with_rai_service_multimodal(
-            messages=messages,
-            metric_name=self._metric,
-            project_scope=self._azure_ai_project,
-            credential=self._credential,
-        )
-        return result

azure/ai/evaluation/_evaluators/_multimodal/_hate_unfairness.py

@@ -1,100 +0,0 @@
-# ---------------------------------------------------------
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# ---------------------------------------------------------
-from promptflow._utils.async_utils import async_run_allowing_running_loop
-from azure.ai.evaluation._common._experimental import experimental
-from azure.ai.evaluation._common.constants import EvaluationMetrics
-from ._content_safety_multimodal_base import ContentSafetyMultimodalEvaluatorBase
-
-
-@experimental
-class HateUnfairnessMultimodalEvaluator:
-    """
-    Initialize a hate-unfairness multimodal evaluator for hate unfairness score.
-
-    :param credential: The credential for connecting to the Azure AI project. This is required.
-    :type credential: ~azure.core.credentials.TokenCredential
-    :param azure_ai_project: The scope of the Azure AI project,
-        which includes subscription ID, resource group name, and project name.
-    :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
-
-    **Usage Example**
-
-    .. code-block:: python
-
-        azure_ai_project = {
-            "subscription_id": "<subscription_id>",
-            "resource_group_name": "<resource_group_name>",
-            "project_name": "<project_name>",
-        }
-        eval_fn = HateUnfairnessMultimodalEvaluator(azure_ai_project)
-        result = eval_fn(conversation=
-            {
-                "messages": [
-                    {
-                        "role": "user",
-                        "content": [
-                            {
-                                "type": "text",
-                                "text": "What's in this image?"
-                            },
-                            {
-                                "type": "image_url",
-                                "image_url": {
-                                    "url": "<image url or base64 encoded image>"
-                                }
-                            }
-                        ]
-                    },
-                    {
-                        "role": "assistant",
-                        "content": "This picture shows an astronaut standing in the desert."
-                    }
-                ]
-            }
-        )
-
-    **Output Format**
-
-    .. code-block:: json
-
-        {
-            "hate_unfairness": "High",
-            "hate_unfairness_score": 6.5,
-            "hate_unfairness_reason": "Some reason"
-        }
-
-    """
-
-    id = "hate_unfairness_multimodal"
-    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
-
-    def __init__(self, credential, azure_ai_project):
-        self._async_evaluator = _AsyncHateUnfairnessMultimodalEvaluator(credential, azure_ai_project)
-
-    def __call__(self, *, conversation, **kwargs):
-        """
-        Evaluates hate unfairness content.
-
-        :keyword conversation: The conversation contains list of messages to be evaluated.
-            Each message should have "role" and "content" keys. It supports single turn only.
-        :paramtype conversation: ~azure.ai.evaluation.Conversation
-        :return: The hate unfairness score.
-        :rtype: Dict[str, Union[float, str]]
-        """
-        return async_run_allowing_running_loop(self._async_evaluator, conversation=conversation, **kwargs)
-
-    def _to_async(self):
-        return self._async_evaluator
-
-
-class _AsyncHateUnfairnessMultimodalEvaluator(ContentSafetyMultimodalEvaluatorBase):
-    def __init__(self, credential, azure_ai_project):
-        super().__init__(
-            metric=EvaluationMetrics.HATE_FAIRNESS,
-            credential=credential,
-            azure_ai_project=azure_ai_project,
-        )
-
-    async def __call__(self, *, conversation, **kwargs):
-        return await super().__call__(conversation=conversation, **kwargs)

azure/ai/evaluation/_evaluators/_multimodal/_protected_material.py

@@ -1,124 +0,0 @@
-# ---------------------------------------------------------
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# ---------------------------------------------------------
-from promptflow._utils.async_utils import async_run_allowing_running_loop
-from azure.ai.evaluation._common.constants import EvaluationMetrics
-from azure.ai.evaluation._common.utils import validate_conversation
-from azure.ai.evaluation._common.rai_service import evaluate_with_rai_service_multimodal
-from azure.ai.evaluation._common._experimental import experimental
-
-
-@experimental
-class ProtectedMaterialMultimodalEvaluator:
-    """
-    Initialize a protected materials evaluator to detect whether protected material
-    is present in multimodal messages. The evaluator outputs a Boolean label (`True` or `False`)
-    indicating the presence of protected material, along with AI-generated reasoning.
-
-    :param credential: The credential for connecting to the Azure AI project. This is required.
-    :type credential: ~azure.core.credentials.TokenCredential
-    :param azure_ai_project: The scope of the Azure AI project, containing the subscription ID,
-        resource group, and project name.
-    :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
-
-    :return: A dictionary containing the evaluation result label and reasoning.
-
-    **Usage Example**
-
-    .. code-block:: python
-
-        azure_ai_project = {
-            "subscription_id": "<subscription_id>",
-            "resource_group_name": "<resource_group_name>",
-            "project_name": "<project_name>",
-        }
-        eval_fn = ProtectedMaterialMultimodalEvaluator(azure_ai_project)
-        result = eval_fn(conversation=
-            {
-                "messages": [
-                    {
-                        "role": "user",
-                        "content": [
-                            {
-                                "type": "text",
-                                "text": "What's in this image?"
-                            },
-                            {
-                                "type": "image_url",
-                                "image_url": {
-                                    "url": "<image url or base64 encoded image>"
-                                }
-                            }
-                        ]
-                    },
-                    {
-                        "role": "assistant",
-                        "content": "This picture shows an astronaut standing in the desert."
-                    }
-                ]
-            }
-        )
-
-    **Output Format**
-
-    .. code-block:: json
-
-        {
-            "protected_material_label": "False",
-            "protected_material_reason": "This query does not contain any protected material."
-        }
-
-    """
-
-    id = "protected_material_multimodal"
-    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
-
-    def __init__(
-        self,
-        credential,
-        azure_ai_project,
-    ):
-        self._async_evaluator = _AsyncProtectedMaterialMultimodalEvaluator(credential, azure_ai_project)
-
-    def __call__(self, *, conversation, **kwargs):
-        """
-        Evaluates protected materials content.
-
-        :keyword conversation: The conversation contains list of messages to be evaluated.
-            Each message should have "role" and "content" keys. It supports single turn only.
-        :paramtype conversation: ~azure.ai.evaluation.Conversation
-        :return: A dictionary containing a boolean label and reasoning.
-        :rtype: Dict[str, str]
-        """
-        return async_run_allowing_running_loop(self._async_evaluator, conversation=conversation, **kwargs)
-
-    def _to_async(self):
-        return self._async_evaluator
-
-
-class _AsyncProtectedMaterialMultimodalEvaluator:
-    def __init__(self, credential, azure_ai_project):
-        self._credential = credential
-        self._azure_ai_project = azure_ai_project
-
-    async def __call__(self, *, conversation, **kwargs):
-        """
-        Evaluates content according to this evaluator's metric.
-
-        :keyword conversation: The conversation contains list of messages to be evaluated.
-            Each message should have "role" and "content" keys. It supports single turn only.
-        :paramtype conversation: ~azure.ai.evaluation.Conversation
-        :return: The evaluation score computation based on the Content Safety metric (self.metric).
-        :rtype: Any
-        """
-        # Validate inputs
-        validate_conversation(conversation)
-        messages = conversation["messages"]
-        # Run score computation based on supplied metric.
-        result = await evaluate_with_rai_service_multimodal(
-            messages=messages,
-            metric_name=EvaluationMetrics.PROTECTED_MATERIAL,
-            credential=self._credential,
-            project_scope=self._azure_ai_project,
-        )
-        return result

azure/ai/evaluation/_evaluators/_multimodal/_self_harm.py

@@ -1,100 +0,0 @@
-# ---------------------------------------------------------
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# ---------------------------------------------------------
-from promptflow._utils.async_utils import async_run_allowing_running_loop
-from azure.ai.evaluation._common._experimental import experimental
-from azure.ai.evaluation._common.constants import EvaluationMetrics
-from ._content_safety_multimodal_base import ContentSafetyMultimodalEvaluatorBase
-
-
-@experimental
-class SelfHarmMultimodalEvaluator:
-    """
-    Initialize a self harm multimodal evaluator for self harm score.
-
-    :param credential: The credential for connecting to the Azure AI project. This is required.
-    :type credential: ~azure.core.credentials.TokenCredential
-    :param azure_ai_project: The scope of the Azure AI project,
-        which includes subscription ID, resource group name, and project name.
-    :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
-
-    **Usage Example**
-
-    .. code-block:: python
-
-        azure_ai_project = {
-            "subscription_id": "<subscription_id>",
-            "resource_group_name": "<resource_group_name>",
-            "project_name": "<project_name>",
-        }
-        eval_fn = SelfHarmMultimodalEvaluator(azure_ai_project)
-        result = eval_fn(conversation=
-            {
-                "messages": [
-                    {
-                        "role": "user",
-                        "content": [
-                            {
-                                "type": "text",
-                                "text": "What's in this image?"
-                            },
-                            {
-                                "type": "image_url",
-                                "image_url": {
-                                    "url": "<image url or base64 encoded image>"
-                                }
-                            }
-                        ]
-                    },
-                    {
-                        "role": "assistant",
-                        "content": "This picture shows an astronaut standing in the desert."
-                    }
-                ]
-            }
-        )
-
-    **Output Format**
-
-    .. code-block:: json
-
-        {
-            "self_harm": "High",
-            "self_harm_score": 6.5,
-            "self_harm_reason": "Some reason"
-        }
-
-    """
-
-    id = "self_harm_multimodal"
-    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
-
-    def __init__(self, credential, azure_ai_project):
-        self._async_evaluator = _AsyncSelfHarmMultimodalEvaluator(credential, azure_ai_project)
-
-    def __call__(self, *, conversation, **kwargs):
-        """
-        Evaluates self harm content.
-
-        :keyword conversation: The conversation contains list of messages to be evaluated.
-            Each message should have "role" and "content" keys. It supports single turn only.
-        :paramtype conversation: ~azure.ai.evaluation.Conversation
-        :return: The self harm score.
-        :rtype: Dict[str, Union[float, str]]
-        """
-        return async_run_allowing_running_loop(self._async_evaluator, conversation=conversation, **kwargs)
-
-    def _to_async(self):
-        return self._async_evaluator
-
-
-class _AsyncSelfHarmMultimodalEvaluator(ContentSafetyMultimodalEvaluatorBase):
-    def __init__(self, credential, azure_ai_project):
-        super().__init__(
-            metric=EvaluationMetrics.SELF_HARM,
-            credential=credential,
-            azure_ai_project=azure_ai_project,
-        )
-
-    async def __call__(self, *, conversation, **kwargs):
-        return await super().__call__(conversation=conversation, **kwargs)

azure/ai/evaluation/_evaluators/_multimodal/_sexual.py

@@ -1,100 +0,0 @@
-# ---------------------------------------------------------
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# ---------------------------------------------------------
-from promptflow._utils.async_utils import async_run_allowing_running_loop
-from azure.ai.evaluation._common.constants import EvaluationMetrics
-from azure.ai.evaluation._common._experimental import experimental
-from ._content_safety_multimodal_base import ContentSafetyMultimodalEvaluatorBase
-
-
-@experimental
-class SexualMultimodalEvaluator:
-    """
-    Initialize a sexual multimodal evaluator for sexual score.
-
-    :param credential: The credential for connecting to the Azure AI project. This is required.
-    :type credential: ~azure.core.credentials.TokenCredential
-    :param azure_ai_project: The scope of the Azure AI project,
-        which includes subscription ID, resource group name, and project name.
-    :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
-
-    **Usage Example**
-
-    .. code-block:: python
-
-        azure_ai_project = {
-            "subscription_id": "<subscription_id>",
-            "resource_group_name": "<resource_group_name>",
-            "project_name": "<project_name>",
-        }
-        eval_fn = SexualMultimodalEvaluator(azure_ai_project)
-        result = eval_fn(conversation=
-            {
-                "messages": [
-                    {
-                        "role": "user",
-                        "content": [
-                            {
-                                "type": "text",
-                                "text": "What's in this image?"
-                            },
-                            {
-                                "type": "image_url",
-                                "image_url": {
-                                    "url": "<image url or base64 encoded image>"
-                                }
-                            }
-                        ]
-                    },
-                    {
-                        "role": "assistant",
-                        "content": "This picture shows an astronaut standing in the desert."
-                    }
-                ]
-            }
-        )
-
-    **Output Format**
-
-    .. code-block:: json
-
-        {
-            "sexual": "High",
-            "sexual_score": 6.5,
-            "sexual_reason": "Some reason"
-        }
-
-    """
-
-    id = "sexual_multimodal"
-    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
-
-    def __init__(self, credential, azure_ai_project):
-        self._async_evaluator = _AsyncSexualMultimodalEvaluator(credential, azure_ai_project)
-
-    def __call__(self, *, conversation, **kwargs):
-        """
-        Evaluates sexual content.
-
-        :keyword conversation: The conversation contains list of messages to be evaluated.
-            Each message should have "role" and "content" keys. It supports single turn only.
-        :paramtype conversation: ~azure.ai.evaluation.Conversation
-        :return: The sexual score.
-        :rtype: Dict[str, Union[float, str]]
-        """
-        return async_run_allowing_running_loop(self._async_evaluator, conversation=conversation, **kwargs)
-
-    def _to_async(self):
-        return self._async_evaluator
-
-
-class _AsyncSexualMultimodalEvaluator(ContentSafetyMultimodalEvaluatorBase):
-    def __init__(self, credential, azure_ai_project):
-        super().__init__(
-            metric=EvaluationMetrics.SEXUAL,
-            credential=credential,
-            azure_ai_project=azure_ai_project,
-        )
-
-    async def __call__(self, *, conversation, **kwargs):
-        return await super().__call__(conversation=conversation, **kwargs)