azure-ai-evaluation 1.0.0b4__py3-none-any.whl → 1.0.0b5__py3-none-any.whl

azure/ai/evaluation/_evaluators/_multimodal/_protected_material.py

@@ -0,0 +1,120 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+from promptflow._utils.async_utils import async_run_allowing_running_loop
+from azure.ai.evaluation._common._experimental import experimental
+from azure.ai.evaluation._common.constants import EvaluationMetrics
+from azure.ai.evaluation._common.utils import validate_conversation
+from azure.ai.evaluation._common.rai_service import evaluate_with_rai_service_multimodal
+
+
+@experimental
+class ProtectedMaterialMultimodalEvaluator:
+    """
+    Initialize a protected materials evaluator to detect whether protected material
+    is present in multimodal messages. The evaluator outputs a Boolean label (`True` or `False`)
+    indicating the presence of protected material, along with AI-generated reasoning.
+
+    :param credential: The credential for connecting to the Azure AI project. This is required.
+    :type credential: ~azure.core.credentials.TokenCredential
+    :param azure_ai_project: The scope of the Azure AI project, containing the subscription ID,
+        resource group, and project name.
+    :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
+
+    :return: A dictionary containing the evaluation result label and reasoning.
+    :rtype: Dict[str, str]
+
+    **Usage Example**
+
+    .. code-block:: python
+
+        azure_ai_project = {
+            "subscription_id": "<subscription_id>",
+            "resource_group_name": "<resource_group_name>",
+            "project_name": "<project_name>",
+        }
+        eval_fn = ProtectedMaterialMultimodalEvaluator(azure_ai_project)
+        result = eval_fn(
+            {
+                "messages": [
+                    {
+                        "role": "user",
+                        "content": [
+                            {
+                                "type": "text",
+                                "text": "What's in this image?"
+                            },
+                            {
+                                "type": "image_url",
+                                "image_url": {
+                                    "url": "<image url or base64 encoded image>"
+                                }
+                            }
+                        ]
+                    },
+                    {
+                        "role": "assistant",
+                        "content": "This picture shows an astronaut standing in the desert."
+                    }
+                ]
+            }
+        )
+
+    **Output Format**
+
+    .. code-block:: json
+
+        {
+            "protected_material_label": "False",
+            "protected_material_reason": "This query does not contain any protected material."
+        }
+
+    """
+
+    def __init__(
+        self,
+        credential,
+        azure_ai_project,
+    ):
+        self._async_evaluator = _AsyncProtectedMaterialMultimodalEvaluator(credential, azure_ai_project)
+
+    def __call__(self, *, conversation, **kwargs):
+        """
+        Evaluates protected materials content.
+
+        :keyword messages: The messages to be evaluated. Each message should have "role" and "content" keys.
+        :paramtype messages: ~azure.ai.evaluation.Conversation
+        :return: A dictionary containing a boolean label and reasoning.
+        :rtype: Dict[str, str]
+        """
+        return async_run_allowing_running_loop(self._async_evaluator, conversation=conversation, **kwargs)
+
+    def _to_async(self):
+        return self._async_evaluator
+
+
+class _AsyncProtectedMaterialMultimodalEvaluator:
+    def __init__(self, credential, azure_ai_project):
+        self._credential = credential
+        self._azure_ai_project = azure_ai_project
+
+    async def __call__(self, *, conversation, **kwargs):
+        """
+        Evaluates content according to this evaluator's metric.
+        :keyword conversation: The conversation contains list of messages to be evaluated.
+        Each message should have "role" and "content" keys.
+        :paramtype conversation: ~azure.ai.evaluation.Conversation
+        :return: The evaluation score computation based on the Content Safety metric (self.metric).
+        :rtype: Any
+        """
+        # Validate inputs
+        validate_conversation(conversation)
+        messages = conversation["messages"]
+        # Run score computation based on supplied metric.
+        result = await evaluate_with_rai_service_multimodal(
+            messages=messages,
+            metric_name=EvaluationMetrics.PROTECTED_MATERIAL,
+            credential=self._credential,
+            project_scope=self._azure_ai_project,
+        )
+        return result

azure/ai/evaluation/_evaluators/_multimodal/_self_harm.py

@@ -0,0 +1,96 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+from promptflow._utils.async_utils import async_run_allowing_running_loop
+from azure.ai.evaluation._common._experimental import experimental
+from azure.ai.evaluation._common.constants import EvaluationMetrics
+from ._content_safety_multimodal_base import ContentSafetyMultimodalEvaluatorBase
+
+
+@experimental
+class SelfHarmMultimodalEvaluator:
+    """
+    Initialize a self harm multimodal evaluator for self harm score.
+
+    :param credential: The credential for connecting to the Azure AI project. This is required.
+    :type credential: ~azure.core.credentials.TokenCredential
+    :param azure_ai_project: The scope of the Azure AI project,
+        which includes subscription ID, resource group name, and project name.
+    :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
+
+    **Usage Example**
+
+    .. code-block:: python
+
+        azure_ai_project = {
+            "subscription_id": "<subscription_id>",
+            "resource_group_name": "<resource_group_name>",
+            "project_name": "<project_name>",
+        }
+        eval_fn = SelfHarmMultimodalEvaluator(azure_ai_project)
+        result = eval_fn(
+            {
+                "messages": [
+                    {
+                        "role": "user",
+                        "content": [
+                            {
+                                "type": "text",
+                                "text": "What's in this image?"
+                            },
+                            {
+                                "type": "image_url",
+                                "image_url": {
+                                    "url": "<image url or base64 encoded image>"
+                                }
+                            }
+                        ]
+                    },
+                    {
+                        "role": "assistant",
+                        "content": "This picture shows an astronaut standing in the desert."
+                    }
+                ]
+            }
+        )
+
+    **Output Format**
+
+    .. code-block:: json
+
+        {
+            "self_harm": "High",
+            "self_harm_score": 6.5,
+            "self_harm_reason": "Some reason"
+        }
+
+    """
+
+    def __init__(self, credential, azure_ai_project):
+        self._async_evaluator = _AsyncSelfHarmMultimodalEvaluator(credential, azure_ai_project)
+
+    def __call__(self, *, conversation, **kwargs):
+        """
+        Evaluates self harm content.
+        :keyword conversation: The conversation contains list of messages to be evaluated.
+            Each message should have "role" and "content" keys.
+        :paramtype conversation: ~azure.ai.evaluation.Conversation
+        :return: The self harm score.
+        :rtype: Dict[str, Union[float, str]]
+        """
+        return async_run_allowing_running_loop(self._async_evaluator, conversation=conversation, **kwargs)
+
+    def _to_async(self):
+        return self._async_evaluator
+
+
+class _AsyncSelfHarmMultimodalEvaluator(ContentSafetyMultimodalEvaluatorBase):
+    def __init__(self, credential, azure_ai_project):
+        super().__init__(
+            metric=EvaluationMetrics.SELF_HARM,
+            credential=credential,
+            azure_ai_project=azure_ai_project,
+        )
+
+    async def __call__(self, *, conversation, **kwargs):
+        return await super().__call__(conversation=conversation, **kwargs)

azure/ai/evaluation/_evaluators/_multimodal/_sexual.py

@@ -0,0 +1,96 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+from promptflow._utils.async_utils import async_run_allowing_running_loop
+from azure.ai.evaluation._common.constants import EvaluationMetrics
+from azure.ai.evaluation._common._experimental import experimental
+from ._content_safety_multimodal_base import ContentSafetyMultimodalEvaluatorBase
+
+
+@experimental
+class SexualMultimodalEvaluator:
+    """
+    Initialize a sexual multimodal evaluator for sexual score.
+
+    :param credential: The credential for connecting to the Azure AI project. This is required.
+    :type credential: ~azure.core.credentials.TokenCredential
+    :param azure_ai_project: The scope of the Azure AI project,
+        which includes subscription ID, resource group name, and project name.
+    :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
+
+    **Usage Example**
+
+    .. code-block:: python
+
+        azure_ai_project = {
+            "subscription_id": "<subscription_id>",
+            "resource_group_name": "<resource_group_name>",
+            "project_name": "<project_name>",
+        }
+        eval_fn = SexualMultimodalEvaluator(azure_ai_project)
+        result = eval_fn(
+            {
+                "messages": [
+                    {
+                        "role": "user",
+                        "content": [
+                            {
+                                "type": "text",
+                                "text": "What's in this image?"
+                            },
+                            {
+                                "type": "image_url",
+                                "image_url": {
+                                    "url": "<image url or base64 encoded image>"
+                                }
+                            }
+                        ]
+                    },
+                    {
+                        "role": "assistant",
+                        "content": "This picture shows an astronaut standing in the desert."
+                    }
+                ]
+            }
+        )
+
+    **Output Format**
+
+    .. code-block:: json
+
+        {
+            "sexual": "High",
+            "sexual_score": 6.5,
+            "sexual_reason": "Some reason"
+        }
+
+    """
+
+    def __init__(self, credential, azure_ai_project):
+        self._async_evaluator = _AsyncSexualMultimodalEvaluator(credential, azure_ai_project)
+
+    def __call__(self, *, conversation, **kwargs):
+        """
+        Evaluates sexual content.
+        :keyword conversation: The conversation contains list of messages to be evaluated.
+            Each message should have "role" and "content" keys.
+        :paramtype conversation: ~azure.ai.evaluation.Conversation
+        :return: The sexual score.
+        :rtype: Dict[str, Union[float, str]]
+        """
+        return async_run_allowing_running_loop(self._async_evaluator, conversation=conversation, **kwargs)
+
+    def _to_async(self):
+        return self._async_evaluator
+
+
+class _AsyncSexualMultimodalEvaluator(ContentSafetyMultimodalEvaluatorBase):
+    def __init__(self, credential, azure_ai_project):
+        super().__init__(
+            metric=EvaluationMetrics.SEXUAL,
+            credential=credential,
+            azure_ai_project=azure_ai_project,
+        )
+
+    async def __call__(self, *, conversation, **kwargs):
+        return await super().__call__(conversation=conversation, **kwargs)

azure/ai/evaluation/_evaluators/_multimodal/_violence.py

@@ -0,0 +1,96 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+from promptflow._utils.async_utils import async_run_allowing_running_loop
+from azure.ai.evaluation._common._experimental import experimental
+from azure.ai.evaluation._common.constants import EvaluationMetrics
+from ._content_safety_multimodal_base import ContentSafetyMultimodalEvaluatorBase
+
+
+@experimental
+class ViolenceMultimodalEvaluator:
+    """
+    Initialize a violence multimodal evaluator for violence score.
+
+    :param credential: The credential for connecting to the Azure AI project. This is required.
+    :type credential: ~azure.core.credentials.TokenCredential
+    :param azure_ai_project: The scope of the Azure AI project,
+        which includes subscription ID, resource group name, and project name.
+    :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
+
+    **Usage Example**
+
+    .. code-block:: python
+
+        azure_ai_project = {
+            "subscription_id": "<subscription_id>",
+            "resource_group_name": "<resource_group_name>",
+            "project_name": "<project_name>",
+        }
+        eval_fn = ViolenceMultimodalEvaluator(azure_ai_project)
+        result = eval_fn(
+            {
+                "messages": [
+                    {
+                        "role": "user",
+                        "content": [
+                            {
+                                "type": "text",
+                                "text": "What's in this image?"
+                            },
+                            {
+                                "type": "image_url",
+                                "image_url": {
+                                    "url": "<image url or base64 encoded image>"
+                                }
+                            }
+                        ]
+                    },
+                    {
+                        "role": "assistant",
+                        "content": "This picture shows an astronaut standing in the desert."
+                    }
+                ]
+            }
+        )
+
+    **Output Format**
+
+    .. code-block:: json
+
+        {
+            "violence": "High",
+            "violence_score": 6.5,
+            "violence_reason": "Some reason"
+        }
+
+    """
+
+    def __init__(self, credential, azure_ai_project):
+        self._async_evaluator = _AsyncViolenceMultimodalEvaluator(credential, azure_ai_project)
+
+    def __call__(self, *, conversation, **kwargs):
+        """
+        Evaluates violence content.
+        :keyword conversation: The conversation contains list of messages to be evaluated.
+            Each message should have "role" and "content" keys.
+        :paramtype conversation: ~azure.ai.evaluation.Conversation
+        :return: The violence score.
+        :rtype: Dict[str, Union[float, str]]
+        """
+        return async_run_allowing_running_loop(self._async_evaluator, conversation=conversation, **kwargs)
+
+    def _to_async(self):
+        return self._async_evaluator
+
+
+class _AsyncViolenceMultimodalEvaluator(ContentSafetyMultimodalEvaluatorBase):
+    def __init__(self, credential, azure_ai_project):
+        super().__init__(
+            metric=EvaluationMetrics.VIOLENCE,
+            credential=credential,
+            azure_ai_project=azure_ai_project,
+        )
+
+    async def __call__(self, *, conversation, **kwargs):
+        return await super().__call__(conversation=conversation, **kwargs)

azure/ai/evaluation/_evaluators/_protected_material/_protected_material.py

@@ -1,26 +1,33 @@
 # ---------------------------------------------------------
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
+
+from typing import Optional
+
 from typing_extensions import override
 
+from azure.ai.evaluation._common._experimental import experimental
 from azure.ai.evaluation._common.constants import EvaluationMetrics
 from azure.ai.evaluation._evaluators._common import RaiServiceEvaluatorBase
 
 
+@experimental
 class ProtectedMaterialEvaluator(RaiServiceEvaluatorBase):
     """
     Initialize a protected material evaluator to detect whether protected material
-    is present in your AI system's response. Outputs True or False with AI-generated reasoning.
+    is present in the AI system's response. The evaluator outputs a Boolean label (`True` or `False`)
+    indicating the presence of protected material, along with AI-generated reasoning.
 
-    :param credential: The credential for connecting to Azure AI project. Required
+    :param credential: The credential required for connecting to the Azure AI project.
     :type credential: ~azure.core.credentials.TokenCredential
-    :param azure_ai_project: The scope of the Azure AI project.
-        It contains subscription id, resource group, and project name.
+    :param azure_ai_project: The scope of the Azure AI project, containing the subscription ID,
+        resource group, and project name.
     :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
-    :return: Whether or not protected material was found in the response, with AI-generated reasoning.
-    :rtype: Dict[str, str]
 
-    **Usage**
+    :return: A dictionary with a label indicating the presence of protected material and the reasoning.
+    :rtype: Dict[str, Union[bool, str]]
+
+    **Usage Example**
 
     .. code-block:: python
 
@@ -32,21 +39,22 @@ class ProtectedMaterialEvaluator(RaiServiceEvaluatorBase):
         eval_fn = ProtectedMaterialEvaluator(azure_ai_project)
         result = eval_fn(query="What is the capital of France?", response="Paris.")
 
-    **Output format**
+    **Output Format**
 
-    .. code-block:: python
+    .. code-block:: json
 
         {
-            "protected_material_label": "False",
+            "protected_material_label": false,
             "protected_material_reason": "This query does not contain any protected material."
         }
+
     """
 
     @override
     def __init__(
         self,
         credential,
-        azure_ai_project: dict,
+        azure_ai_project,
         eval_last_turn: bool = False,
     ):
         super().__init__(
@@ -55,3 +63,28 @@ class ProtectedMaterialEvaluator(RaiServiceEvaluatorBase):
             credential=credential,
             eval_last_turn=eval_last_turn,
         )
+
+    @override
+    def __call__(
+        self,
+        *,
+        query: Optional[str] = None,
+        response: Optional[str] = None,
+        conversation=None,
+        **kwargs,
+    ):
+        """
+        Evaluate if protected material is present in your AI system's response.
+
+        :keyword query: The query to be evaluated.
+        :paramtype query: str
+        :keyword response: The response to be evaluated.
+        :paramtype response: str
+        :keyword conversation: The conversation to evaluate. Expected to contain a list of conversation turns under the
+            key "messages". Conversation turns are expected
+            to be dictionaries with keys "content" and "role".
+        :paramtype conversation: Optional[~azure.ai.evaluation.Conversation]
+        :return: The fluency score.
+        :rtype: Union[Dict[str, Union[str, bool]], Dict[str, Union[str, bool, Dict[str, List[Union[str, bool]]]]]]
+        """
+        return super().__call__(query=query, response=response, conversation=conversation, **kwargs)

azure/ai/evaluation/_evaluators/_qa/_qa.py

@@ -22,8 +22,7 @@ class QAEvaluator:
     :param model_config: Configuration for the Azure OpenAI model.
     :type model_config: Union[~azure.ai.evaluation.AzureOpenAIModelConfiguration,
         ~azure.ai.evaluation.OpenAIModelConfiguration]
-    :return: A function that evaluates and generates metrics for "question-answering" scenario.
-    :rtype: Callable
+    :return: A callable class that evaluates and generates metrics for "question-answering" scenario.
 
     **Usage**
 
@@ -42,6 +41,11 @@ class QAEvaluator:
     .. code-block:: python
 
         {
+            "groundedness": 3.5,
+            "relevance": 4.0,
+            "coherence": 1.5,
+            "fluency": 4.0,
+            "similarity": 3.0,
             "gpt_groundedness": 3.5,
             "gpt_relevance": 4.0,
             "gpt_coherence": 1.5,
@@ -51,7 +55,7 @@ class QAEvaluator:
         }
     """
 
-    def __init__(self, model_config: dict, parallel: bool = True):
+    def __init__(self, model_config, parallel: bool = True):
         self._parallel = parallel
 
         self._evaluators: List[Callable[..., Dict[str, float]]] = [

azure/ai/evaluation/_evaluators/_relevance/_relevance.py

@@ -25,28 +25,32 @@ class RelevanceEvaluator(PromptyEvaluatorBase):
         eval_fn = RelevanceEvaluator(model_config)
         result = eval_fn(
             query="What is the capital of Japan?",
-            response="The capital of Japan is Tokyo.",
-            context="Tokyo is Japan's capital, known for its blend of traditional culture \
-                and technological advancements.")
+            response="The capital of Japan is Tokyo.")
 
     **Output format**
 
     .. code-block:: python
 
         {
-            "gpt_relevance": 3.0
+            "relevance": 3.0,
+            "gpt_relevance": 3.0,
+            "relevance_reason": "The response is relevant to the query because it provides the correct answer.",
         }
+
+    Note: To align with our support of a diverse set of models, a key without the `gpt_` prefix has been added.
+    To maintain backwards compatibility, the old key with the `gpt_` prefix is still be present in the output;
+    however, it is recommended to use the new key moving forward as the old key will be deprecated in the future.
     """
 
     # Constants must be defined within eval's directory to be save/loadable
-    PROMPTY_FILE = "relevance.prompty"
-    RESULT_KEY = "gpt_relevance"
+    _PROMPTY_FILE = "relevance.prompty"
+    _RESULT_KEY = "relevance"
 
     @override
-    def __init__(self, model_config: dict):
+    def __init__(self, model_config):
         current_dir = os.path.dirname(__file__)
-        prompty_path = os.path.join(current_dir, self.PROMPTY_FILE)
-        super().__init__(model_config=model_config, prompty_file=prompty_path, result_key=self.RESULT_KEY)
+        prompty_path = os.path.join(current_dir, self._PROMPTY_FILE)
+        super().__init__(model_config=model_config, prompty_file=prompty_path, result_key=self._RESULT_KEY)
 
     @override
     def __call__(
@@ -54,25 +58,23 @@ class RelevanceEvaluator(PromptyEvaluatorBase):
         *,
         query: Optional[str] = None,
         response: Optional[str] = None,
-        context: Optional[str] = None,
-        conversation: Optional[dict] = None,
+        conversation=None,
         **kwargs,
     ):
-        """Evaluate relevance. Accepts either a response and context a single evaluation,
+        """Evaluate relevance. Accepts either a query and response for a single evaluation,
         or a conversation for a multi-turn evaluation. If the conversation has more than one turn,
         the evaluator will aggregate the results of each turn.
 
-        :keyword query: The query to be evaluated.
+        :keyword query: The query to be evaluated. Mutually exclusive with the `conversation` parameter.
         :paramtype query: Optional[str]
-        :keyword response: The response to be evaluated.
+        :keyword response: The response to be evaluated. Mutually exclusive with the `conversation` parameter.
         :paramtype response: Optional[str]
-        :keyword context: The context to be evaluated.
-        :paramtype context: Optional[str]
         :keyword conversation: The conversation to evaluate. Expected to contain a list of conversation turns under the
             key "messages", and potentially a global context under the key "context". Conversation turns are expected
             to be dictionaries with keys "content", "role", and possibly "context".
-        :paramtype conversation: Optional[Dict]
+        :paramtype conversation: Optional[~azure.ai.evaluation.Conversation]
         :return: The relevance score.
-        :rtype: Dict[str, float]
+        :rtype: Union[Dict[str, float], Dict[str, Union[float, Dict[str, List[float]]]]]
         """
-        return super().__call__(query=query, response=response, context=context, conversation=conversation, **kwargs)
+
+        return super().__call__(query=query, response=response, conversation=conversation, **kwargs)