azure-ai-evaluation 1.0.0__py3-none-any.whl → 1.0.0b2__py3-none-any.whl

azure/ai/evaluation/_evaluators/_common/_base_rai_svc_eval.py

@@ -1,133 +0,0 @@
-# ---------------------------------------------------------
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# ---------------------------------------------------------
-from typing import Dict, TypeVar, Union
-
-from typing_extensions import override
-
-from azure.ai.evaluation._common.constants import (
-    EvaluationMetrics,
-    _InternalEvaluationMetrics,
-    Tasks,
-    _InternalAnnotationTasks,
-)
-from azure.ai.evaluation._common.rai_service import evaluate_with_rai_service
-from azure.ai.evaluation._common.utils import validate_azure_ai_project
-from azure.ai.evaluation._exceptions import EvaluationException
-from azure.core.credentials import TokenCredential
-
-from . import EvaluatorBase
-
-T = TypeVar("T")
-
-
-class RaiServiceEvaluatorBase(EvaluatorBase[T]):
-    """Base class for all evaluators that require the use of the Azure AI RAI service for evaluation.
-    This includes content safety evaluators, protected material evaluators, and others. These evaluators
-    are all assumed to be of the "query and response or conversation" input variety.
-
-    :param eval_metric: The evaluation metric to be used for evaluation. This is used by the API call logic
-        to specify which evaluation to perform.
-    :type eval_metric: ~azure.ai.evaluation._common.constants.EvaluationMetrics
-    :param eval_last_turn: If True, only the last turn of the conversation will be evaluated, and no
-        aggregation will be performed. If False, all turns will be evaluated and the numeric results will be,
-        aggregated. Per-turn results are still be available in the output via the "evaluation_per_turn" key
-        when this occurs. Default is False, resulting full conversation evaluation and aggregation.
-    :type eval_last_turn: bool
-    """
-
-    @override
-    def __init__(
-        self,
-        eval_metric: Union[EvaluationMetrics, _InternalEvaluationMetrics],
-        azure_ai_project: dict,
-        credential: TokenCredential,
-        eval_last_turn: bool = False,
-    ):
-        super().__init__(eval_last_turn=eval_last_turn)
-        self._eval_metric = eval_metric
-        self._azure_ai_project = validate_azure_ai_project(azure_ai_project)
-        self._credential = credential
-
-    @override
-    def __call__(  # pylint: disable=docstring-missing-param
-        self,
-        *args,
-        **kwargs,
-    ):
-        """Evaluate either a query and response or a conversation. Must supply either a query AND response,
-        or a conversation, but not both.
-
-        :keyword query: The query to evaluate.
-        :paramtype query: Optional[str]
-        :keyword response: The response to evaluate.
-        :paramtype response: Optional[str]
-        :keyword conversation: The conversation to evaluate. Expected to contain a list of conversation turns under the
-            key "messages", and potentially a global context under the key "context". Conversation turns are expected
-            to be dictionaries with keys "content", "role", and possibly "context".
-        :paramtype conversation: Optional[~azure.ai.evaluation.Conversation]
-        :rtype: Union[Dict[str, T], Dict[str, Union[float, Dict[str, List[T]]]]]
-        """
-        return super().__call__(*args, **kwargs)
-
-    @override
-    async def _do_eval(self, eval_input: Dict) -> Dict[str, T]:
-        """Perform the evaluation using the Azure AI RAI service.
-        The exact evaluation performed is determined by the evaluation metric supplied
-        by the child class initializer.
-
-        :param eval_input: The input to the evaluation function.
-        :type eval_input: Dict
-        :return: The evaluation result.
-        :rtype: Dict
-        """
-        query = eval_input.get("query", None)
-        response = eval_input.get("response", None)
-        if query is None or response is None:
-            raise EvaluationException(
-                message="Not implemented",
-                internal_message=(
-                    "Reached query/response evaluation without supplying query or response."
-                    + " This should have failed earlier."
-                ),
-            )
-        input_data = {"query": query, "response": response}
-
-        if "context" in self._singleton_inputs:
-            context = eval_input.get("context", None)
-            if context is None:
-                raise EvaluationException(
-                    message="Not implemented",
-                    internal_message=(
-                        "Attempted context-based evaluation without supplying context."
-                        + " This should have failed earlier."
-                    ),
-                )
-            input_data["context"] = context
-
-        return await evaluate_with_rai_service(  # type: ignore
-            metric_name=self._eval_metric,
-            data=input_data,
-            project_scope=self._azure_ai_project,
-            credential=self._credential,
-            annotation_task=self._get_task(),
-        )
-
-    def _get_task(self):
-        """Get the annotation task for the current evaluation metric.
-        The annotation task is used by the RAI service script to determine a the message format
-        of the API call, and how the output is processed, among other things.
-
-        :return: The annotation task for the evaluator's self._eval_metric value.
-        :rtype: ~azure.ai.evaluation._common.constants.Tasks
-
-        """
-        if self._eval_metric == EvaluationMetrics.GROUNDEDNESS:
-            return Tasks.GROUNDEDNESS
-        if self._eval_metric == EvaluationMetrics.XPIA:
-            return Tasks.XPIA
-        if self._eval_metric == _InternalEvaluationMetrics.ECI:
-            return _InternalAnnotationTasks.ECI
-        if self._eval_metric == EvaluationMetrics.PROTECTED_MATERIAL:
-            return Tasks.PROTECTED_MATERIAL
-        return Tasks.CONTENT_HARM

azure/ai/evaluation/_evaluators/_groundedness/groundedness_with_query.prompty

@@ -1,113 +0,0 @@
----
-name: Groundedness
-description: Evaluates groundedness score for RAG scenario
-model:
-  api: chat
-  parameters:
-    temperature: 0.0
-    max_tokens: 800
-    top_p: 1.0
-    presence_penalty: 0
-    frequency_penalty: 0
-    response_format:
-      type: text
-
-inputs:
-  query:
-    type: string
-  response:
-    type: string
-  context:
-    type: string
-
-
----
-system:
-# Instruction
-## Goal
-### You are an expert in evaluating the quality of a RESPONSE from an intelligent system based on provided definition and data. Your goal will involve answering the questions below using the information provided.
-- **Definition**: You are given a definition of the communication trait that is being evaluated to help guide your Score.
-- **Data**: Your input data include CONTEXT, QUERY, and RESPONSE.
-- **Tasks**: To complete your evaluation you will be asked to evaluate the Data in different ways.
-
-user:
-# Definition
-**Groundedness** refers to how well an answer is anchored in the provided context, evaluating its relevance, accuracy, and completeness based exclusively on that context. It assesses the extent to which the answer directly and fully addresses the question without introducing unrelated or incorrect information. The scale ranges from 1 to 5, with higher numbers indicating greater groundedness.
-
-# Ratings
-## [Groundedness: 1] (Completely Unrelated Response)
-**Definition:** An answer that does not relate to the question or the context in any way. It fails to address the topic, provides irrelevant information, or introduces completely unrelated subjects.
-
-**Examples:**
-  **Context:** The company's annual meeting will be held next Thursday.
-  **Query:** When is the company's annual meeting?
-  **Response:** I enjoy hiking in the mountains during summer.
-
-  **Context:** The new policy aims to reduce carbon emissions by 20% over the next five years.
-  **Query:** What is the goal of the new policy?
-  **Response:** My favorite color is blue.
-
-## [Groundedness: 2] (Related Topic but Does Not Respond to the Query)
-**Definition:** An answer that relates to the general topic of the context but does not answer the specific question asked. It may mention concepts from the context but fails to provide a direct or relevant response.
-
-**Examples:**
-  **Context:** The museum will exhibit modern art pieces from various local artists.
-  **Query:** What kind of art will be exhibited at the museum?
-  **Response:** Museums are important cultural institutions.
-
-  **Context:** The new software update improves battery life and performance.
-  **Query:** What does the new software update improve?
-  **Response:** Software updates can sometimes fix bugs.
-
-## [Groundedness: 3] (Attempts to Respond but Contains Incorrect Information)
-**Definition:** An answer that attempts to respond to the question but includes incorrect information not supported by the context. It may misstate facts, misinterpret the context, or provide erroneous details.
-
-**Examples:**
-  **Context:** The festival starts on June 5th and features international musicians.
-  **Query:** When does the festival start?
-  **Response:** The festival starts on July 5th and features local artists.
-
-  **Context:** The recipe requires two eggs and one cup of milk.
-  **Query:** How many eggs are needed for the recipe?
-  **Response:** You need three eggs for the recipe.
-
-## [Groundedness: 4] (Partially Correct Response)
-**Definition:** An answer that provides a correct response to the question but is incomplete or lacks specific details mentioned in the context. It captures some of the necessary information but omits key elements needed for a full understanding.
-
-**Examples:**
-  **Context:** The bookstore offers a 15% discount to students and a 10% discount to senior citizens.
-  **Query:** What discount does the bookstore offer to students?
-  **Response:** Students get a discount at the bookstore.
-
-  **Context:** The company's headquarters are located in Berlin, Germany.
-  **Query:** Where are the company's headquarters?
-  **Response:** The company's headquarters are in Germany.
-
-## [Groundedness: 5] (Fully Correct and Complete Response)
-**Definition:** An answer that thoroughly and accurately responds to the question, including all relevant details from the context. It directly addresses the question with precise information, demonstrating complete understanding without adding extraneous information.
-
-**Examples:**
-  **Context:** The author released her latest novel, 'The Silent Echo', on September 1st.
-  **Query:** When was 'The Silent Echo' released?
-  **Response:** 'The Silent Echo' was released on September 1st.
-
-  **Context:** Participants must register by May 31st to be eligible for early bird pricing.
-  **Query:** By what date must participants register to receive early bird pricing?
-  **Response:** Participants must register by May 31st to receive early bird pricing.
-
-
-# Data
-CONTEXT: {{context}}
-QUERY: {{query}}
-RESPONSE: {{response}}
-
-
-# Tasks
-## Please provide your assessment Score for the previous RESPONSE in relation to the CONTEXT and QUERY based on the Definitions above. Your output should include the following information:
-- **ThoughtChain**: To improve the reasoning process, think step by step and include a step-by-step explanation of your thought process as you analyze the data based on the definitions. Keep it brief and start your ThoughtChain with "Let's think step by step:".
-- **Explanation**: a very short explanation of why you think the input Data should get that Score.
-- **Score**: based on your previous analysis, provide your Score. The Score you give MUST be a integer score (i.e., "1", "2"...) based on the levels of the definitions.
-
-
-## Please provide your answers between the tags: <S0>your chain of thoughts</S0>, <S1>your explanation</S1>, <S2>your Score</S2>.
-# Output

azure/ai/evaluation/_evaluators/_groundedness/groundedness_without_query.prompty

@@ -1,99 +0,0 @@
----
-name: Groundedness
-description: Evaluates groundedness score for RAG scenario
-model:
-  api: chat
-  parameters:
-    temperature: 0.0
-    max_tokens: 800
-    top_p: 1.0
-    presence_penalty: 0
-    frequency_penalty: 0
-    response_format:
-      type: text
-
-inputs:
-  response:
-    type: string
-  context:
-    type: string
-
----
-system:
-# Instruction
-## Goal
-### You are an expert in evaluating the quality of a RESPONSE from an intelligent system based on provided definition and data. Your goal will involve answering the questions below using the information provided.
-- **Definition**: You are given a definition of the communication trait that is being evaluated to help guide your Score.
-- **Data**: Your input data include CONTEXT and RESPONSE.
-- **Tasks**: To complete your evaluation you will be asked to evaluate the Data in different ways.
-
-user:
-# Definition
-**Groundedness** refers to how faithfully a response adheres to the information provided in the CONTEXT, ensuring that all content is directly supported by the context without introducing unsupported information or omitting critical details. It evaluates the fidelity and precision of the response in relation to the source material.
-
-# Ratings
-## [Groundedness: 1] (Completely Ungrounded Response)
-**Definition:** The response is entirely unrelated to the CONTEXT, introducing topics or information that have no connection to the provided material.
-
-**Examples:**
-  **Context:** The company's profits increased by 20% in the last quarter.
-  **Response:** I enjoy playing soccer on weekends with my friends.
-
-  **Context:** The new smartphone model features a larger display and improved battery life.
-  **Response:** The history of ancient Egypt is fascinating and full of mysteries.
-
-## [Groundedness: 2] (Contradictory Response)
-**Definition:** The response directly contradicts or misrepresents the information provided in the CONTEXT.
-
-**Examples:**
-  **Context:** The company's profits increased by 20% in the last quarter.
-  **Response:** The company's profits decreased by 20% in the last quarter.
-
-  **Context:** The new smartphone model features a larger display and improved battery life.
-  **Response:** The new smartphone model has a smaller display and shorter battery life.
-
-## [Groundedness: 3] (Accurate Response with Unsupported Additions)
-**Definition:** The response accurately includes information from the CONTEXT but adds details, opinions, or explanations that are not supported by the provided material.
-
-**Examples:**
-  **Context:** The company's profits increased by 20% in the last quarter.
-  **Response:** The company's profits increased by 20% in the last quarter due to their aggressive marketing strategy.
-
-  **Context:** The new smartphone model features a larger display and improved battery life.
-  **Response:** The new smartphone model features a larger display, improved battery life, and comes with a free case.
-
-## [Groundedness: 4] (Incomplete Response Missing Critical Details)
-**Definition:** The response contains information from the CONTEXT but omits essential details that are necessary for a comprehensive understanding of the main point.
-
-**Examples:**
-  **Context:** The company's profits increased by 20% in the last quarter, marking the highest growth rate in its history.      
-  **Response:** The company's profits increased by 20% in the last quarter.
-
-  **Context:** The new smartphone model features a larger display, improved battery life, and an upgraded camera system.        
-  **Response:** The new smartphone model features a larger display and improved battery life.
-
-## [Groundedness: 5] (Fully Grounded and Complete Response)
-**Definition:** The response is entirely based on the CONTEXT, accurately and thoroughly conveying all essential information without introducing unsupported details or omitting critical points.
-
-**Examples:**
-  **Context:** The company's profits increased by 20% in the last quarter, marking the highest growth rate in its history.      
-  **Response:** The company's profits increased by 20% in the last quarter, marking the highest growth rate in its history.     
-
-  **Context:** The new smartphone model features a larger display, improved battery life, and an upgraded camera system.        
-  **Response:** The new smartphone model features a larger display, improved battery life, and an upgraded camera system.  
-
-
-# Data
-CONTEXT: {{context}}
-RESPONSE: {{response}}
-
-
-# Tasks
-## Please provide your assessment Score for the previous RESPONSE in relation to the CONTEXT based on the Definitions above. Your output should include the following information:
-- **ThoughtChain**: To improve the reasoning process, think step by step and include a step-by-step explanation of your thought process as you analyze the data based on the definitions. Keep it brief and start your ThoughtChain with "Let's think step by step:".
-- **Explanation**: a very short explanation of why you think the input Data should get that Score.
-- **Score**: based on your previous analysis, provide your Score. The Score you give MUST be a integer score (i.e., "1", "2"...) based on the levels of the definitions.
-
-
-## Please provide your answers between the tags: <S0>your chain of thoughts</S0>, <S1>your explanation</S1>, <S2>your Score</S2>.
-# Output

azure/ai/evaluation/_evaluators/_multimodal/__init__.py

@@ -1,20 +0,0 @@
-# ---------------------------------------------------------
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# ---------------------------------------------------------
-from ._content_safety_multimodal import ContentSafetyMultimodalEvaluator
-from ._content_safety_multimodal_base import ContentSafetyMultimodalEvaluatorBase
-from ._hate_unfairness import HateUnfairnessMultimodalEvaluator
-from ._self_harm import SelfHarmMultimodalEvaluator
-from ._sexual import SexualMultimodalEvaluator
-from ._violence import ViolenceMultimodalEvaluator
-from ._protected_material import ProtectedMaterialMultimodalEvaluator
-
-__all__ = [
-    "ContentSafetyMultimodalEvaluator",
-    "ContentSafetyMultimodalEvaluatorBase",
-    "ViolenceMultimodalEvaluator",
-    "SexualMultimodalEvaluator",
-    "SelfHarmMultimodalEvaluator",
-    "HateUnfairnessMultimodalEvaluator",
-    "ProtectedMaterialMultimodalEvaluator",
-]

azure/ai/evaluation/_evaluators/_multimodal/_content_safety_multimodal.py

@@ -1,132 +0,0 @@
-# ---------------------------------------------------------
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# ---------------------------------------------------------
-import logging
-from concurrent.futures import as_completed
-from typing import Callable, Dict, List, Union
-
-from promptflow.tracing import ThreadPoolExecutorWithContext as ThreadPoolExecutor
-from azure.ai.evaluation._common._experimental import experimental
-from azure.ai.evaluation._common.utils import validate_conversation
-from ._hate_unfairness import HateUnfairnessMultimodalEvaluator
-from ._self_harm import SelfHarmMultimodalEvaluator
-from ._sexual import SexualMultimodalEvaluator
-from ._violence import ViolenceMultimodalEvaluator
-
-
-logger = logging.getLogger(__name__)
-
-
-@experimental
-class ContentSafetyMultimodalEvaluator:
-    """
-    Initialize a content safety multimodal evaluator configured to evaluate content safety metrics
-    in a multimodal scenario.
-
-    :param credential: The credential required for connecting to the Azure AI project.
-    :type credential: ~azure.core.credentials.TokenCredential
-    :param azure_ai_project: The scope of the Azure AI project, containing the subscription ID,
-        resource group, and project name.
-    :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
-    :param kwargs: Additional arguments to pass to the evaluator.
-    :type kwargs: Any
-
-    :return: A function that evaluates multimodal chat messages and generates content safety metrics.
-
-    **Usage Example**
-
-    .. code-block:: python
-
-        azure_ai_project = {
-            "subscription_id": "<subscription_id>",
-            "resource_group_name": "<resource_group_name>",
-            "project_name": "<project_name>",
-        }
-        eval_fn = ContentSafetyMultimodalEvaluator(azure_ai_project)
-        result = eval_fn(conversation=
-            {
-                "messages": [
-                    {
-                        "role": "user",
-                        "content": [
-                            {
-                                "type": "text",
-                                "text": "What's in this image?"
-                            },
-                            {
-                                "type": "image_url",
-                                "image_url": {
-                                    "url": "<image url or base64 encoded image>"
-                                }
-                            }
-                        ]
-                    },
-                    {
-                        "role": "assistant",
-                        "content": "This picture shows an astronaut standing in the desert."
-                    }
-                ]
-            }
-        )
-
-    **Output Format**
-
-    .. code-block:: json
-
-        {
-            "violence": "Medium",
-            "violence_score": 5.0,
-            "violence_reason": "Some reason",
-            "sexual": "Medium",
-            "sexual_score": 5.0,
-            "sexual_reason": "Some reason",
-            "self_harm": "Medium",
-            "self_harm_score": 5.0,
-            "self_harm_reason": "Some reason",
-            "hate_unfairness": "Medium",
-            "hate_unfairness_score": 5.0,
-            "hate_unfairness_reason": "Some reason"
-        }
-
-    """
-
-    id = "content_safety_multimodal"
-    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
-
-    def __init__(self, credential, azure_ai_project, **kwargs):
-        self._parallel = kwargs.pop("_parallel", False)
-        self._evaluators: List[Callable[..., Dict[str, Union[str, float]]]] = [
-            ViolenceMultimodalEvaluator(credential=credential, azure_ai_project=azure_ai_project),
-            SexualMultimodalEvaluator(credential=credential, azure_ai_project=azure_ai_project),
-            SelfHarmMultimodalEvaluator(credential=credential, azure_ai_project=azure_ai_project),
-            HateUnfairnessMultimodalEvaluator(credential=credential, azure_ai_project=azure_ai_project),
-        ]
-
-    def __call__(self, *, conversation, **kwargs):
-        """
-        Evaluates content-safety metrics for list of messages.
-
-        :keyword conversation: The conversation contains list of messages to be evaluated.
-            Each message should have "role" and "content" keys. It supports single turn only.
-        :paramtype conversation: ~azure.ai.evaluation.Conversation
-        :return: The evaluation score based on the Content Safety Metrics.
-        :rtype: Dict[str, Union[float, str]]
-        """
-        # validate inputs
-        validate_conversation(conversation)
-        results: Dict[str, Union[str, float]] = {}
-        if self._parallel:
-            with ThreadPoolExecutor() as executor:
-                futures = {
-                    executor.submit(evaluator, conversation=conversation, **kwargs): evaluator
-                    for evaluator in self._evaluators
-                }
-
-                for future in as_completed(futures):
-                    results.update(future.result())
-        else:
-            for evaluator in self._evaluators:
-                result = evaluator(conversation=conversation, **kwargs)
-                results.update(result)
-
-        return results

azure/ai/evaluation/_evaluators/_multimodal/_hate_unfairness.py

@@ -1,100 +0,0 @@
-# ---------------------------------------------------------
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# ---------------------------------------------------------
-from promptflow._utils.async_utils import async_run_allowing_running_loop
-from azure.ai.evaluation._common._experimental import experimental
-from azure.ai.evaluation._common.constants import EvaluationMetrics
-from ._content_safety_multimodal_base import ContentSafetyMultimodalEvaluatorBase
-
-
-@experimental
-class HateUnfairnessMultimodalEvaluator:
-    """
-    Initialize a hate-unfairness multimodal evaluator for hate unfairness score.
-
-    :param credential: The credential for connecting to the Azure AI project. This is required.
-    :type credential: ~azure.core.credentials.TokenCredential
-    :param azure_ai_project: The scope of the Azure AI project,
-        which includes subscription ID, resource group name, and project name.
-    :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
-
-    **Usage Example**
-
-    .. code-block:: python
-
-        azure_ai_project = {
-            "subscription_id": "<subscription_id>",
-            "resource_group_name": "<resource_group_name>",
-            "project_name": "<project_name>",
-        }
-        eval_fn = HateUnfairnessMultimodalEvaluator(azure_ai_project)
-        result = eval_fn(conversation=
-            {
-                "messages": [
-                    {
-                        "role": "user",
-                        "content": [
-                            {
-                                "type": "text",
-                                "text": "What's in this image?"
-                            },
-                            {
-                                "type": "image_url",
-                                "image_url": {
-                                    "url": "<image url or base64 encoded image>"
-                                }
-                            }
-                        ]
-                    },
-                    {
-                        "role": "assistant",
-                        "content": "This picture shows an astronaut standing in the desert."
-                    }
-                ]
-            }
-        )
-
-    **Output Format**
-
-    .. code-block:: json
-
-        {
-            "hate_unfairness": "High",
-            "hate_unfairness_score": 6.5,
-            "hate_unfairness_reason": "Some reason"
-        }
-
-    """
-
-    id = "hate_unfairness_multimodal"
-    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
-
-    def __init__(self, credential, azure_ai_project):
-        self._async_evaluator = _AsyncHateUnfairnessMultimodalEvaluator(credential, azure_ai_project)
-
-    def __call__(self, *, conversation, **kwargs):
-        """
-        Evaluates hate unfairness content.
-
-        :keyword conversation: The conversation contains list of messages to be evaluated.
-            Each message should have "role" and "content" keys. It supports single turn only.
-        :paramtype conversation: ~azure.ai.evaluation.Conversation
-        :return: The hate unfairness score.
-        :rtype: Dict[str, Union[float, str]]
-        """
-        return async_run_allowing_running_loop(self._async_evaluator, conversation=conversation, **kwargs)
-
-    def _to_async(self):
-        return self._async_evaluator
-
-
-class _AsyncHateUnfairnessMultimodalEvaluator(ContentSafetyMultimodalEvaluatorBase):
-    def __init__(self, credential, azure_ai_project):
-        super().__init__(
-            metric=EvaluationMetrics.HATE_FAIRNESS,
-            credential=credential,
-            azure_ai_project=azure_ai_project,
-        )
-
-    async def __call__(self, *, conversation, **kwargs):
-        return await super().__call__(conversation=conversation, **kwargs)