azure-ai-evaluation 1.0.0b2__py3-none-any.whl → 1.13.3__py3-none-any.whl

azure/ai/evaluation/_evaluators/_chat/_chat.py

@@ -1,357 +0,0 @@
-# ---------------------------------------------------------
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# ---------------------------------------------------------
-import json
-import logging
-from concurrent.futures import as_completed
-from typing import Dict, List
-
-import numpy as np
-from promptflow.tracing import ThreadPoolExecutorWithContext as ThreadPoolExecutor
-
-from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
-
-from .._coherence import CoherenceEvaluator
-from .._fluency import FluencyEvaluator
-from .._groundedness import GroundednessEvaluator
-from .._relevance import RelevanceEvaluator
-from .retrieval import RetrievalChatEvaluator
-
-logger = logging.getLogger(__name__)
-
-
-class ChatEvaluator:
-    """
-    Initialize a chat evaluator configured for a specific Azure OpenAI model.
-
-    :param model_config: Configuration for the Azure OpenAI model.
-    :type model_config: Union[~azure.ai.evaluation.AzureOpenAIModelConfiguration,
-        ~azure.ai.evaluation.OpenAIModelConfiguration]
-    :param eval_last_turn: Set to True to evaluate only the most recent exchange in the dialogue,
-        focusing on the latest user inquiry and the assistant's corresponding response. Defaults to False
-    :type eval_last_turn: bool
-    :param parallel: If True, use parallel execution for evaluators. Else, use sequential execution.
-        Default is True.
-    :type parallel: bool
-    :return: A function that evaluates and generates metrics for "chat" scenario.
-    :rtype: Callable
-
-    **Usage**
-
-    .. code-block:: python
-
-        chat_eval = ChatEvaluator(model_config)
-        conversation = [
-            {"role": "user", "content": "What is the value of 2 + 2?"},
-            {"role": "assistant", "content": "2 + 2 = 4", "context": {
-                "citations": [
-                        {"id": "math_doc.md", "content": "Information about additions: 1 + 2 = 3, 2 + 2 = 4"}
-                    ]
-                }
-            }
-        ]
-        result = chat_eval(conversation=conversation)
-
-    **Output format**
-
-    .. code-block:: python
-
-        {
-            "evaluation_per_turn": {
-                "gpt_retrieval": [1.0, 2.0],
-                "gpt_groundedness": [5.0, 2.0],
-                "gpt_relevance": [3.0, 5.0],
-                "gpt_coherence": [1.0, 2.0],
-                "gpt_fluency": [3.0, 5.0]
-            }
-            "gpt_retrieval": 1.5,
-            "gpt_groundedness": 3.5,
-            "gpt_relevance": 4.0,
-            "gpt_coherence": 1.5,
-            "gpt_fluency": 4.0
-        }
-    """
-
-    def __init__(
-        self,
-        model_config: dict,
-        eval_last_turn: bool = False,
-        parallel: bool = True,
-    ):
-        self._eval_last_turn = eval_last_turn
-        self._parallel = parallel
-
-        # TODO: Need a built-in evaluator for retrieval. It needs to be added to `self._rag_evaluators` collection
-        self._rag_evaluators = [
-            GroundednessEvaluator(model_config),
-            RelevanceEvaluator(model_config),
-        ]
-        self._non_rag_evaluators = [
-            CoherenceEvaluator(model_config),
-            FluencyEvaluator(model_config),
-        ]
-        # TODO: Temporary workaround to close the gap of missing retrieval score
-        # https://msdata.visualstudio.com/Vienna/_workitems/edit/3186644
-        # For long term, we need to add a built-in evaluator for retrieval after prompt is generalized for QA and Chat
-        self._retrieval_chat_evaluator = RetrievalChatEvaluator(model_config)
-
-    def __call__(self, *, conversation, **kwargs):
-        """
-        Evaluates chat scenario.
-
-        :keyword conversation: The conversation to be evaluated. Each turn should have "role" and "content" keys.
-            "context" key is optional for assistant's turn and should have "citations" key with list of citations.
-        :paramtype conversation: List[Dict]
-        :return: The scores for Chat scenario.
-        :rtype: dict
-        """
-        self._validate_conversation(conversation)
-
-        # Extract queries, responses and contexts from conversation
-        queries = []
-        responses = []
-        contexts = []
-
-        if self._eval_last_turn:
-            # Process only the last two turns if _eval_last_turn is True
-            conversation_slice = conversation[-2:] if len(conversation) >= 2 else conversation
-        else:
-            conversation_slice = conversation
-
-        for each_turn in conversation_slice:
-            role = each_turn["role"]
-            if role == "user":
-                queries.append(each_turn["content"])
-            elif role == "assistant":
-                responses.append(each_turn["content"])
-                if "context" in each_turn and "citations" in each_turn["context"]:
-                    citations = json.dumps(each_turn["context"]["citations"])
-                    contexts.append(citations)
-
-        # Select evaluators to be used for evaluation
-        compute_rag_based_metrics = True
-        if len(responses) != len(contexts):
-            safe_message = (
-                "Skipping rag based metrics as we need citations or "
-                "retrieved_documents in context key of every assistant's turn"
-            )
-            logger.warning(safe_message)
-            compute_rag_based_metrics = False
-
-        selected_evaluators = []
-        selected_evaluators.extend(self._non_rag_evaluators)
-        if compute_rag_based_metrics:
-            selected_evaluators.extend(self._rag_evaluators)
-
-        # Evaluate each turn
-        per_turn_results = []
-        for turn_num in range(len(queries)):
-            current_turn_result = {}
-
-            if self._parallel:
-                # Parallel execution
-                with ThreadPoolExecutor() as executor:
-                    future_to_evaluator = {
-                        executor.submit(
-                            self._evaluate_turn, turn_num, queries, responses, contexts, evaluator
-                        ): evaluator
-                        for evaluator in selected_evaluators
-                    }
-
-                    for future in as_completed(future_to_evaluator):
-                        result = future.result()
-                        current_turn_result.update(result)
-            else:
-                # Sequential execution
-                for evaluator in selected_evaluators:
-                    async_evaluator = evaluator._to_async()
-                    result = self._evaluate_turn(turn_num, queries, responses, contexts, async_evaluator)
-                    current_turn_result.update(result)
-
-            per_turn_results.append(current_turn_result)
-
-        # Aggregate results
-        # Final aggregated results for a conversation will look like:
-        #     "gpt_groundedness": 2.0, # Mean of all groundedness scores
-        #     "evaluation_per_turn": {
-        #         "gpt_groundedness": {
-        #             "score": [1.0, ...],
-        #             "reason": ["reason1", ...],
-        #         },
-        #     },
-        # }
-        aggregated = self._aggregate_results(per_turn_results)
-
-        # Run RetrievalChatEvaluator and merge the results
-        if compute_rag_based_metrics:
-            retrieval_score = self._retrieval_chat_evaluator(conversation=conversation_slice)
-            aggregated["gpt_retrieval"] = retrieval_score["gpt_retrieval"]
-            aggregated["evaluation_per_turn"]["gpt_retrieval"] = retrieval_score["evaluation_per_turn"]["gpt_retrieval"]
-            aggregated = dict(sorted(aggregated.items()))
-
-        return aggregated
-
-    def _evaluate_turn(self, turn_num, queries, responses, contexts, evaluator):
-        try:
-            query = queries[turn_num] if turn_num < len(queries) else ""
-            response = responses[turn_num] if turn_num < len(responses) else ""
-            context = contexts[turn_num] if turn_num < len(contexts) else ""
-
-            score = evaluator(query=query, response=response, context=context)
-
-            return score
-        except Exception as e:  # pylint: disable=broad-exception-caught
-            logger.warning(
-                "Evaluator %s failed for turn %s with exception: %s", evaluator.__class__.__name__, turn_num + 1, e
-            )
-            return {}
-
-    def _aggregate_results(self, per_turn_results: List[Dict]):
-        scores = {}
-        reasons = {}
-
-        for turn in per_turn_results:
-            for metric, value in turn.items():
-                if "reason" in metric:
-                    if metric not in reasons:
-                        reasons[metric] = []
-                    reasons[metric].append(value)
-                else:
-                    if metric not in scores:
-                        scores[metric] = []
-                    scores[metric].append(value)
-
-        aggregated = {}
-        evaluation_per_turn = {}
-
-        for metric, values in scores.items():
-            aggregated[metric] = np.nanmean(values)
-
-            # Prepare per-turn evaluations
-            evaluation_per_turn[metric] = {"score": values}
-            reason_key = f"{metric}_reason"
-            if reason_key in reasons:
-                evaluation_per_turn[metric]["reason"] = reasons[reason_key]
-
-        aggregated["evaluation_per_turn"] = evaluation_per_turn
-
-        return aggregated
-
-    def _validate_conversation(self, conversation: List[Dict]):
-        if conversation is None or not isinstance(conversation, list):
-            msg = "conversation must be a list of dictionaries"
-            raise EvaluationException(
-                message=msg,
-                internal_message=msg,
-                target=ErrorTarget.CHAT_EVALUATOR,
-                category=ErrorCategory.INVALID_VALUE,
-                blame=ErrorBlame.USER_ERROR,
-            )
-
-        expected_role = "user"
-        for turn_num, turn in enumerate(conversation):
-            one_based_turn_num = turn_num + 1
-
-            if not isinstance(turn, dict):
-                msg = f"Each turn in 'conversation' must be a dictionary. Turn number: {one_based_turn_num}"
-                raise EvaluationException(
-                    message=msg,
-                    internal_message=msg,
-                    target=ErrorTarget.CHAT_EVALUATOR,
-                    category=ErrorCategory.INVALID_VALUE,
-                    blame=ErrorBlame.USER_ERROR,
-                )
-
-            if "role" not in turn or "content" not in turn:
-                msg = (
-                    "Each turn in 'conversation' must have 'role' and 'content' keys. "
-                    + f"Turn number: {one_based_turn_num}"
-                )
-                raise EvaluationException(
-                    message=msg,
-                    internal_message=msg,
-                    target=ErrorTarget.CHAT_EVALUATOR,
-                    category=ErrorCategory.INVALID_VALUE,
-                    blame=ErrorBlame.USER_ERROR,
-                )
-
-            if turn["role"] != expected_role:
-                msg = f"Expected role {expected_role} but got {turn['role']}. Turn number: {one_based_turn_num}"
-                raise EvaluationException(
-                    message=msg,
-                    internal_message=msg,
-                    target=ErrorTarget.CHAT_EVALUATOR,
-                    category=ErrorCategory.INVALID_VALUE,
-                    blame=ErrorBlame.USER_ERROR,
-                )
-
-            if not isinstance(turn["content"], str):
-                msg = f"Content in each turn must be a string. Turn number: {one_based_turn_num}"
-                raise EvaluationException(
-                    message=msg,
-                    internal_message=msg,
-                    target=ErrorTarget.CHAT_EVALUATOR,
-                    category=ErrorCategory.INVALID_VALUE,
-                    blame=ErrorBlame.USER_ERROR,
-                )
-
-            if turn["role"] == "assistant" and "context" in turn:
-                if not isinstance(turn["context"], dict):
-                    msg = f"Context in each assistant's turn must be a dictionary. Turn number: {one_based_turn_num}"
-                    raise EvaluationException(
-                        message=msg,
-                        internal_message=msg,
-                        target=ErrorTarget.CHAT_EVALUATOR,
-                        category=ErrorCategory.INVALID_VALUE,
-                        blame=ErrorBlame.USER_ERROR,
-                    )
-
-                if "citations" not in turn["context"]:
-                    msg = (
-                        f"Context in each assistant's turn must have 'citations' key. Turn number: {one_based_turn_num}"
-                    )
-                    raise EvaluationException(
-                        message=msg,
-                        internal_message=msg,
-                        target=ErrorTarget.CHAT_EVALUATOR,
-                        category=ErrorCategory.MISSING_FIELD,
-                        blame=ErrorBlame.USER_ERROR,
-                    )
-
-                if not isinstance(turn["context"]["citations"], list):
-                    msg = f"'citations' in context must be a list. Turn number: {one_based_turn_num}"
-                    raise EvaluationException(
-                        message=msg,
-                        internal_message=msg,
-                        target=ErrorTarget.CHAT_EVALUATOR,
-                        category=ErrorCategory.INVALID_VALUE,
-                        blame=ErrorBlame.USER_ERROR,
-                    )
-
-                for citation_num, citation in enumerate(turn["context"]["citations"]):
-                    if not isinstance(citation, dict):
-                        msg = (
-                            "Each citation in 'citations' must be a dictionary. "
-                            + f"Turn number: {one_based_turn_num}, Citation number: {citation_num + 1}"
-                        )
-                        raise EvaluationException(
-                            message=msg,
-                            internal_message=msg,
-                            target=ErrorTarget.CHAT_EVALUATOR,
-                            category=ErrorCategory.INVALID_VALUE,
-                            blame=ErrorBlame.USER_ERROR,
-                        )
-
-            # Toggle expected role for the next turn
-            expected_role = "user" if expected_role == "assistant" else "assistant"
-
-        # Ensure the conversation ends with an assistant's turn
-        if expected_role != "user":
-            msg = "The conversation must end with an assistant's turn."
-            raise EvaluationException(
-                message=msg,
-                internal_message=msg,
-                target=ErrorTarget.CHAT_EVALUATOR,
-                category=ErrorCategory.INVALID_VALUE,
-                blame=ErrorBlame.USER_ERROR,
-            )

azure/ai/evaluation/_evaluators/_chat/retrieval/_retrieval.py

@@ -1,157 +0,0 @@
-# ---------------------------------------------------------
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# ---------------------------------------------------------
-
-import json
-import logging
-import os
-import re
-
-import numpy as np
-from promptflow._utils.async_utils import async_run_allowing_running_loop
-from promptflow.core import AsyncPrompty
-
-from ...._common.utils import ensure_api_version_in_aoai_model_config, ensure_user_agent_in_aoai_model_config
-
-logger = logging.getLogger(__name__)
-
-try:
-    from ...._user_agent import USER_AGENT
-except ImportError:
-    USER_AGENT = None
-
-
-class _AsyncRetrievalChatEvaluator:
-    # Constants must be defined within eval's directory to be save/loadable
-    PROMPTY_FILE = "retrieval.prompty"
-    LLM_CALL_TIMEOUT = 600
-    DEFAULT_OPEN_API_VERSION = "2024-02-15-preview"
-
-    def __init__(self, model_config: dict):
-        ensure_api_version_in_aoai_model_config(model_config, self.DEFAULT_OPEN_API_VERSION)
-
-        prompty_model_config = {"configuration": model_config, "parameters": {"extra_headers": {}}}
-
-        # Handle "RuntimeError: Event loop is closed" from httpx AsyncClient
-        # https://github.com/encode/httpx/discussions/2959
-        prompty_model_config["parameters"]["extra_headers"].update({"Connection": "close"})
-
-        ensure_user_agent_in_aoai_model_config(
-            model_config,
-            prompty_model_config,
-            USER_AGENT,
-        )
-
-        current_dir = os.path.dirname(__file__)
-        prompty_path = os.path.join(current_dir, self.PROMPTY_FILE)
-        self._flow = AsyncPrompty.load(source=prompty_path, model=prompty_model_config)
-
-    async def __call__(self, *, conversation, **kwargs):
-        # Extract queries, responses and contexts from conversation
-        queries = []
-        responses = []
-        contexts = []
-
-        for each_turn in conversation:
-            role = each_turn["role"]
-            if role == "user":
-                queries.append(each_turn["content"])
-            elif role == "assistant":
-                responses.append(each_turn["content"])
-                if "context" in each_turn and "citations" in each_turn["context"]:
-                    citations = json.dumps(each_turn["context"]["citations"])
-                    contexts.append(citations)
-
-        # Evaluate each turn
-        per_turn_scores = []
-        history = []
-        for turn_num, query in enumerate(queries):
-            try:
-                query = query if turn_num < len(queries) else ""
-                answer = responses[turn_num] if turn_num < len(responses) else ""
-                context = contexts[turn_num] if turn_num < len(contexts) else ""
-
-                history.append({"user": query, "assistant": answer})
-
-                llm_output = await self._flow(
-                    query=query, history=history, documents=context, timeout=self.LLM_CALL_TIMEOUT, **kwargs
-                )
-                score = np.nan
-                if llm_output:
-                    parsed_score_response = re.findall(r"\d+", llm_output.split("# Result")[-1].strip())
-                    if len(parsed_score_response) > 0:
-                        score = float(parsed_score_response[0].replace("'", "").strip())
-
-                per_turn_scores.append(score)
-
-            except Exception as e:  # pylint: disable=broad-exception-caught
-                logger.warning(
-                    "Evaluator %s failed for turn %s with exception: %s", self.__class__.__name__, turn_num + 1, e
-                )
-
-                per_turn_scores.append(np.nan)
-
-        return {
-            "gpt_retrieval": np.nanmean(per_turn_scores),
-            "evaluation_per_turn": {
-                "gpt_retrieval": {
-                    "score": per_turn_scores,
-                }
-            },
-        }
-
-
-class RetrievalChatEvaluator:
-    """
-    Initialize an evaluator configured for a specific Azure OpenAI model.
-
-    :param model_config: Configuration for the Azure OpenAI model.
-    :type model_config: Union[~azure.ai.evaluation.AzureOpenAIModelConfiguration,
-        ~azure.ai.evaluation.OpenAIModelConfiguration]
-    :return: A function that evaluates and generates metrics for "chat" scenario.
-    :rtype: Callable
-    **Usage**
-
-    .. code-block:: python
-
-        chat_eval = RetrievalChatEvaluator(model_config)
-        conversation = [
-            {"role": "user", "content": "What is the value of 2 + 2?"},
-            {"role": "assistant", "content": "2 + 2 = 4", "context": {
-                "citations": [
-                        {"id": "math_doc.md", "content": "Information about additions: 1 + 2 = 3, 2 + 2 = 4"}
-                        ]
-                }
-            }
-        ]
-        result = chat_eval(conversation=conversation)
-
-    **Output format**
-
-    .. code-block:: python
-
-    {
-        "gpt_retrieval": 3.0
-        "evaluation_per_turn": {
-            "gpt_retrieval": {
-                "score": [1.0, 2.0, 3.0]
-            }
-        }
-    }
-    """
-
-    def __init__(self, model_config: dict):
-        self._async_evaluator = _AsyncRetrievalChatEvaluator(model_config)
-
-    def __call__(self, *, conversation, **kwargs):
-        """Evaluates retrieval score chat scenario.
-
-        :keyword conversation: The conversation to be evaluated.
-        :paramtype conversation: List[Dict]
-        :return: The scores for Chat scenario.
-        :rtype: dict
-        """
-        return async_run_allowing_running_loop(self._async_evaluator, conversation=conversation, **kwargs)
-
-    def _to_async(self):
-        return self._async_evaluator

azure/ai/evaluation/_evaluators/_chat/retrieval/retrieval.prompty

@@ -1,48 +0,0 @@
----
-name: Retrieval
-description: Evaluates retrieval score for Chat scenario
-model:
-  api: chat
-  configuration:
-    type: azure_openai
-    azure_deployment: ${env:AZURE_DEPLOYMENT}
-    api_key: ${env:AZURE_OPENAI_API_KEY}
-    azure_endpoint: ${env:AZURE_OPENAI_ENDPOINT}
-  parameters:
-    temperature: 0.0
-    top_p: 1.0
-    presence_penalty: 0
-    frequency_penalty: 0
-    response_format:
-      type: text
-
-inputs:
-  query:
-    type: string
-  history:
-    type: string
-  documents:
-    type: string
-
----
-system:
-A chat history between user and bot is shown below
-A list of documents is shown below in json format, and each document has one unique id.
-These listed documents are used as context to answer the given question.
-The task is to score the relevance between the documents and the potential answer to the given question in the range of 1 to 5.
-1 means none of the documents is relevant to the question at all. 5 means either one of the document or combination of a few documents is ideal for answering the given question.
-Think through step by step:
-- Summarize each given document first
-- Determine the underlying intent of the given question, when the question is ambiguous, refer to the given chat history
-- Measure how suitable each document to the given question, list the document id and the corresponding relevance score.
-- Summarize the overall relevance of given list of documents to the given question after # Overall Reason, note that the answer to the question can solely from single document or a combination of multiple documents.
-- Finally, output "# Result" followed by a score from 1 to 5.
-
-# Question
-{{ query }}
-# Chat History
-{{ history }}
-# Documents
-===BEGIN RETRIEVED DOCUMENTS===
-{{ documents }}
-===END RETRIEVED DOCUMENTS===

azure/ai/evaluation/_evaluators/_content_safety/_content_safety_base.py

@@ -1,65 +0,0 @@
-# ---------------------------------------------------------
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# ---------------------------------------------------------
-
-from abc import ABC
-
-from azure.ai.evaluation._common.constants import EvaluationMetrics
-from azure.ai.evaluation._common.rai_service import evaluate_with_rai_service
-from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
-
-
-class ContentSafetyEvaluatorBase(ABC):
-    """
-    Initialize a evaluator for a specified Evaluation Metric. Base class that is not
-    meant to be instantiated by users.
-
-
-    :param metric: The metric to be evaluated.
-    :type metric: ~azure.ai.evaluation._evaluators._content_safety.flow.constants.EvaluationMetrics
-    :param azure_ai_project: The scope of the Azure AI project.
-        It contains subscription id, resource group, and project name.
-    :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
-    :param credential: The credential for connecting to Azure AI project.
-    :type credential: ~azure.core.credentials.TokenCredential
-    """
-
-    def __init__(self, metric: EvaluationMetrics, azure_ai_project: dict, credential=None):
-        self._metric = metric
-        self._azure_ai_project = azure_ai_project
-        self._credential = credential
-
-    async def __call__(self, *, query: str, response: str, **kwargs):
-        """
-        Evaluates content according to this evaluator's metric.
-
-        :keyword query: The query to be evaluated.
-        :paramtype query: str
-        :keyword response: The response to be evaluated.
-        :paramtype response: str
-        :return: The evaluation score computation based on the Content Safety metric (self.metric).
-        :rtype: Any
-        """
-        # Validate inputs
-        # Raises value error if failed, so execution alone signifies success.
-        if not (query and query.strip() and query != "None") or not (
-            response and response.strip() and response != "None"
-        ):
-            msg = "Both 'query' and 'response' must be non-empty strings."
-            raise EvaluationException(
-                message=msg,
-                internal_message=msg,
-                error_category=ErrorCategory.MISSING_FIELD,
-                error_blame=ErrorBlame.USER_ERROR,
-                error_target=ErrorTarget.CONTENT_SAFETY_CHAT_EVALUATOR,
-            )
-
-        # Run score computation based on supplied metric.
-        result = await evaluate_with_rai_service(
-            metric_name=self._metric,
-            query=query,
-            response=response,
-            project_scope=self._azure_ai_project,
-            credential=self._credential,
-        )
-        return result