azure-ai-evaluation 1.0.0b3__py3-none-any.whl → 1.0.0b5__py3-none-any.whl

azure/ai/evaluation/_evaluators/_qa/_qa.py

@@ -3,6 +3,7 @@
 # ---------------------------------------------------------
 
 from concurrent.futures import as_completed
+from typing import Callable, Dict, List
 
 from promptflow.tracing import ThreadPoolExecutorWithContext as ThreadPoolExecutor
 
@@ -21,8 +22,7 @@ class QAEvaluator:
     :param model_config: Configuration for the Azure OpenAI model.
     :type model_config: Union[~azure.ai.evaluation.AzureOpenAIModelConfiguration,
         ~azure.ai.evaluation.OpenAIModelConfiguration]
-    :return: A function that evaluates and generates metrics for "question-answering" scenario.
-    :rtype: Callable
+    :return: A callable class that evaluates and generates metrics for "question-answering" scenario.
 
     **Usage**
 
@@ -41,6 +41,11 @@ class QAEvaluator:
     .. code-block:: python
 
         {
+            "groundedness": 3.5,
+            "relevance": 4.0,
+            "coherence": 1.5,
+            "fluency": 4.0,
+            "similarity": 3.0,
             "gpt_groundedness": 3.5,
             "gpt_relevance": 4.0,
             "gpt_coherence": 1.5,
@@ -50,10 +55,10 @@ class QAEvaluator:
         }
     """
 
-    def __init__(self, model_config: dict, parallel: bool = True):
+    def __init__(self, model_config, parallel: bool = True):
         self._parallel = parallel
 
-        self._evaluators = [
+        self._evaluators: List[Callable[..., Dict[str, float]]] = [
             GroundednessEvaluator(model_config),
             RelevanceEvaluator(model_config),
             CoherenceEvaluator(model_config),
@@ -77,9 +82,9 @@ class QAEvaluator:
         :keyword parallel: Whether to evaluate in parallel. Defaults to True.
         :paramtype parallel: bool
         :return: The scores for QA scenario.
-        :rtype: dict
+        :rtype: Dict[str, float]
         """
-        results = {}
+        results: Dict[str, float] = {}
         if self._parallel:
             with ThreadPoolExecutor() as executor:
                 futures = {

azure/ai/evaluation/_evaluators/_relevance/_relevance.py

@@ -4,6 +4,7 @@
 
 import os
 from typing import Optional
+
 from typing_extensions import override
 
 from azure.ai.evaluation._evaluators._common import PromptyEvaluatorBase
@@ -24,28 +25,32 @@ class RelevanceEvaluator(PromptyEvaluatorBase):
         eval_fn = RelevanceEvaluator(model_config)
         result = eval_fn(
             query="What is the capital of Japan?",
-            response="The capital of Japan is Tokyo.",
-            context="Tokyo is Japan's capital, known for its blend of traditional culture \
-                and technological advancements.")
+            response="The capital of Japan is Tokyo.")
 
     **Output format**
 
     .. code-block:: python
 
         {
-            "gpt_relevance": 3.0
+            "relevance": 3.0,
+            "gpt_relevance": 3.0,
+            "relevance_reason": "The response is relevant to the query because it provides the correct answer.",
         }
+
+    Note: To align with our support of a diverse set of models, a key without the `gpt_` prefix has been added.
+    To maintain backwards compatibility, the old key with the `gpt_` prefix is still be present in the output;
+    however, it is recommended to use the new key moving forward as the old key will be deprecated in the future.
     """
 
     # Constants must be defined within eval's directory to be save/loadable
-    PROMPTY_FILE = "relevance.prompty"
-    RESULT_KEY = "gpt_relevance"
+    _PROMPTY_FILE = "relevance.prompty"
+    _RESULT_KEY = "relevance"
 
     @override
-    def __init__(self, model_config: dict):
+    def __init__(self, model_config):
         current_dir = os.path.dirname(__file__)
-        prompty_path = os.path.join(current_dir, self.PROMPTY_FILE)
-        super().__init__(model_config=model_config, prompty_file=prompty_path, result_key=self.RESULT_KEY)
+        prompty_path = os.path.join(current_dir, self._PROMPTY_FILE)
+        super().__init__(model_config=model_config, prompty_file=prompty_path, result_key=self._RESULT_KEY)
 
     @override
     def __call__(
@@ -53,25 +58,23 @@ class RelevanceEvaluator(PromptyEvaluatorBase):
         *,
         query: Optional[str] = None,
         response: Optional[str] = None,
-        context: Optional[str] = None,
-        conversation: Optional[dict] = None,
-        **kwargs
+        conversation=None,
+        **kwargs,
     ):
-        """Evaluate relevance. Accepts either a response and context a single evaluation,
+        """Evaluate relevance. Accepts either a query and response for a single evaluation,
         or a conversation for a multi-turn evaluation. If the conversation has more than one turn,
         the evaluator will aggregate the results of each turn.
 
-        :keyword query: The query to be evaluated.
+        :keyword query: The query to be evaluated. Mutually exclusive with the `conversation` parameter.
         :paramtype query: Optional[str]
-        :keyword response: The response to be evaluated.
+        :keyword response: The response to be evaluated. Mutually exclusive with the `conversation` parameter.
         :paramtype response: Optional[str]
-        :keyword context: The context to be evaluated.
-        :paramtype context: Optional[str]
         :keyword conversation: The conversation to evaluate. Expected to contain a list of conversation turns under the
             key "messages", and potentially a global context under the key "context". Conversation turns are expected
             to be dictionaries with keys "content", "role", and possibly "context".
-        :paramtype conversation: Optional[Dict]
+        :paramtype conversation: Optional[~azure.ai.evaluation.Conversation]
         :return: The relevance score.
-        :rtype: dict
+        :rtype: Union[Dict[str, float], Dict[str, Union[float, Dict[str, List[float]]]]]
         """
-        return super().__call__(query=query, response=response, context=context, conversation=conversation, **kwargs)
+
+        return super().__call__(query=query, response=response, conversation=conversation, **kwargs)

azure/ai/evaluation/_evaluators/_relevance/relevance.prompty

@@ -5,7 +5,7 @@ model:
   api: chat
   parameters:
     temperature: 0.0
-    max_tokens: 1
+    max_tokens: 800
     top_p: 1.0
     presence_penalty: 0
     frequency_penalty: 0
@@ -17,48 +17,84 @@ inputs:
     type: string
   response:
     type: string
-  context:
-    type: string
 
 ---
 system:
-You are an AI assistant. You will be given the definition of an evaluation metric for assessing the quality of an answer in a question-answering task. Your job is to compute an accurate evaluation score using the provided evaluation metric. You should return a single integer value between 1 to 5 representing the evaluation metric. You will include no other text or information.
+# Instruction
+## Goal
+### You are an expert in evaluating the quality of a RESPONSE from an intelligent system based on provided definition and data. Your goal will involve answering the questions below using the information provided.
+- **Definition**: You are given a definition of the communication trait that is being evaluated to help guide your Score.
+- **Data**: Your input data include QUERY and RESPONSE.
+- **Tasks**: To complete your evaluation you will be asked to evaluate the Data in different ways.
+
 user:
-Relevance measures how well the answer addresses the main aspects of the question, based on the context. Consider whether all and only the important aspects are contained in the answer when evaluating relevance. Given the context and question, score the relevance of the answer between one to five stars using the following rating scale:
-One star: the answer completely lacks relevance
-Two stars: the answer mostly lacks relevance
-Three stars: the answer is partially relevant
-Four stars: the answer is mostly relevant
-Five stars: the answer has perfect relevance
-
-This rating value should always be an integer between 1 and 5. So the rating produced should be 1 or 2 or 3 or 4 or 5.
-
-context: Marie Curie was a Polish-born physicist and chemist who pioneered research on radioactivity and was the first woman to win a Nobel Prize.
-question: What field did Marie Curie excel in?
-answer: Marie Curie was a renowned painter who focused mainly on impressionist styles and techniques.
-stars: 1
-
-context: The Beatles were an English rock band formed in Liverpool in 1960, and they are widely regarded as the most influential music band in history.
-question: Where were The Beatles formed?
-answer: The band The Beatles began their journey in London, England, and they changed the history of music.
-stars: 2
-
-context: The recent Mars rover, Perseverance, was launched in 2020 with the main goal of searching for signs of ancient life on Mars. The rover also carries an experiment called MOXIE, which aims to generate oxygen from the Martian atmosphere.
-question: What are the main goals of Perseverance Mars rover mission?
-answer: The Perseverance Mars rover mission focuses on searching for signs of ancient life on Mars.
-stars: 3
-
-context: The Mediterranean diet is a commonly recommended dietary plan that emphasizes fruits, vegetables, whole grains, legumes, lean proteins, and healthy fats. Studies have shown that it offers numerous health benefits, including a reduced risk of heart disease and improved cognitive health.
-question: What are the main components of the Mediterranean diet?
-answer: The Mediterranean diet primarily consists of fruits, vegetables, whole grains, and legumes.
-stars: 4
-
-context: The Queen's Royal Castle is a well-known tourist attraction in the United Kingdom. It spans over 500 acres and contains extensive gardens and parks. The castle was built in the 15th century and has been home to generations of royalty.
-question: What are the main attractions of the Queen's Royal Castle?
-answer: The main attractions of the Queen's Royal Castle are its expansive 500-acre grounds, extensive gardens, parks, and the historical castle itself, which dates back to the 15th century and has housed generations of royalty.
-stars: 5
-
-context: {{context}}
-question: {{query}}
-answer: {{response}}
-stars:
+# Definition
+**Relevance** refers to how effectively a response addresses a question. It assesses the accuracy, completeness, and direct relevance of the response based solely on the given information.
+
+# Ratings
+## [Relevance: 1] (Irrelevant Response)
+**Definition:** The response is unrelated to the question. It provides information that is off-topic and does not attempt to address the question posed.
+
+**Examples:**
+  **Query:** What is the team preparing for?
+  **Response:** I went grocery shopping yesterday evening.
+
+  **Query:** When will the company's new product line launch?
+  **Response:** International travel can be very rewarding and educational.
+
+## [Relevance: 2] (Incorrect Response)
+**Definition:** The response attempts to address the question but includes incorrect information. It provides a response that is factually wrong based on the provided information.
+
+**Examples:**
+  **Query:** When was the merger between the two firms finalized?
+  **Response:** The merger was finalized on April 10th.     
+
+  **Query:** Where and when will the solar eclipse be visible?
+  **Response:** The solar eclipse will be visible in Asia on December 14th.
+
+## [Relevance: 3] (Incomplete Response)
+**Definition:** The response addresses the question but omits key details necessary for a full understanding. It provides a partial response that lacks essential information.      
+
+**Examples:**
+  **Query:** What type of food does the new restaurant offer?
+  **Response:** The restaurant offers Italian food like pasta.
+
+  **Query:** What topics will the conference cover?
+  **Response:** The conference will cover renewable energy and climate change.
+
+## [Relevance: 4] (Complete Response)
+**Definition:** The response fully addresses the question with accurate and complete information. It includes all essential details required for a comprehensive understanding, without adding any extraneous information.
+
+**Examples:**
+  **Query:** What type of food does the new restaurant offer?
+  **Response:** The new restaurant offers Italian cuisine, featuring dishes like pasta, pizza, and risotto.
+
+  **Query:** What topics will the conference cover?
+  **Response:** The conference will cover renewable energy, climate change, and sustainability practices.
+
+## [Relevance: 5] (Comprehensive Response with Insights)    
+**Definition:** The response not only fully and accurately addresses the question but also includes additional relevant insights or elaboration. It may explain the significance, implications, or provide minor inferences that enhance understanding.
+
+**Examples:**
+  **Query:** What type of food does the new restaurant offer?
+  **Response:** The new restaurant offers Italian cuisine, featuring dishes like pasta, pizza, and risotto, aiming to provide customers with an authentic Italian dining experience.
+
+  **Query:** What topics will the conference cover?
+  **Response:** The conference will cover renewable energy, climate change, and sustainability practices, bringing together global experts to discuss these critical issues. 
+
+
+
+# Data
+QUERY: {{query}}
+RESPONSE: {{response}}
+
+
+# Tasks
+## Please provide your assessment Score for the previous RESPONSE in relation to the QUERY based on the Definitions above. Your output should include the following information:
+- **ThoughtChain**: To improve the reasoning process, think step by step and include a step-by-step explanation of your thought process as you analyze the data based on the definitions. Keep it brief and start your ThoughtChain with "Let's think step by step:".
+- **Explanation**: a very short explanation of why you think the input Data should get that Score.
+- **Score**: based on your previous analysis, provide your Score. The Score you give MUST be a integer score (i.e., "1", "2"...) based on the levels of the definitions.
+
+
+## Please provide your answers between the tags: <S0>your chain of thoughts</S0>, <S1>your explanation</S1>, <S2>your Score</S2>.
+# Output

azure/ai/evaluation/_evaluators/_retrieval/_retrieval.py

@@ -4,93 +4,106 @@
 
 import json
 import logging
+import math
 import os
-import re
+from typing import Optional
 
-import numpy as np
 from promptflow._utils.async_utils import async_run_allowing_running_loop
 from promptflow.core import AsyncPrompty
 
-
-from ..._common.utils import construct_prompty_model_config
+from azure.ai.evaluation._exceptions import EvaluationException, ErrorBlame, ErrorCategory, ErrorTarget
+from ..._common.math import list_mean_nan_safe
+from ..._common.utils import construct_prompty_model_config, validate_model_config, parse_quality_evaluator_reason_score
 
 logger = logging.getLogger(__name__)
 
 try:
     from .._user_agent import USER_AGENT
 except ImportError:
-    USER_AGENT = None
+    USER_AGENT = "None"
 
 
 class _AsyncRetrievalScoreEvaluator:
     # Constants must be defined within eval's directory to be save/loadable
-    PROMPTY_FILE = "retrieval.prompty"
-    LLM_CALL_TIMEOUT = 600
-    DEFAULT_OPEN_API_VERSION = "2024-02-15-preview"
+    _PROMPTY_FILE = "retrieval.prompty"
+    _LLM_CALL_TIMEOUT = 600
+    _DEFAULT_OPEN_API_VERSION = "2024-02-15-preview"
 
     def __init__(self, model_config: dict):
         prompty_model_config = construct_prompty_model_config(
-            model_config,
-            self.DEFAULT_OPEN_API_VERSION,
+            validate_model_config(model_config),
+            self._DEFAULT_OPEN_API_VERSION,
             USER_AGENT,
         )
 
         current_dir = os.path.dirname(__file__)
-        prompty_path = os.path.join(current_dir, self.PROMPTY_FILE)
+        prompty_path = os.path.join(current_dir, self._PROMPTY_FILE)
         self._flow = AsyncPrompty.load(source=prompty_path, model=prompty_model_config)
 
-    async def __call__(self, *, conversation, **kwargs):
-        # Extract queries, responses and contexts from conversation
-        queries = []
-        responses = []
-        contexts = []
-
-        for each_turn in conversation:
-            role = each_turn["role"]
-            if role == "user":
-                queries.append(each_turn["content"])
-            elif role == "assistant":
-                responses.append(each_turn["content"])
-                if "context" in each_turn and "citations" in each_turn["context"]:
-                    citations = json.dumps(each_turn["context"]["citations"])
-                    contexts.append(citations)
-
-        # Evaluate each turn
-        per_turn_scores = []
-        history = []
-        for turn_num, query in enumerate(queries):
-            try:
-                query = query if turn_num < len(queries) else ""
-                answer = responses[turn_num] if turn_num < len(responses) else ""
-                context = contexts[turn_num] if turn_num < len(contexts) else ""
-
-                history.append({"user": query, "assistant": answer})
-
-                llm_output = await self._flow(
-                    query=query, history=history, documents=context, timeout=self.LLM_CALL_TIMEOUT, **kwargs
-                )
-                score = np.nan
-                if llm_output:
-                    parsed_score_response = re.findall(r"\d+", llm_output.split("# Result")[-1].strip())
-                    if len(parsed_score_response) > 0:
-                        score = float(parsed_score_response[0].replace("'", "").strip())
-
-                per_turn_scores.append(score)
-
-            except Exception as e:  # pylint: disable=broad-exception-caught
-                logger.warning(
-                    "Evaluator %s failed for turn %s with exception: %s", self.__class__.__name__, turn_num + 1, e
-                )
-
-                per_turn_scores.append(np.nan)
+    async def __call__(self, *, query, context, conversation, **kwargs):
+        if conversation:
+            # Extract queries, responses and contexts from conversation
+            queries = []
+            responses = []
+            contexts = []
+
+            conversation = conversation.get("messages", None)
+
+            for each_turn in conversation:
+                role = each_turn["role"]
+                if role == "user":
+                    queries.append(each_turn["content"])
+                elif role == "assistant":
+                    responses.append(each_turn["content"])
+                    if "context" in each_turn:
+                        if "citations" in each_turn["context"]:
+                            citations = json.dumps(each_turn["context"]["citations"])
+                            contexts.append(citations)
+                        elif isinstance(each_turn["context"], str):
+                            contexts.append(each_turn["context"])
+
+            # Evaluate each turn
+            per_turn_scores = []
+            per_turn_reasons = []
+            for turn_num, turn_query in enumerate(queries):
+                try:
+                    if turn_num >= len(queries):
+                        turn_query = ""
+                    context = contexts[turn_num] if turn_num < len(contexts) else ""
+
+                    llm_output = await self._flow(
+                        query=turn_query, context=context, timeout=self._LLM_CALL_TIMEOUT, **kwargs
+                    )
+                    score, reason = parse_quality_evaluator_reason_score(llm_output)
+                    per_turn_scores.append(score)
+                    per_turn_reasons.append(reason)
+
+                except Exception as e:  # pylint: disable=broad-exception-caught
+                    logger.warning(
+                        "Evaluator %s failed for turn %s with exception: %s", self.__class__.__name__, turn_num + 1, e
+                    )
+
+                    per_turn_scores.append(math.nan)
+                    per_turn_reasons.append("")
+
+            mean_per_turn_score = list_mean_nan_safe(per_turn_scores)
+
+            return {
+                "retrieval": mean_per_turn_score,
+                "gpt_retrieval": mean_per_turn_score,
+                "evaluation_per_turn": {
+                    "gpt_retrieval": per_turn_scores,
+                    "retrieval": per_turn_scores,
+                    "retrieval_reason": per_turn_reasons,
+                },
+            }
+        llm_output = await self._flow(query=query, context=context, timeout=self._LLM_CALL_TIMEOUT, **kwargs)
+        score, reason = parse_quality_evaluator_reason_score(llm_output)
 
         return {
-            "gpt_retrieval": np.nanmean(per_turn_scores),
-            "evaluation_per_turn": {
-                "gpt_retrieval": {
-                    "score": per_turn_scores,
-                }
-            },
+            "retrieval": score,
+            "retrieval_reason": reason,
+            "gpt_retrieval": score,
         }
 
 
@@ -108,16 +121,16 @@ class RetrievalEvaluator:
 
     .. code-block:: python
 
-        chat_eval = RetrievalScoreEvaluator(model_config)
-        conversation = [
-            {"role": "user", "content": "What is the value of 2 + 2?"},
-            {"role": "assistant", "content": "2 + 2 = 4", "context": {
-                "citations": [
-                        {"id": "math_doc.md", "content": "Information about additions: 1 + 2 = 3, 2 + 2 = 4"}
-                        ]
+        chat_eval = RetrievalEvaluator(model_config)
+        conversation = {
+            "messages": [
+                {"role": "user", "content": "What is the value of 2 + 2?"},
+                {
+                    "role": "assistant", "content": "2 + 2 = 4",
+                    "context": "From 'math_doc.md': Information about additions: 1 + 2 = 3, 2 + 2 = 4"
                 }
-            }
-        ]
+            ]
+        }
         result = chat_eval(conversation=conversation)
 
     **Output format**
@@ -125,27 +138,60 @@ class RetrievalEvaluator:
     .. code-block:: python
 
         {
-            "gpt_retrieval": 3.0
+            "gpt_retrieval": 3.0,
+            "retrieval": 3.0,
             "evaluation_per_turn": {
-                "gpt_retrieval": {
-                    "score": [1.0, 2.0, 3.0]
-                }
+                "gpt_retrieval": [1.0, 2.0, 3.0],
+                "retrieval": [1.0, 2.0, 3.0],
+                "retrieval_reason": ["<reasoning for score 1>", "<reasoning for score 2>", "<reasoning for score 3>"]
             }
         }
+
+    Note: To align with our support of a diverse set of models, a key without the `gpt_` prefix has been added.
+    To maintain backwards compatibility, the old key with the `gpt_` prefix is still be present in the output;
+    however, it is recommended to use the new key moving forward as the old key will be deprecated in the future.
     """
 
-    def __init__(self, model_config: dict):
+    def __init__(self, model_config):
         self._async_evaluator = _AsyncRetrievalScoreEvaluator(model_config)
 
-    def __call__(self, *, conversation, **kwargs):
-        """Evaluates retrieval score chat scenario.
+    def __call__(self, *, query: Optional[str] = None, context: Optional[str] = None, conversation=None, **kwargs):
+        """Evaluates retrieval score chat scenario. Accepts either a query and context for a single evaluation,
+        or a conversation for a multi-turn evaluation. If the conversation has more than one turn,
+        the evaluator will aggregate the results of each turn.
 
+        :keyword query: The query to be evaluated. Mutually exclusive with `conversation` parameter.
+        :paramtype query: Optional[str]
+        :keyword context: The context to be evaluated. Mutually exclusive with `conversation` parameter.
+        :paramtype context: Optional[str]
         :keyword conversation: The conversation to be evaluated.
-        :paramtype conversation: List[Dict]
+        :paramtype conversation: Optional[~azure.ai.evaluation.Conversation]
         :return: The scores for Chat scenario.
-        :rtype: dict
+        :rtype: :rtype: Dict[str, Union[float, Dict[str, List[float]]]]
         """
-        return async_run_allowing_running_loop(self._async_evaluator, conversation=conversation, **kwargs)
+        if (query is None or context is None) and conversation is None:
+            msg = "Either a pair of 'query'/'context' or 'conversation' must be provided."
+            raise EvaluationException(
+                message=msg,
+                internal_message=msg,
+                blame=ErrorBlame.USER_ERROR,
+                category=ErrorCategory.MISSING_FIELD,
+                target=ErrorTarget.RETRIEVAL_EVALUATOR,
+            )
+
+        if (query or context) and conversation:
+            msg = "Either a pair of 'query'/'context' or 'conversation' must be provided, but not both."
+            raise EvaluationException(
+                message=msg,
+                internal_message=msg,
+                blame=ErrorBlame.USER_ERROR,
+                category=ErrorCategory.INVALID_VALUE,
+                target=ErrorTarget.RETRIEVAL_EVALUATOR,
+            )
+
+        return async_run_allowing_running_loop(
+            self._async_evaluator, query=query, context=context, conversation=conversation, **kwargs
+        )
 
     def _to_async(self):
         return self._async_evaluator