azure-ai-evaluation 1.12.0__py3-none-any.whl → 1.13.0__py3-none-any.whl

azure/ai/evaluation/_evaluators/_common/_base_prompty_eval.py

@@ -5,7 +5,8 @@
 import math
 import re
 import os
-from typing import Dict, Optional, TypeVar, Union
+from itertools import chain
+from typing import Dict, Optional, TypeVar, Union, List
 
 if os.getenv("AI_EVALS_USE_PF_PROMPTY", "false").lower() == "true":
     from promptflow.core._flow import AsyncPrompty
@@ -132,10 +133,19 @@ class PromptyEvaluatorBase(EvaluatorBase[T]):
                 category=ErrorCategory.INVALID_VALUE,
                 target=ErrorTarget.CONVERSATION,
             )
-        llm_output = await self._flow(timeout=self._LLM_CALL_TIMEOUT, **eval_input)
+        # Call the prompty flow to get the evaluation result.
+        prompty_output_dict = await self._flow(timeout=self._LLM_CALL_TIMEOUT, **eval_input)
 
         score = math.nan
-        if llm_output:
+        if prompty_output_dict:
+            llm_output = prompty_output_dict.get("llm_output", "")
+            input_token_count = prompty_output_dict.get("input_token_count", 0)
+            output_token_count = prompty_output_dict.get("output_token_count", 0)
+            total_token_count = prompty_output_dict.get("total_token_count", 0)
+            finish_reason = prompty_output_dict.get("finish_reason", "")
+            model_id = prompty_output_dict.get("model_id", "")
+            sample_input = prompty_output_dict.get("sample_input", "")
+            sample_output = prompty_output_dict.get("sample_output", "")
             # Parse out score and reason from evaluators known to possess them.
             if self._result_key in PROMPT_BASED_REASON_EVALUATORS:
                 score, reason = parse_quality_evaluator_reason_score(llm_output)
@@ -146,6 +156,13 @@ class PromptyEvaluatorBase(EvaluatorBase[T]):
                     f"{self._result_key}_reason": reason,
                     f"{self._result_key}_result": binary_result,
                     f"{self._result_key}_threshold": self._threshold,
+                    f"{self._result_key}_prompt_tokens": input_token_count,
+                    f"{self._result_key}_completion_tokens": output_token_count,
+                    f"{self._result_key}_total_tokens": total_token_count,
+                    f"{self._result_key}_finish_reason": finish_reason,
+                    f"{self._result_key}_model": model_id,
+                    f"{self._result_key}_sample_input": sample_input,
+                    f"{self._result_key}_sample_output": sample_output,
                 }
             match = re.search(r"\d", llm_output)
             if match:
@@ -156,6 +173,13 @@ class PromptyEvaluatorBase(EvaluatorBase[T]):
                 f"gpt_{self._result_key}": float(score),
                 f"{self._result_key}_result": binary_result,
                 f"{self._result_key}_threshold": self._threshold,
+                f"{self._result_key}_prompt_tokens": input_token_count,
+                f"{self._result_key}_completion_tokens": output_token_count,
+                f"{self._result_key}_total_tokens": total_token_count,
+                f"{self._result_key}_finish_reason": finish_reason,
+                f"{self._result_key}_model": model_id,
+                f"{self._result_key}_sample_input": sample_input,
+                f"{self._result_key}_sample_output": sample_output,
             }
 
         binary_result = self._get_binary_result(score)
@@ -165,3 +189,157 @@ class PromptyEvaluatorBase(EvaluatorBase[T]):
             f"{self._result_key}_result": binary_result,
             f"{self._result_key}_threshold": self._threshold,
         }
+
+    @staticmethod
+    def _get_built_in_tool_definition(tool_name: str):
+        """Get the definition for the built-in tool."""
+        try:
+            from ..._converters._models import _BUILT_IN_DESCRIPTIONS, _BUILT_IN_PARAMS
+
+            if tool_name in _BUILT_IN_DESCRIPTIONS:
+                return {
+                    "type": tool_name,
+                    "description": _BUILT_IN_DESCRIPTIONS[tool_name],
+                    "name": tool_name,
+                    "parameters": _BUILT_IN_PARAMS.get(tool_name, {}),
+                }
+        except ImportError:
+            pass
+        return None
+
+    def _get_needed_built_in_tool_definitions(self, tool_calls: List[Dict]) -> List[Dict]:
+        """Extract tool definitions needed for the given built-in tool calls."""
+        needed_definitions = []
+        for tool_call in tool_calls:
+            if isinstance(tool_call, dict):
+                tool_type = tool_call.get("type")
+
+                # Only support converter format: {type: "tool_call", name: "bing_custom_search", arguments: {...}}
+                if tool_type == "tool_call":
+                    tool_name = tool_call.get("name")
+                    if tool_name:
+                        definition = self._get_built_in_tool_definition(tool_name)
+                        if definition and definition not in needed_definitions:
+                            needed_definitions.append(definition)
+
+        return needed_definitions
+
+    def _extract_tool_names_from_calls(self, tool_calls: List[Dict]) -> List[str]:
+        """Extract just the tool names from tool calls, removing parameters."""
+        tool_names = []
+        for tool_call in tool_calls:
+            if isinstance(tool_call, dict):
+                tool_type = tool_call.get("type")
+                if tool_type == "tool_call":
+                    tool_name = tool_call.get("name")
+                    if tool_name:
+                        tool_names.append(tool_name)
+                elif tool_call.get("function", {}).get("name"):
+                    # Handle function call format
+                    tool_names.append(tool_call["function"]["name"])
+                elif tool_call.get("name"):
+                    # Handle direct name format
+                    tool_names.append(tool_call["name"])
+        return tool_names
+
+    def _extract_needed_tool_definitions(
+        self, tool_calls: List[Dict], tool_definitions: List[Dict], error_target: ErrorTarget
+    ) -> List[Dict]:
+        """Extract the tool definitions that are needed for the provided tool calls.
+
+        :param tool_calls: The tool calls that need definitions
+        :type tool_calls: List[Dict]
+        :param tool_definitions: User-provided tool definitions
+        :type tool_definitions: List[Dict]
+        :param error_target: The evaluator-specific error target for exceptions
+        :type error_target: ErrorTarget
+        :return: List of needed tool definitions
+        :rtype: List[Dict]
+        :raises EvaluationException: If validation fails
+        """
+        needed_tool_definitions = []
+
+        # Add all user-provided tool definitions
+        needed_tool_definitions.extend(tool_definitions)
+
+        # Add the needed built-in tool definitions (if they are called)
+        built_in_definitions = self._get_needed_built_in_tool_definitions(tool_calls)
+        needed_tool_definitions.extend(built_in_definitions)
+
+        # OpenAPI tool is a collection of functions, so we need to expand it
+        tool_definitions_expanded = list(
+            chain.from_iterable(
+                tool.get("functions", []) if tool.get("type") == "openapi" else [tool]
+                for tool in needed_tool_definitions
+            )
+        )
+
+        # Validate that all tool calls have corresponding definitions
+        for tool_call in tool_calls:
+            if isinstance(tool_call, dict):
+                tool_type = tool_call.get("type")
+
+                if tool_type == "tool_call":
+                    tool_name = tool_call.get("name")
+                    if tool_name and self._get_built_in_tool_definition(tool_name):
+                        # This is a built-in tool from converter, already handled above
+                        continue
+                    elif tool_name:
+                        # This is a regular function tool from converter
+                        tool_definition_exists = any(
+                            tool.get("name") == tool_name and tool.get("type", "function") == "function"
+                            for tool in tool_definitions_expanded
+                        )
+                        if not tool_definition_exists:
+                            raise EvaluationException(
+                                message=f"Tool definition for {tool_name} not found",
+                                blame=ErrorBlame.USER_ERROR,
+                                category=ErrorCategory.INVALID_VALUE,
+                                target=error_target,
+                            )
+                    else:
+                        raise EvaluationException(
+                            message=f"Tool call missing name: {tool_call}",
+                            blame=ErrorBlame.USER_ERROR,
+                            category=ErrorCategory.INVALID_VALUE,
+                            target=error_target,
+                        )
+                else:
+                    # Unsupported tool format - only converter format is supported
+                    raise EvaluationException(
+                        message=f"Unsupported tool call format. Only converter format is supported: {tool_call}",
+                        blame=ErrorBlame.USER_ERROR,
+                        category=ErrorCategory.INVALID_VALUE,
+                        target=error_target,
+                    )
+            else:
+                # Tool call is not a dictionary
+                raise EvaluationException(
+                    message=f"Tool call is not a dictionary: {tool_call}",
+                    blame=ErrorBlame.USER_ERROR,
+                    category=ErrorCategory.INVALID_VALUE,
+                    target=error_target,
+                )
+
+        return needed_tool_definitions
+
+    def _not_applicable_result(
+        self, error_message: str, threshold: Union[int, float]
+    ) -> Dict[str, Union[str, float, Dict]]:
+        """Return a result indicating that the evaluation is not applicable.
+
+        :param error_message: The error message explaining why evaluation is not applicable.
+        :type error_message: str
+        :param threshold: The threshold value for the evaluator.
+        :type threshold: Union[int, float]
+        :return: A dictionary containing the result of the evaluation.
+        :rtype: Dict[str, Union[str, float, Dict]]
+        """
+        # If no tool calls were made or tool call type is not supported, return not applicable result
+        return {
+            self._result_key: self._NOT_APPLICABLE_RESULT,
+            f"{self._result_key}_result": "pass",
+            f"{self._result_key}_threshold": threshold,
+            f"{self._result_key}_reason": error_message,
+            f"{self._result_key}_details": {},
+        }

azure/ai/evaluation/_evaluators/_groundedness/_groundedness.py

@@ -2,10 +2,10 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
 import os, logging
-from typing import Dict, List, Optional, Union
+from typing import Dict, List, Optional, Union, Any, Tuple
 
 from typing_extensions import overload, override
-from azure.ai.evaluation._legacy._adapters._flows import AsyncPrompty
+from azure.ai.evaluation._legacy.prompty import AsyncPrompty
 
 from azure.ai.evaluation._evaluators._common import PromptyEvaluatorBase
 from azure.ai.evaluation._model_configurations import Conversation
@@ -16,6 +16,7 @@ from ..._common.utils import (
     ErrorCategory,
     construct_prompty_model_config,
     validate_model_config,
+    simplify_messages,
 )
 
 try:
@@ -32,8 +33,7 @@ logger = logging.getLogger(__name__)
 
 
 class GroundednessEvaluator(PromptyEvaluatorBase[Union[str, float]]):
-    """
-    Evaluates groundedness score for a given query (optional), response, and context or a multi-turn conversation,
+    """Evaluates groundedness score for a given query (optional), response, and context or a multi-turn conversation,
     including reasoning.
 
     The groundedness measure assesses the correspondence between claims in an AI-generated answer and the source
@@ -65,6 +65,7 @@ class GroundednessEvaluator(PromptyEvaluatorBase[Union[str, float]]):
             :caption: Initialize and call a GroundednessEvaluator.
 
     .. admonition:: Example with Threshold:
+
         .. literalinclude:: ../samples/evaluation_samples_threshold.py
             :start-after: [START threshold_groundedness_evaluator]
             :end-before: [END threshold_groundedness_evaluator]
@@ -201,18 +202,60 @@ class GroundednessEvaluator(PromptyEvaluatorBase[Union[str, float]]):
         """
 
         if kwargs.get("query", None):
-            current_dir = os.path.dirname(__file__)
-            prompty_path = os.path.join(current_dir, self._PROMPTY_FILE_WITH_QUERY)
-            self._prompty_file = prompty_path
-            prompty_model_config = construct_prompty_model_config(
-                validate_model_config(self._model_config),
-                self._DEFAULT_OPEN_API_VERSION,
-                UserAgentSingleton().value,
-            )
-            self._flow = AsyncPrompty.load(source=self._prompty_file, model=prompty_model_config)
+            self._ensure_query_prompty_loaded()
 
         return super().__call__(*args, **kwargs)
 
+    def _ensure_query_prompty_loaded(self):
+        """Switch to the query prompty file if not already loaded."""
+
+        current_dir = os.path.dirname(__file__)
+        prompty_path = os.path.join(current_dir, self._PROMPTY_FILE_WITH_QUERY)
+
+        self._prompty_file = prompty_path
+        prompty_model_config = construct_prompty_model_config(
+            validate_model_config(self._model_config),
+            self._DEFAULT_OPEN_API_VERSION,
+            UserAgentSingleton().value,
+        )
+        self._flow = AsyncPrompty.load(source=self._prompty_file, model=prompty_model_config)
+
+    def _has_context(self, eval_input: dict) -> bool:
+        """
+        Return True if eval_input contains a non-empty 'context' field.
+        Treats None, empty strings, empty lists, and lists of empty strings as no context.
+        """
+        context = eval_input.get("context", None)
+        if not context:
+            return False
+        if context == "<>":  # Special marker for no context
+            return False
+        if isinstance(context, list):
+            return any(str(c).strip() for c in context)
+        if isinstance(context, str):
+            return bool(context.strip())
+        return True
+
+    @override
+    async def _do_eval(self, eval_input: Dict) -> Dict[str, Union[float, str]]:
+        if eval_input.get("query", None) is None:
+            return await super()._do_eval(eval_input)
+
+        contains_context = self._has_context(eval_input)
+
+        simplified_query = simplify_messages(eval_input["query"], drop_tool_calls=contains_context)
+        simplified_response = simplify_messages(eval_input["response"], drop_tool_calls=False)
+
+        # Build simplified input
+        simplified_eval_input = {
+            "query": simplified_query,
+            "response": simplified_response,
+            "context": eval_input["context"],
+        }
+
+        # Replace and call the parent method
+        return await super()._do_eval(simplified_eval_input)
+
     async def _real_call(self, **kwargs):
         """The asynchronous call where real end-to-end evaluation logic is performed.
 
@@ -236,57 +279,76 @@ class GroundednessEvaluator(PromptyEvaluatorBase[Union[str, float]]):
                 raise ex
 
     def _convert_kwargs_to_eval_input(self, **kwargs):
-        if "context" in kwargs or "conversation" in kwargs:
+        if kwargs.get("context") or kwargs.get("conversation"):
             return super()._convert_kwargs_to_eval_input(**kwargs)
-
         query = kwargs.get("query")
         response = kwargs.get("response")
         tool_definitions = kwargs.get("tool_definitions")
 
-        if not query or not response or not tool_definitions:
-            msg = f"{type(self).__name__}: Either 'conversation' or individual inputs must be provided. For Agent groundedness 'query', 'response' and 'tool_definitions' are required."
+        if query and self._prompty_file != self._PROMPTY_FILE_WITH_QUERY:
+            self._ensure_query_prompty_loaded()
+
+        if (not query) or (not response):  # or not tool_definitions:
+            msg = f"{type(self).__name__}: Either 'conversation' or individual inputs must be provided. For Agent groundedness 'query' and 'response' are required."
             raise EvaluationException(
                 message=msg,
                 blame=ErrorBlame.USER_ERROR,
                 category=ErrorCategory.INVALID_VALUE,
                 target=ErrorTarget.GROUNDEDNESS_EVALUATOR,
             )
-
         context = self._get_context_from_agent_response(response, tool_definitions)
-        if not context:
-            raise EvaluationException(
-                message=f"Context could not be extracted from agent response. Supported tools for groundedness are {self._SUPPORTED_TOOLS}. If supported tools are not used groundedness is not calculated.",
-                blame=ErrorBlame.USER_ERROR,
-                category=ErrorCategory.NOT_APPLICABLE,
-                target=ErrorTarget.GROUNDEDNESS_EVALUATOR,
-            )
 
-        return super()._convert_kwargs_to_eval_input(response=response[-1], context=context, query=query)
+        filtered_response = self._filter_file_search_results(response)
+        return super()._convert_kwargs_to_eval_input(response=filtered_response, context=context, query=query)
+
+    def _filter_file_search_results(self, messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        """Filter out file_search tool results from the messages."""
+        file_search_ids = self._get_file_search_tool_call_ids(messages)
+        return [
+            msg for msg in messages if not (msg.get("role") == "tool" and msg.get("tool_call_id") in file_search_ids)
+        ]
 
     def _get_context_from_agent_response(self, response, tool_definitions):
+        """Extract context text from file_search tool results in the agent response."""
+        NO_CONTEXT = "<>"
         context = ""
         try:
             logger.debug("Extracting context from response")
             tool_calls = self._parse_tools_from_response(response=response)
-            logger.debug(f"Tool Calls parsed successfully : {tool_calls}")
-            if tool_calls:
-                for tool_call in tool_calls:
-                    if isinstance(tool_call, dict) and tool_call.get("type") == "tool_call":
-                        tool_name = tool_call.get("name")
-                        for tool in tool_definitions:
-                            if tool.get("name") == tool_name and tool.get("type") in self._SUPPORTED_TOOLS:
-                                if tool_name == "file_search":
-                                    tool_result = tool_call.get("tool_result")
-                                    if tool_result:
-                                        for result in tool_result:
-                                            content_list = result.get("content")
-                                            if content_list:
-                                                for content in content_list:
-                                                    text = content.get("text")
-                                                    if text:
-                                                        context = context + "\n" + str(text)
+            logger.debug(f"Tool Calls parsed successfully: {tool_calls}")
+
+            if not tool_calls:
+                return NO_CONTEXT
+
+            context_lines = []
+            for tool_call in tool_calls:
+                if not isinstance(tool_call, dict) or tool_call.get("type") != "tool_call":
+                    continue
+
+                tool_name = tool_call.get("name")
+                if tool_name != "file_search":
+                    continue
+
+                # Extract tool results
+                for result in tool_call.get("tool_result", []):
+                    results = result if isinstance(result, list) else [result]
+                    for r in results:
+                        file_name = r.get("file_name", "Unknown file name")
+                        for content in r.get("content", []):
+                            text = content.get("text")
+                            if text:
+                                context_lines.append(f"{file_name}:\n- {text}---\n\n")
+
+            context = "\n".join(context_lines) if len(context_lines) > 0 else None
+
         except Exception as ex:
             logger.debug(f"Error extracting context from agent response : {str(ex)}")
-            context = ""
+            context = None
+
+        context = context if context else NO_CONTEXT
+        return context
 
-        return context if context else None
+    def _get_file_search_tool_call_ids(self, query_or_response):
+        """Return a list of tool_call_ids for file search tool calls."""
+        tool_calls = self._parse_tools_from_response(query_or_response)
+        return [tc.get("tool_call_id") for tc in tool_calls if tc.get("name") == "file_search"]

azure/ai/evaluation/_evaluators/_groundedness/groundedness_with_query.prompty

@@ -32,52 +32,53 @@ system:
 
 user:
 # Definition
-**Groundedness** refers to how well an answer is anchored in the provided context, evaluating its relevance, accuracy, and completeness based exclusively on that context. It assesses the extent to which the answer directly and fully addresses the question without introducing unrelated or incorrect information. The scale ranges from 1 to 5, with higher numbers indicating greater groundedness.
+**Groundedness** refers to how well an answer is anchored in the provided context, evaluating its relevance, accuracy, and completeness based exclusively on that context. It assesses the extent to which the answer directly and fully addresses the question without introducing unrelated or incorrect information.
+
+> Context is the source of truth for evaluating the response. If it's empty, rely on the tool results in the response and query.
+> Evaluate the groundedness of the response message, not the chat history.
 
 # Ratings
 ## [Groundedness: 1] (Completely Unrelated Response)
-**Definition:** An answer that does not relate to the question or the context in any way. It fails to address the topic, provides irrelevant information, or introduces completely unrelated subjects.
+**Definition:** An answer that does not relate to the question or the context in any way.
+  - Does not relate to the question or context at all.
+  - Talks about the general topic but does not respond to the query.
 
 **Examples:**
   **Context:** The company's annual meeting will be held next Thursday.
   **Query:** When is the company's annual meeting?
   **Response:** I enjoy hiking in the mountains during summer.
 
-  **Context:** The new policy aims to reduce carbon emissions by 20% over the next five years.
-  **Query:** What is the goal of the new policy?
-  **Response:** My favorite color is blue.
-
-## [Groundedness: 2] (Related Topic but Does Not Respond to the Query)
-**Definition:** An answer that relates to the general topic of the context but does not answer the specific question asked. It may mention concepts from the context but fails to provide a direct or relevant response.
-
-**Examples:**
   **Context:** The museum will exhibit modern art pieces from various local artists.
   **Query:** What kind of art will be exhibited at the museum?
   **Response:** Museums are important cultural institutions.
 
-  **Context:** The new software update improves battery life and performance.
-  **Query:** What does the new software update improve?
-  **Response:** Software updates can sometimes fix bugs.
-
-## [Groundedness: 3] (Attempts to Respond but Contains Incorrect Information)
-**Definition:** An answer that attempts to respond to the question but includes incorrect information not supported by the context. It may misstate facts, misinterpret the context, or provide erroneous details.
+## [Groundedness: 2] (Attempts to Respond but Contains Incorrect Information)
+**Definition:** An answer that attempts to respond to the question but includes incorrect information not supported by the context. It may misstate facts, misinterpret the context, or provide erroneous details. Even if some points are correct, the presence of inaccuracies makes the response unreliable.
 
 **Examples:**
-  **Context:** The festival starts on June 5th and features international musicians.
+  **Context:** - The festival starts on June 5th and features international musicians.
   **Query:** When does the festival start?
   **Response:** The festival starts on July 5th and features local artists.
 
-  **Context:** The recipe requires two eggs and one cup of milk.
-  **Query:** How many eggs are needed for the recipe?
-  **Response:** You need three eggs for the recipe.
+  **Context:** bakery_menu.txt: - Croissant au Beurre — flaky, buttery croissant
+  **Query:** [{"role":"user","content":"Are there croissants?"}]
+  **Response:** [{"role":"assistant","content":"Yes, Croissant au Beurre is on the menu, served with jam."}]
+
+## [Groundedness: 3] (Nothing to be Grounded)
+Definition: An answer that does not provide any information that can be evaluated against the context. This includes responses that are asking for clarification, providing polite fillers, or follow-up questions.
+
+**Examples:**
+  **Context:**
+  **Query:** [{"role":"user","content":"How many eggs are needed for the recipe?"}, {"role":"tool","content":"tool_result": [{"file_name": "recipe.txt", "content": "The recipe requires two eggs and one cup of milk."}]}, {"role":"assistant","content":"You need three eggs for the recipe."}, {"role":"user","content":"Thank you."}]
+  **Response:** [{"role":"assistant","content":"You're welcome, anything else I can help with?"}]
 
 ## [Groundedness: 4] (Partially Correct Response)
 **Definition:** An answer that provides a correct response to the question but is incomplete or lacks specific details mentioned in the context. It captures some of the necessary information but omits key elements needed for a full understanding.
 
 **Examples:**
-  **Context:** The bookstore offers a 15% discount to students and a 10% discount to senior citizens.
-  **Query:** What discount does the bookstore offer to students?
-  **Response:** Students get a discount at the bookstore.
+  **Context:** - store_details.txt: The bookstore offers a 15% discount to students and a 10% discount to senior citizens.
+  **Query:** [{"role":"user","content":"What discount does the bookstore offer to students, if any?"}]
+  **Response:** [{"role":"assistant","content":"Yes, students get a discount at the bookstore."}]
 
   **Context:** The company's headquarters are located in Berlin, Germany.
   **Query:** Where are the company's headquarters?
@@ -87,13 +88,13 @@ user:
 **Definition:** An answer that thoroughly and accurately responds to the question, including all relevant details from the context. It directly addresses the question with precise information, demonstrating complete understanding without adding extraneous information.
 
 **Examples:**
-  **Context:** The author released her latest novel, 'The Silent Echo', on September 1st.
-  **Query:** When was 'The Silent Echo' released?
-  **Response:** 'The Silent Echo' was released on September 1st.
+  **CONTEXT:** The author released her latest novel, 'The Silent Echo', on September 1st.
+  **QUERY:** [{"role":"user","content":"When was 'The Silent Echo' released?"}]
+  **RESPONSE:** [{"role":"assistant","content":"The 'Silent Echo' was released on September 1st."}]
 
-  **Context:** Participants must register by May 31st to be eligible for early bird pricing.
+  **Context:** 
   **Query:** By what date must participants register to receive early bird pricing?
-  **Response:** Participants must register by May 31st to receive early bird pricing.
+  **Response:** [{"role":"tool","content":"tool_result": [{"file_name": "store_guidelines.txt", "content": "Participants registering before and including May 31st will be eligible for early bird pricing."}]}, {"role":"assistant","content":"Participants must register by May 31st to receive early bird pricing."}]
 
 
 # Data
@@ -103,7 +104,7 @@ RESPONSE: {{response}}
 
 
 # Tasks
-## Please provide your assessment Score for the previous RESPONSE in relation to the CONTEXT and QUERY based on the Definitions above. Your output should include the following information:
+## Please provide your assessment Score for the previous RESPONSE message in relation to the CONTEXT, QUERY and RESPONSE tools based on the Definitions above. Your output should include the following information:
 - **ThoughtChain**: To improve the reasoning process, think step by step and include a step-by-step explanation of your thought process as you analyze the data based on the definitions. Keep it brief and start your ThoughtChain with "Let's think step by step:".
 - **Explanation**: a very short explanation of why you think the input Data should get that Score.
 - **Score**: based on your previous analysis, provide your Score. The Score you give MUST be a integer score (i.e., "1", "2"...) based on the levels of the definitions.

azure/ai/evaluation/_evaluators/_groundedness/groundedness_without_query.prompty

@@ -29,11 +29,16 @@ system:
 
 user:
 # Definition
-**Groundedness** refers to how faithfully a response adheres to the information provided in the CONTEXT, ensuring that all content is directly supported by the context without introducing unsupported information or omitting critical details. It evaluates the fidelity and precision of the response in relation to the source material.
+**Groundedness** refers to how well a response is anchored in the provided context, evaluating its relevance, accuracy, and completeness based exclusively on that context. It assesses the extent to which the response directly and fully addresses the information without introducing unrelated or incorrect information.
+
+> Context is the source of truth for evaluating the response. 
+> Evaluate the groundedness of the response message based on the provided context.
 
 # Ratings
-## [Groundedness: 1] (Completely Ungrounded Response)
-**Definition:** The response is entirely unrelated to the CONTEXT, introducing topics or information that have no connection to the provided material.
+## [Groundedness: 1] (Completely Unrelated Response)
+**Definition:** A response that does not relate to the context in any way.
+  - Does not relate to the context at all.
+  - Talks about the general topic but does not respond to the context.
 
 **Examples:**
   **Context:** The company's profits increased by 20% in the last quarter.
@@ -42,8 +47,8 @@ user:
   **Context:** The new smartphone model features a larger display and improved battery life.
   **Response:** The history of ancient Egypt is fascinating and full of mysteries.
 
-## [Groundedness: 2] (Contradictory Response)
-**Definition:** The response directly contradicts or misrepresents the information provided in the CONTEXT.
+## [Groundedness: 2] (Attempts to Respond but Contains Incorrect Information)
+**Definition:** A response that attempts to relate to the context but includes incorrect information not supported by the context. It may misstate facts, misinterpret the context, or provide erroneous details. Even if some points are correct, the presence of inaccuracies makes the response unreliable.
 
 **Examples:**
   **Context:** The company's profits increased by 20% in the last quarter.
@@ -52,18 +57,18 @@ user:
   **Context:** The new smartphone model features a larger display and improved battery life.
   **Response:** The new smartphone model has a smaller display and shorter battery life.
 
-## [Groundedness: 3] (Accurate Response with Unsupported Additions)
-**Definition:** The response accurately includes information from the CONTEXT but adds details, opinions, or explanations that are not supported by the provided material.
+## [Groundedness: 3] (Accurate but Vague Response)
+**Definition:** A response that provides accurate information from the context but is overly generic or vague, not meaningfully engaging with the specific details in the context. The information is correct but lacks specificity and detail.
 
 **Examples:**
-  **Context:** The company's profits increased by 20% in the last quarter.
-  **Response:** The company's profits increased by 20% in the last quarter due to their aggressive marketing strategy.
+  **Context:** The company's profits increased by 20% in the last quarter, marking the highest growth rate in its history.
+  **Response:** The company is doing well financially.
 
-  **Context:** The new smartphone model features a larger display and improved battery life.
-  **Response:** The new smartphone model features a larger display, improved battery life, and comes with a free case.
+  **Context:** The new smartphone model features a larger display, improved battery life, and an upgraded camera system.
+  **Response:** The smartphone has some nice features.
 
-## [Groundedness: 4] (Incomplete Response Missing Critical Details)
-**Definition:** The response contains information from the CONTEXT but omits essential details that are necessary for a comprehensive understanding of the main point.
+## [Groundedness: 4] (Partially Correct Response)
+**Definition:** A response that provides correct information from the context but is incomplete or lacks specific details mentioned in the context. It captures some of the necessary information but omits key elements needed for a full understanding.
 
 **Examples:**
   **Context:** The company's profits increased by 20% in the last quarter, marking the highest growth rate in its history.      
@@ -73,7 +78,7 @@ user:
   **Response:** The new smartphone model features a larger display and improved battery life.
 
 ## [Groundedness: 5] (Fully Grounded and Complete Response)
-**Definition:** The response is entirely based on the CONTEXT, accurately and thoroughly conveying all essential information without introducing unsupported details or omitting critical points.
+**Definition:** A response that thoroughly and accurately conveys information from the context, including all relevant details. It directly addresses the context with precise information, demonstrating complete understanding without adding extraneous information.
 
 **Examples:**
   **Context:** The company's profits increased by 20% in the last quarter, marking the highest growth rate in its history.