azure-ai-evaluation 1.0.1__py3-none-any.whl → 1.13.3__py3-none-any.whl

azure/ai/evaluation/_evaluators/_fluency/_fluency.py

@@ -23,6 +23,13 @@ class FluencyEvaluator(PromptyEvaluatorBase[Union[str, float]]):
     :param model_config: Configuration for the Azure OpenAI model.
     :type model_config: Union[~azure.ai.evaluation.AzureOpenAIModelConfiguration,
         ~azure.ai.evaluation.OpenAIModelConfiguration]
+    :param threshold: The threshold for the fluency evaluator. Default is 3.
+    :type threshold: int
+    :param credential: The credential for authenticating to Azure AI service.
+    :type credential: ~azure.core.credentials.TokenCredential
+    :keyword is_reasoning_model: If True, the evaluator will use reasoning model configuration (o1/o3 models).
+        This will adjust parameters like max_completion_tokens and remove unsupported parameters. Default is False.
+    :paramtype is_reasoning_model: bool
 
     .. admonition:: Example:
 
@@ -33,6 +40,25 @@ class FluencyEvaluator(PromptyEvaluatorBase[Union[str, float]]):
             :dedent: 8
             :caption: Initialize and call a FluencyEvaluator.
 
+    .. admonition:: Example with Threshold:
+
+        .. literalinclude:: ../samples/evaluation_samples_threshold.py
+            :start-after: [START threshold_fluency_evaluator]
+            :end-before: [END threshold_fluency_evaluator]
+            :language: python
+            :dedent: 8
+            :caption: Initialize with threshold and call a FluencyEvaluator.
+
+    .. admonition:: Example using Azure AI Project URL:
+
+        .. literalinclude:: ../samples/evaluation_samples_evaluate_fdp.py
+            :start-after: [START fluency_evaluator]
+            :end-before: [END fluency_evaluator]
+            :language: python
+            :dedent: 8
+            :caption: Initialize and call FluencyEvaluator using Azure AI Project URL in the following format
+                https://{resource_name}.services.ai.azure.com/api/projects/{project_name}
+
     .. note::
 
         To align with our support of a diverse set of models, an output key without the `gpt_` prefix has been added.
@@ -43,14 +69,24 @@ class FluencyEvaluator(PromptyEvaluatorBase[Union[str, float]]):
     _PROMPTY_FILE = "fluency.prompty"
     _RESULT_KEY = "fluency"
 
-    id = "azureml://registries/azureml/models/Fluency-Evaluator/versions/4"
+    id = "azureai://built-in/evaluators/fluency"
     """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
 
     @override
-    def __init__(self, model_config):
+    def __init__(self, model_config, *, credential=None, threshold=3, **kwargs):
         current_dir = os.path.dirname(__file__)
         prompty_path = os.path.join(current_dir, self._PROMPTY_FILE)
-        super().__init__(model_config=model_config, prompty_file=prompty_path, result_key=self._RESULT_KEY)
+        self._threshold = threshold
+        self._higher_is_better = True
+        super().__init__(
+            model_config=model_config,
+            prompty_file=prompty_path,
+            result_key=self._RESULT_KEY,
+            threshold=threshold,
+            credential=credential,
+            _higher_is_better=self._higher_is_better,
+            **kwargs,
+        )
 
     @overload
     def __call__(

azure/ai/evaluation/_evaluators/_gleu/_gleu.py

@@ -1,28 +1,17 @@
 # ---------------------------------------------------------
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
+from typing import Dict
 from nltk.translate.gleu_score import sentence_gleu
-from promptflow._utils.async_utils import async_run_allowing_running_loop
+from typing_extensions import overload, override
 
 from azure.ai.evaluation._common.utils import nltk_tokenize
 
+from azure.ai.evaluation._evaluators._common import EvaluatorBase
+from azure.ai.evaluation._constants import EVALUATION_PASS_FAIL_MAPPING
 
-class _AsyncGleuScoreEvaluator:
-    def __init__(self):
-        pass
 
-    async def __call__(self, *, ground_truth: str, response: str, **kwargs):
-        reference_tokens = nltk_tokenize(ground_truth)
-        hypothesis_tokens = nltk_tokenize(response)
-
-        score = sentence_gleu([reference_tokens], hypothesis_tokens)
-
-        return {
-            "gleu_score": score,
-        }
-
-
-class GleuScoreEvaluator:
+class GleuScoreEvaluator(EvaluatorBase):
     """
     Calculates the GLEU (Google-BLEU) score between a response and the ground truth.
 
@@ -34,6 +23,9 @@ class GleuScoreEvaluator:
     GLEU scores range from 0 to 1, where a value of 1 represents perfect overlap between the response and
     the ground truth and a value of 0 indicates no overlap.
 
+    :param threshold: The threshold for the GLEU evaluator. Default is 0.5.
+    :type threshold: float
+
     .. admonition:: Example:
 
         .. literalinclude:: ../samples/evaluation_samples_evaluate.py
@@ -42,15 +34,66 @@ class GleuScoreEvaluator:
             :language: python
             :dedent: 8
             :caption: Initialize and call a GleuScoreEvaluator.
+
+    .. admonition:: Example with Threshold:
+
+        .. literalinclude:: ../samples/evaluation_samples_threshold.py
+            :start-after: [START threshold_gleu_score_evaluator]
+            :end-before: [END threshold_gleu_score_evaluator]
+            :language: python
+            :dedent: 8
+            :caption: Initialize with threshold and call a GleuScoreEvaluator.
+
+    .. admonition:: Example using Azure AI Project URL:
+
+        .. literalinclude:: ../samples/evaluation_samples_evaluate_fdp.py
+            :start-after: [START gleu_score_evaluator]
+            :end-before: [END gleu_score_evaluator]
+            :language: python
+            :dedent: 8
+            :caption: Initialize and call GleuScoreEvaluator using Azure AI Project URL in the following format
+                https://{resource_name}.services.ai.azure.com/api/projects/{project_name}
     """
 
-    id = "azureml://registries/azureml/models/Gleu-Score-Evaluator/versions/3"
+    id = "azureai://built-in/evaluators/gleu_score"
     """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
 
-    def __init__(self):
-        self._async_evaluator = _AsyncGleuScoreEvaluator()
+    @override
+    def __init__(self, *, threshold=0.5):
+        self._threshold = threshold
+        self._higher_is_better = True
+        super().__init__(threshold=threshold, _higher_is_better=self._higher_is_better)
 
-    def __call__(self, *, ground_truth: str, response: str, **kwargs):
+    @override
+    async def _do_eval(self, eval_input: Dict) -> Dict[str, float]:
+        """Produce a glue score evaluation result.
+
+        :param eval_input: The input to the evaluation function.
+        :type eval_input: Dict
+        :return: The evaluation result.
+        :rtype: Dict
+        """
+        ground_truth = eval_input["ground_truth"]
+        response = eval_input["response"]
+        reference_tokens = nltk_tokenize(ground_truth)
+        hypothesis_tokens = nltk_tokenize(response)
+
+        score = sentence_gleu([reference_tokens], hypothesis_tokens)
+        binary_result = False
+        if self._higher_is_better:
+            if score >= self._threshold:
+                binary_result = True
+        else:
+            if score <= self._threshold:
+                binary_result = True
+        return {
+            "gleu_score": score,
+            "gleu_result": EVALUATION_PASS_FAIL_MAPPING[binary_result],
+            "gleu_threshold": self._threshold,
+        }
+
+    @overload  # type: ignore
+    def __call__(self, *, ground_truth: str, response: str):
         """
         Evaluate the GLEU score between the response and the ground truth.
 
@@ -61,9 +104,21 @@ class GleuScoreEvaluator:
         :return: The GLEU score.
         :rtype: Dict[str, float]
         """
-        return async_run_allowing_running_loop(
-            self._async_evaluator, ground_truth=ground_truth, response=response, **kwargs
-        )
 
-    def _to_async(self):
-        return self._async_evaluator
+    @override
+    def __call__(  # pylint: disable=docstring-missing-param
+        self,
+        *args,
+        **kwargs,
+    ):
+        """
+        Evaluate the GLEU score between the response and the ground truth.
+
+        :keyword response: The response to be evaluated.
+        :paramtype response: str
+        :keyword ground_truth: The ground truth to be compared against.
+        :paramtype ground_truth: str
+        :return: The GLEU score.
+        :rtype: Dict[str, float]
+        """
+        return super().__call__(*args, **kwargs)

azure/ai/evaluation/_evaluators/_groundedness/_groundedness.py

@@ -1,25 +1,39 @@
 # ---------------------------------------------------------
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
-import os
-from typing import Dict, List, Optional, Union
+import os, logging
+from typing import Dict, List, Optional, Union, Any, Tuple
 
 from typing_extensions import overload, override
-from promptflow.core import AsyncPrompty
+from azure.ai.evaluation._legacy.prompty import AsyncPrompty
 
 from azure.ai.evaluation._evaluators._common import PromptyEvaluatorBase
 from azure.ai.evaluation._model_configurations import Conversation
-from ..._common.utils import construct_prompty_model_config, validate_model_config
+from ..._common.utils import (
+    ErrorBlame,
+    ErrorTarget,
+    EvaluationException,
+    ErrorCategory,
+    construct_prompty_model_config,
+    validate_model_config,
+    simplify_messages,
+)
 
 try:
-    from ..._user_agent import USER_AGENT
+    from ..._user_agent import UserAgentSingleton
 except ImportError:
-    USER_AGENT = "None"
+
+    class UserAgentSingleton:
+        @property
+        def value(self) -> str:
+            return "None"
+
+
+logger = logging.getLogger(__name__)
 
 
 class GroundednessEvaluator(PromptyEvaluatorBase[Union[str, float]]):
-    """
-    Evaluates groundedness score for a given query (optional), response, and context or a multi-turn conversation,
+    """Evaluates groundedness score for a given query (optional), response, and context or a multi-turn conversation,
     including reasoning.
 
     The groundedness measure assesses the correspondence between claims in an AI-generated answer and the source
@@ -33,6 +47,13 @@ class GroundednessEvaluator(PromptyEvaluatorBase[Union[str, float]]):
     :param model_config: Configuration for the Azure OpenAI model.
     :type model_config: Union[~azure.ai.evaluation.AzureOpenAIModelConfiguration,
         ~azure.ai.evaluation.OpenAIModelConfiguration]
+    :param threshold: The threshold for the groundedness evaluator. Default is 3.
+    :type threshold: int
+    :param credential: The credential for authenticating to Azure AI service.
+    :type credential: ~azure.core.credentials.TokenCredential
+    :keyword is_reasoning_model: If True, the evaluator will use reasoning model configuration (o1/o3 models).
+        This will adjust parameters like max_completion_tokens and remove unsupported parameters. Default is False.
+    :paramtype is_reasoning_model: bool
 
     .. admonition:: Example:
 
@@ -43,6 +64,25 @@ class GroundednessEvaluator(PromptyEvaluatorBase[Union[str, float]]):
             :dedent: 8
             :caption: Initialize and call a GroundednessEvaluator.
 
+    .. admonition:: Example with Threshold:
+
+        .. literalinclude:: ../samples/evaluation_samples_threshold.py
+            :start-after: [START threshold_groundedness_evaluator]
+            :end-before: [END threshold_groundedness_evaluator]
+            :language: python
+            :dedent: 8
+            :caption: Initialize with threshold and call a GroundednessEvaluator.
+
+    .. admonition:: Example using Azure AI Project URL:
+
+        .. literalinclude:: ../samples/evaluation_samples_evaluate_fdp.py
+            :start-after: [START groundedness_evaluator]
+            :end-before: [END groundedness_evaluator]
+            :language: python
+            :dedent: 8
+            :caption: Initialize and call GroundednessEvaluator using Azure AI Project URL in the following format
+                https://{resource_name}.services.ai.azure.com/api/projects/{project_name}
+
     .. note::
 
         To align with our support of a diverse set of models, an output key without the `gpt_` prefix has been added.
@@ -54,17 +94,28 @@ class GroundednessEvaluator(PromptyEvaluatorBase[Union[str, float]]):
     _PROMPTY_FILE_WITH_QUERY = "groundedness_with_query.prompty"
     _RESULT_KEY = "groundedness"
     _OPTIONAL_PARAMS = ["query"]
+    _SUPPORTED_TOOLS = ["file_search"]
 
-    id = "azureml://registries/azureml/models/Groundedness-Evaluator/versions/4"
+    id = "azureai://built-in/evaluators/groundedness"
     """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
 
     @override
-    def __init__(self, model_config):
+    def __init__(self, model_config, *, threshold=3, credential=None, **kwargs):
         current_dir = os.path.dirname(__file__)
         prompty_path = os.path.join(current_dir, self._PROMPTY_FILE_NO_QUERY)  # Default to no query
 
-        super().__init__(model_config=model_config, prompty_file=prompty_path, result_key=self._RESULT_KEY)
+        self._higher_is_better = True
+        super().__init__(
+            model_config=model_config,
+            prompty_file=prompty_path,
+            result_key=self._RESULT_KEY,
+            threshold=threshold,
+            credential=credential,
+            _higher_is_better=self._higher_is_better,
+            **kwargs,
+        )
         self._model_config = model_config
+        self.threshold = threshold
         # Needs to be set because it's used in call method to re-validate prompt if `query` is provided
 
     @overload
@@ -88,6 +139,26 @@ class GroundednessEvaluator(PromptyEvaluatorBase[Union[str, float]]):
         :rtype: Dict[str, float]
         """
 
+    @overload
+    def __call__(
+        self,
+        *,
+        query: str,
+        response: List[dict],
+        tool_definitions: List[dict],
+    ) -> Dict[str, Union[str, float]]:
+        """Evaluate groundedness for agent response with tool calls. Only file_search tool is supported.
+
+        :keyword query: The query to be evaluated.
+        :paramtype query: str
+        :keyword response: The response from the agent to be evaluated.
+        :paramtype response: List[dict]
+        :keyword tool_definitions: The tool definitions used by the agent.
+        :paramtype tool_definitions: List[dict]
+        :return: The groundedness score.
+        :rtype: Dict[str, Union[str, float]]
+        """
+
     @overload
     def __call__(
         self,
@@ -131,14 +202,153 @@ class GroundednessEvaluator(PromptyEvaluatorBase[Union[str, float]]):
         """
 
         if kwargs.get("query", None):
-            current_dir = os.path.dirname(__file__)
-            prompty_path = os.path.join(current_dir, self._PROMPTY_FILE_WITH_QUERY)
-            self._prompty_file = prompty_path
-            prompty_model_config = construct_prompty_model_config(
-                validate_model_config(self._model_config),
-                self._DEFAULT_OPEN_API_VERSION,
-                USER_AGENT,
-            )
-            self._flow = AsyncPrompty.load(source=self._prompty_file, model=prompty_model_config)
+            self._ensure_query_prompty_loaded()
 
         return super().__call__(*args, **kwargs)
+
+    def _ensure_query_prompty_loaded(self):
+        """Switch to the query prompty file if not already loaded."""
+
+        current_dir = os.path.dirname(__file__)
+        prompty_path = os.path.join(current_dir, self._PROMPTY_FILE_WITH_QUERY)
+
+        self._prompty_file = prompty_path
+        prompty_model_config = construct_prompty_model_config(
+            validate_model_config(self._model_config),
+            self._DEFAULT_OPEN_API_VERSION,
+            UserAgentSingleton().value,
+        )
+        self._flow = AsyncPrompty.load(source=self._prompty_file, model=prompty_model_config)
+
+    def _has_context(self, eval_input: dict) -> bool:
+        """
+        Return True if eval_input contains a non-empty 'context' field.
+        Treats None, empty strings, empty lists, and lists of empty strings as no context.
+        """
+        context = eval_input.get("context", None)
+        if not context:
+            return False
+        if context == "<>":  # Special marker for no context
+            return False
+        if isinstance(context, list):
+            return any(str(c).strip() for c in context)
+        if isinstance(context, str):
+            return bool(context.strip())
+        return True
+
+    @override
+    async def _do_eval(self, eval_input: Dict) -> Dict[str, Union[float, str]]:
+        if eval_input.get("query", None) is None:
+            return await super()._do_eval(eval_input)
+
+        contains_context = self._has_context(eval_input)
+
+        simplified_query = simplify_messages(eval_input["query"], drop_tool_calls=contains_context)
+        simplified_response = simplify_messages(eval_input["response"], drop_tool_calls=False)
+
+        # Build simplified input
+        simplified_eval_input = {
+            "query": simplified_query,
+            "response": simplified_response,
+            "context": eval_input["context"],
+        }
+
+        # Replace and call the parent method
+        return await super()._do_eval(simplified_eval_input)
+
+    async def _real_call(self, **kwargs):
+        """The asynchronous call where real end-to-end evaluation logic is performed.
+
+        :keyword kwargs: The inputs to evaluate.
+        :type kwargs: Dict
+        :return: The evaluation result.
+        :rtype: Union[DoEvalResult[T_EvalValue], AggregateResult[T_EvalValue]]
+        """
+        # Convert inputs into list of evaluable inputs.
+        try:
+            return await super()._real_call(**kwargs)
+        except EvaluationException as ex:
+            if ex.category == ErrorCategory.NOT_APPLICABLE:
+                return {
+                    self._result_key: self._NOT_APPLICABLE_RESULT,
+                    f"{self._result_key}_result": "pass",
+                    f"{self._result_key}_threshold": self.threshold,
+                    f"{self._result_key}_reason": f"Supported tools were not called. Supported tools for groundedness are {self._SUPPORTED_TOOLS}.",
+                }
+            else:
+                raise ex
+
+    def _convert_kwargs_to_eval_input(self, **kwargs):
+        if kwargs.get("context") or kwargs.get("conversation"):
+            return super()._convert_kwargs_to_eval_input(**kwargs)
+        query = kwargs.get("query")
+        response = kwargs.get("response")
+        tool_definitions = kwargs.get("tool_definitions")
+
+        if query and self._prompty_file != self._PROMPTY_FILE_WITH_QUERY:
+            self._ensure_query_prompty_loaded()
+
+        if (not query) or (not response):  # or not tool_definitions:
+            msg = f"{type(self).__name__}: Either 'conversation' or individual inputs must be provided. For Agent groundedness 'query' and 'response' are required."
+            raise EvaluationException(
+                message=msg,
+                blame=ErrorBlame.USER_ERROR,
+                category=ErrorCategory.INVALID_VALUE,
+                target=ErrorTarget.GROUNDEDNESS_EVALUATOR,
+            )
+        context = self._get_context_from_agent_response(response, tool_definitions)
+
+        filtered_response = self._filter_file_search_results(response)
+        return super()._convert_kwargs_to_eval_input(response=filtered_response, context=context, query=query)
+
+    def _filter_file_search_results(self, messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        """Filter out file_search tool results from the messages."""
+        file_search_ids = self._get_file_search_tool_call_ids(messages)
+        return [
+            msg for msg in messages if not (msg.get("role") == "tool" and msg.get("tool_call_id") in file_search_ids)
+        ]
+
+    def _get_context_from_agent_response(self, response, tool_definitions):
+        """Extract context text from file_search tool results in the agent response."""
+        NO_CONTEXT = "<>"
+        context = ""
+        try:
+            logger.debug("Extracting context from response")
+            tool_calls = self._parse_tools_from_response(response=response)
+            logger.debug(f"Tool Calls parsed successfully: {tool_calls}")
+
+            if not tool_calls:
+                return NO_CONTEXT
+
+            context_lines = []
+            for tool_call in tool_calls:
+                if not isinstance(tool_call, dict) or tool_call.get("type") != "tool_call":
+                    continue
+
+                tool_name = tool_call.get("name")
+                if tool_name != "file_search":
+                    continue
+
+                # Extract tool results
+                for result in tool_call.get("tool_result", []):
+                    results = result if isinstance(result, list) else [result]
+                    for r in results:
+                        file_name = r.get("file_name", "Unknown file name")
+                        for content in r.get("content", []):
+                            text = content.get("text")
+                            if text:
+                                context_lines.append(f"{file_name}:\n- {text}---\n\n")
+
+            context = "\n".join(context_lines) if len(context_lines) > 0 else None
+
+        except Exception as ex:
+            logger.debug(f"Error extracting context from agent response : {str(ex)}")
+            context = None
+
+        context = context if context else NO_CONTEXT
+        return context
+
+    def _get_file_search_tool_call_ids(self, query_or_response):
+        """Return a list of tool_call_ids for file search tool calls."""
+        tool_calls = self._parse_tools_from_response(query_or_response)
+        return [tc.get("tool_call_id") for tc in tool_calls if tc.get("name") == "file_search"]

azure/ai/evaluation/_evaluators/_groundedness/groundedness_with_query.prompty

@@ -32,52 +32,53 @@ system:
 
 user:
 # Definition
-**Groundedness** refers to how well an answer is anchored in the provided context, evaluating its relevance, accuracy, and completeness based exclusively on that context. It assesses the extent to which the answer directly and fully addresses the question without introducing unrelated or incorrect information. The scale ranges from 1 to 5, with higher numbers indicating greater groundedness.
+**Groundedness** refers to how well an answer is anchored in the provided context, evaluating its relevance, accuracy, and completeness based exclusively on that context. It assesses the extent to which the answer directly and fully addresses the question without introducing unrelated or incorrect information.
+
+> Context is the source of truth for evaluating the response. If it's empty, rely on the tool results in the response and query.
+> Evaluate the groundedness of the response message, not the chat history.
 
 # Ratings
 ## [Groundedness: 1] (Completely Unrelated Response)
-**Definition:** An answer that does not relate to the question or the context in any way. It fails to address the topic, provides irrelevant information, or introduces completely unrelated subjects.
+**Definition:** An answer that does not relate to the question or the context in any way.
+  - Does not relate to the question or context at all.
+  - Talks about the general topic but does not respond to the query.
 
 **Examples:**
   **Context:** The company's annual meeting will be held next Thursday.
   **Query:** When is the company's annual meeting?
   **Response:** I enjoy hiking in the mountains during summer.
 
-  **Context:** The new policy aims to reduce carbon emissions by 20% over the next five years.
-  **Query:** What is the goal of the new policy?
-  **Response:** My favorite color is blue.
-
-## [Groundedness: 2] (Related Topic but Does Not Respond to the Query)
-**Definition:** An answer that relates to the general topic of the context but does not answer the specific question asked. It may mention concepts from the context but fails to provide a direct or relevant response.
-
-**Examples:**
   **Context:** The museum will exhibit modern art pieces from various local artists.
   **Query:** What kind of art will be exhibited at the museum?
   **Response:** Museums are important cultural institutions.
 
-  **Context:** The new software update improves battery life and performance.
-  **Query:** What does the new software update improve?
-  **Response:** Software updates can sometimes fix bugs.
-
-## [Groundedness: 3] (Attempts to Respond but Contains Incorrect Information)
-**Definition:** An answer that attempts to respond to the question but includes incorrect information not supported by the context. It may misstate facts, misinterpret the context, or provide erroneous details.
+## [Groundedness: 2] (Attempts to Respond but Contains Incorrect Information)
+**Definition:** An answer that attempts to respond to the question but includes incorrect information not supported by the context. It may misstate facts, misinterpret the context, or provide erroneous details. Even if some points are correct, the presence of inaccuracies makes the response unreliable.
 
 **Examples:**
-  **Context:** The festival starts on June 5th and features international musicians.
+  **Context:** - The festival starts on June 5th and features international musicians.
   **Query:** When does the festival start?
   **Response:** The festival starts on July 5th and features local artists.
 
-  **Context:** The recipe requires two eggs and one cup of milk.
-  **Query:** How many eggs are needed for the recipe?
-  **Response:** You need three eggs for the recipe.
+  **Context:** bakery_menu.txt: - Croissant au Beurre — flaky, buttery croissant
+  **Query:** [{"role":"user","content":"Are there croissants?"}]
+  **Response:** [{"role":"assistant","content":"Yes, Croissant au Beurre is on the menu, served with jam."}]
+
+## [Groundedness: 3] (Nothing to be Grounded)
+Definition: An answer that does not provide any information that can be evaluated against the context. This includes responses that are asking for clarification, providing polite fillers, or follow-up questions.
+
+**Examples:**
+  **Context:**
+  **Query:** [{"role":"user","content":"How many eggs are needed for the recipe?"}, {"role":"tool","content":"tool_result": [{"file_name": "recipe.txt", "content": "The recipe requires two eggs and one cup of milk."}]}, {"role":"assistant","content":"You need three eggs for the recipe."}, {"role":"user","content":"Thank you."}]
+  **Response:** [{"role":"assistant","content":"You're welcome, anything else I can help with?"}]
 
 ## [Groundedness: 4] (Partially Correct Response)
 **Definition:** An answer that provides a correct response to the question but is incomplete or lacks specific details mentioned in the context. It captures some of the necessary information but omits key elements needed for a full understanding.
 
 **Examples:**
-  **Context:** The bookstore offers a 15% discount to students and a 10% discount to senior citizens.
-  **Query:** What discount does the bookstore offer to students?
-  **Response:** Students get a discount at the bookstore.
+  **Context:** - store_details.txt: The bookstore offers a 15% discount to students and a 10% discount to senior citizens.
+  **Query:** [{"role":"user","content":"What discount does the bookstore offer to students, if any?"}]
+  **Response:** [{"role":"assistant","content":"Yes, students get a discount at the bookstore."}]
 
   **Context:** The company's headquarters are located in Berlin, Germany.
   **Query:** Where are the company's headquarters?
@@ -87,13 +88,13 @@ user:
 **Definition:** An answer that thoroughly and accurately responds to the question, including all relevant details from the context. It directly addresses the question with precise information, demonstrating complete understanding without adding extraneous information.
 
 **Examples:**
-  **Context:** The author released her latest novel, 'The Silent Echo', on September 1st.
-  **Query:** When was 'The Silent Echo' released?
-  **Response:** 'The Silent Echo' was released on September 1st.
+  **CONTEXT:** The author released her latest novel, 'The Silent Echo', on September 1st.
+  **QUERY:** [{"role":"user","content":"When was 'The Silent Echo' released?"}]
+  **RESPONSE:** [{"role":"assistant","content":"The 'Silent Echo' was released on September 1st."}]
 
-  **Context:** Participants must register by May 31st to be eligible for early bird pricing.
+  **Context:** 
   **Query:** By what date must participants register to receive early bird pricing?
-  **Response:** Participants must register by May 31st to receive early bird pricing.
+  **Response:** [{"role":"tool","content":"tool_result": [{"file_name": "store_guidelines.txt", "content": "Participants registering before and including May 31st will be eligible for early bird pricing."}]}, {"role":"assistant","content":"Participants must register by May 31st to receive early bird pricing."}]
 
 
 # Data
@@ -103,7 +104,7 @@ RESPONSE: {{response}}
 
 
 # Tasks
-## Please provide your assessment Score for the previous RESPONSE in relation to the CONTEXT and QUERY based on the Definitions above. Your output should include the following information:
+## Please provide your assessment Score for the previous RESPONSE message in relation to the CONTEXT, QUERY and RESPONSE tools based on the Definitions above. Your output should include the following information:
 - **ThoughtChain**: To improve the reasoning process, think step by step and include a step-by-step explanation of your thought process as you analyze the data based on the definitions. Keep it brief and start your ThoughtChain with "Let's think step by step:".
 - **Explanation**: a very short explanation of why you think the input Data should get that Score.
 - **Score**: based on your previous analysis, provide your Score. The Score you give MUST be a integer score (i.e., "1", "2"...) based on the levels of the definitions.