azure-ai-evaluation 1.11.2__py3-none-any.whl → 1.13.0__py3-none-any.whl

azure/ai/evaluation/_evaluators/_groundedness/groundedness_without_query.prompty

@@ -29,11 +29,16 @@ system:
 
 user:
 # Definition
-**Groundedness** refers to how faithfully a response adheres to the information provided in the CONTEXT, ensuring that all content is directly supported by the context without introducing unsupported information or omitting critical details. It evaluates the fidelity and precision of the response in relation to the source material.
+**Groundedness** refers to how well a response is anchored in the provided context, evaluating its relevance, accuracy, and completeness based exclusively on that context. It assesses the extent to which the response directly and fully addresses the information without introducing unrelated or incorrect information.
+
+> Context is the source of truth for evaluating the response. 
+> Evaluate the groundedness of the response message based on the provided context.
 
 # Ratings
-## [Groundedness: 1] (Completely Ungrounded Response)
-**Definition:** The response is entirely unrelated to the CONTEXT, introducing topics or information that have no connection to the provided material.
+## [Groundedness: 1] (Completely Unrelated Response)
+**Definition:** A response that does not relate to the context in any way.
+  - Does not relate to the context at all.
+  - Talks about the general topic but does not respond to the context.
 
 **Examples:**
   **Context:** The company's profits increased by 20% in the last quarter.
@@ -42,8 +47,8 @@ user:
   **Context:** The new smartphone model features a larger display and improved battery life.
   **Response:** The history of ancient Egypt is fascinating and full of mysteries.
 
-## [Groundedness: 2] (Contradictory Response)
-**Definition:** The response directly contradicts or misrepresents the information provided in the CONTEXT.
+## [Groundedness: 2] (Attempts to Respond but Contains Incorrect Information)
+**Definition:** A response that attempts to relate to the context but includes incorrect information not supported by the context. It may misstate facts, misinterpret the context, or provide erroneous details. Even if some points are correct, the presence of inaccuracies makes the response unreliable.
 
 **Examples:**
   **Context:** The company's profits increased by 20% in the last quarter.
@@ -52,18 +57,18 @@ user:
   **Context:** The new smartphone model features a larger display and improved battery life.
   **Response:** The new smartphone model has a smaller display and shorter battery life.
 
-## [Groundedness: 3] (Accurate Response with Unsupported Additions)
-**Definition:** The response accurately includes information from the CONTEXT but adds details, opinions, or explanations that are not supported by the provided material.
+## [Groundedness: 3] (Accurate but Vague Response)
+**Definition:** A response that provides accurate information from the context but is overly generic or vague, not meaningfully engaging with the specific details in the context. The information is correct but lacks specificity and detail.
 
 **Examples:**
-  **Context:** The company's profits increased by 20% in the last quarter.
-  **Response:** The company's profits increased by 20% in the last quarter due to their aggressive marketing strategy.
+  **Context:** The company's profits increased by 20% in the last quarter, marking the highest growth rate in its history.
+  **Response:** The company is doing well financially.
 
-  **Context:** The new smartphone model features a larger display and improved battery life.
-  **Response:** The new smartphone model features a larger display, improved battery life, and comes with a free case.
+  **Context:** The new smartphone model features a larger display, improved battery life, and an upgraded camera system.
+  **Response:** The smartphone has some nice features.
 
-## [Groundedness: 4] (Incomplete Response Missing Critical Details)
-**Definition:** The response contains information from the CONTEXT but omits essential details that are necessary for a comprehensive understanding of the main point.
+## [Groundedness: 4] (Partially Correct Response)
+**Definition:** A response that provides correct information from the context but is incomplete or lacks specific details mentioned in the context. It captures some of the necessary information but omits key elements needed for a full understanding.
 
 **Examples:**
   **Context:** The company's profits increased by 20% in the last quarter, marking the highest growth rate in its history.      
@@ -73,7 +78,7 @@ user:
   **Response:** The new smartphone model features a larger display and improved battery life.
 
 ## [Groundedness: 5] (Fully Grounded and Complete Response)
-**Definition:** The response is entirely based on the CONTEXT, accurately and thoroughly conveying all essential information without introducing unsupported details or omitting critical points.
+**Definition:** A response that thoroughly and accurately conveys information from the context, including all relevant details. It directly addresses the context with precise information, demonstrating complete understanding without adding extraneous information.
 
 **Examples:**
   **Context:** The company's profits increased by 20% in the last quarter, marking the highest growth rate in its history.      

azure/ai/evaluation/_evaluators/_intent_resolution/_intent_resolution.py

@@ -69,7 +69,9 @@ class IntentResolutionEvaluator(PromptyEvaluatorBase[Union[str, float]]):
             model_config=model_config,
             prompty_file=prompty_path,
             result_key=self._RESULT_KEY,
+            threshold=threshold,
             credential=credential,
+            _higher_is_better=True,
             **kwargs,
         )
 
@@ -145,8 +147,10 @@ class IntentResolutionEvaluator(PromptyEvaluatorBase[Union[str, float]]):
         eval_input["query"] = reformat_conversation_history(eval_input["query"], logger)
         eval_input["response"] = reformat_agent_response(eval_input["response"], logger)
 
-        llm_output = await self._flow(timeout=self._LLM_CALL_TIMEOUT, **eval_input)
+        prompty_output_dict = await self._flow(timeout=self._LLM_CALL_TIMEOUT, **eval_input)
+        llm_output = prompty_output_dict["llm_output"]
         # llm_output should always be a dictionary because the response_format of prompty is set to json_object, but checking anyway
+        score = math.nan
         if isinstance(llm_output, dict):
             score = llm_output.get("score", math.nan)
             if not check_score_is_valid(
@@ -162,16 +166,31 @@ class IntentResolutionEvaluator(PromptyEvaluatorBase[Union[str, float]]):
                 )
             reason = llm_output.get("explanation", "")
             score = float(score)
-            score_result = "pass" if score >= self.threshold else "fail"
+            score_result = "pass" if score >= self._threshold else "fail"
 
             response_dict = {
                 f"{self._result_key}": score,
+                f"gpt_{self._result_key}": score,
                 f"{self._result_key}_result": score_result,
-                f"{self._result_key}_threshold": self.threshold,
+                f"{self._result_key}_threshold": self._threshold,
                 f"{self._result_key}_reason": reason,
+                f"{self._result_key}_prompt_tokens": prompty_output_dict.get("input_token_count", 0),
+                f"{self._result_key}_completion_tokens": prompty_output_dict.get("output_token_count", 0),
+                f"{self._result_key}_total_tokens": prompty_output_dict.get("total_token_count", 0),
+                f"{self._result_key}_finish_reason": prompty_output_dict.get("finish_reason", ""),
+                f"{self._result_key}_model": prompty_output_dict.get("model_id", ""),
+                f"{self._result_key}_sample_input": prompty_output_dict.get("sample_input", ""),
+                f"{self._result_key}_sample_output": prompty_output_dict.get("sample_output", ""),
             }
             return response_dict
         # If llm_output is not a dictionary, return NaN for the score. This should never happen
         if logger:
             logger.warning("LLM output is not a dictionary, returning NaN for the score.")
-        return {self._result_key: math.nan}
+
+        binary_result = self._get_binary_result(score)
+        return {
+            self._result_key: float(score),
+            f"gpt_{self._result_key}": float(score),
+            f"{self._result_key}_result": binary_result,
+            f"{self._result_key}_threshold": self._threshold,
+        }

azure/ai/evaluation/_evaluators/_relevance/_relevance.py

@@ -35,6 +35,11 @@ class RelevanceEvaluator(PromptyEvaluatorBase):
         ~azure.ai.evaluation.OpenAIModelConfiguration]
     :param threshold: The threshold for the relevance evaluator. Default is 3.
     :type threshold: int
+    :param credential: The credential for authenticating to Azure AI service.
+    :type credential: ~azure.core.credentials.TokenCredential
+    :keyword is_reasoning_model: If True, the evaluator will use reasoning model configuration (o1/o3 models).
+        This will adjust parameters like max_completion_tokens and remove unsupported parameters. Default is False.
+    :paramtype is_reasoning_model: bool
 
     .. admonition:: Example:
 
@@ -79,18 +84,17 @@ class RelevanceEvaluator(PromptyEvaluatorBase):
     """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
 
     @override
-    def __init__(self, model_config, *, credential=None, threshold=3):
+    def __init__(self, model_config, *, credential=None, threshold=3, **kwargs):
         current_dir = os.path.dirname(__file__)
         prompty_path = os.path.join(current_dir, self._PROMPTY_FILE)
-        self._threshold = threshold
-        self._higher_is_better = True
         super().__init__(
             model_config=model_config,
             prompty_file=prompty_path,
             result_key=self._RESULT_KEY,
             threshold=threshold,
             credential=credential,
-            _higher_is_better=self._higher_is_better,
+            _higher_is_better=True,
+            **kwargs,
         )
 
     @overload
@@ -171,7 +175,8 @@ class RelevanceEvaluator(PromptyEvaluatorBase):
             eval_input["query"] = reformat_conversation_history(eval_input["query"], logger)
         if not isinstance(eval_input["response"], str):
             eval_input["response"] = reformat_agent_response(eval_input["response"], logger)
-        llm_output = await self._flow(timeout=self._LLM_CALL_TIMEOUT, **eval_input)
+        result = await self._flow(timeout=self._LLM_CALL_TIMEOUT, **eval_input)
+        llm_output = result.get("llm_output")
         score = math.nan
 
         if isinstance(llm_output, dict):
@@ -182,15 +187,24 @@ class RelevanceEvaluator(PromptyEvaluatorBase):
             return {
                 self._result_key: float(score),
                 f"gpt_{self._result_key}": float(score),
-                f"{self._result_key}_reason": reason,
                 f"{self._result_key}_result": binary_result,
                 f"{self._result_key}_threshold": self._threshold,
+                f"{self._result_key}_reason": reason,
+                f"{self._result_key}_prompt_tokens": result.get("input_token_count", 0),
+                f"{self._result_key}_completion_tokens": result.get("output_token_count", 0),
+                f"{self._result_key}_total_tokens": result.get("total_token_count", 0),
+                f"{self._result_key}_finish_reason": result.get("finish_reason", ""),
+                f"{self._result_key}_model": result.get("model_id", ""),
+                f"{self._result_key}_sample_input": result.get("sample_input", ""),
+                f"{self._result_key}_sample_output": result.get("sample_output", ""),
             }
 
+        if logger:
+            logger.warning("LLM output is not a dictionary, returning NaN for the score.")
+
         binary_result = self._get_binary_result(score)
         return {
             self._result_key: float(score),
-            f"gpt_{self._result_key}": float(score),
             f"{self._result_key}_result": binary_result,
             f"{self._result_key}_threshold": self._threshold,
         }

azure/ai/evaluation/_evaluators/_relevance/relevance.prompty

@@ -20,22 +20,25 @@ inputs:
 ---
 
 system:
-You are a Relevance-Judge, an impartial evaluator that scores how well the RESPONSE addresses the QUERY using the definitions provided.
+You are a Relevance-Judge, an impartial evaluator that scores how well the RESPONSE addresses the user's queries in the CONVERSATION_HISTORY using the definitions provided.
 
 user:
 ROLE
 ====
-You are a Relevance Evaluator. Your task is to judge how relevant a RESPONSE is to a QUERY using the Relevance definitions provided.
+You are a Relevance Evaluator. Your task is to judge how relevant a RESPONSE is to the CONVERSATION_HISTORY using the Relevance definitions provided.
 
 INPUT
 =====
-QUERY: {{query}}
+CONVERSATION_HISTORY: {{query}}
 RESPONSE: {{response}}
 
+CONVERSATION_HISTORY is the full dialogue between the user and the agent up to the user's latest message. For single-turn interactions, this will be just the user's query.
+RESPONSE is the agent's reply to the user's latest message.
+
 TASK
 ====
 Output a JSON object with:
-    1) a concise explanation of 15-60 words justifying your score based on how well the response is relevant to the query.
+    1) a concise explanation of 15-60 words justifying your score based on how well the response is relevant to the user's queries in the CONVERSATION_HISTORY.
     2) an integer score from 1 (very poor) to 5 (excellent) using the rubric below.
 
 The explanation should always precede the score and should clearly justify the score based on the rubric definitions.
@@ -49,13 +52,14 @@ Response format exactly as follows:
 
 EVALUATION STEPS
 ================
-A. Read the QUERY and RESPONSE carefully.
-B. Compare the RESPONSE against the rubric below:
-    - Does the response directly address the query?
+A. Read the CONVERSATION_HISTORY and RESPONSE carefully.
+B. Identify the user's query from the latest message (use conversation history for context if needed).
+C. Compare the RESPONSE against the rubric below:
+    - Does the response directly address the user's query?
     - Is the information complete, partial, or off-topic?
     - Is it vague, generic, or insightful?
-C. Match the response to the best score from the rubric.
-D. Provide a short explanation and the score using the required format.
+D. Match the response to the best score from the rubric.
+E. Provide a short explanation and the score using the required format.
 
 SCORING RUBRIC
 ==============
@@ -64,7 +68,7 @@ SCORING RUBRIC
 Definition: The response is unrelated to the question. It provides off-topic information and does not attempt to address the question posed.
 
 **Example A**
-QUERY: What is the team preparing for?
+CONVERSATION_HISTORY: What is the team preparing for?
 RESPONSE: I went grocery shopping yesterday evening.
 
 Expected Output:
@@ -75,7 +79,7 @@ Expected Output:
 
 
 **Example B**
-QUERY: When will the company's new product line launch?
+CONVERSATION_HISTORY: When will the company's new product line launch?
 RESPONSE: International travel can be very rewarding and educational.
 
 Expected Output:
@@ -89,7 +93,7 @@ Expected Output:
 Definition: The response is loosely or formally related to the query but fails to deliver any meaningful, specific, or useful information. This includes vague phrases, non-answers, or failure/error messages.
 
 **Example A**
-QUERY: What is the event about?
+CONVERSATION_HISTORY: What is the event about?
 RESPONSE: It’s something important.
 
 Expected Output:
@@ -99,7 +103,7 @@ Expected Output:
 }
 
 **Example B**
-QUERY: What’s the weather in Paris?
+CONVERSATION_HISTORY: What’s the weather in Paris?
 RESPONSE: I tried to find the forecast but the query failed.
 
 Expected Output:
@@ -112,7 +116,7 @@ Expected Output:
 Definition: The response addresses the query and includes relevant information, but omits essential components or detail. The answer is on-topic but insufficient to fully satisfy the request.
 
 **Example A**
-QUERY: What amenities does the new apartment complex provide?
+CONVERSATION_HISTORY: What amenities does the new apartment complex provide?
 RESPONSE: The apartment complex has a gym.
 
 Expected Output:
@@ -122,7 +126,7 @@ Expected Output:
 }
 
 **Example B**
-QUERY: What services does the premium membership include?
+CONVERSATION_HISTORY: What services does the premium membership include?
 RESPONSE: It includes priority customer support.
 
 Expected Output:
@@ -137,7 +141,7 @@ Expected Output:
 Definition: The response fully addresses the question with accurate and sufficient information, covering all essential aspects. Very minor omissions are acceptable as long as the core information is intact and the intent is clearly conveyed.
 
 **Example A**
-QUERY: What amenities does the new apartment complex provide?
+CONVERSATION_HISTORY: What amenities does the new apartment complex provide?
 RESPONSE: The apartment complex provides a gym, swimming pool, and 24/7 security.
 
 Expected Output:
@@ -147,7 +151,7 @@ Expected Output:
 }
 
 **Example B**
-QUERY: What services does the premium membership include?
+CONVERSATION_HISTORY: What services does the premium membership include?
 RESPONSE: The premium membership includes priority customer support, exclusive content access, and early product releases.
 
 Expected Output:
@@ -161,7 +165,7 @@ Expected Output:
 Definition: The response not only fully and accurately answers the question, but also adds meaningful elaboration, interpretation, or context that enhances the user's understanding. This goes beyond just listing relevant details — it offers insight into why the information matters, how it's useful, or what impact it has.
 
 **Example A**
-QUERY: What amenities does the new apartment complex provide?
+CONVERSATION_HISTORY: What amenities does the new apartment complex provide?
 RESPONSE: The apartment complex provides a gym, swimming pool, and 24/7 security, designed to offer residents a comfortable and active lifestyle while ensuring their safety.
 
 Expected Output:
@@ -171,7 +175,7 @@ Expected Output:
 }
 
 **Example B**
-QUERY: What services does the premium membership include?
+CONVERSATION_HISTORY: What services does the premium membership include?
 RESPONSE: The premium membership includes priority customer support, exclusive content access, and early product releases — tailored for users who want quicker resolutions and first access to new features.
 
 Expected Output:
@@ -179,3 +183,16 @@ Expected Output:
     "explanation": "The response covers all essential services and adds valuable insight about the target user and benefits, enriching the response beyond basic listing.",
     "score": 5
 }
+
+### Multi-turn Conversation Example
+When evaluating responses in a multi-turn conversation, consider the conversation context to understand the user's intent:
+
+**Example - Multi-turn Context**
+CONVERSATION_HISTORY: [{"role":"user","content":"I'm planning a vacation to Europe."},{"role":"assistant","content":"That sounds exciting! What time of year are you thinking of traveling?"},{"role":"user","content":"Probably in July. What's the weather like then?"}]
+RESPONSE: [{"role":"assistant","content":"July is summer in Europe with generally warm and pleasant weather. Most countries have temperatures between 20-25°C (68-77°F). It's a popular travel time, so expect crowds at major tourist attractions and higher accommodation prices."}]
+
+Expected Output:
+{
+    "explanation": "The response directly addresses the weather question while providing valuable context about crowds and pricing that's relevant to vacation planning established in the conversation.",
+    "score": 5
+}

azure/ai/evaluation/_evaluators/_response_completeness/_response_completeness.py

@@ -3,6 +3,7 @@
 # ---------------------------------------------------------
 
 import os
+import logging
 import math
 from typing import Dict, List, Union, Optional
 
@@ -14,39 +15,37 @@ from azure.ai.evaluation._common.utils import parse_quality_evaluator_reason_sco
 from azure.ai.evaluation._model_configurations import Conversation, Message
 from azure.ai.evaluation._common._experimental import experimental
 
+logger = logging.getLogger(__name__)
+
 
 @experimental
 class ResponseCompletenessEvaluator(PromptyEvaluatorBase[Union[str, float]]):
-    """
-    Evaluates the extent to which a given response contains all necessary and relevant information with respect to the
-     provided ground truth.
+    """Evaluates the extent to which a given response contains all necessary and relevant information with respect to the
+    provided ground truth.
+
     The completeness measure assesses how thoroughly an AI model's generated response aligns with the key information,
     claims, and statements established in the ground truth. This evaluation considers the presence, accuracy,
     and relevance of the content provided.
+
     The assessment spans multiple levels, ranging from fully incomplete to fully complete, ensuring a comprehensive
     evaluation of the response's content quality.
+
     Use this metric when you need to evaluate an AI model's ability to deliver comprehensive and accurate information,
     particularly in text generation tasks where conveying all essential details is crucial for clarity,
     context, and correctness.
+
     Completeness scores range from 1 to 5:
+
     1: Fully incomplete — Contains none of the necessary information.
     2: Barely complete — Contains only a small portion of the required information.
     3: Moderately complete — Covers about half of the required content.
     4: Mostly complete — Includes most of the necessary details with minimal omissions.
     5: Fully complete — Contains all key information without any omissions.
+
     :param model_config: Configuration for the Azure OpenAI model.
     :type model_config: Union[~azure.ai.evaluation.AzureOpenAIModelConfiguration,
         ~azure.ai.evaluation.OpenAIModelConfiguration]
 
-    .. admonition:: Example:
-
-        .. literalinclude:: ../samples/evaluation_samples_evaluate.py
-            :start-after: [START completeness_evaluator]
-            :end-before: [END completeness_evaluator]
-            :language: python
-            :dedent: 8
-            :caption: Initialize and call a CompletenessEvaluator with a response and groundtruth.
-
     .. admonition:: Example using Azure AI Project URL:
 
         .. literalinclude:: ../samples/evaluation_samples_evaluate_fdp.py
@@ -78,12 +77,14 @@ class ResponseCompletenessEvaluator(PromptyEvaluatorBase[Union[str, float]]):
     ):
         current_dir = os.path.dirname(__file__)
         prompty_path = os.path.join(current_dir, self._PROMPTY_FILE)
-        self.threshold = threshold
+        self.threshold = threshold  # to be removed in favor of _threshold
         super().__init__(
             model_config=model_config,
             prompty_file=prompty_path,
             result_key=self._RESULT_KEY,
+            threshold=threshold,
             credential=credential,
+            _higher_is_better=True,
             **kwargs,
         )
 
@@ -160,20 +161,42 @@ class ResponseCompletenessEvaluator(PromptyEvaluatorBase[Union[str, float]]):
                 target=ErrorTarget.COMPLETENESS_EVALUATOR,
             )
 
-        llm_output = await self._flow(timeout=self._LLM_CALL_TIMEOUT, **eval_input)
+        result = await self._flow(timeout=self._LLM_CALL_TIMEOUT, **eval_input)
+        llm_output = result.get("llm_output") if isinstance(result, dict) else result
 
         score = math.nan
-        if llm_output:
-            score, reason = parse_quality_evaluator_reason_score(llm_output, valid_score_range="[1-5]")
+        llm_output_is_dict = isinstance(llm_output, dict)
+        if llm_output_is_dict or isinstance(llm_output, str):
+            reason = ""
+            if llm_output_is_dict:
+                score = float(llm_output.get("score", math.nan))
+                reason = llm_output.get("explanation", "")
+            else:
+                score, reason = parse_quality_evaluator_reason_score(llm_output, valid_score_range="[1-5]")
 
-            score_result = "pass" if score >= self.threshold else "fail"
+            binary_result = self._get_binary_result(score)
 
             # updating the result key and threshold to int based on the schema
             return {
                 f"{self._result_key}": int(score),
-                f"{self._result_key}_result": score_result,
-                f"{self._result_key}_threshold": int(self.threshold),
+                f"{self._result_key}_result": binary_result,
+                f"{self._result_key}_threshold": int(self._threshold),
                 f"{self._result_key}_reason": reason,
+                f"{self._result_key}_prompt_tokens": result.get("input_token_count", 0),
+                f"{self._result_key}_completion_tokens": result.get("output_token_count", 0),
+                f"{self._result_key}_total_tokens": result.get("total_token_count", 0),
+                f"{self._result_key}_finish_reason": result.get("finish_reason", ""),
+                f"{self._result_key}_model": result.get("model_id", ""),
+                f"{self._result_key}_sample_input": result.get("sample_input", ""),
+                f"{self._result_key}_sample_output": result.get("sample_output", ""),
             }
 
-        return {self._result_key: math.nan}
+        if logger:
+            logger.warning("LLM output is not a dictionary, returning NaN for the score.")
+
+        binary_result = self._get_binary_result(score)
+        return {
+            self._result_key: float(score),
+            f"{self._result_key}_result": binary_result,
+            f"{self._result_key}_threshold": self._threshold,
+        }

azure/ai/evaluation/_evaluators/_retrieval/_retrieval.py

@@ -33,6 +33,11 @@ class RetrievalEvaluator(PromptyEvaluatorBase[Union[str, float]]):
         ~azure.ai.evaluation.OpenAIModelConfiguration]
     :param threshold: The threshold for the evaluation. Default is 3.
     :type threshold: float
+    :param credential: The credential for authenticating to Azure AI service.
+    :type credential: ~azure.core.credentials.TokenCredential
+    :keyword is_reasoning_model: If True, the evaluator will use reasoning model configuration (o1/o3 models).
+        This will adjust parameters like max_completion_tokens and remove unsupported parameters. Default is False.
+    :paramtype is_reasoning_model: bool
     :return: A function that evaluates and generates metrics for "chat" scenario.
     :rtype: Callable
 
@@ -78,7 +83,7 @@ class RetrievalEvaluator(PromptyEvaluatorBase[Union[str, float]]):
     """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
 
     @override
-    def __init__(self, model_config, *, threshold: float = 3, credential=None):
+    def __init__(self, model_config, *, threshold: float = 3, credential=None, **kwargs):
         current_dir = os.path.dirname(__file__)
         prompty_path = os.path.join(current_dir, self._PROMPTY_FILE)
         self._threshold = threshold
@@ -90,6 +95,7 @@ class RetrievalEvaluator(PromptyEvaluatorBase[Union[str, float]]):
             threshold=threshold,
             credential=credential,
             _higher_is_better=self._higher_is_better,
+            **kwargs,
         )
 
     @overload

azure/ai/evaluation/_evaluators/_similarity/_similarity.py

@@ -30,6 +30,11 @@ class SimilarityEvaluator(PromptyEvaluatorBase):
         ~azure.ai.evaluation.OpenAIModelConfiguration]
     :param threshold: The threshold for the similarity evaluator. Default is 3.
     :type threshold: int
+    :param credential: The credential for authenticating to Azure AI service.
+    :type credential: ~azure.core.credentials.TokenCredential
+    :keyword is_reasoning_model: If True, the evaluator will use reasoning model configuration (o1/o3 models).
+        This will adjust parameters like max_completion_tokens and remove unsupported parameters. Default is False.
+    :paramtype is_reasoning_model: bool
 
     .. admonition:: Example:
 
@@ -75,7 +80,7 @@ class SimilarityEvaluator(PromptyEvaluatorBase):
     """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
 
     @override
-    def __init__(self, model_config, *, threshold=3, credential=None):
+    def __init__(self, model_config, *, threshold=3, credential=None, **kwargs):
         current_dir = os.path.dirname(__file__)
         prompty_path = os.path.join(current_dir, self._PROMPTY_FILE)
         self._threshold = threshold
@@ -87,6 +92,7 @@ class SimilarityEvaluator(PromptyEvaluatorBase):
             threshold=threshold,
             credential=credential,
             _higher_is_better=self._higher_is_better,
+            **kwargs,
         )
 
     # Ignoring a mypy error about having only 1 overload function.

azure/ai/evaluation/_evaluators/_task_adherence/_task_adherence.py

@@ -10,7 +10,11 @@ from typing_extensions import overload, override
 
 from azure.ai.evaluation._exceptions import EvaluationException, ErrorBlame, ErrorCategory, ErrorTarget
 from azure.ai.evaluation._evaluators._common import PromptyEvaluatorBase
-from ..._common.utils import reformat_conversation_history, reformat_agent_response, reformat_tool_definitions
+from ..._common.utils import (
+    reformat_conversation_history,
+    reformat_agent_response,
+    reformat_tool_definitions,
+)
 from azure.ai.evaluation._model_configurations import Message
 from azure.ai.evaluation._common._experimental import experimental
 
@@ -40,6 +44,7 @@ class TaskAdherenceEvaluator(PromptyEvaluatorBase[Union[str, float]]):
         ~azure.ai.evaluation.OpenAIModelConfiguration]
 
     .. admonition:: Example:
+
         .. literalinclude:: ../samples/evaluation_samples_evaluate.py
             :start-after: [START task_adherence_evaluator]
             :end-before: [END task_adherence_evaluator]
@@ -72,12 +77,14 @@ class TaskAdherenceEvaluator(PromptyEvaluatorBase[Union[str, float]]):
     def __init__(self, model_config, *, threshold=_DEFAULT_TASK_ADHERENCE_SCORE, credential=None, **kwargs):
         current_dir = os.path.dirname(__file__)
         prompty_path = os.path.join(current_dir, self._PROMPTY_FILE)
-        self.threshold = threshold
+        self.threshold = threshold  # to be removed in favor of _threshold
         super().__init__(
             model_config=model_config,
             prompty_file=prompty_path,
             result_key=self._RESULT_KEY,
+            threshold=threshold,
             credential=credential,
+            _higher_is_better=True,
             **kwargs,
         )
 
@@ -153,19 +160,38 @@ class TaskAdherenceEvaluator(PromptyEvaluatorBase[Union[str, float]]):
         eval_input["response"] = reformat_agent_response(eval_input["response"], logger, include_tool_messages=True)
         if "tool_definitions" in eval_input and eval_input["tool_definitions"] is not None:
             eval_input["tool_definitions"] = reformat_tool_definitions(eval_input["tool_definitions"], logger)
-        llm_output = await self._flow(timeout=self._LLM_CALL_TIMEOUT, **eval_input)
+
+        prompty_output_dict = await self._flow(timeout=self._LLM_CALL_TIMEOUT, **eval_input)
+        llm_output = prompty_output_dict["llm_output"]
+
+        score = math.nan
         if isinstance(llm_output, dict):
             score = float(llm_output.get("score", math.nan))
-            score_result = "pass" if score >= self.threshold else "fail"
+            score_result = "pass" if score >= self._threshold else "fail"
             reason = llm_output.get("explanation", "")
             return {
                 f"{self._result_key}": score,
+                f"gpt_{self._result_key}": score,
                 f"{self._result_key}_result": score_result,
-                f"{self._result_key}_threshold": self.threshold,
+                f"{self._result_key}_threshold": self._threshold,
                 f"{self._result_key}_reason": reason,
                 # Uncomment the following line in the next iteration after UI contracts are validated.
                 # f"{self._result_key}_additional_details": llm_output
+                f"{self._result_key}_prompt_tokens": prompty_output_dict.get("input_token_count", 0),
+                f"{self._result_key}_completion_tokens": prompty_output_dict.get("output_token_count", 0),
+                f"{self._result_key}_total_tokens": prompty_output_dict.get("total_token_count", 0),
+                f"{self._result_key}_finish_reason": prompty_output_dict.get("finish_reason", ""),
+                f"{self._result_key}_model": prompty_output_dict.get("model_id", ""),
+                f"{self._result_key}_sample_input": prompty_output_dict.get("sample_input", ""),
+                f"{self._result_key}_sample_output": prompty_output_dict.get("sample_output", ""),
             }
         if logger:
             logger.warning("LLM output is not a dictionary, returning NaN for the score.")
-        return {self._result_key: math.nan}
+
+        binary_result = self._get_binary_result(score)
+        return {
+            self._result_key: float(score),
+            f"gpt_{self._result_key}": float(score),
+            f"{self._result_key}_result": binary_result,
+            f"{self._result_key}_threshold": self._threshold,
+        }

azure/ai/evaluation/_evaluators/_task_completion/__init__.py

@@ -0,0 +1,7 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from ._task_completion import _TaskCompletionEvaluator
+
+__all__ = ["_TaskCompletionEvaluator"]