azure-ai-evaluation 1.12.0__py3-none-any.whl → 1.13.0__py3-none-any.whl

azure/ai/evaluation/_evaluators/_intent_resolution/_intent_resolution.py

@@ -69,7 +69,9 @@ class IntentResolutionEvaluator(PromptyEvaluatorBase[Union[str, float]]):
             model_config=model_config,
             prompty_file=prompty_path,
             result_key=self._RESULT_KEY,
+            threshold=threshold,
             credential=credential,
+            _higher_is_better=True,
             **kwargs,
         )
 
@@ -145,8 +147,10 @@ class IntentResolutionEvaluator(PromptyEvaluatorBase[Union[str, float]]):
         eval_input["query"] = reformat_conversation_history(eval_input["query"], logger)
         eval_input["response"] = reformat_agent_response(eval_input["response"], logger)
 
-        llm_output = await self._flow(timeout=self._LLM_CALL_TIMEOUT, **eval_input)
+        prompty_output_dict = await self._flow(timeout=self._LLM_CALL_TIMEOUT, **eval_input)
+        llm_output = prompty_output_dict["llm_output"]
         # llm_output should always be a dictionary because the response_format of prompty is set to json_object, but checking anyway
+        score = math.nan
         if isinstance(llm_output, dict):
             score = llm_output.get("score", math.nan)
             if not check_score_is_valid(
@@ -162,16 +166,31 @@ class IntentResolutionEvaluator(PromptyEvaluatorBase[Union[str, float]]):
                 )
             reason = llm_output.get("explanation", "")
             score = float(score)
-            score_result = "pass" if score >= self.threshold else "fail"
+            score_result = "pass" if score >= self._threshold else "fail"
 
             response_dict = {
                 f"{self._result_key}": score,
+                f"gpt_{self._result_key}": score,
                 f"{self._result_key}_result": score_result,
-                f"{self._result_key}_threshold": self.threshold,
+                f"{self._result_key}_threshold": self._threshold,
                 f"{self._result_key}_reason": reason,
+                f"{self._result_key}_prompt_tokens": prompty_output_dict.get("input_token_count", 0),
+                f"{self._result_key}_completion_tokens": prompty_output_dict.get("output_token_count", 0),
+                f"{self._result_key}_total_tokens": prompty_output_dict.get("total_token_count", 0),
+                f"{self._result_key}_finish_reason": prompty_output_dict.get("finish_reason", ""),
+                f"{self._result_key}_model": prompty_output_dict.get("model_id", ""),
+                f"{self._result_key}_sample_input": prompty_output_dict.get("sample_input", ""),
+                f"{self._result_key}_sample_output": prompty_output_dict.get("sample_output", ""),
             }
             return response_dict
         # If llm_output is not a dictionary, return NaN for the score. This should never happen
         if logger:
             logger.warning("LLM output is not a dictionary, returning NaN for the score.")
-        return {self._result_key: math.nan}
+
+        binary_result = self._get_binary_result(score)
+        return {
+            self._result_key: float(score),
+            f"gpt_{self._result_key}": float(score),
+            f"{self._result_key}_result": binary_result,
+            f"{self._result_key}_threshold": self._threshold,
+        }

azure/ai/evaluation/_evaluators/_relevance/_relevance.py

@@ -87,15 +87,13 @@ class RelevanceEvaluator(PromptyEvaluatorBase):
     def __init__(self, model_config, *, credential=None, threshold=3, **kwargs):
         current_dir = os.path.dirname(__file__)
         prompty_path = os.path.join(current_dir, self._PROMPTY_FILE)
-        self._threshold = threshold
-        self._higher_is_better = True
         super().__init__(
             model_config=model_config,
             prompty_file=prompty_path,
             result_key=self._RESULT_KEY,
             threshold=threshold,
             credential=credential,
-            _higher_is_better=self._higher_is_better,
+            _higher_is_better=True,
             **kwargs,
         )
 
@@ -177,7 +175,8 @@ class RelevanceEvaluator(PromptyEvaluatorBase):
             eval_input["query"] = reformat_conversation_history(eval_input["query"], logger)
         if not isinstance(eval_input["response"], str):
             eval_input["response"] = reformat_agent_response(eval_input["response"], logger)
-        llm_output = await self._flow(timeout=self._LLM_CALL_TIMEOUT, **eval_input)
+        result = await self._flow(timeout=self._LLM_CALL_TIMEOUT, **eval_input)
+        llm_output = result.get("llm_output")
         score = math.nan
 
         if isinstance(llm_output, dict):
@@ -188,15 +187,24 @@ class RelevanceEvaluator(PromptyEvaluatorBase):
             return {
                 self._result_key: float(score),
                 f"gpt_{self._result_key}": float(score),
-                f"{self._result_key}_reason": reason,
                 f"{self._result_key}_result": binary_result,
                 f"{self._result_key}_threshold": self._threshold,
+                f"{self._result_key}_reason": reason,
+                f"{self._result_key}_prompt_tokens": result.get("input_token_count", 0),
+                f"{self._result_key}_completion_tokens": result.get("output_token_count", 0),
+                f"{self._result_key}_total_tokens": result.get("total_token_count", 0),
+                f"{self._result_key}_finish_reason": result.get("finish_reason", ""),
+                f"{self._result_key}_model": result.get("model_id", ""),
+                f"{self._result_key}_sample_input": result.get("sample_input", ""),
+                f"{self._result_key}_sample_output": result.get("sample_output", ""),
             }
 
+        if logger:
+            logger.warning("LLM output is not a dictionary, returning NaN for the score.")
+
         binary_result = self._get_binary_result(score)
         return {
             self._result_key: float(score),
-            f"gpt_{self._result_key}": float(score),
             f"{self._result_key}_result": binary_result,
             f"{self._result_key}_threshold": self._threshold,
         }

azure/ai/evaluation/_evaluators/_relevance/relevance.prompty

@@ -20,22 +20,25 @@ inputs:
 ---
 
 system:
-You are a Relevance-Judge, an impartial evaluator that scores how well the RESPONSE addresses the QUERY using the definitions provided.
+You are a Relevance-Judge, an impartial evaluator that scores how well the RESPONSE addresses the user's queries in the CONVERSATION_HISTORY using the definitions provided.
 
 user:
 ROLE
 ====
-You are a Relevance Evaluator. Your task is to judge how relevant a RESPONSE is to a QUERY using the Relevance definitions provided.
+You are a Relevance Evaluator. Your task is to judge how relevant a RESPONSE is to the CONVERSATION_HISTORY using the Relevance definitions provided.
 
 INPUT
 =====
-QUERY: {{query}}
+CONVERSATION_HISTORY: {{query}}
 RESPONSE: {{response}}
 
+CONVERSATION_HISTORY is the full dialogue between the user and the agent up to the user's latest message. For single-turn interactions, this will be just the user's query.
+RESPONSE is the agent's reply to the user's latest message.
+
 TASK
 ====
 Output a JSON object with:
-    1) a concise explanation of 15-60 words justifying your score based on how well the response is relevant to the query.
+    1) a concise explanation of 15-60 words justifying your score based on how well the response is relevant to the user's queries in the CONVERSATION_HISTORY.
     2) an integer score from 1 (very poor) to 5 (excellent) using the rubric below.
 
 The explanation should always precede the score and should clearly justify the score based on the rubric definitions.
@@ -49,13 +52,14 @@ Response format exactly as follows:
 
 EVALUATION STEPS
 ================
-A. Read the QUERY and RESPONSE carefully.
-B. Compare the RESPONSE against the rubric below:
-    - Does the response directly address the query?
+A. Read the CONVERSATION_HISTORY and RESPONSE carefully.
+B. Identify the user's query from the latest message (use conversation history for context if needed).
+C. Compare the RESPONSE against the rubric below:
+    - Does the response directly address the user's query?
     - Is the information complete, partial, or off-topic?
     - Is it vague, generic, or insightful?
-C. Match the response to the best score from the rubric.
-D. Provide a short explanation and the score using the required format.
+D. Match the response to the best score from the rubric.
+E. Provide a short explanation and the score using the required format.
 
 SCORING RUBRIC
 ==============
@@ -64,7 +68,7 @@ SCORING RUBRIC
 Definition: The response is unrelated to the question. It provides off-topic information and does not attempt to address the question posed.
 
 **Example A**
-QUERY: What is the team preparing for?
+CONVERSATION_HISTORY: What is the team preparing for?
 RESPONSE: I went grocery shopping yesterday evening.
 
 Expected Output:
@@ -75,7 +79,7 @@ Expected Output:
 
 
 **Example B**
-QUERY: When will the company's new product line launch?
+CONVERSATION_HISTORY: When will the company's new product line launch?
 RESPONSE: International travel can be very rewarding and educational.
 
 Expected Output:
@@ -89,7 +93,7 @@ Expected Output:
 Definition: The response is loosely or formally related to the query but fails to deliver any meaningful, specific, or useful information. This includes vague phrases, non-answers, or failure/error messages.
 
 **Example A**
-QUERY: What is the event about?
+CONVERSATION_HISTORY: What is the event about?
 RESPONSE: It’s something important.
 
 Expected Output:
@@ -99,7 +103,7 @@ Expected Output:
 }
 
 **Example B**
-QUERY: What’s the weather in Paris?
+CONVERSATION_HISTORY: What’s the weather in Paris?
 RESPONSE: I tried to find the forecast but the query failed.
 
 Expected Output:
@@ -112,7 +116,7 @@ Expected Output:
 Definition: The response addresses the query and includes relevant information, but omits essential components or detail. The answer is on-topic but insufficient to fully satisfy the request.
 
 **Example A**
-QUERY: What amenities does the new apartment complex provide?
+CONVERSATION_HISTORY: What amenities does the new apartment complex provide?
 RESPONSE: The apartment complex has a gym.
 
 Expected Output:
@@ -122,7 +126,7 @@ Expected Output:
 }
 
 **Example B**
-QUERY: What services does the premium membership include?
+CONVERSATION_HISTORY: What services does the premium membership include?
 RESPONSE: It includes priority customer support.
 
 Expected Output:
@@ -137,7 +141,7 @@ Expected Output:
 Definition: The response fully addresses the question with accurate and sufficient information, covering all essential aspects. Very minor omissions are acceptable as long as the core information is intact and the intent is clearly conveyed.
 
 **Example A**
-QUERY: What amenities does the new apartment complex provide?
+CONVERSATION_HISTORY: What amenities does the new apartment complex provide?
 RESPONSE: The apartment complex provides a gym, swimming pool, and 24/7 security.
 
 Expected Output:
@@ -147,7 +151,7 @@ Expected Output:
 }
 
 **Example B**
-QUERY: What services does the premium membership include?
+CONVERSATION_HISTORY: What services does the premium membership include?
 RESPONSE: The premium membership includes priority customer support, exclusive content access, and early product releases.
 
 Expected Output:
@@ -161,7 +165,7 @@ Expected Output:
 Definition: The response not only fully and accurately answers the question, but also adds meaningful elaboration, interpretation, or context that enhances the user's understanding. This goes beyond just listing relevant details — it offers insight into why the information matters, how it's useful, or what impact it has.
 
 **Example A**
-QUERY: What amenities does the new apartment complex provide?
+CONVERSATION_HISTORY: What amenities does the new apartment complex provide?
 RESPONSE: The apartment complex provides a gym, swimming pool, and 24/7 security, designed to offer residents a comfortable and active lifestyle while ensuring their safety.
 
 Expected Output:
@@ -171,7 +175,7 @@ Expected Output:
 }
 
 **Example B**
-QUERY: What services does the premium membership include?
+CONVERSATION_HISTORY: What services does the premium membership include?
 RESPONSE: The premium membership includes priority customer support, exclusive content access, and early product releases — tailored for users who want quicker resolutions and first access to new features.
 
 Expected Output:
@@ -179,3 +183,16 @@ Expected Output:
     "explanation": "The response covers all essential services and adds valuable insight about the target user and benefits, enriching the response beyond basic listing.",
     "score": 5
 }
+
+### Multi-turn Conversation Example
+When evaluating responses in a multi-turn conversation, consider the conversation context to understand the user's intent:
+
+**Example - Multi-turn Context**
+CONVERSATION_HISTORY: [{"role":"user","content":"I'm planning a vacation to Europe."},{"role":"assistant","content":"That sounds exciting! What time of year are you thinking of traveling?"},{"role":"user","content":"Probably in July. What's the weather like then?"}]
+RESPONSE: [{"role":"assistant","content":"July is summer in Europe with generally warm and pleasant weather. Most countries have temperatures between 20-25°C (68-77°F). It's a popular travel time, so expect crowds at major tourist attractions and higher accommodation prices."}]
+
+Expected Output:
+{
+    "explanation": "The response directly addresses the weather question while providing valuable context about crowds and pricing that's relevant to vacation planning established in the conversation.",
+    "score": 5
+}

azure/ai/evaluation/_evaluators/_response_completeness/_response_completeness.py

@@ -3,6 +3,7 @@
 # ---------------------------------------------------------
 
 import os
+import logging
 import math
 from typing import Dict, List, Union, Optional
 
@@ -14,39 +15,37 @@ from azure.ai.evaluation._common.utils import parse_quality_evaluator_reason_sco
 from azure.ai.evaluation._model_configurations import Conversation, Message
 from azure.ai.evaluation._common._experimental import experimental
 
+logger = logging.getLogger(__name__)
+
 
 @experimental
 class ResponseCompletenessEvaluator(PromptyEvaluatorBase[Union[str, float]]):
-    """
-    Evaluates the extent to which a given response contains all necessary and relevant information with respect to the
-     provided ground truth.
+    """Evaluates the extent to which a given response contains all necessary and relevant information with respect to the
+    provided ground truth.
+
     The completeness measure assesses how thoroughly an AI model's generated response aligns with the key information,
     claims, and statements established in the ground truth. This evaluation considers the presence, accuracy,
     and relevance of the content provided.
+
     The assessment spans multiple levels, ranging from fully incomplete to fully complete, ensuring a comprehensive
     evaluation of the response's content quality.
+
     Use this metric when you need to evaluate an AI model's ability to deliver comprehensive and accurate information,
     particularly in text generation tasks where conveying all essential details is crucial for clarity,
     context, and correctness.
+
     Completeness scores range from 1 to 5:
+
     1: Fully incomplete — Contains none of the necessary information.
     2: Barely complete — Contains only a small portion of the required information.
     3: Moderately complete — Covers about half of the required content.
     4: Mostly complete — Includes most of the necessary details with minimal omissions.
     5: Fully complete — Contains all key information without any omissions.
+
     :param model_config: Configuration for the Azure OpenAI model.
     :type model_config: Union[~azure.ai.evaluation.AzureOpenAIModelConfiguration,
         ~azure.ai.evaluation.OpenAIModelConfiguration]
 
-    .. admonition:: Example:
-
-        .. literalinclude:: ../samples/evaluation_samples_evaluate.py
-            :start-after: [START completeness_evaluator]
-            :end-before: [END completeness_evaluator]
-            :language: python
-            :dedent: 8
-            :caption: Initialize and call a CompletenessEvaluator with a response and groundtruth.
-
     .. admonition:: Example using Azure AI Project URL:
 
         .. literalinclude:: ../samples/evaluation_samples_evaluate_fdp.py
@@ -78,12 +77,14 @@ class ResponseCompletenessEvaluator(PromptyEvaluatorBase[Union[str, float]]):
     ):
         current_dir = os.path.dirname(__file__)
         prompty_path = os.path.join(current_dir, self._PROMPTY_FILE)
-        self.threshold = threshold
+        self.threshold = threshold  # to be removed in favor of _threshold
         super().__init__(
             model_config=model_config,
             prompty_file=prompty_path,
             result_key=self._RESULT_KEY,
+            threshold=threshold,
             credential=credential,
+            _higher_is_better=True,
             **kwargs,
         )
 
@@ -160,20 +161,42 @@ class ResponseCompletenessEvaluator(PromptyEvaluatorBase[Union[str, float]]):
                 target=ErrorTarget.COMPLETENESS_EVALUATOR,
             )
 
-        llm_output = await self._flow(timeout=self._LLM_CALL_TIMEOUT, **eval_input)
+        result = await self._flow(timeout=self._LLM_CALL_TIMEOUT, **eval_input)
+        llm_output = result.get("llm_output") if isinstance(result, dict) else result
 
         score = math.nan
-        if llm_output:
-            score, reason = parse_quality_evaluator_reason_score(llm_output, valid_score_range="[1-5]")
+        llm_output_is_dict = isinstance(llm_output, dict)
+        if llm_output_is_dict or isinstance(llm_output, str):
+            reason = ""
+            if llm_output_is_dict:
+                score = float(llm_output.get("score", math.nan))
+                reason = llm_output.get("explanation", "")
+            else:
+                score, reason = parse_quality_evaluator_reason_score(llm_output, valid_score_range="[1-5]")
 
-            score_result = "pass" if score >= self.threshold else "fail"
+            binary_result = self._get_binary_result(score)
 
             # updating the result key and threshold to int based on the schema
             return {
                 f"{self._result_key}": int(score),
-                f"{self._result_key}_result": score_result,
-                f"{self._result_key}_threshold": int(self.threshold),
+                f"{self._result_key}_result": binary_result,
+                f"{self._result_key}_threshold": int(self._threshold),
                 f"{self._result_key}_reason": reason,
+                f"{self._result_key}_prompt_tokens": result.get("input_token_count", 0),
+                f"{self._result_key}_completion_tokens": result.get("output_token_count", 0),
+                f"{self._result_key}_total_tokens": result.get("total_token_count", 0),
+                f"{self._result_key}_finish_reason": result.get("finish_reason", ""),
+                f"{self._result_key}_model": result.get("model_id", ""),
+                f"{self._result_key}_sample_input": result.get("sample_input", ""),
+                f"{self._result_key}_sample_output": result.get("sample_output", ""),
             }
 
-        return {self._result_key: math.nan}
+        if logger:
+            logger.warning("LLM output is not a dictionary, returning NaN for the score.")
+
+        binary_result = self._get_binary_result(score)
+        return {
+            self._result_key: float(score),
+            f"{self._result_key}_result": binary_result,
+            f"{self._result_key}_threshold": self._threshold,
+        }

azure/ai/evaluation/_evaluators/_task_adherence/_task_adherence.py

@@ -10,7 +10,11 @@ from typing_extensions import overload, override
 
 from azure.ai.evaluation._exceptions import EvaluationException, ErrorBlame, ErrorCategory, ErrorTarget
 from azure.ai.evaluation._evaluators._common import PromptyEvaluatorBase
-from ..._common.utils import reformat_conversation_history, reformat_agent_response, reformat_tool_definitions
+from ..._common.utils import (
+    reformat_conversation_history,
+    reformat_agent_response,
+    reformat_tool_definitions,
+)
 from azure.ai.evaluation._model_configurations import Message
 from azure.ai.evaluation._common._experimental import experimental
 
@@ -40,6 +44,7 @@ class TaskAdherenceEvaluator(PromptyEvaluatorBase[Union[str, float]]):
         ~azure.ai.evaluation.OpenAIModelConfiguration]
 
     .. admonition:: Example:
+
         .. literalinclude:: ../samples/evaluation_samples_evaluate.py
             :start-after: [START task_adherence_evaluator]
             :end-before: [END task_adherence_evaluator]
@@ -72,12 +77,14 @@ class TaskAdherenceEvaluator(PromptyEvaluatorBase[Union[str, float]]):
     def __init__(self, model_config, *, threshold=_DEFAULT_TASK_ADHERENCE_SCORE, credential=None, **kwargs):
         current_dir = os.path.dirname(__file__)
         prompty_path = os.path.join(current_dir, self._PROMPTY_FILE)
-        self.threshold = threshold
+        self.threshold = threshold  # to be removed in favor of _threshold
         super().__init__(
             model_config=model_config,
             prompty_file=prompty_path,
             result_key=self._RESULT_KEY,
+            threshold=threshold,
             credential=credential,
+            _higher_is_better=True,
             **kwargs,
         )
 
@@ -153,19 +160,38 @@ class TaskAdherenceEvaluator(PromptyEvaluatorBase[Union[str, float]]):
         eval_input["response"] = reformat_agent_response(eval_input["response"], logger, include_tool_messages=True)
         if "tool_definitions" in eval_input and eval_input["tool_definitions"] is not None:
             eval_input["tool_definitions"] = reformat_tool_definitions(eval_input["tool_definitions"], logger)
-        llm_output = await self._flow(timeout=self._LLM_CALL_TIMEOUT, **eval_input)
+
+        prompty_output_dict = await self._flow(timeout=self._LLM_CALL_TIMEOUT, **eval_input)
+        llm_output = prompty_output_dict["llm_output"]
+
+        score = math.nan
         if isinstance(llm_output, dict):
             score = float(llm_output.get("score", math.nan))
-            score_result = "pass" if score >= self.threshold else "fail"
+            score_result = "pass" if score >= self._threshold else "fail"
             reason = llm_output.get("explanation", "")
             return {
                 f"{self._result_key}": score,
+                f"gpt_{self._result_key}": score,
                 f"{self._result_key}_result": score_result,
-                f"{self._result_key}_threshold": self.threshold,
+                f"{self._result_key}_threshold": self._threshold,
                 f"{self._result_key}_reason": reason,
                 # Uncomment the following line in the next iteration after UI contracts are validated.
                 # f"{self._result_key}_additional_details": llm_output
+                f"{self._result_key}_prompt_tokens": prompty_output_dict.get("input_token_count", 0),
+                f"{self._result_key}_completion_tokens": prompty_output_dict.get("output_token_count", 0),
+                f"{self._result_key}_total_tokens": prompty_output_dict.get("total_token_count", 0),
+                f"{self._result_key}_finish_reason": prompty_output_dict.get("finish_reason", ""),
+                f"{self._result_key}_model": prompty_output_dict.get("model_id", ""),
+                f"{self._result_key}_sample_input": prompty_output_dict.get("sample_input", ""),
+                f"{self._result_key}_sample_output": prompty_output_dict.get("sample_output", ""),
             }
         if logger:
             logger.warning("LLM output is not a dictionary, returning NaN for the score.")
-        return {self._result_key: math.nan}
+
+        binary_result = self._get_binary_result(score)
+        return {
+            self._result_key: float(score),
+            f"gpt_{self._result_key}": float(score),
+            f"{self._result_key}_result": binary_result,
+            f"{self._result_key}_threshold": self._threshold,
+        }

azure/ai/evaluation/_evaluators/{_path_efficiency → _task_completion}/__init__.py

@@ -2,6 +2,6 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
 
-from ._path_efficiency import PathEfficiencyEvaluator
+from ._task_completion import _TaskCompletionEvaluator
 
-__all__ = ["PathEfficiencyEvaluator"]
+__all__ = ["_TaskCompletionEvaluator"]

azure/ai/evaluation/_evaluators/{_task_success/_task_success.py → _task_completion/_task_completion.py}

@@ -18,8 +18,8 @@ logger = logging.getLogger(__name__)
 
 
 @experimental
-class TaskSuccessEvaluator(PromptyEvaluatorBase[Union[str, bool]]):
-    """The Task Success evaluator determines whether an AI agent successfully completed the requested task based on:
+class _TaskCompletionEvaluator(PromptyEvaluatorBase[Union[str, float]]):
+    """The Task Completion evaluator determines whether an AI agent successfully completed the requested task based on:
 
         - Final outcome and deliverable of the task
         - Completeness of task requirements
@@ -27,8 +27,8 @@ class TaskSuccessEvaluator(PromptyEvaluatorBase[Union[str, bool]]):
     This evaluator focuses solely on task completion and success, not on task adherence or intent understanding.
 
     Scoring is binary:
-    - TRUE: Task fully completed with usable deliverable that meets all user requirements
-    - FALSE: Task incomplete, partially completed, or deliverable does not meet requirements
+    - 1 (pass): Task fully completed with usable deliverable that meets all user requirements
+    - 0 (fail): Task incomplete, partially completed, or deliverable does not meet requirements
 
     The evaluation includes task requirement analysis, outcome assessment, and completion gap identification.
 
@@ -39,29 +39,29 @@ class TaskSuccessEvaluator(PromptyEvaluatorBase[Union[str, bool]]):
 
     .. admonition:: Example:
         .. literalinclude:: ../samples/evaluation_samples_evaluate.py
-            :start-after: [START task_success_evaluator]
-            :end-before: [END task_success_evaluator]
+            :start-after: [START task_completion_evaluator]
+            :end-before: [END task_completion_evaluator]
             :language: python
             :dedent: 8
-            :caption: Initialize and call a TaskSuccessEvaluator with a query and response.
+            :caption: Initialize and call a _TaskCompletionEvaluator with a query and response.
 
     .. admonition:: Example using Azure AI Project URL:
 
         .. literalinclude:: ../samples/evaluation_samples_evaluate_fdp.py
-            :start-after: [START task_success_evaluator]
-            :end-before: [END task_success_evaluator]
+            :start-after: [START task_completion_evaluator]
+            :end-before: [END task_completion_evaluator]
             :language: python
             :dedent: 8
-            :caption: Initialize and call TaskSuccessEvaluator using Azure AI Project URL in the following format
+            :caption: Initialize and call a _TaskCompletionEvaluator using Azure AI Project URL in the following format
                 https://{resource_name}.services.ai.azure.com/api/projects/{project_name}
 
     """
 
-    _PROMPTY_FILE = "task_success.prompty"
-    _RESULT_KEY = "task_success"
+    _PROMPTY_FILE = "task_completion.prompty"
+    _RESULT_KEY = "task_completion"
     _OPTIONAL_PARAMS = ["tool_definitions"]
 
-    id = "azureai://built-in/evaluators/task_success"
+    id = "azureai://built-in/evaluators/task_completion"
     """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
 
     @override
@@ -83,20 +83,20 @@ class TaskSuccessEvaluator(PromptyEvaluatorBase[Union[str, bool]]):
         query: Union[str, List[dict]],
         response: Union[str, List[dict]],
         tool_definitions: Optional[Union[dict, List[dict]]] = None,
-    ) -> Dict[str, Union[str, bool]]:
-        """Evaluate task success for a given query, response, and optionally tool definitions.
+    ) -> Dict[str, Union[str, float]]:
+        """Evaluate task completion for a given query, response, and optionally tool definitions.
         The query and response can be either a string or a list of messages.
 
 
         Example with string inputs and no tools:
-            evaluator = TaskSuccessEvaluator(model_config)
+            evaluator = _TaskCompletionEvaluator(model_config)
             query = "Plan a 3-day itinerary for Paris with cultural landmarks and local cuisine."
             response = "**Day 1:** Morning: Louvre Museum, Lunch: Le Comptoir du Relais..."
 
             result = evaluator(query=query, response=response)
 
         Example with list of messages:
-            evaluator = TaskSuccessEvaluator(model_config)
+            evaluator = _TaskCompletionEvaluator(model_config)
             query = [{'role': 'system', 'content': 'You are a helpful travel planning assistant.'}, {'createdAt': 1700000060, 'role': 'user', 'content': [{'type': 'text', 'text': 'Plan a 3-day Paris itinerary with cultural landmarks and cuisine'}]}]
             response = [{'createdAt': 1700000070, 'run_id': '0', 'role': 'assistant', 'content': [{'type': 'text', 'text': '**Day 1:** Morning: Visit Louvre Museum (9 AM - 12 PM)...'}]}]
             tool_definitions = [{'name': 'get_attractions', 'description': 'Get tourist attractions for a city.', 'parameters': {'type': 'object', 'properties': {'city': {'type': 'string', 'description': 'The city name.'}}}}]
@@ -109,8 +109,8 @@ class TaskSuccessEvaluator(PromptyEvaluatorBase[Union[str, bool]]):
         :paramtype response: Union[str, List[dict]]
         :keyword tool_definitions: An optional list of messages containing the tool definitions the agent is aware of.
         :paramtype tool_definitions: Optional[Union[dict, List[dict]]]
-        :return: A dictionary with the task success evaluation results.
-        :rtype: Dict[str, Union[str, bool]]
+        :return: A dictionary with the task completion evaluation results.
+        :rtype: Dict[str, Union[str, float]]
         """
 
     @override
@@ -127,8 +127,8 @@ class TaskSuccessEvaluator(PromptyEvaluatorBase[Union[str, bool]]):
         return super().__call__(*args, **kwargs)
 
     @override
-    async def _do_eval(self, eval_input: Dict) -> Dict[str, Union[bool, str]]:  # type: ignore[override]
-        """Do Task Success evaluation.
+    async def _do_eval(self, eval_input: Dict) -> Dict[str, Union[float, str]]:  # type: ignore[override]
+        """Do Task Completion evaluation.
         :param eval_input: The input to the evaluator. Expected to contain whatever inputs are needed for the _flow method
         :type eval_input: Dict
         :return: The evaluation result.
@@ -138,31 +138,40 @@ class TaskSuccessEvaluator(PromptyEvaluatorBase[Union[str, bool]]):
         # which is a different schema than _base_prompty_eval.py
         if "query" not in eval_input and "response" not in eval_input:
             raise EvaluationException(
-                message=f"Both query and response must be provided as input to the Task Success evaluator.",
-                internal_message=f"Both query and response must be provided as input to the Task Success evaluator.",
+                message=f"Both query and response must be provided as input to the Task Completion evaluator.",
+                internal_message=f"Both query and response must be provided as input to the Task Completion evaluator.",
                 blame=ErrorBlame.USER_ERROR,
                 category=ErrorCategory.MISSING_FIELD,
-                target=ErrorTarget.TASK_SUCCESS_EVALUATOR,
+                target=ErrorTarget.TASK_COMPLETION_EVALUATOR,
             )
         eval_input["query"] = reformat_conversation_history(eval_input["query"], logger, include_system_messages=True)
         eval_input["response"] = reformat_agent_response(eval_input["response"], logger, include_tool_messages=True)
         if "tool_definitions" in eval_input and eval_input["tool_definitions"] is not None:
             eval_input["tool_definitions"] = reformat_tool_definitions(eval_input["tool_definitions"], logger)
 
-        llm_output = await self._flow(timeout=self._LLM_CALL_TIMEOUT, **eval_input)
+        prompty_output_dict = await self._flow(timeout=self._LLM_CALL_TIMEOUT, **eval_input)
+        llm_output = prompty_output_dict.get("llm_output", {})
+
         if isinstance(llm_output, dict):
-            success = llm_output.get("success", False)
+            success = llm_output.get("success", 0)
             if isinstance(success, str):
-                success = success.upper() == "TRUE"
+                success = 1 if success.upper() == "TRUE" else 0
 
-            success_result = "pass" if success == True else "fail"
+            success_result = "pass" if success == 1 else "fail"
             reason = llm_output.get("explanation", "")
             return {
                 f"{self._result_key}": success,
                 f"{self._result_key}_result": success_result,
                 f"{self._result_key}_reason": reason,
                 f"{self._result_key}_details": llm_output.get("details", ""),
+                f"{self._result_key}_prompt_tokens": prompty_output_dict.get("input_token_count", 0),
+                f"{self._result_key}_completion_tokens": prompty_output_dict.get("output_token_count", 0),
+                f"{self._result_key}_total_tokens": prompty_output_dict.get("total_token_count", 0),
+                f"{self._result_key}_finish_reason": prompty_output_dict.get("finish_reason", ""),
+                f"{self._result_key}_model": prompty_output_dict.get("model_id", ""),
+                f"{self._result_key}_sample_input": prompty_output_dict.get("sample_input", ""),
+                f"{self._result_key}_sample_output": prompty_output_dict.get("sample_output", ""),
             }
         if logger:
-            logger.warning("LLM output is not a dictionary, returning False for the success.")
-        return {self._result_key: False}
+            logger.warning("LLM output is not a dictionary, returning 0 for the success.")
+        return {self._result_key: 0}

azure/ai/evaluation/_evaluators/{_task_success/task_success.prompty → _task_completion/task_completion.prompty}

@@ -1,5 +1,5 @@
 ---
-name: Task Success
+name: Task Completion
 description: Evaluates whether a task was successfully completed
 model:
   api: chat
@@ -27,7 +27,7 @@ You are an expert evaluator who determines if an agent has successfully complete
 user:
 ROLE
 ====
-You are a judge on Task Success who assesses the final outcome of a user-agent interaction. Your single focus is: **Was the user's task successfully and completely accomplished?**
+You are a judge on Task Completion who assesses the final outcome of a user-agent interaction. Your single focus is: **Was the user's task successfully and completely accomplished?**
 
 You are NOT evaluating:
 - How well the agent followed instructions