azure-ai-evaluation 1.11.2__py3-none-any.whl → 1.13.0__py3-none-any.whl

azure/ai/evaluation/_evaluators/_task_completion/_task_completion.py

@@ -0,0 +1,177 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+import os
+import math
+import logging
+from typing import Dict, Union, List, Optional
+
+from typing_extensions import overload, override
+
+from azure.ai.evaluation._exceptions import EvaluationException, ErrorBlame, ErrorCategory, ErrorTarget
+from azure.ai.evaluation._evaluators._common import PromptyEvaluatorBase
+from ..._common.utils import reformat_conversation_history, reformat_agent_response, reformat_tool_definitions
+from azure.ai.evaluation._model_configurations import Message
+from azure.ai.evaluation._common._experimental import experimental
+
+logger = logging.getLogger(__name__)
+
+
+@experimental
+class _TaskCompletionEvaluator(PromptyEvaluatorBase[Union[str, float]]):
+    """The Task Completion evaluator determines whether an AI agent successfully completed the requested task based on:
+
+        - Final outcome and deliverable of the task
+        - Completeness of task requirements
+
+    This evaluator focuses solely on task completion and success, not on task adherence or intent understanding.
+
+    Scoring is binary:
+    - 1 (pass): Task fully completed with usable deliverable that meets all user requirements
+    - 0 (fail): Task incomplete, partially completed, or deliverable does not meet requirements
+
+    The evaluation includes task requirement analysis, outcome assessment, and completion gap identification.
+
+
+    :param model_config: Configuration for the Azure OpenAI model.
+    :type model_config: Union[~azure.ai.evaluation.AzureOpenAIModelConfiguration,
+        ~azure.ai.evaluation.OpenAIModelConfiguration]
+
+    .. admonition:: Example:
+        .. literalinclude:: ../samples/evaluation_samples_evaluate.py
+            :start-after: [START task_completion_evaluator]
+            :end-before: [END task_completion_evaluator]
+            :language: python
+            :dedent: 8
+            :caption: Initialize and call a _TaskCompletionEvaluator with a query and response.
+
+    .. admonition:: Example using Azure AI Project URL:
+
+        .. literalinclude:: ../samples/evaluation_samples_evaluate_fdp.py
+            :start-after: [START task_completion_evaluator]
+            :end-before: [END task_completion_evaluator]
+            :language: python
+            :dedent: 8
+            :caption: Initialize and call a _TaskCompletionEvaluator using Azure AI Project URL in the following format
+                https://{resource_name}.services.ai.azure.com/api/projects/{project_name}
+
+    """
+
+    _PROMPTY_FILE = "task_completion.prompty"
+    _RESULT_KEY = "task_completion"
+    _OPTIONAL_PARAMS = ["tool_definitions"]
+
+    id = "azureai://built-in/evaluators/task_completion"
+    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+
+    @override
+    def __init__(self, model_config, *, credential=None, **kwargs):
+        current_dir = os.path.dirname(__file__)
+        prompty_path = os.path.join(current_dir, self._PROMPTY_FILE)
+        super().__init__(
+            model_config=model_config,
+            prompty_file=prompty_path,
+            result_key=self._RESULT_KEY,
+            credential=credential,
+            **kwargs,
+        )
+
+    @overload
+    def __call__(
+        self,
+        *,
+        query: Union[str, List[dict]],
+        response: Union[str, List[dict]],
+        tool_definitions: Optional[Union[dict, List[dict]]] = None,
+    ) -> Dict[str, Union[str, float]]:
+        """Evaluate task completion for a given query, response, and optionally tool definitions.
+        The query and response can be either a string or a list of messages.
+
+
+        Example with string inputs and no tools:
+            evaluator = _TaskCompletionEvaluator(model_config)
+            query = "Plan a 3-day itinerary for Paris with cultural landmarks and local cuisine."
+            response = "**Day 1:** Morning: Louvre Museum, Lunch: Le Comptoir du Relais..."
+
+            result = evaluator(query=query, response=response)
+
+        Example with list of messages:
+            evaluator = _TaskCompletionEvaluator(model_config)
+            query = [{'role': 'system', 'content': 'You are a helpful travel planning assistant.'}, {'createdAt': 1700000060, 'role': 'user', 'content': [{'type': 'text', 'text': 'Plan a 3-day Paris itinerary with cultural landmarks and cuisine'}]}]
+            response = [{'createdAt': 1700000070, 'run_id': '0', 'role': 'assistant', 'content': [{'type': 'text', 'text': '**Day 1:** Morning: Visit Louvre Museum (9 AM - 12 PM)...'}]}]
+            tool_definitions = [{'name': 'get_attractions', 'description': 'Get tourist attractions for a city.', 'parameters': {'type': 'object', 'properties': {'city': {'type': 'string', 'description': 'The city name.'}}}}]
+
+            result = evaluator(query=query, response=response, tool_definitions=tool_definitions)
+
+        :keyword query: The query being evaluated, either a string or a list of messages.
+        :paramtype query: Union[str, List[dict]]
+        :keyword response: The response being evaluated, either a string or a list of messages (full agent response potentially including tool calls)
+        :paramtype response: Union[str, List[dict]]
+        :keyword tool_definitions: An optional list of messages containing the tool definitions the agent is aware of.
+        :paramtype tool_definitions: Optional[Union[dict, List[dict]]]
+        :return: A dictionary with the task completion evaluation results.
+        :rtype: Dict[str, Union[str, float]]
+        """
+
+    @override
+    def __call__(  # pylint: disable=docstring-missing-param
+        self,
+        *args,
+        **kwargs,
+    ):
+        """
+        Invokes the instance using the overloaded __call__ signature.
+
+        For detailed parameter types and return value documentation, see the overloaded __call__ definition.
+        """
+        return super().__call__(*args, **kwargs)
+
+    @override
+    async def _do_eval(self, eval_input: Dict) -> Dict[str, Union[float, str]]:  # type: ignore[override]
+        """Do Task Completion evaluation.
+        :param eval_input: The input to the evaluator. Expected to contain whatever inputs are needed for the _flow method
+        :type eval_input: Dict
+        :return: The evaluation result.
+        :rtype: Dict
+        """
+        # we override the _do_eval method as we want the output to be a dictionary,
+        # which is a different schema than _base_prompty_eval.py
+        if "query" not in eval_input and "response" not in eval_input:
+            raise EvaluationException(
+                message=f"Both query and response must be provided as input to the Task Completion evaluator.",
+                internal_message=f"Both query and response must be provided as input to the Task Completion evaluator.",
+                blame=ErrorBlame.USER_ERROR,
+                category=ErrorCategory.MISSING_FIELD,
+                target=ErrorTarget.TASK_COMPLETION_EVALUATOR,
+            )
+        eval_input["query"] = reformat_conversation_history(eval_input["query"], logger, include_system_messages=True)
+        eval_input["response"] = reformat_agent_response(eval_input["response"], logger, include_tool_messages=True)
+        if "tool_definitions" in eval_input and eval_input["tool_definitions"] is not None:
+            eval_input["tool_definitions"] = reformat_tool_definitions(eval_input["tool_definitions"], logger)
+
+        prompty_output_dict = await self._flow(timeout=self._LLM_CALL_TIMEOUT, **eval_input)
+        llm_output = prompty_output_dict.get("llm_output", {})
+
+        if isinstance(llm_output, dict):
+            success = llm_output.get("success", 0)
+            if isinstance(success, str):
+                success = 1 if success.upper() == "TRUE" else 0
+
+            success_result = "pass" if success == 1 else "fail"
+            reason = llm_output.get("explanation", "")
+            return {
+                f"{self._result_key}": success,
+                f"{self._result_key}_result": success_result,
+                f"{self._result_key}_reason": reason,
+                f"{self._result_key}_details": llm_output.get("details", ""),
+                f"{self._result_key}_prompt_tokens": prompty_output_dict.get("input_token_count", 0),
+                f"{self._result_key}_completion_tokens": prompty_output_dict.get("output_token_count", 0),
+                f"{self._result_key}_total_tokens": prompty_output_dict.get("total_token_count", 0),
+                f"{self._result_key}_finish_reason": prompty_output_dict.get("finish_reason", ""),
+                f"{self._result_key}_model": prompty_output_dict.get("model_id", ""),
+                f"{self._result_key}_sample_input": prompty_output_dict.get("sample_input", ""),
+                f"{self._result_key}_sample_output": prompty_output_dict.get("sample_output", ""),
+            }
+        if logger:
+            logger.warning("LLM output is not a dictionary, returning 0 for the success.")
+        return {self._result_key: 0}

azure/ai/evaluation/_evaluators/_task_completion/task_completion.prompty

@@ -0,0 +1,220 @@
+---
+name: Task Completion
+description: Evaluates whether a task was successfully completed
+model:
+  api: chat
+  parameters:
+    temperature: 0.0
+    max_tokens: 1500
+    top_p: 1.0
+    presence_penalty: 0
+    frequency_penalty: 0
+    response_format:
+      type: json_object
+inputs:
+  query:
+    type: string
+  response:
+    type: string
+  tool_definitions:
+    type: Dict
+    optional: true
+    default: {}
+---
+system:
+You are an expert evaluator who determines if an agent has successfully completed the task required by the user based on the final outcome.
+
+user:
+ROLE
+====
+You are a judge on Task Completion who assesses the final outcome of a user-agent interaction. Your single focus is: **Was the user's task successfully and completely accomplished?**
+
+You are NOT evaluating:
+- How well the agent followed instructions
+- How well the agent understood the user's intent
+
+You ARE evaluating:
+- Whether the task is actually completed in the final outcome
+- Whether the deliverable meets the user's requirements
+- Whether the end result is actionable and usable
+
+INPUT
+=====
+CONVERSATION_HISTORY: {{query}}
+AGENT_RESPONSE: {{response}}
+
+CONVERSATION_HISTORY includes the full dialogue. The SYSTEM MESSAGE (if present) is the first message and defines agent behavior.
+AGENT_RESPONSE is the agent's reply to the latest user query.
+Tool calls and tool results are not visible to the user. The user only sees the agent's final response.
+
+EVALUATION FRAMEWORK
+====================
+
+A. Identify the Task Requirements:
+- What specific outcome did the user request?
+- What deliverables were expected?
+- What criteria define "completion" for this task?
+
+B. Analyze the Final Outcome:
+- What did the agent actually deliver?
+- Is the deliverable complete and usable?
+- Does it meet the user's specified requirements?
+
+C. Assess Task Completion:
+- **Fully Complete**: All requirements met, deliverable is ready for use
+- **Partially Complete**: Some requirements met, but missing key components
+- **Incomplete**: No usable deliverable or major requirements unmet
+
+D. Assign a Score:
+   - **TRUE**: The agent delivered a complete and correct solution that accomplishes the user's entire goal. The user does not need to take further action or ask follow-up questions to get what they originally asked for.
+   - **FALSE**: The agent failed to complete one or more parts of the task, provided an incorrect/incomplete result, or left the user's goal unresolved.
+
+OUTPUT FORMAT
+=============
+Output a JSON object with these keys:
+{
+  "explanation": "<15-60 words explaining the completion status>",
+  "details": {
+    "task_requirements": "<15-60 words on what the user specifically requested>",
+    "delivered_outcome": "<15-60 words on what the agent actually provided>",
+    "completion_gaps": "<15-60 words on missing elements if task is incomplete>"
+  },
+  "success": <TRUE or FALSE>
+}
+
+SCORING EXAMPLES
+================
+
+### SUCCESS: TRUE - Example A
+
+CONVERSATION_HISTORY:
+User: Plan a 3-day itinerary for Paris with cultural landmarks and local cuisine.
+
+AGENT_RESPONSE:
+**Day 1:**
+- Morning: Visit Louvre Museum (9 AM - 12 PM)
+- Lunch: Le Comptoir du Relais (traditional French bistro)
+- Afternoon: Walk through Tuileries Garden
+- Dinner: L'As du Fallafel (famous in Marais district)
+
+**Day 2:**
+- Morning: Tour Eiffel Tower and Trocadéro
+- Lunch: Du Pain et des Idées (artisanal bakery)
+- Afternoon: Explore Montmartre and Sacré-Cœur
+- Dinner: Le Procope (historic restaurant)
+
+**Day 3:**
+- Morning: Visit Notre-Dame Cathedral area
+- Lunch: Marché des Enfants Rouges (food market)
+- Afternoon: Musée d'Orsay
+- Dinner: Guy Savoy (Michelin-starred French cuisine)
+
+Transportation: Metro Day Pass recommended. Book Louvre tickets online in advance.
+
+EXPECTED OUTPUT:
+{
+  "explanation": "Complete 3-day Paris itinerary delivered with cultural landmarks and local cuisine for each day, including practical transportation advice.",
+  "details": {
+    "task_requirements": "3-day Paris itinerary with cultural landmarks and local cuisine",
+    "delivered_outcome": "Detailed 3-day schedule with specific landmarks, restaurants, and practical tips",
+    "completion_gaps": "None"
+  },
+  "success": TRUE
+}
+
+### SUCCESS: TRUE - Example B
+
+CONVERSATION_HISTORY:
+User: My internet connection keeps dropping. Help me troubleshoot and then upgrade my data plan to unlimited.
+
+AGENT_RESPONSE:
+[TOOL_CALL] run_network_diagnostic()
+[TOOL_RESULT] Issue found: Router firmware outdated, DNS servers slow
+
+[TOOL_CALL] upgrade_plan(customer_id="12345", plan="unlimited")
+[TOOL_RESULT] Plan upgraded successfully. Unlimited data active immediately.
+
+Your internet issues are resolved - update firmware and change DNS as instructed. Your unlimited data plan is now active. You should see stable connection and no data limits immediately.
+
+EXPECTED OUTPUT:
+{
+  "explanation": "Both issues resolved: specific troubleshooting steps provided for internet drops, and unlimited data plan successfully activated.",
+  "details": {
+    "task_requirements": "Fix internet connection issues and upgrade to unlimited data plan",
+    "delivered_outcome": "Diagnostic completed with specific fix instructions, data plan upgrade confirmed active",
+    "completion_gaps": "None"
+  },
+  "success": TRUE
+}
+
+### SUCCESS: FALSE - Example A
+
+CONVERSATION_HISTORY:
+User: Plan a 3-day itinerary for Paris with cultural landmarks and local cuisine.
+
+AGENT_RESPONSE:
+I'd recommend visiting the Louvre and Eiffel Tower. Paris has great food - try French pastries and wine. The Metro is convenient for getting around.
+
+EXPECTED OUTPUT:
+{
+  "explanation": "Vague suggestions provided but no actual day-by-day itinerary created. Missing specific landmarks, restaurants, and detailed schedule.",
+  "details": {
+    "task_requirements": "3-day Paris itinerary with cultural landmarks and local cuisine",
+    "delivered_outcome": "General recommendations without structured itinerary",
+    "completion_gaps": "No day-by-day schedule, no specific restaurant recommendations, no detailed cultural landmarks"
+  },
+  "success": FALSE
+}
+
+### SUCCESS: FALSE - Example B
+
+CONVERSATION_HISTORY:
+User: My internet connection keeps dropping. Help me troubleshoot and then upgrade my data plan to unlimited.
+
+AGENT_RESPONSE:
+[TOOL_CALL] run_network_diagnostic()
+[TOOL_RESULT] Issue found: Router firmware outdated
+I found that your router firmware needs updating. You can usually do this through your router's admin panel. Let me know if you need help with the data plan upgrade after you fix the connection.
+
+EXPECTED OUTPUT:
+{
+  "explanation": "Internet troubleshooting started but not completed with specific steps. Data plan upgrade not attempted. User cannot proceed without more help.",
+  "details": {
+    "task_requirements": "Fix internet connection issues and upgrade to unlimited data plan",
+    "delivered_outcome": "Identified router firmware issue but no specific fix instructions, data upgrade not completed",
+    "completion_gaps": "No specific firmware update steps, data plan upgrade not processed"
+  },
+  "success": FALSE
+}
+
+### SUCCESS: FALSE - Example C
+
+CONVERSATION_HISTORY:
+User: Analyze our Q3 sales data and generate a summary report.
+
+AGENT_RESPONSE:
+I can help you analyze sales data! Sales analysis is important for business growth. You should look at metrics like revenue, customer acquisition, and seasonal trends. Would you like me to explain different types of sales analysis techniques?
+
+EXPECTED OUTPUT:
+{
+  "explanation": "No actual analysis performed and no report generated. Agent provided general information about sales analysis instead of completing the requested task.",
+  "details": {
+    "task_requirements": "Analyze Q3 sales data and generate summary report",
+    "delivered_outcome": "General information about sales analysis concepts",
+    "completion_gaps": "No data analysis performed, no summary report created"
+  },
+  "success": FALSE
+}
+
+KEY PRINCIPLES
+==============
+
+1. **Outcome Focus**: Judge only the final deliverable, not the process
+2. **User Readiness**: Can the user proceed with what was delivered?
+3. **Requirement Matching**: Does the outcome match what was specifically requested?
+4. **Completeness**: Are all components of the task addressed?
+5. **Actionability**: Is the deliverable usable in its current form?
+
+Remember: A task can be understood correctly and approached properly but still fail if the final outcome doesn't meet requirements.
+
+# Output

azure/ai/evaluation/_evaluators/_task_navigation_efficiency/__init__.py

@@ -0,0 +1,7 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from ._task_navigation_efficiency import _TaskNavigationEfficiencyEvaluator, _TaskNavigationEfficiencyMatchingMode
+
+__all__ = ["_TaskNavigationEfficiencyEvaluator", "_TaskNavigationEfficiencyMatchingMode"]