azure-ai-evaluation 1.10.0__py3-none-any.whl → 1.11.0__py3-none-any.whl

azure/ai/evaluation/_evaluators/_tool_call_accuracy/_tool_call_accuracy.py

@@ -1,11 +1,12 @@
 # ---------------------------------------------------------
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
+from itertools import chain
 import math
 import os
 import logging
 import re
-from typing import Dict, List, Union, TypeVar, cast
+from typing import Dict, List, Union, TypeVar, Optional
 from typing_extensions import overload, override
 from azure.ai.evaluation._evaluators._common import PromptyEvaluatorBase
 from azure.ai.evaluation._exceptions import (
@@ -16,12 +17,46 @@ from azure.ai.evaluation._exceptions import (
 )
 from ..._common.utils import check_score_is_valid
 from azure.ai.evaluation._common._experimental import experimental
+from ..._converters._models import (
+    _BUILT_IN_DESCRIPTIONS,
+    _BUILT_IN_PARAMS,
+)
 
 logger = logging.getLogger(__name__)
 
 T_EvalValue = TypeVar("T_EvalValue")
 
 
+def _get_built_in_definition(tool_name: str):
+    """Get the definition for the built-in tool."""
+    if tool_name in _BUILT_IN_DESCRIPTIONS:
+        return {
+            "type": tool_name,
+            "description": _BUILT_IN_DESCRIPTIONS[tool_name],
+            "name": tool_name,
+            "parameters": _BUILT_IN_PARAMS.get(tool_name, {}),
+        }
+    return None
+
+
+def _get_needed_built_in_definitions(tool_calls: List[Dict]) -> List[Dict]:
+    """Extract tool definitions needed for the given built-in tool calls."""
+    needed_definitions = []
+    for tool_call in tool_calls:
+        if isinstance(tool_call, dict):
+            tool_type = tool_call.get("type")
+
+            # Only support converter format: {type: "tool_call", name: "bing_custom_search", arguments: {...}}
+            if tool_type == "tool_call":
+                tool_name = tool_call.get("name")
+                if tool_name in _BUILT_IN_DESCRIPTIONS:
+                    built_in_def = _get_built_in_definition(tool_name)
+                    if built_in_def and built_in_def not in needed_definitions:
+                        needed_definitions.append(built_in_def)
+
+    return needed_definitions
+
+
 @experimental
 class ToolCallAccuracyEvaluator(PromptyEvaluatorBase[Union[str, float]]):
     """The Tool Call Accuracy evaluator assesses how accurately an AI uses tools by examining:
@@ -88,7 +123,7 @@ class ToolCallAccuracyEvaluator(PromptyEvaluatorBase[Union[str, float]]):
     """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
 
     @override
-    def __init__(self, model_config, *, threshold=_DEFAULT_TOOL_CALL_ACCURACY_SCORE, **kwargs):
+    def __init__(self, model_config, *, threshold=_DEFAULT_TOOL_CALL_ACCURACY_SCORE, credential=None, **kwargs):
         current_dir = os.path.dirname(__file__)
         prompty_path = os.path.join(current_dir, self._PROMPTY_FILE)
         self.threshold = threshold
@@ -96,6 +131,7 @@ class ToolCallAccuracyEvaluator(PromptyEvaluatorBase[Union[str, float]]):
             model_config=model_config,
             prompty_file=prompty_path,
             result_key=self._RESULT_KEY,
+            credential=credential,
             **kwargs,
         )
 
@@ -153,10 +189,9 @@ class ToolCallAccuracyEvaluator(PromptyEvaluatorBase[Union[str, float]]):
         # TODO add warning that only tool calls of type function are supported
         # Collect inputs
         tool_calls = kwargs.get("tool_calls")
-        tool_definitions = kwargs.get("tool_definitions")
+        tool_definitions = kwargs.get("tool_definitions", [])  # Default to empty list
         query = kwargs.get("query")
         response = kwargs.get("response")
-
         # TODO : Support classes that represents tool calls, messages etc once client side definitions are available
         if response:
             parsed_tool_calls = self._parse_tools_from_response(response)
@@ -165,20 +200,23 @@ class ToolCallAccuracyEvaluator(PromptyEvaluatorBase[Union[str, float]]):
 
         if not tool_calls:
             return {"error_message": self._NO_TOOL_CALLS_MESSAGE}
-        if not tool_definitions or len(tool_definitions) == 0:
-            return {"error_message": self._NO_TOOL_DEFINITIONS_MESSAGE}
 
         if not isinstance(tool_calls, list):
             tool_calls = [tool_calls]
         if not isinstance(tool_definitions, list):
-            tool_definitions = [tool_definitions]
+            tool_definitions = [tool_definitions] if tool_definitions else []
 
         try:
             needed_tool_definitions = self._extract_needed_tool_definitions(tool_calls, tool_definitions)
         except EvaluationException as e:
-            return {"error_message": self._TOOL_DEFINITIONS_MISSING_MESSAGE}
+            # Check if this is because no tool definitions were provided at all
+            if len(tool_definitions) == 0:
+                return {"error_message": self._NO_TOOL_DEFINITIONS_MESSAGE}
+            else:
+                return {"error_message": self._TOOL_DEFINITIONS_MISSING_MESSAGE}
+
         if len(needed_tool_definitions) == 0:
-            return {"error_message": self._TOOL_DEFINITIONS_MISSING_MESSAGE}
+            return {"error_message": self._NO_TOOL_DEFINITIONS_MESSAGE}
 
         return {
             "query": query,
@@ -268,66 +306,72 @@ class ToolCallAccuracyEvaluator(PromptyEvaluatorBase[Union[str, float]]):
             "details": {},
         }
 
-    def _parse_tools_from_response(self, response):
-        """Parse the response to extract tool calls and results.
-        :param response: The response to parse.
-        :type response: Union[str, List[dict]]
-        :return: List of tool calls extracted from the response.
-        :rtype: List[dict]
-        """
-        tool_calls = []
-        tool_results_map = {}
-        if isinstance(response, list):
-            for message in response:
-                # Extract tool calls from assistant messages
-                if message.get("role") == "assistant" and isinstance(message.get("content"), list):
-                    for content_item in message.get("content"):
-                        if isinstance(content_item, dict) and content_item.get("type") == "tool_call":
-                            tool_calls.append(content_item)
-
-                # Extract tool results from tool messages
-                elif message.get("role") == "tool" and message.get("tool_call_id"):
-                    tool_call_id = message.get("tool_call_id")
-                    if isinstance(message.get("content"), list) and len(message.get("content")) > 0:
-                        result_content = message.get("content")[0]
-                        if isinstance(result_content, dict) and result_content.get("type") == "tool_result":
-                            tool_results_map[tool_call_id] = result_content
-
-        # Attach results to their corresponding calls
-        for tool_call in tool_calls:
-            tool_call_id = tool_call.get("tool_call_id")
-            if tool_call_id in tool_results_map:
-                tool_call["tool_result"] = tool_results_map[tool_call_id]["tool_result"]
-
-        return tool_calls
-
     def _extract_needed_tool_definitions(self, tool_calls, tool_definitions):
-        """Extract the tool definitions that are needed for the provided tool calls.
-        :param tool_calls: List of tool calls to evaluate.
-        :type tool_calls: List[dict]
-        :param tool_definitions: List of tool definitions to use for evaluation.
-        :type tool_definitions: List[dict]
-        :return: List of tool definitions that are needed for the provided tool calls.
-        :rtype: List[dict]
-        """
+        """Extract the tool definitions that are needed for the provided tool calls."""
         needed_tool_definitions = []
+
+        # Add all user-provided tool definitions
+        needed_tool_definitions.extend(tool_definitions)
+
+        # Add the needed built-in tool definitions (if they are called)
+        built_in_definitions = _get_needed_built_in_definitions(tool_calls)
+        needed_tool_definitions.extend(built_in_definitions)
+
+        # OpenAPI tool is a collection of functions, so we need to expand it
+        tool_definitions_expanded = list(
+            chain.from_iterable(
+                tool.get("functions", []) if tool.get("type") == "openapi" else [tool]
+                for tool in needed_tool_definitions
+            )
+        )
+
+        # Validate that all tool calls have corresponding definitions
         for tool_call in tool_calls:
-            if isinstance(tool_call, dict) and tool_call.get("type") == "tool_call":
-                tool_name = tool_call.get("name")
-                tool_definition = [
-                    tool
-                    for tool in tool_definitions
-                    if tool.get("name") == tool_name and tool.get("type", "function") == "function"
-                ]
-                if len(tool_definition) > 0:
-                    needed_tool_definitions.extend(tool_definition)
+            if isinstance(tool_call, dict):
+                tool_type = tool_call.get("type")
+
+                if tool_type == "tool_call":
+                    tool_name = tool_call.get("name")
+                    if tool_name and tool_name in _BUILT_IN_DESCRIPTIONS:
+                        # This is a built-in tool from converter, already handled above
+                        continue
+                    elif tool_name:
+                        # This is a regular function tool from converter
+                        tool_definition_exists = any(
+                            tool.get("name") == tool_name and tool.get("type", "function") == "function"
+                            for tool in tool_definitions_expanded
+                        )
+                        if not tool_definition_exists:
+                            raise EvaluationException(
+                                message=f"Tool definition for {tool_name} not found",
+                                blame=ErrorBlame.USER_ERROR,
+                                category=ErrorCategory.INVALID_VALUE,
+                                target=ErrorTarget.TOOL_CALL_ACCURACY_EVALUATOR,
+                            )
+                    else:
+                        raise EvaluationException(
+                            message=f"Tool call missing name: {tool_call}",
+                            blame=ErrorBlame.USER_ERROR,
+                            category=ErrorCategory.INVALID_VALUE,
+                            target=ErrorTarget.TOOL_CALL_ACCURACY_EVALUATOR,
+                        )
                 else:
+                    # Unsupported tool format - only converter format is supported
                     raise EvaluationException(
-                        message=f"Tool definition for {tool_name} not found",
+                        message=f"Unsupported tool call format. Only converter format is supported: {tool_call}",
                         blame=ErrorBlame.USER_ERROR,
                         category=ErrorCategory.INVALID_VALUE,
                         target=ErrorTarget.TOOL_CALL_ACCURACY_EVALUATOR,
                     )
+            else:
+                # Tool call is not a dictionary
+                raise EvaluationException(
+                    message=f"Tool call is not a dictionary: {tool_call}",
+                    blame=ErrorBlame.USER_ERROR,
+                    category=ErrorCategory.INVALID_VALUE,
+                    target=ErrorTarget.TOOL_CALL_ACCURACY_EVALUATOR,
+                )
+
         return needed_tool_definitions
 
     @override

azure/ai/evaluation/_evaluators/_tool_call_accuracy/tool_call_accuracy.prompty

@@ -31,99 +31,116 @@ system:
 
 user:
 # Definition
-**Tool Call Accuracy** refers to the relevance and potential usefulness of a TOOL CALL in the context of an ongoing CONVERSATION and EXTRACTION of RIGHT PARAMETER VALUES from the CONVERSATION.It assesses how likely the TOOL CALL is to contribute meaningfully to the CONVERSATION and help address the user's needs. Focus on evaluating the potential value of the TOOL CALL within the specific context of the given CONVERSATION, without making assumptions beyond the provided information.
-  Consider the following factors in your evaluation:
+# Definition
+**Tool Call Accuracy** refers to the overall effectiveness of ALL TOOL CALLS made by an agent in response to a user's query within an ongoing CONVERSATION.
+
+# EVALUATION CRITERIA
+Evaluate based on these factors:
+
+1. **Collective Relevance**: Do the tool calls, taken together, appropriately address the user's query?
+2. **Parameter Correctness**: Are all parameter values extracted from or reasonably inferred from the CONVERSATION? 
+   - *Fabricated parameters automatically result in Level 2*
+3. **Completeness**: Did the agent make all necessary tool calls available in the tool definitions?
+   - *Failed calls don't count as missing*
+4. **Efficiency**: Did the agent avoid unnecessary duplicate tool calls with identical parameters?
+   - *Don't penalize single tools returning multiple results (like file_search)*
+5. **Execution Success**: Were tool calls executed successfully or recovered from errors appropriately?
+6. **Scope Limitation**: ONLY evaluate tool calls in the "TOOL CALLS TO BE EVALUATED" section.
+   - Tool calls in the CONVERSATION section are for context only
+   - Focus exclusively on the agent's response to the user's LAST query
+   - Use conversation history only to verify parameter correctness and context
+
+**Success Criteria**: Tools should retrieve relevant data to help answer the query. Complete final answers are not required from individual tools.
 
-  1. Relevance: How well does the proposed tool call align with the current topic and flow of the conversation?
-  2. Parameter Appropriateness: Do the parameters used in the TOOL CALL match the TOOL DEFINITION and are the parameters relevant to the latest user's query?
-  3. Parameter Value Correctness: Are the parameters values used in the TOOL CALL present or inferred by CONVERSATION and relevant to the latest user's query?
-  4. Potential Value: Is the information this tool call might provide likely to be useful in advancing the conversation or addressing the user expressed or implied needs?
-  5. Context Appropriateness: Does the tool call make sense at this point in the conversation, given what has been discussed so far?
+**Tool Assessment**: Focus solely on appropriate use of available tools, not on capabilities beyond what tools can provide.
 
 
 # Ratings
 ## [Tool Call Accuracy: 1] (Irrelevant)
 **Definition:**
 Tool calls were not relevant to the user's query, resulting in an irrelevant or unhelpful final output.
-This level is a 'fail'.
 
 **Example:**
- The user's query is asking for most popular hotels in New York, but the agent calls a tool that does search in local files on a machine. This tool is not relevant to the user query, so this case is a Level 1 'fail'.
+ User asks for distance between two cities -> Agent calls a weather function to get the weather in the two cities.
 
 
-## [Tool Call Accuracy: 2] (Partially Relevant - No correct output)
+## [Tool Call Accuracy: 2] (Partially Relevant - Wrong Execution)
 **Definition:**
-Tool calls were somewhat related to the user's query, but the agent was not able to reach a final output that addresses the user query due to one or more of the following:
-•	Tools returned errors, and no retrials for the tool call were successful.
-•	Parameters passed to the tool were incorrect.
-•	Not enough tools were called to fully address the query (missing tool calls).
-This level is a 'fail'.
+Tool calls were somewhat related to the user's query, but the agent was not able to reach information that helps address the user query due to one or more of the following:
+  •	Parameters passed to the tool were incorrect.
+  •	Not enough tools (available in the tool definitions) were called to fully help address the query (missing tool calls).
+  •	Tools returned errors, and no retrials for the tool call were successful.
+
 
 **Example:**
-  The user asks for the coordinates of Chicago. The agent calls the correct tool that retrieves the coordinates -which is the relevant tool for the user query- but passes 'New York' instead of 'Chicago' as the parameter to the tool. So this is a Level 2 'fail'.
+  The user asks for the coordinates of Chicago. The agent calls the tool that gets the coordinates but passes 'New York' instead of Chicago as parameter.
 
 **Example:**
-  The user asks for the coordinates of Chicago. The agent calls the correct tool that retrieves the coordinates -which is the relevant tool for the user query- and passes 'Chicago' as the parameter to the tool which is also correct, but the tool returns an error so the agent can't reach the correct answer to the user's query. This is a Level 2 'fail'.
+  The user asks for the coordinates of Chicago. The agent calls the tool that gets the coordinates and passes 'Chicago' as the tool parameter, but the tool returns an error.
 
 **Example:**
-  The user asks a question that needs 3 tool calls for it to be answered. The agent calls only one of the three required tool calls. So this case is a Level 2 'fail'.
+  The user asks a question that needs 3 tool calls for it to be answered. The agent calls only one of the three required tool calls. So this case is a Level 2.
 
 
-## [Tool Call Accuracy: 3] (Slightly Correct - Reached Output)
+## [Tool Call Accuracy: 3] (Relevant but Inefficient)
 **Definition:**
 Tool calls were relevant, correct and grounded parameters were passed so that led to a correct output. However, multiple excessive, unnecessary tool calls were made.
-This level is a 'pass'.
+
+**Important**: Do NOT penalize built-in tools like file_search that naturally return multiple results in a single call. Only penalize when there are actually multiple separate tool call objects.
 
 **Example:**
-  The user asked to do a modification in the database. The agent called the tool multiple times, resulting in multiple modifications in the database instead of one. This is a level 3 'pass'.
+  The user asked to do a modification in the database. The agent called the tool multiple times, resulting in multiple modifications in the database instead of one.
 
 **Example:**
-  The user asked for popular hotels in a certain place. The agent calls the same tool with the same parameters multiple times, even though a single tool call that returns an output is sufficient. So there were unnecessary tool calls. This is a Level 3 'pass'.
+  The user asked for popular hotels in a certain place. The agent calls the same tool with the same parameters multiple times, even though a single tool call that returns an output is sufficient. So there were unnecessary tool calls.
 
 
-## [Tool Call Accuracy: 4] (Mostly Correct - Reached output)
+## [Tool Call Accuracy: 4] (Correct with Retrials)
 **Definition:**
 Tool calls were fully relevant and efficient:
 •	Correct tools were called with the correct and grounded parameters, whether they are extracted from the conversation history or the current user query.
 •	A tool returned an error, but the agent retried calling the tool and successfully got an output.
-This level is a 'pass'.
 
 **Example:**
-  The user asks for the weather forecast in a certain place. The agent calls the correct tool that retrieves the weather forecast with the correct parameters, but the tool returns an error. The agent re-calls the tool once again and it returns the correct output. This is a Level 4 'pass'.
+  The user asks for the weather forecast in a certain place. The agent calls the correct tool that retrieves the weather forecast, but the tool returns an error. The agent re-calls the tool once again and it returns the correct output. This is a Level 4.
 
 
-## [Tool Call Accuracy: 5] (Optimal Solution - Reached output)
+## [Tool Call Accuracy: 5] (Optimal Solution)
 **Definition:**
 Tool calls were fully relevant and efficient:
-•	Correct tools were called with the correct and grounded parameters, whether they are extracted from the conversation history or the current user query.
-•	No unnecessary or excessive tool calls were made.
-•	No errors occurred in any of the tools.
-•	The agent was able to reach the final output that addresses the user's query without facing any issues.
-This level is a 'pass'.
+  •	Correct tools were called with the correct and grounded parameters, whether they are extracted from the conversation history or the current user query.
+  •	No unnecessary or excessive tool calls were made.
+  •	No errors occurred in any of the tools.
+  •	The tool calls made helped the agent address the user's query without facing any issues.
 
 **Example:**
-  The user asks for the distance between two places. The agent correctly calls the tools that retrieve the coordinates for the two places respectively, then calls the tool that calculates the distance between the two sets of coordinates, passing the correct arguments to all the tools, without calling other tools excessively or unnecessarily. This is the optimal solution for the user's query. This is a Level 5 'pass'.
+  The user asks for the distance between two places. The agent correctly calls the tools that retrieve the coordinates for the two places respectively, then calls the tool that calculates the distance between the two sets of coordinates, passing the correct arguments to all the tools, without calling other tools excessively or unnecessarily. This is the optimal solution for the user's query.
 
 **Example:**
-  The user asks for the distance between two places. The agent retrieves the needed coordinates from the outputs of the tool calls in the conversation history, and then correctly passes these coordinates to the tool that calculates the distance to output it to the user. This is also an optimal solution for the user's query. This is a Level 5 'pass'.
+  The user asks for the distance between two places. The agent retrieves the needed coordinates from the outputs of the tool calls in the conversation history, and then correctly passes these coordinates to the tool that calculates the distance to output it to the user. This is also an optimal solution for the user's query.
 
+**Example:**
+  The user asked to summarize a file on their SharePoint. The agent calls the sharepoint_grounding tool to retrieve the file. This retrieved file will help the agent fulfill the task of summarization. This is a Level 5.
 
 
-# IMPORTANT NOTES
-- There is a clear distinction between 'pass' levels and 'fail' levels. The distinction is that the tools are called correctly in order to reach the required output. If the agent was not able to reach the final output that addresses the user query, it cannot be either of the 'pass' levels, and vice versa. It is crucial that you ensure you are rating the agent's response with the correct level based on the tool calls made to address the user's query.
-- "Correct output" means correct tool with the correct, grounded parameters. You are NOT concerned with the correctness of the result of the tool. As long as the parameters passed were correct and the tool did not return an error, then the tool output is correct and accurate.
-- Ensure that every single parameter that is passed to the tools is correct and grounded from the user query or the conversation history. If the agent passes incorrect parameters or completely makes them up, then this is a fail, even if somehow the agent reaches a correct result.
+## Chain of Thought Structure
+Structure your reasoning as follows:
+1. **Start with the user's last query**: Understand well what the last message that is sent by the user is.
+2. **Identify relevant available tools**: Look into the TOOL DEFINITIONS and analyze which tools could help answer the user's last query in the conversation.
+3. **Analyze the actual tool calls made**: Compare what was done in the TOOL CALLS TO BE EVALUATED section vs. What should've been done by the agent.
+4. **Check parameter grounding** - Ensure all parameters are grounded from the CONVERSATION section and are not hallucinated.
+5. **Determine the appropriate level** - Be VERY precise and follow the level definitions exactly.
 
 # Data
 CONVERSATION : {{query}}
-TOOL CALLS: {{tool_calls}}
-TOOL DEFINITION: {{tool_definition}}
+TOOL CALLS TO BE EVALUATED: {{tool_calls}}
+TOOL DEFINITIONS: {{tool_definitions}}
 
 
 # Tasks
 ## Please provide your evaluation for the assistant RESPONSE in relation to the user QUERY and tool definitions based on the Definitions and examples above.
 Your output should consist only of a JSON object, as provided in the examples, that has the following keys:
-  - chain_of_thought: a string that explains your thought process to decide on the tool call accuracy level. Start this string with 'Let's think step by step:', and think deeply and precisely about which level should be chosen based on the agent's tool calls and how they were able to address the user's query.
+  - chain_of_thought: a string that explains your thought process to decide on the tool call accuracy level, based on the Chain of Thought structure. Start this string with 'Let's think step by step:'.
   - tool_calls_success_level: a integer value between 1 and 5 that represents the level of tool call success, based on the level definitions mentioned before. You need to be very precise when deciding on this level. Ensure you are correctly following the rating system based on the description of each level.
   - details: a dictionary that contains the following keys:
         - tool_calls_made_by_agent: total number of tool calls made by the agent
@@ -141,7 +158,7 @@ Your output should consist only of a JSON object, as provided in the examples, t
               - tool_name: name of the tool
               - excess_count: number of excess calls made for this query
         - missing_tool_calls: a dictionary with the following keys:
-            - total: total number of missing tool calls that should have been made by the agent to be able to answer the query
+            - total: total number of missing tool calls that should have been made by the agent to be able to answer the query, but were not made by the agent at all.
             - details: a list of dictionaries, each containing:
               - tool_name: name of the tool
               - missing_count: number of missing calls for this query

azure/ai/evaluation/_exceptions.py

@@ -48,6 +48,7 @@ class ErrorCategory(Enum):
     PROJECT_ACCESS_ERROR = "PROJECT ACCESS ERROR"
     UNKNOWN = "UNKNOWN"
     UPLOAD_ERROR = "UPLOAD ERROR"
+    NOT_APPLICABLE = "NOT APPLICABLE"
 
 
 class ErrorBlame(Enum):

azure/ai/evaluation/_version.py

@@ -3,4 +3,4 @@
 # ---------------------------------------------------------
 # represents upcoming version
 
-VERSION = "1.10.0"
+VERSION = "1.11.0"

azure/ai/evaluation/red_team/__init__.py

@@ -5,7 +5,7 @@
 try:
     from ._red_team import RedTeam
     from ._attack_strategy import AttackStrategy
-    from ._attack_objective_generator import RiskCategory
+    from ._attack_objective_generator import RiskCategory, SupportedLanguages
     from ._red_team_result import RedTeamResult
 except ImportError:
     raise ImportError(
@@ -18,4 +18,5 @@ __all__ = [
     "AttackStrategy",
     "RiskCategory",
     "RedTeamResult",
+    "SupportedLanguages",
 ]

azure/ai/evaluation/red_team/_attack_objective_generator.py

@@ -20,6 +20,23 @@ class RiskCategory(str, Enum):
     SelfHarm = "self_harm"
     ProtectedMaterial = "protected_material"
     CodeVulnerability = "code_vulnerability"
+    UngroundedAttributes = "ungrounded_attributes"
+    IndirectAttack = "indirect_attack"
+
+
+@experimental
+class SupportedLanguages(Enum):
+    """Supported languages for attack objectives, using ISO standard language codes."""
+
+    Spanish = "es"
+    Italian = "it"
+    French = "fr"
+    German = "de"
+    SimplifiedChinese = "zh-cn"
+    Portuguese = "pt"
+    Japanese = "ja"
+    English = "en"
+    Korean = "ko"
 
 
 @experimental

azure/ai/evaluation/red_team/_callback_chat_target.py

@@ -19,6 +19,7 @@ class _CallbackChatTarget(PromptChatTarget):
         *,
         callback: Callable[[List[Dict], bool, Optional[str], Optional[Dict[str, Any]]], Dict],
         stream: bool = False,
+        prompt_to_context: Optional[Dict[str, str]] = None,
     ) -> None:
         """
         Initializes an instance of the _CallbackChatTarget class.
@@ -32,10 +33,12 @@ class _CallbackChatTarget(PromptChatTarget):
         Args:
             callback (Callable): The callback function that sends a prompt to a target and receives a response.
             stream (bool, optional): Indicates whether the target supports streaming. Defaults to False.
+            prompt_to_context (Optional[Dict[str, str]], optional): Mapping from prompt content to context. Defaults to None.
         """
         PromptChatTarget.__init__(self)
         self._callback = callback
         self._stream = stream
+        self._prompt_to_context = prompt_to_context or {}
 
     async def send_prompt_async(self, *, prompt_request: PromptRequestResponse) -> PromptRequestResponse:
 
@@ -48,8 +51,18 @@ class _CallbackChatTarget(PromptChatTarget):
 
         logger.info(f"Sending the following prompt to the prompt target: {request}")
 
+        # Get context for the current prompt if available
+        current_prompt_content = request.converted_value
+        context_data = self._prompt_to_context.get(current_prompt_content, "")
+        context_dict = {"context": context_data} if context_data else {}
+
+        # If context is not available via prompt_to_context, it can be fetched from the memory
+        if not context_dict:
+            memory_label_context = request.labels.get("context", None)
+            context_dict = {"context": memory_label_context} if memory_label_context else {}
+
         # response_context contains "messages", "stream", "session_state, "context"
-        response_context = await self._callback(messages=messages, stream=self._stream, session_state=None, context=None)  # type: ignore
+        response_context = await self._callback(messages=messages, stream=self._stream, session_state=None, context=context_dict)  # type: ignore
 
         response_text = response_context["messages"][-1]["content"]
         response_entry = construct_response_from_request(request=request, response_text_pieces=[response_text])