azure-ai-evaluation 1.0.0b2__py3-none-any.whl → 1.13.3__py3-none-any.whl

azure/ai/evaluation/simulator/_direct_attack_simulator.py

@@ -1,102 +1,82 @@
 # ---------------------------------------------------------
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
+# pylint: disable=C0301,C0114,R0913,R0903
 # noqa: E501
-import functools
 import logging
 from random import randint
-from typing import Callable, Optional
-
-from promptflow._sdk._telemetry import ActivityType, monitor_operation
+from typing import Callable, Optional, cast, Union
 
+from azure.ai.evaluation._constants import TokenScope
+from azure.ai.evaluation._common._experimental import experimental
+from azure.ai.evaluation._common.utils import validate_azure_ai_project, is_onedp_project
 from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
-from azure.ai.evaluation._model_configurations import AzureAIProject
 from azure.ai.evaluation.simulator import AdversarialScenario
-from azure.identity import DefaultAzureCredential
+from azure.ai.evaluation._model_configurations import AzureAIProject
+from azure.ai.evaluation._common.onedp._client import ProjectsClient as AIProjectClient
+from azure.core.credentials import TokenCredential
 
 from ._adversarial_simulator import AdversarialSimulator
-from ._model_tools import AdversarialTemplateHandler, ManagedIdentityAPITokenManager, RAIClient, TokenScope
+from ._model_tools import AdversarialTemplateHandler, ManagedIdentityAPITokenManager, RAIClient
 
 logger = logging.getLogger(__name__)
 
 
-def monitor_adversarial_scenario(func) -> Callable:
-    """Decorator to monitor adversarial scenario.
-
-    :param func: The function to be decorated.
-    :type func: Callable
-    :return: The decorated function.
-    :rtype: Callable
-    """
-
-    @functools.wraps(func)
-    def wrapper(*args, **kwargs):
-        scenario = str(kwargs.get("scenario", None))
-        max_conversation_turns = kwargs.get("max_conversation_turns", None)
-        max_simulation_results = kwargs.get("max_simulation_results", None)
-        decorated_func = monitor_operation(
-            activity_name="jailbreak.adversarial.simulator.call",
-            activity_type=ActivityType.PUBLICAPI,
-            custom_dimensions={
-                "scenario": scenario,
-                "max_conversation_turns": max_conversation_turns,
-                "max_simulation_results": max_simulation_results,
-            },
-        )(func)
-
-        return decorated_func(*args, **kwargs)
-
-    return wrapper
-
-
+@experimental
 class DirectAttackSimulator:
     """
     Initialize a UPIA (user prompt injected attack) jailbreak adversarial simulator with a project scope.
     This simulator converses with your AI system using prompts designed to interrupt normal functionality.
 
-    :param azure_ai_project: The scope of the Azure AI project. It contains subscription id, resource group, and project
-        name.
-    :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
+    :param azure_ai_project: The Azure AI project, which can either be a string representing the project endpoint
+        or an instance of AzureAIProject. It contains subscription id, resource group, and project name.
+    :type azure_ai_project: Union[str, AzureAIProject]
     :param credential: The credential for connecting to Azure AI project.
     :type credential: ~azure.core.credentials.TokenCredential
+
+    .. admonition:: Example:
+
+        .. literalinclude:: ../samples/evaluation_samples_simulate.py
+            :start-after: [START direct_attack_simulator]
+            :end-before: [END direct_attack_simulator]
+            :language: python
+            :dedent: 8
+            :caption: Run the DirectAttackSimulator to produce 2 results with 3 conversation turns each (6 messages in each result).
     """
 
-    def __init__(self, *, azure_ai_project: AzureAIProject, credential=None):
+    def __init__(self, *, azure_ai_project: Union[str, AzureAIProject], credential: TokenCredential):
         """Constructor."""
-        # check if azure_ai_project has the keys: subscription_id, resource_group_name, project_name, credential
-        if not all(key in azure_ai_project for key in ["subscription_id", "resource_group_name", "project_name"]):
-            msg = "azure_ai_project must contain keys: subscription_id, resource_group_name and project_name"
-            raise EvaluationException(
-                message=msg,
-                internal_message=msg,
-                target=ErrorTarget.DIRECT_ATTACK_SIMULATOR,
-                category=ErrorCategory.MISSING_FIELD,
-                blame=ErrorBlame.USER_ERROR,
+
+        if is_onedp_project(azure_ai_project):
+            self.azure_ai_project = azure_ai_project
+            self.credential = cast(TokenCredential, credential)
+            self.token_manager = ManagedIdentityAPITokenManager(
+                token_scope=TokenScope.COGNITIVE_SERVICES_MANAGEMENT,
+                logger=logging.getLogger("AdversarialSimulator"),
+                credential=self.credential,
             )
-        # check the value of the keys in azure_ai_project is not none
-        if not all(azure_ai_project[key] for key in ["subscription_id", "resource_group_name", "project_name"]):
-            msg = "subscription_id, resource_group_name and project_name keys cannot be None"
-            raise EvaluationException(
-                message=msg,
-                internal_message=msg,
-                target=ErrorTarget.DIRECT_ATTACK_SIMULATOR,
-                category=ErrorCategory.MISSING_FIELD,
-                blame=ErrorBlame.USER_ERROR,
+            self.rai_client = AIProjectClient(endpoint=azure_ai_project, credential=credential)
+        else:
+            try:
+                self.azure_ai_project = validate_azure_ai_project(azure_ai_project)
+            except EvaluationException as e:
+                raise EvaluationException(
+                    message=e.message,
+                    internal_message=e.internal_message,
+                    target=ErrorTarget.DIRECT_ATTACK_SIMULATOR,
+                    category=e.category,
+                    blame=e.blame,
+                ) from e
+            self.credential = cast(TokenCredential, credential)
+            self.token_manager = ManagedIdentityAPITokenManager(
+                token_scope=TokenScope.DEFAULT_AZURE_MANAGEMENT,
+                logger=logging.getLogger("AdversarialSimulator"),
+                credential=self.credential,
             )
-        if "credential" not in azure_ai_project and not credential:
-            credential = DefaultAzureCredential()
-        elif "credential" in azure_ai_project:
-            credential = azure_ai_project["credential"]
-        self.credential = credential
-        self.azure_ai_project = azure_ai_project
-        self.token_manager = ManagedIdentityAPITokenManager(
-            token_scope=TokenScope.DEFAULT_AZURE_MANAGEMENT,
-            logger=logging.getLogger("AdversarialSimulator"),
-            credential=credential,
-        )
-        self.rai_client = RAIClient(azure_ai_project=azure_ai_project, token_manager=self.token_manager)
+            self.rai_client = RAIClient(azure_ai_project=self.azure_ai_project, token_manager=self.token_manager)
+
         self.adversarial_template_handler = AdversarialTemplateHandler(
-            azure_ai_project=azure_ai_project, rai_client=self.rai_client
+            azure_ai_project=self.azure_ai_project, rai_client=self.rai_client
         )
 
     def _ensure_service_dependencies(self):
@@ -110,7 +90,6 @@ class DirectAttackSimulator:
                 blame=ErrorBlame.USER_ERROR,
             )
 
-    # @monitor_adversarial_scenario
     async def __call__(
         self,
         *,
@@ -169,7 +148,7 @@ class DirectAttackSimulator:
          - '**$schema**': A string indicating the schema URL for the conversation format.
 
          The 'content' for 'assistant' role messages may includes the messages that your callback returned.
-        :rtype: Dict[str, [List[Dict[str, Any]]]] with two elements
+        :rtype: Dict[str, [List[Dict[str, Any]]]]
 
         **Output format**
 
@@ -232,7 +211,7 @@ class DirectAttackSimulator:
             api_call_retry_sleep_sec=api_call_retry_sleep_sec,
             api_call_delay_sec=api_call_delay_sec,
             concurrent_async_task=concurrent_async_task,
-            randomize_order=True,
+            randomize_order=False,
             randomization_seed=randomization_seed,
         )
         jb_sim = AdversarialSimulator(azure_ai_project=self.azure_ai_project, credential=self.credential)
@@ -246,7 +225,7 @@ class DirectAttackSimulator:
             api_call_delay_sec=api_call_delay_sec,
             concurrent_async_task=concurrent_async_task,
             _jailbreak_type="upia",
-            randomize_order=True,
+            randomize_order=False,
             randomization_seed=randomization_seed,
         )
         return {"jailbreak": jb_sim_results, "regular": regular_sim_results}

azure/ai/evaluation/simulator/_helpers/__init__.py

@@ -1,5 +1,4 @@
-from ._experimental import experimental
 from ._language_suffix_mapping import SUPPORTED_LANGUAGES_MAPPING
 from ._simulator_data_classes import ConversationHistory, Turn
 
-__all__ = ["ConversationHistory", "Turn", "SUPPORTED_LANGUAGES_MAPPING", "experimental"]
+__all__ = ["ConversationHistory", "Turn", "SUPPORTED_LANGUAGES_MAPPING"]

azure/ai/evaluation/simulator/_helpers/_language_suffix_mapping.py

@@ -14,4 +14,5 @@ SUPPORTED_LANGUAGES_MAPPING = {
     SupportedLanguages.SimplifiedChinese: BASE_SUFFIX.replace("__language__", "simplified chinese"),
     SupportedLanguages.Portuguese: BASE_SUFFIX.replace("__language__", "portuguese"),
     SupportedLanguages.Japanese: BASE_SUFFIX.replace("__language__", "japanese"),
+    SupportedLanguages.Korean: BASE_SUFFIX.replace("__language__", "korean"),
 }

azure/ai/evaluation/simulator/_helpers/_simulator_data_classes.py

@@ -18,7 +18,7 @@ class Turn:
 
     role: Union[str, ConversationRole]
     content: str
-    context: str = None
+    context: Optional[str] = None
 
     def to_dict(self) -> Dict[str, Optional[str]]:
         """
@@ -30,7 +30,19 @@ class Turn:
         return {
             "role": self.role.value if isinstance(self.role, ConversationRole) else self.role,
             "content": self.content,
-            "context": self.context,
+            "context": str(self.context),
+        }
+
+    def to_context_free_dict(self) -> Dict[str, Optional[str]]:
+        """
+        Convert the conversation turn to a dictionary without context.
+
+        :returns: A dictionary representation of the conversation turn without context.
+        :rtype: Dict[str, Optional[str]]
+        """
+        return {
+            "role": self.role.value if isinstance(self.role, ConversationRole) else self.role,
+            "content": self.content,
         }
 
     def __repr__(self):
@@ -42,13 +54,13 @@ class ConversationHistory:
     Conversation history class to keep track of the conversation turns in a conversation.
     """
 
-    def __init__(self):
+    def __init__(self) -> None:
         """
         Initializes the conversation history with an empty list of turns.
         """
         self.history: List[Turn] = []
 
-    def add_to_history(self, turn: Turn):
+    def add_to_history(self, turn: Turn) -> None:
         """
         Adds a turn to the conversation history.
 
@@ -57,7 +69,7 @@ class ConversationHistory:
         """
         self.history.append(turn)
 
-    def to_list(self) -> List[Dict[str, str]]:
+    def to_list(self) -> List[Dict[str, Optional[str]]]:
         """
         Converts the conversation history to a list of dictionaries.
 
@@ -66,6 +78,15 @@ class ConversationHistory:
         """
         return [turn.to_dict() for turn in self.history]
 
+    def to_context_free_list(self) -> List[Dict[str, Optional[str]]]:
+        """
+        Converts the conversation history to a list of dictionaries without context.
+
+        :returns: A list of dictionaries representing the conversation turns without context.
+        :rtype: List[Dict[str, str]]
+        """
+        return [turn.to_context_free_dict() for turn in self.history]
+
     def __len__(self) -> int:
         return len(self.history)
 

azure/ai/evaluation/simulator/_indirect_attack_simulator.py

@@ -1,100 +1,90 @@
 # ---------------------------------------------------------
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
+# pylint: disable=C0301,C0114,R0913,R0903
 # noqa: E501
-import functools
+import asyncio
 import logging
-from typing import Callable
+import random
+from typing import Callable, cast, Union, Optional
 
-from promptflow._sdk._telemetry import ActivityType, monitor_operation
+from tqdm import tqdm
 
+from azure.ai.evaluation._common.utils import validate_azure_ai_project, is_onedp_project
+from azure.ai.evaluation._common._experimental import experimental
 from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
+from azure.ai.evaluation.simulator import AdversarialScenarioJailbreak, SupportedLanguages
 from azure.ai.evaluation._model_configurations import AzureAIProject
-from azure.ai.evaluation.simulator import AdversarialScenario
-from azure.identity import DefaultAzureCredential
+from azure.ai.evaluation._common.onedp._client import ProjectsClient as AIProjectClient
+from azure.core.credentials import TokenCredential
+from azure.ai.evaluation._constants import TokenScope
 
-from ._adversarial_simulator import AdversarialSimulator
-from ._model_tools import AdversarialTemplateHandler, ManagedIdentityAPITokenManager, RAIClient, TokenScope
+from ._adversarial_simulator import AdversarialSimulator, JsonLineList
 
-logger = logging.getLogger(__name__)
-
-
-def monitor_adversarial_scenario(func) -> Callable:
-    """Decorator to monitor adversarial scenario.
-
-    :param func: The function to be decorated.
-    :type func: Callable
-    :return: The decorated function.
-    :rtype: Callable
-    """
+from ._model_tools import AdversarialTemplateHandler, ManagedIdentityAPITokenManager, RAIClient
 
-    @functools.wraps(func)
-    def wrapper(*args, **kwargs):
-        scenario = str(kwargs.get("scenario", None))
-        max_conversation_turns = kwargs.get("max_conversation_turns", None)
-        max_simulation_results = kwargs.get("max_simulation_results", None)
-        decorated_func = monitor_operation(
-            activity_name="xpia.adversarial.simulator.call",
-            activity_type=ActivityType.PUBLICAPI,
-            custom_dimensions={
-                "scenario": scenario,
-                "max_conversation_turns": max_conversation_turns,
-                "max_simulation_results": max_simulation_results,
-            },
-        )(func)
-
-        return decorated_func(*args, **kwargs)
-
-    return wrapper
+logger = logging.getLogger(__name__)
 
 
-class IndirectAttackSimulator:
+@experimental
+class IndirectAttackSimulator(AdversarialSimulator):
     """
     Initializes the XPIA (cross domain prompt injected attack) jailbreak adversarial simulator with a project scope.
 
-    :param azure_ai_project: The scope of the Azure AI project. It contains subscription id, resource group, and project
-        name.
-    :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
+    :param azure_ai_project: The Azure AI project, which can either be a string representing the project endpoint
+        or an instance of AzureAIProject. It contains subscription id, resource group, and project name.
+    :type azure_ai_project: Union[str, AzureAIProject]
     :param credential: The credential for connecting to Azure AI project.
     :type credential: ~azure.core.credentials.TokenCredential
+
+    .. admonition:: Example:
+
+        .. literalinclude:: ../samples/evaluation_samples_simulate.py
+            :start-after: [START indirect_attack_simulator]
+            :end-before: [END indirect_attack_simulator]
+            :language: python
+            :dedent: 8
+            :caption: Run the IndirectAttackSimulator to produce 1 result with 1 conversation turn (2 messages in the result).
     """
 
-    def __init__(self, *, azure_ai_project: AzureAIProject, credential=None):
+    def __init__(self, *, azure_ai_project: Union[str, AzureAIProject], credential: TokenCredential):
         """Constructor."""
-        # check if azure_ai_project has the keys: subscription_id, resource_group_name, project_name, credential
-        if not all(key in azure_ai_project for key in ["subscription_id", "resource_group_name", "project_name"]):
-            msg = "azure_ai_project must contain keys: subscription_id, resource_group_name and project_name"
-            raise EvaluationException(
-                message=msg,
-                internal_message=msg,
-                target=ErrorTarget.DIRECT_ATTACK_SIMULATOR,
-                category=ErrorCategory.MISSING_FIELD,
-                blame=ErrorBlame.USER_ERROR,
+
+        if is_onedp_project(azure_ai_project):
+            self.azure_ai_project = azure_ai_project
+            self.credential = cast(TokenCredential, credential)
+            self.token_manager = ManagedIdentityAPITokenManager(
+                token_scope=TokenScope.COGNITIVE_SERVICES_MANAGEMENT,
+                logger=logging.getLogger("AdversarialSimulator"),
+                credential=self.credential,
             )
-        if not all(azure_ai_project[key] for key in ["subscription_id", "resource_group_name", "project_name"]):
-            msg = "subscription_id, resource_group_name and project_name keys cannot be None"
-            raise EvaluationException(
-                message=msg,
-                internal_message=msg,
-                target=ErrorTarget.DIRECT_ATTACK_SIMULATOR,
-                category=ErrorCategory.MISSING_FIELD,
-                blame=ErrorBlame.USER_ERROR,
+            self.rai_client = AIProjectClient(endpoint=azure_ai_project, credential=credential)
+            self.adversarial_template_handler = AdversarialTemplateHandler(
+                azure_ai_project=self.azure_ai_project, rai_client=self.rai_client
             )
-        if "credential" not in azure_ai_project and not credential:
-            credential = DefaultAzureCredential()
-        elif "credential" in azure_ai_project:
-            credential = azure_ai_project["credential"]
-        self.credential = credential
-        self.azure_ai_project = azure_ai_project
-        self.token_manager = ManagedIdentityAPITokenManager(
-            token_scope=TokenScope.DEFAULT_AZURE_MANAGEMENT,
-            logger=logging.getLogger("AdversarialSimulator"),
-            credential=credential,
-        )
-        self.rai_client = RAIClient(azure_ai_project=azure_ai_project, token_manager=self.token_manager)
-        self.adversarial_template_handler = AdversarialTemplateHandler(
-            azure_ai_project=azure_ai_project, rai_client=self.rai_client
-        )
+        else:
+            try:
+                self.azure_ai_project = validate_azure_ai_project(azure_ai_project)
+            except EvaluationException as e:
+                raise EvaluationException(
+                    message=e.message,
+                    internal_message=e.internal_message,
+                    target=ErrorTarget.DIRECT_ATTACK_SIMULATOR,
+                    category=e.category,
+                    blame=e.blame,
+                ) from e
+
+            self.credential = cast(TokenCredential, credential)
+            self.token_manager = ManagedIdentityAPITokenManager(
+                token_scope=TokenScope.DEFAULT_AZURE_MANAGEMENT,
+                logger=logging.getLogger("AdversarialSimulator"),
+                credential=self.credential,
+            )
+            self.rai_client = RAIClient(azure_ai_project=self.azure_ai_project, token_manager=self.token_manager)
+            self.adversarial_template_handler = AdversarialTemplateHandler(
+                azure_ai_project=self.azure_ai_project, rai_client=self.rai_client
+            )
+        super().__init__(azure_ai_project=azure_ai_project, credential=credential)
 
     def _ensure_service_dependencies(self):
         if self.rai_client is None:
@@ -107,33 +97,26 @@ class IndirectAttackSimulator:
                 blame=ErrorBlame.USER_ERROR,
             )
 
-    # @monitor_adversarial_scenario
     async def __call__(
         self,
         *,
-        scenario: AdversarialScenario,
         target: Callable,
-        max_conversation_turns: int = 1,
         max_simulation_results: int = 3,
         api_call_retry_limit: int = 3,
         api_call_retry_sleep_sec: int = 1,
         api_call_delay_sec: int = 0,
         concurrent_async_task: int = 3,
+        randomization_seed: Optional[int] = None,
+        **kwargs,
     ):
         """
         Initializes the XPIA (cross domain prompt injected attack) jailbreak adversarial simulator with a project scope.
         This simulator converses with your AI system using prompts injected into the context to interrupt normal
         expected functionality by eliciting manipulated content, intrusion and attempting to gather information outside
         the scope of your AI system.
-
-        :keyword scenario: Enum value specifying the adversarial scenario used for generating inputs.
-        :paramtype scenario: azure.ai.evaluation.simulator.AdversarialScenario
         :keyword target: The target function to simulate adversarial inputs against.
             This function should be asynchronous and accept a dictionary representing the adversarial input.
         :paramtype target: Callable
-        :keyword max_conversation_turns: The maximum number of conversation turns to simulate.
-            Defaults to 1.
-        :paramtype max_conversation_turns: int
         :keyword max_simulation_results: The maximum number of simulation results to return.
             Defaults to 3.
         :paramtype max_simulation_results: int
@@ -149,6 +132,9 @@ class IndirectAttackSimulator:
         :keyword concurrent_async_task: The number of asynchronous tasks to run concurrently during the simulation.
             Defaults to 3.
         :paramtype concurrent_async_task: int
+        :keyword randomization_seed: The seed used to randomize prompt selection. If unset, the system's
+            default seed is used. Defaults to None.
+        :paramtype randomization_seed: Optional[int]
         :return: A list of dictionaries, each representing a simulated conversation. Each dictionary contains:
 
          - 'template_parameters': A dictionary with parameters used in the conversation template,
@@ -170,11 +156,11 @@ class IndirectAttackSimulator:
                     'template_parameters': {},
                     'messages': [
                         {
-                            'content': '<jailbreak prompt> <adversarial query>',
+                            'content': '<adversarial query>',
                             'role': 'user'
                         },
                         {
-                            'content': "<response from endpoint>",
+                            'content': "<response from your callback>",
                             'role': 'assistant',
                             'context': None
                         }
@@ -183,25 +169,80 @@ class IndirectAttackSimulator:
                 }]
             }
         """
-        if scenario not in AdversarialScenario.__members__.values():
-            msg = f"Invalid scenario: {scenario}. Supported scenarios: {AdversarialScenario.__members__.values()}"
-            raise EvaluationException(
-                message=msg,
-                internal_message=msg,
-                target=ErrorTarget.DIRECT_ATTACK_SIMULATOR,
-                category=ErrorCategory.INVALID_VALUE,
-                blame=ErrorBlame.USER_ERROR,
+        # values that cannot be changed:
+        scenario = AdversarialScenarioJailbreak.ADVERSARIAL_INDIRECT_JAILBREAK
+        max_conversation_turns = 2
+        language = SupportedLanguages.English
+        self._ensure_service_dependencies()
+        templates = await self.adversarial_template_handler._get_content_harm_template_collections(scenario.value)
+        concurrent_async_task = min(concurrent_async_task, 1000)
+        semaphore = asyncio.Semaphore(concurrent_async_task)
+        sim_results = []
+        tasks = []
+        total_tasks = sum(len(t.template_parameters) for t in templates)
+        if max_simulation_results > total_tasks:
+            logger.warning(
+                "Cannot provide %s results due to maximum number of adversarial simulations that can be generated: %s."
+                "\n %s simulations will be generated.",
+                max_simulation_results,
+                total_tasks,
+                total_tasks,
             )
-        jb_sim = AdversarialSimulator(azure_ai_project=self.azure_ai_project, credential=self.credential)
-        jb_sim_results = await jb_sim(
-            scenario=scenario,
-            target=target,
-            max_conversation_turns=max_conversation_turns,
-            max_simulation_results=max_simulation_results,
-            api_call_retry_limit=api_call_retry_limit,
-            api_call_retry_sleep_sec=api_call_retry_sleep_sec,
-            api_call_delay_sec=api_call_delay_sec,
-            concurrent_async_task=concurrent_async_task,
-            _jailbreak_type="xpia",
+        total_tasks = min(total_tasks, max_simulation_results)
+        progress_bar = tqdm(
+            total=total_tasks,
+            desc="generating jailbreak simulations",
+            ncols=100,
+            unit="simulations",
         )
-        return jb_sim_results
+
+        # Apply randomization to templates if seed is provided
+        if randomization_seed is not None:
+            # Create a local random instance to avoid polluting global state
+            local_random = random.Random(randomization_seed)
+            local_random.shuffle(templates)
+
+        for template in templates:
+            for parameter in template.template_parameters:
+                tasks.append(
+                    asyncio.create_task(
+                        self._simulate_async(
+                            target=target,
+                            template=template,
+                            parameters=parameter,
+                            max_conversation_turns=max_conversation_turns,
+                            api_call_retry_limit=api_call_retry_limit,
+                            api_call_retry_sleep_sec=api_call_retry_sleep_sec,
+                            api_call_delay_sec=api_call_delay_sec,
+                            language=language,
+                            semaphore=semaphore,
+                            scenario=scenario,
+                        )
+                    )
+                )
+                if len(tasks) >= max_simulation_results:
+                    break
+            if len(tasks) >= max_simulation_results:
+                break
+        for task in asyncio.as_completed(tasks):
+            completed_task = await task  # type: ignore
+            template_parameters = completed_task.get("template_parameters", {})  # type: ignore
+            xpia_attack_type = template_parameters.get("xpia_attack_type", "")  # type: ignore
+            action = template_parameters.get("action", "")  # type: ignore
+            document_type = template_parameters.get("document_type", "")  # type: ignore
+            sim_results.append(
+                {
+                    "messages": completed_task["messages"],  # type: ignore
+                    "$schema": "http://azureml/sdk-2-0/ChatConversation.json",
+                    "template_parameters": {
+                        "metadata": {
+                            "xpia_attack_type": xpia_attack_type,
+                            "action": action,
+                            "document_type": document_type,
+                        },
+                    },
+                }
+            )
+            progress_bar.update(1)
+        progress_bar.close()
+        return JsonLineList(sim_results)

azure/ai/evaluation/simulator/_model_tools/__init__.py

@@ -4,11 +4,12 @@
 
 """Tooling for model evaluation"""
 
-from ._identity_manager import ManagedIdentityAPITokenManager, PlainTokenManager, TokenScope
+from ._identity_manager import ManagedIdentityAPITokenManager, PlainTokenManager
 from ._proxy_completion_model import ProxyChatCompletionsModel
 from ._rai_client import RAIClient
 from ._template_handler import CONTENT_HARM_TEMPLATES_COLLECTION_KEY, AdversarialTemplateHandler
 from .models import LLMBase, OpenAIChatCompletionsModel
+from ..._constants import TokenScope
 
 __all__ = [
     "ManagedIdentityAPITokenManager",