azure-ai-evaluation 1.0.0__py3-none-any.whl → 1.0.0b1__py3-none-any.whl

azure/ai/evaluation/simulator/_direct_attack_simulator.py

@@ -1,26 +1,54 @@
 # ---------------------------------------------------------
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
-# pylint: disable=C0301,C0114,R0913,R0903
 # noqa: E501
+import functools
 import logging
 from random import randint
-from typing import Callable, Optional, cast
+from typing import Any, Callable, Dict, Optional
 
-from azure.ai.evaluation._common._experimental import experimental
-from azure.ai.evaluation._common.utils import validate_azure_ai_project
-from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
+from azure.identity import DefaultAzureCredential
+
+from promptflow._sdk._telemetry import ActivityType, monitor_operation
+from azure.ai.evaluation._exceptions import EvaluationException, ErrorBlame, ErrorCategory, ErrorTarget
 from azure.ai.evaluation.simulator import AdversarialScenario
 from azure.ai.evaluation._model_configurations import AzureAIProject
-from azure.core.credentials import TokenCredential
 
-from ._adversarial_simulator import AdversarialSimulator
 from ._model_tools import AdversarialTemplateHandler, ManagedIdentityAPITokenManager, RAIClient, TokenScope
+from ._adversarial_simulator import AdversarialSimulator
 
 logger = logging.getLogger(__name__)
 
 
-@experimental
+def monitor_adversarial_scenario(func) -> Callable:
+    """Decorator to monitor adversarial scenario.
+
+    :param func: The function to be decorated.
+    :type func: Callable
+    :return: The decorated function.
+    :rtype: Callable
+    """
+
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        scenario = str(kwargs.get("scenario", None))
+        max_conversation_turns = kwargs.get("max_conversation_turns", None)
+        max_simulation_results = kwargs.get("max_simulation_results", None)
+        decorated_func = monitor_operation(
+            activity_name="jailbreak.adversarial.simulator.call",
+            activity_type=ActivityType.PUBLICAPI,
+            custom_dimensions={
+                "scenario": scenario,
+                "max_conversation_turns": max_conversation_turns,
+                "max_simulation_results": max_simulation_results,
+            },
+        )(func)
+
+        return decorated_func(*args, **kwargs)
+
+    return wrapper
+
+
 class DirectAttackSimulator:
     """
     Initialize a UPIA (user prompt injected attack) jailbreak adversarial simulator with a project scope.
@@ -31,39 +59,44 @@ class DirectAttackSimulator:
     :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
     :param credential: The credential for connecting to Azure AI project.
     :type credential: ~azure.core.credentials.TokenCredential
-
-    .. admonition:: Example:
-
-        .. literalinclude:: ../samples/evaluation_samples_simulate.py
-            :start-after: [START direct_attack_simulator]
-            :end-before: [END direct_attack_simulator]
-            :language: python
-            :dedent: 8
-            :caption: Run the DirectAttackSimulator to produce 2 results with 3 conversation turns each (6 messages in each result).
     """
 
-    def __init__(self, *, azure_ai_project: AzureAIProject, credential: TokenCredential):
+    def __init__(self, *, azure_ai_project: AzureAIProject, credential=None):
         """Constructor."""
-
-        try:
-            self.azure_ai_project = validate_azure_ai_project(azure_ai_project)
-        except EvaluationException as e:
+        # check if azure_ai_project has the keys: subscription_id, resource_group_name, project_name, credential
+        if not all(key in azure_ai_project for key in ["subscription_id", "resource_group_name", "project_name"]):
+            msg = "azure_ai_project must contain keys: subscription_id, resource_group_name and project_name"
             raise EvaluationException(
-                message=e.message,
-                internal_message=e.internal_message,
+                message=msg,
+                internal_message=msg,
+                target=ErrorTarget.DIRECT_ATTACK_SIMULATOR,
+                category=ErrorCategory.MISSING_FIELD,
+                blame=ErrorBlame.USER_ERROR,
+            )
+        # check the value of the keys in azure_ai_project is not none
+        if not all(azure_ai_project[key] for key in ["subscription_id", "resource_group_name", "project_name"]):
+            msg = "subscription_id, resource_group_name and project_name keys cannot be None"
+            raise EvaluationException(
+                message=msg,
+                internal_message=msg,
                 target=ErrorTarget.DIRECT_ATTACK_SIMULATOR,
-                category=e.category,
-                blame=e.blame,
-            ) from e
-        self.credential = cast(TokenCredential, credential)
+                category=ErrorCategory.MISSING_FIELD,
+                blame=ErrorBlame.USER_ERROR,
+            )
+        if "credential" not in azure_ai_project and not credential:
+            credential = DefaultAzureCredential()
+        elif "credential" in azure_ai_project:
+            credential = azure_ai_project["credential"]
+        self.credential = credential
+        self.azure_ai_project = azure_ai_project
         self.token_manager = ManagedIdentityAPITokenManager(
             token_scope=TokenScope.DEFAULT_AZURE_MANAGEMENT,
             logger=logging.getLogger("AdversarialSimulator"),
-            credential=self.credential,
+            credential=credential,
         )
-        self.rai_client = RAIClient(azure_ai_project=self.azure_ai_project, token_manager=self.token_manager)
+        self.rai_client = RAIClient(azure_ai_project=azure_ai_project, token_manager=self.token_manager)
         self.adversarial_template_handler = AdversarialTemplateHandler(
-            azure_ai_project=self.azure_ai_project, rai_client=self.rai_client
+            azure_ai_project=azure_ai_project, rai_client=self.rai_client
         )
 
     def _ensure_service_dependencies(self):
@@ -77,6 +110,7 @@ class DirectAttackSimulator:
                 blame=ErrorBlame.USER_ERROR,
             )
 
+    # @monitor_adversarial_scenario
     async def __call__(
         self,
         *,
@@ -135,7 +169,7 @@ class DirectAttackSimulator:
          - '**$schema**': A string indicating the schema URL for the conversation format.
 
          The 'content' for 'assistant' role messages may includes the messages that your callback returned.
-        :rtype: Dict[str, [List[Dict[str, Any]]]]
+        :rtype: Dict[str, [List[Dict[str, Any]]]] with two elements
 
         **Output format**
 
@@ -198,7 +232,7 @@ class DirectAttackSimulator:
             api_call_retry_sleep_sec=api_call_retry_sleep_sec,
             api_call_delay_sec=api_call_delay_sec,
             concurrent_async_task=concurrent_async_task,
-            randomize_order=False,
+            randomize_order=True,
             randomization_seed=randomization_seed,
         )
         jb_sim = AdversarialSimulator(azure_ai_project=self.azure_ai_project, credential=self.credential)
@@ -212,7 +246,7 @@ class DirectAttackSimulator:
             api_call_delay_sec=api_call_delay_sec,
             concurrent_async_task=concurrent_async_task,
             _jailbreak_type="upia",
-            randomize_order=False,
+            randomize_order=True,
             randomization_seed=randomization_seed,
         )
         return {"jailbreak": jb_sim_results, "regular": regular_sim_results}

azure/ai/evaluation/simulator/_helpers/__init__.py

@@ -1,4 +1,4 @@
-from ._language_suffix_mapping import SUPPORTED_LANGUAGES_MAPPING
 from ._simulator_data_classes import ConversationHistory, Turn
+from ._language_suffix_mapping import SUPPORTED_LANGUAGES_MAPPING
 
 __all__ = ["ConversationHistory", "Turn", "SUPPORTED_LANGUAGES_MAPPING"]

azure/ai/evaluation/simulator/_helpers/_simulator_data_classes.py

@@ -3,7 +3,7 @@
 # ---------------------------------------------------------
 # pylint: disable=C0103,C0114,C0116
 from dataclasses import dataclass
-from typing import Dict, List, Optional, Union
+from typing import Union
 
 from azure.ai.evaluation.simulator._conversation.constants import ConversationRole
 
@@ -18,34 +18,28 @@ class Turn:
 
     role: Union[str, ConversationRole]
     content: str
-    context: Optional[str] = None
+    context: str = None
 
-    def to_dict(self) -> Dict[str, Optional[str]]:
+    def to_dict(self):
         """
         Convert the conversation turn to a dictionary.
 
-        :returns: A dictionary representation of the conversation turn.
-        :rtype: Dict[str, Optional[str]]
+        Returns:
+            dict: A dictionary representation of the conversation turn.
         """
         return {
             "role": self.role.value if isinstance(self.role, ConversationRole) else self.role,
             "content": self.content,
-            "context": str(self.context),
+            "context": self.context,
         }
 
-    def to_context_free_dict(self) -> Dict[str, Optional[str]]:
+    def __repr__(self):
         """
-        Convert the conversation turn to a dictionary without context.
+        Return the string representation of the conversation turn.
 
-        :returns: A dictionary representation of the conversation turn without context.
-        :rtype: Dict[str, Optional[str]]
+        Returns:
+            str: A string representation of the conversation turn.
         """
-        return {
-            "role": self.role.value if isinstance(self.role, ConversationRole) else self.role,
-            "content": self.content,
-        }
-
-    def __repr__(self):
         return f"Turn(role={self.role}, content={self.content})"
 
 
@@ -54,43 +48,46 @@ class ConversationHistory:
     Conversation history class to keep track of the conversation turns in a conversation.
     """
 
-    def __init__(self) -> None:
+    def __init__(self):
         """
         Initializes the conversation history with an empty list of turns.
         """
-        self.history: List[Turn] = []
+        self.history = []
 
-    def add_to_history(self, turn: Turn) -> None:
+    def add_to_history(self, turn: Turn):
         """
         Adds a turn to the conversation history.
 
-        :param turn: The conversation turn to add.
-        :type turn: Turn
+        Args:
+            turn (Turn): The conversation turn to add.
         """
         self.history.append(turn)
 
-    def to_list(self) -> List[Dict[str, Optional[str]]]:
+    def to_list(self):
         """
         Converts the conversation history to a list of dictionaries.
 
-        :returns: A list of dictionaries representing the conversation turns.
-        :rtype: List[Dict[str, str]]
+        Returns:
+            list: A list of dictionaries representing the conversation turns.
         """
         return [turn.to_dict() for turn in self.history]
 
-    def to_context_free_list(self) -> List[Dict[str, Optional[str]]]:
+    def get_length(self):
         """
-        Converts the conversation history to a list of dictionaries without context.
+        Returns the length of the conversation.
 
-        :returns: A list of dictionaries representing the conversation turns without context.
-        :rtype: List[Dict[str, str]]
+        Returns:
+            int: The number of turns in the conversation history.
         """
-        return [turn.to_context_free_dict() for turn in self.history]
-
-    def __len__(self) -> int:
         return len(self.history)
 
     def __repr__(self):
+        """
+        Returns the string representation of the conversation history.
+
+        Returns:
+            str: A string representation of the conversation history.
+        """
         for turn in self.history:
             print(turn)
         return ""

azure/ai/evaluation/simulator/_indirect_attack_simulator.py

@@ -1,30 +1,54 @@
 # ---------------------------------------------------------
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
-# pylint: disable=C0301,C0114,R0913,R0903
 # noqa: E501
-import asyncio
+import functools
 import logging
-from typing import Callable, cast
+from typing import Any, Callable, Dict
 
-from tqdm import tqdm
+from azure.identity import DefaultAzureCredential
 
-from azure.ai.evaluation._common.utils import validate_azure_ai_project
-from azure.ai.evaluation._common._experimental import experimental
-from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
-from azure.ai.evaluation.simulator import AdversarialScenarioJailbreak, SupportedLanguages
+from promptflow._sdk._telemetry import ActivityType, monitor_operation
+from azure.ai.evaluation.simulator import AdversarialScenario
 from azure.ai.evaluation._model_configurations import AzureAIProject
-from azure.core.credentials import TokenCredential
-
-from ._adversarial_simulator import AdversarialSimulator, JsonLineList
 
 from ._model_tools import AdversarialTemplateHandler, ManagedIdentityAPITokenManager, RAIClient, TokenScope
+from azure.ai.evaluation._exceptions import EvaluationException, ErrorBlame, ErrorCategory, ErrorTarget
+from ._adversarial_simulator import AdversarialSimulator
 
 logger = logging.getLogger(__name__)
 
 
-@experimental
-class IndirectAttackSimulator(AdversarialSimulator):
+def monitor_adversarial_scenario(func) -> Callable:
+    """Decorator to monitor adversarial scenario.
+
+    :param func: The function to be decorated.
+    :type func: Callable
+    :return: The decorated function.
+    :rtype: Callable
+    """
+
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        scenario = str(kwargs.get("scenario", None))
+        max_conversation_turns = kwargs.get("max_conversation_turns", None)
+        max_simulation_results = kwargs.get("max_simulation_results", None)
+        decorated_func = monitor_operation(
+            activity_name="xpia.adversarial.simulator.call",
+            activity_type=ActivityType.PUBLICAPI,
+            custom_dimensions={
+                "scenario": scenario,
+                "max_conversation_turns": max_conversation_turns,
+                "max_simulation_results": max_simulation_results,
+            },
+        )(func)
+
+        return decorated_func(*args, **kwargs)
+
+    return wrapper
+
+
+class IndirectAttackSimulator:
     """
     Initializes the XPIA (cross domain prompt injected attack) jailbreak adversarial simulator with a project scope.
 
@@ -33,42 +57,44 @@ class IndirectAttackSimulator(AdversarialSimulator):
     :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
     :param credential: The credential for connecting to Azure AI project.
     :type credential: ~azure.core.credentials.TokenCredential
-
-    .. admonition:: Example:
-
-        .. literalinclude:: ../samples/evaluation_samples_simulate.py
-            :start-after: [START indirect_attack_simulator]
-            :end-before: [END indirect_attack_simulator]
-            :language: python
-            :dedent: 8
-            :caption: Run the IndirectAttackSimulator to produce 1 result with 1 conversation turn (2 messages in the result).
     """
 
-    def __init__(self, *, azure_ai_project: AzureAIProject, credential: TokenCredential):
+    def __init__(self, *, azure_ai_project: AzureAIProject, credential=None):
         """Constructor."""
-
-        try:
-            self.azure_ai_project = validate_azure_ai_project(azure_ai_project)
-        except EvaluationException as e:
+        # check if azure_ai_project has the keys: subscription_id, resource_group_name, project_name, credential
+        if not all(key in azure_ai_project for key in ["subscription_id", "resource_group_name", "project_name"]):
+            msg = "azure_ai_project must contain keys: subscription_id, resource_group_name and project_name"
             raise EvaluationException(
-                message=e.message,
-                internal_message=e.internal_message,
+                message=msg,
+                internal_message=msg,
                 target=ErrorTarget.DIRECT_ATTACK_SIMULATOR,
-                category=e.category,
-                blame=e.blame,
-            ) from e
-
-        self.credential = cast(TokenCredential, credential)
+                category=ErrorCategory.MISSING_FIELD,
+                blame=ErrorBlame.USER_ERROR,
+            )
+        if not all(azure_ai_project[key] for key in ["subscription_id", "resource_group_name", "project_name"]):
+            msg = "subscription_id, resource_group_name and project_name keys cannot be None"
+            raise EvaluationException(
+                message=msg,
+                internal_message=msg,
+                target=ErrorTarget.DIRECT_ATTACK_SIMULATOR,
+                category=ErrorCategory.MISSING_FIELD,
+                blame=ErrorBlame.USER_ERROR,
+            )
+        if "credential" not in azure_ai_project and not credential:
+            credential = DefaultAzureCredential()
+        elif "credential" in azure_ai_project:
+            credential = azure_ai_project["credential"]
+        self.credential = credential
+        self.azure_ai_project = azure_ai_project
         self.token_manager = ManagedIdentityAPITokenManager(
             token_scope=TokenScope.DEFAULT_AZURE_MANAGEMENT,
             logger=logging.getLogger("AdversarialSimulator"),
-            credential=self.credential,
+            credential=credential,
         )
-        self.rai_client = RAIClient(azure_ai_project=self.azure_ai_project, token_manager=self.token_manager)
+        self.rai_client = RAIClient(azure_ai_project=azure_ai_project, token_manager=self.token_manager)
         self.adversarial_template_handler = AdversarialTemplateHandler(
-            azure_ai_project=self.azure_ai_project, rai_client=self.rai_client
+            azure_ai_project=azure_ai_project, rai_client=self.rai_client
         )
-        super().__init__(azure_ai_project=azure_ai_project, credential=credential)
 
     def _ensure_service_dependencies(self):
         if self.rai_client is None:
@@ -81,25 +107,33 @@ class IndirectAttackSimulator(AdversarialSimulator):
                 blame=ErrorBlame.USER_ERROR,
             )
 
+    # @monitor_adversarial_scenario
     async def __call__(
         self,
         *,
+        scenario: AdversarialScenario,
         target: Callable,
+        max_conversation_turns: int = 1,
         max_simulation_results: int = 3,
         api_call_retry_limit: int = 3,
         api_call_retry_sleep_sec: int = 1,
         api_call_delay_sec: int = 0,
         concurrent_async_task: int = 3,
-        **kwargs,
     ):
         """
         Initializes the XPIA (cross domain prompt injected attack) jailbreak adversarial simulator with a project scope.
         This simulator converses with your AI system using prompts injected into the context to interrupt normal
         expected functionality by eliciting manipulated content, intrusion and attempting to gather information outside
         the scope of your AI system.
+
+        :keyword scenario: Enum value specifying the adversarial scenario used for generating inputs.
+        :paramtype scenario: azure.ai.evaluation.simulator.AdversarialScenario
         :keyword target: The target function to simulate adversarial inputs against.
             This function should be asynchronous and accept a dictionary representing the adversarial input.
         :paramtype target: Callable
+        :keyword max_conversation_turns: The maximum number of conversation turns to simulate.
+            Defaults to 1.
+        :paramtype max_conversation_turns: int
         :keyword max_simulation_results: The maximum number of simulation results to return.
             Defaults to 3.
         :paramtype max_simulation_results: int
@@ -136,11 +170,11 @@ class IndirectAttackSimulator(AdversarialSimulator):
                     'template_parameters': {},
                     'messages': [
                         {
-                            'content': '<adversarial query>',
+                            'content': '<jailbreak prompt> <adversarial query>',
                             'role': 'user'
                         },
                         {
-                            'content': "<response from your callback>",
+                            'content': "<response from endpoint>",
                             'role': 'assistant',
                             'context': None
                         }
@@ -149,72 +183,25 @@ class IndirectAttackSimulator(AdversarialSimulator):
                 }]
             }
         """
-        # values that cannot be changed:
-        scenario = AdversarialScenarioJailbreak.ADVERSARIAL_INDIRECT_JAILBREAK
-        max_conversation_turns = 2
-        language = SupportedLanguages.English
-        self._ensure_service_dependencies()
-        templates = await self.adversarial_template_handler._get_content_harm_template_collections(scenario.value)
-        concurrent_async_task = min(concurrent_async_task, 1000)
-        semaphore = asyncio.Semaphore(concurrent_async_task)
-        sim_results = []
-        tasks = []
-        total_tasks = sum(len(t.template_parameters) for t in templates)
-        if max_simulation_results > total_tasks:
-            logger.warning(
-                "Cannot provide %s results due to maximum number of adversarial simulations that can be generated: %s."
-                "\n %s simulations will be generated.",
-                max_simulation_results,
-                total_tasks,
-                total_tasks,
+        if scenario not in AdversarialScenario.__members__.values():
+            msg = f"Invalid scenario: {scenario}. Supported scenarios: {AdversarialScenario.__members__.values()}"
+            raise EvaluationException(
+                message=msg,
+                internal_message=msg,
+                target=ErrorTarget.DIRECT_ATTACK_SIMULATOR,
+                category=ErrorCategory.INVALID_VALUE,
+                blame=ErrorBlame.USER_ERROR,
             )
-        total_tasks = min(total_tasks, max_simulation_results)
-        progress_bar = tqdm(
-            total=total_tasks,
-            desc="generating jailbreak simulations",
-            ncols=100,
-            unit="simulations",
+        jb_sim = AdversarialSimulator(azure_ai_project=self.azure_ai_project, credential=self.credential)
+        jb_sim_results = await jb_sim(
+            scenario=scenario,
+            target=target,
+            max_conversation_turns=max_conversation_turns,
+            max_simulation_results=max_simulation_results,
+            api_call_retry_limit=api_call_retry_limit,
+            api_call_retry_sleep_sec=api_call_retry_sleep_sec,
+            api_call_delay_sec=api_call_delay_sec,
+            concurrent_async_task=concurrent_async_task,
+            _jailbreak_type="xpia",
         )
-        for template in templates:
-            for parameter in template.template_parameters:
-                tasks.append(
-                    asyncio.create_task(
-                        self._simulate_async(
-                            target=target,
-                            template=template,
-                            parameters=parameter,
-                            max_conversation_turns=max_conversation_turns,
-                            api_call_retry_limit=api_call_retry_limit,
-                            api_call_retry_sleep_sec=api_call_retry_sleep_sec,
-                            api_call_delay_sec=api_call_delay_sec,
-                            language=language,
-                            semaphore=semaphore,
-                        )
-                    )
-                )
-                if len(tasks) >= max_simulation_results:
-                    break
-            if len(tasks) >= max_simulation_results:
-                break
-        for task in asyncio.as_completed(tasks):
-            completed_task = await task  # type: ignore
-            template_parameters = completed_task.get("template_parameters", {})  # type: ignore
-            xpia_attack_type = template_parameters.get("xpia_attack_type", "")  # type: ignore
-            action = template_parameters.get("action", "")  # type: ignore
-            document_type = template_parameters.get("document_type", "")  # type: ignore
-            sim_results.append(
-                {
-                    "messages": completed_task["messages"],  # type: ignore
-                    "$schema": "http://azureml/sdk-2-0/ChatConversation.json",
-                    "template_parameters": {
-                        "metadata": {
-                            "xpia_attack_type": xpia_attack_type,
-                            "action": action,
-                            "document_type": document_type,
-                        },
-                    },
-                }
-            )
-            progress_bar.update(1)
-        progress_bar.close()
-        return JsonLineList(sim_results)
+        return jb_sim_results