azure-ai-evaluation 1.0.0__py3-none-any.whl → 1.0.0b1__py3-none-any.whl

azure/ai/evaluation/simulator/_adversarial_simulator.py

@@ -6,23 +6,19 @@
 import asyncio
 import logging
 import random
-from typing import Any, Callable, Dict, List, Literal, Optional, Union, cast
-from itertools import zip_longest
+from typing import Any, Callable, Dict, List, Optional
 
+from azure.core.pipeline.policies import AsyncRetryPolicy, RetryMode
+from azure.identity import DefaultAzureCredential
 from tqdm import tqdm
 
-from azure.ai.evaluation._common._experimental import experimental
-from azure.ai.evaluation._common.utils import validate_azure_ai_project
-from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
 from azure.ai.evaluation._http_utils import get_async_http_client
+from azure.ai.evaluation._exceptions import EvaluationException, ErrorBlame, ErrorCategory, ErrorTarget
 from azure.ai.evaluation._model_configurations import AzureAIProject
 from azure.ai.evaluation.simulator import AdversarialScenario
 from azure.ai.evaluation.simulator._adversarial_scenario import _UnstableAdversarialScenario
-from azure.core.credentials import TokenCredential
-from azure.core.pipeline.policies import AsyncRetryPolicy, RetryMode
 
-from ._constants import SupportedLanguages
-from ._conversation import CallbackConversationBot, ConversationBot, ConversationRole, ConversationTurn
+from ._conversation import CallbackConversationBot, ConversationBot, ConversationRole
 from ._conversation._conversation import simulate_conversation
 from ._model_tools import (
     AdversarialTemplateHandler,
@@ -31,13 +27,13 @@ from ._model_tools import (
     RAIClient,
     TokenScope,
 )
-from ._model_tools._template_handler import AdversarialTemplate, TemplateParameters
+from ._tracing import monitor_adversarial_scenario
 from ._utils import JsonLineList
+from ._constants import SupportedLanguages
 
 logger = logging.getLogger(__name__)
 
 
-@experimental
 class AdversarialSimulator:
     """
     Initializes the adversarial simulator with a project scope.
@@ -47,40 +43,43 @@ class AdversarialSimulator:
     :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
     :param credential: The credential for connecting to Azure AI project.
     :type credential: ~azure.core.credentials.TokenCredential
-
-    .. admonition:: Example:
-
-        .. literalinclude:: ../samples/evaluation_samples_simulate.py
-            :start-after: [START adversarial_scenario]
-            :end-before: [END adversarial_scenario]
-            :language: python
-            :dedent: 8
-            :caption: Run the AdversarialSimulator with an AdversarialConversation scenario to produce 2 results with
-                2 conversation turns each (4 messages per result).
     """
 
-    def __init__(self, *, azure_ai_project: AzureAIProject, credential: TokenCredential):
+    def __init__(self, *, azure_ai_project: AzureAIProject, credential=None):
         """Constructor."""
-
-        try:
-            self.azure_ai_project = validate_azure_ai_project(azure_ai_project)
-        except EvaluationException as e:
+        # check if azure_ai_project has the keys: subscription_id, resource_group_name and project_name
+        if not all(key in azure_ai_project for key in ["subscription_id", "resource_group_name", "project_name"]):
+            msg = "azure_ai_project must contain keys: subscription_id, resource_group_name, project_name"
             raise EvaluationException(
-                message=e.message,
-                internal_message=e.internal_message,
+                message=msg,
+                internal_message=msg,
                 target=ErrorTarget.ADVERSARIAL_SIMULATOR,
-                category=e.category,
-                blame=e.blame,
-            ) from e
-
+                category=ErrorCategory.MISSING_FIELD,
+                blame=ErrorBlame.USER_ERROR,
+            )
+        # check the value of the keys in azure_ai_project is not none
+        if not all(azure_ai_project[key] for key in ["subscription_id", "resource_group_name", "project_name"]):
+            msg = "subscription_id, resource_group_name and project_name cannot be None"
+            raise EvaluationException(
+                message=msg,
+                internal_message=msg,
+                target=ErrorTarget.ADVERSARIAL_SIMULATOR,
+                category=ErrorCategory.MISSING_FIELD,
+                blame=ErrorBlame.USER_ERROR,
+            )
+        if "credential" not in azure_ai_project and not credential:
+            credential = DefaultAzureCredential()
+        elif "credential" in azure_ai_project:
+            credential = azure_ai_project["credential"]
+        self.azure_ai_project = azure_ai_project
         self.token_manager = ManagedIdentityAPITokenManager(
             token_scope=TokenScope.DEFAULT_AZURE_MANAGEMENT,
             logger=logging.getLogger("AdversarialSimulator"),
-            credential=cast(TokenCredential, credential),
+            credential=credential,
         )
-        self.rai_client = RAIClient(azure_ai_project=self.azure_ai_project, token_manager=self.token_manager)
+        self.rai_client = RAIClient(azure_ai_project=azure_ai_project, token_manager=self.token_manager)
         self.adversarial_template_handler = AdversarialTemplateHandler(
-            azure_ai_project=self.azure_ai_project, rai_client=self.rai_client
+            azure_ai_project=azure_ai_project, rai_client=self.rai_client
         )
 
     def _ensure_service_dependencies(self):
@@ -94,7 +93,7 @@ class AdversarialSimulator:
                 blame=ErrorBlame.USER_ERROR,
             )
 
-    # pylint: disable=too-many-locals
+    # @monitor_adversarial_scenario
     async def __call__(
         self,
         *,
@@ -108,10 +107,10 @@ class AdversarialSimulator:
         api_call_retry_sleep_sec: int = 1,
         api_call_delay_sec: int = 0,
         concurrent_async_task: int = 3,
+        _jailbreak_type: Optional[str] = None,
         language: SupportedLanguages = SupportedLanguages.English,
         randomize_order: bool = True,
         randomization_seed: Optional[int] = None,
-        **kwargs,
     ):
         """
         Executes the adversarial simulation against a specified target function asynchronously.
@@ -161,6 +160,28 @@ class AdversarialSimulator:
 
          The 'content' for 'assistant' role messages may includes the messages that your callback returned.
         :rtype: List[Dict[str, Any]]
+
+        **Output format**
+
+        .. code-block:: python
+
+            return_value = [
+                {
+                    'template_parameters': {},
+                    'messages': [
+                        {
+                            'content': '<jailbreak prompt> <adversarial query>',
+                            'role': 'user'
+                        },
+                        {
+                            'content': "<response from endpoint>",
+                            'role': 'assistant',
+                            'context': None
+                        }
+                    ],
+                    '$schema': 'http://azureml/sdk-2-0/ChatConversation.json'
+                }
+            ]
         """
 
         # validate the inputs
@@ -196,7 +217,6 @@ class AdversarialSimulator:
                 total_tasks,
             )
         total_tasks = min(total_tasks, max_simulation_results)
-        _jailbreak_type = kwargs.get("_jailbreak_type", None)
         if _jailbreak_type:
             jailbreak_dataset = await self.rai_client.get_jailbreaks_dataset(type=_jailbreak_type)
         progress_bar = tqdm(
@@ -205,18 +225,17 @@ class AdversarialSimulator:
             ncols=100,
             unit="simulations",
         )
-
-        if randomize_order:
-            # The template parameter lists are persistent across sim runs within a session,
-            # So randomize a the selection instead of the parameter list directly,
-            # or a potentially large deep copy.
-            if randomization_seed is not None:
-                random.seed(randomization_seed)
-            random.shuffle(templates)
-        parameter_lists = [t.template_parameters for t in templates]
-        zipped_parameters = list(zip_longest(*parameter_lists))
-        for param_group in zipped_parameters:
-            for template, parameter in zip(templates, param_group):
+        for template in templates:
+            parameter_order = list(range(len(template.template_parameters)))
+            if randomize_order:
+                # The template parameter lists are persistent across sim runs within a session,
+                # So randomize a the selection instead of the parameter list directly,
+                # or a potentially large deep copy.
+                if randomization_seed is not None:
+                    random.seed(randomization_seed)
+                random.shuffle(parameter_order)
+            for index in parameter_order:
+                parameter = template.template_parameters[index].copy()
                 if _jailbreak_type == "upia":
                     parameter = self._join_conversation_starter(parameter, random.choice(jailbreak_dataset))
                 tasks.append(
@@ -245,21 +264,16 @@ class AdversarialSimulator:
 
         return JsonLineList(sim_results)
 
-    def _to_chat_protocol(
-        self,
-        *,
-        conversation_history: List[ConversationTurn],
-        template_parameters: Optional[Dict[str, Union[str, Dict[str, str]]]] = None,
-    ):
+    def _to_chat_protocol(self, *, conversation_history, template_parameters: Dict = None):
         if template_parameters is None:
             template_parameters = {}
         messages = []
         for _, m in enumerate(conversation_history):
             message = {"content": m.message, "role": m.role.value}
-            if m.full_response is not None and "context" in m.full_response:
+            if "context" in m.full_response:
                 message["context"] = m.full_response["context"]
             messages.append(message)
-        conversation_category = cast(Dict[str, str], template_parameters.pop("metadata", {})).get("Category")
+        conversation_category = template_parameters.pop("metadata", {}).get("Category")
         template_parameters["metadata"] = {}
         for key in (
             "conversation_starter",
@@ -267,9 +281,6 @@ class AdversarialSimulator:
             "target_population",
             "topic",
             "ch_template_placeholder",
-            "chatbot_name",
-            "name",
-            "group",
         ):
             template_parameters.pop(key, None)
         if conversation_category:
@@ -284,14 +295,14 @@ class AdversarialSimulator:
         self,
         *,
         target: Callable,
-        template: AdversarialTemplate,
-        parameters: TemplateParameters,
-        max_conversation_turns: int,
-        api_call_retry_limit: int,
-        api_call_retry_sleep_sec: int,
-        api_call_delay_sec: int,
-        language: SupportedLanguages,
-        semaphore: asyncio.Semaphore,
+        template,
+        parameters,
+        max_conversation_turns,
+        api_call_retry_limit,
+        api_call_retry_sleep_sec,
+        api_call_delay_sec,
+        language,
+        semaphore,
     ) -> List[Dict]:
         user_bot = self._setup_bot(role=ConversationRole.USER, template=template, parameters=parameters)
         system_bot = self._setup_bot(
@@ -314,15 +325,9 @@ class AdversarialSimulator:
                 api_call_delay_sec=api_call_delay_sec,
                 language=language,
             )
+        return self._to_chat_protocol(conversation_history=conversation_history, template_parameters=parameters)
 
-        return self._to_chat_protocol(
-            conversation_history=conversation_history,
-            template_parameters=cast(Dict[str, Union[str, Dict[str, str]]], parameters),
-        )
-
-    def _get_user_proxy_completion_model(
-        self, template_key: str, template_parameters: TemplateParameters
-    ) -> ProxyChatCompletionsModel:
+    def _get_user_proxy_completion_model(self, template_key, template_parameters):
         return ProxyChatCompletionsModel(
             name="raisvc_proxy_model",
             template_key=template_key,
@@ -334,15 +339,8 @@ class AdversarialSimulator:
             temperature=0.0,
         )
 
-    def _setup_bot(
-        self,
-        *,
-        role: ConversationRole,
-        template: AdversarialTemplate,
-        parameters: TemplateParameters,
-        target: Optional[Callable] = None,
-    ) -> ConversationBot:
-        if role is ConversationRole.USER:
+    def _setup_bot(self, *, role, template, parameters, target: Callable = None):
+        if role == ConversationRole.USER:
             model = self._get_user_proxy_completion_model(
                 template_key=template.template_name, template_parameters=parameters
             )
@@ -353,46 +351,27 @@ class AdversarialSimulator:
                 instantiation_parameters=parameters,
             )
 
-        if role is ConversationRole.ASSISTANT:
-            if target is None:
-                msg = "Cannot setup system bot. Target is None"
-
-                raise EvaluationException(
-                    message=msg,
-                    internal_message=msg,
-                    target=ErrorTarget.ADVERSARIAL_SIMULATOR,
-                    error_category=ErrorCategory.INVALID_VALUE,
-                    blame=ErrorBlame.SYSTEM_ERROR,
-                )
-
-            class DummyModel:
-                def __init__(self):
-                    self.name = "dummy_model"
-
-                def __call__(self) -> None:
-                    pass
-
+        if role == ConversationRole.ASSISTANT:
+            dummy_model = lambda: None  # noqa: E731
+            dummy_model.name = "dummy_model"
             return CallbackConversationBot(
                 callback=target,
                 role=role,
-                model=DummyModel(),
+                model=dummy_model,
                 user_template=str(template),
                 user_template_parameters=parameters,
                 conversation_template="",
                 instantiation_parameters={},
             )
-
-        msg = "Invalid value for enum ConversationRole. This should never happen."
-        raise EvaluationException(
-            message=msg,
-            internal_message=msg,
-            target=ErrorTarget.ADVERSARIAL_SIMULATOR,
-            category=ErrorCategory.INVALID_VALUE,
-            blame=ErrorBlame.SYSTEM_ERROR,
+        return ConversationBot(
+            role=role,
+            model=model,
+            conversation_template=template,
+            instantiation_parameters=parameters,
         )
 
-    def _join_conversation_starter(self, parameters: TemplateParameters, to_join: str) -> TemplateParameters:
-        key: Literal["conversation_starter"] = "conversation_starter"
+    def _join_conversation_starter(self, parameters, to_join):
+        key = "conversation_starter"
         if key in parameters.keys():
             parameters[key] = f"{to_join} {parameters[key]}"
         else:

azure/ai/evaluation/simulator/_constants.py

@@ -5,17 +5,7 @@ from enum import Enum
 
 
 class SupportedLanguages(Enum):
-    """Supported languages for evaluation, using ISO standard language codes.
-
-    .. admonition:: Example:
-
-        .. literalinclude:: ../samples/evaluation_samples_simulate.py
-            :start-after: [START supported_languages]
-            :end-before: [END supported_languages]
-            :language: python
-            :dedent: 8
-            :caption: Run the AdversarialSimulator with Simplified Chinese language support for evaluation.
-    """
+    """Supported languages for evaluation, using ISO standard language codes."""
 
     Spanish = "es"
     Italian = "it"

azure/ai/evaluation/simulator/_conversation/__init__.py

@@ -7,15 +7,14 @@ import copy
 import logging
 import time
 from dataclasses import dataclass
-from typing import Any, Callable, Dict, List, Optional, Tuple, Union, cast
+from typing import Any, Callable, Dict, List, Optional, Tuple, Union
 
 import jinja2
 
-from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
 from azure.ai.evaluation._http_utils import AsyncHttpPipeline
+from azure.ai.evaluation._exceptions import EvaluationException, ErrorBlame, ErrorCategory, ErrorTarget
 
 from .._model_tools import LLMBase, OpenAIChatCompletionsModel
-from .._model_tools._template_handler import TemplateParameters
 from .constants import ConversationRole
 
 
@@ -41,7 +40,7 @@ class ConversationTurn:
     role: "ConversationRole"
     name: Optional[str] = None
     message: str = ""
-    full_response: Optional[Dict[str, Any]] = None
+    full_response: Optional[Any] = None
     request: Optional[Any] = None
 
     def to_openai_chat_format(self, reverse: bool = False) -> Dict[str, str]:
@@ -110,7 +109,7 @@ class ConversationBot:
         role: ConversationRole,
         model: Union[LLMBase, OpenAIChatCompletionsModel],
         conversation_template: str,
-        instantiation_parameters: TemplateParameters,
+        instantiation_parameters: Dict[str, str],
     ) -> None:
         self.role = role
         self.conversation_template_orig = conversation_template
@@ -119,13 +118,13 @@ class ConversationBot:
         )
         self.persona_template_args = instantiation_parameters
         if self.role == ConversationRole.USER:
-            self.name: str = cast(str, self.persona_template_args.get("name", role.value))
+            self.name = self.persona_template_args.get("name", role.value)
         else:
-            self.name = cast(str, self.persona_template_args.get("chatbot_name", role.value)) or model.name
+            self.name = self.persona_template_args.get("chatbot_name", role.value) or model.name
         self.model = model
 
         self.logger = logging.getLogger(repr(self))
-        self.conversation_starter: Optional[Union[str, jinja2.Template, Dict]] = None
+        self.conversation_starter = None  # can either be a dictionary or jinja template
         if role == ConversationRole.USER:
             if "conversation_starter" in self.persona_template_args:
                 conversation_starter_content = self.persona_template_args["conversation_starter"]
@@ -149,7 +148,7 @@ class ConversationBot:
         conversation_history: List[ConversationTurn],
         max_history: int,
         turn_number: int = 0,
-    ) -> Tuple[dict, dict, float, dict]:
+    ) -> Tuple[dict, dict, int, dict]:
         """
         Prompt the ConversationBot for a response.
 
@@ -162,7 +161,7 @@ class ConversationBot:
         :param turn_number: Parameters used to query GPT-4 model.
         :type turn_number: int
         :return: The response from the ConversationBot.
-        :rtype: Tuple[dict, dict, float, dict]
+        :rtype: Tuple[dict, dict, int, dict]
         """
 
         # check if this is the first turn and the conversation_starter is not None,
@@ -170,11 +169,11 @@ class ConversationBot:
         if turn_number == 0 and self.conversation_starter is not None:
             # if conversation_starter is a dictionary, pass it into samples as is
             if isinstance(self.conversation_starter, dict):
-                samples: List[Union[str, jinja2.Template, Dict]] = [self.conversation_starter]
+                samples = [self.conversation_starter]
             if isinstance(self.conversation_starter, jinja2.Template):
                 samples = [self.conversation_starter.render(**self.persona_template_args)]
             else:
-                samples = [self.conversation_starter]
+                samples = [self.conversation_starter]  # type: ignore[attr-defined]
             time_taken = 0
 
             finish_reason = ["stop"]
@@ -239,7 +238,7 @@ class CallbackConversationBot(ConversationBot):
         self,
         callback: Callable,
         user_template: str,
-        user_template_parameters: TemplateParameters,
+        user_template_parameters: Dict,
         *args,
         **kwargs,
     ) -> None:
@@ -255,7 +254,7 @@ class CallbackConversationBot(ConversationBot):
         conversation_history: List[Any],
         max_history: int,
         turn_number: int = 0,
-    ) -> Tuple[dict, dict, float, dict]:
+    ) -> Tuple[dict, dict, int, dict]:
         chat_protocol_message = self._to_chat_protocol(
             self.user_template, conversation_history, self.user_template_parameters
         )

azure/ai/evaluation/simulator/_conversation/_conversation.py

@@ -4,14 +4,14 @@
 
 import asyncio
 import logging
-from typing import Callable, Dict, List, Optional, Tuple, Union
+from typing import Callable, Dict, List, Tuple, Union
 
-from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
-from azure.ai.evaluation.simulator._constants import SupportedLanguages
 from azure.ai.evaluation.simulator._helpers._language_suffix_mapping import SUPPORTED_LANGUAGES_MAPPING
+from azure.ai.evaluation.simulator._constants import SupportedLanguages
 
 from ..._http_utils import AsyncHttpPipeline
 from . import ConversationBot, ConversationTurn
+from azure.ai.evaluation._exceptions import EvaluationException, ErrorBlame, ErrorCategory, ErrorTarget
 
 
 def is_closing_message(response: Union[Dict, str], recursion_depth: int = 0) -> bool:
@@ -80,26 +80,26 @@ async def simulate_conversation(
     history_limit: int = 5,
     api_call_delay_sec: float = 0,
     logger: logging.Logger = logging.getLogger(__name__),
-) -> Tuple[Optional[str], List[ConversationTurn]]:
+) -> Tuple:
     """
     Simulate a conversation between the given bots.
 
-    :keyword bots: List of ConversationBot instances participating in the conversation.
-    :paramtype bots: List[ConversationBot]
-    :keyword session: The session to use for making API calls.
-    :paramtype session: AsyncHttpPipeline
-    :keyword stopping_criteria: A callable that determines when the conversation should stop.
-    :paramtype stopping_criteria: Callable[[str], bool]
-    :keyword turn_limit: The maximum number of turns in the conversation. Defaults to 10.
-    :paramtype turn_limit: int
-    :keyword history_limit: The maximum number of turns to keep in the conversation history. Defaults to 5.
-    :paramtype history_limit: int
-    :keyword api_call_delay_sec: Delay between API calls in seconds. Defaults to 0.
-    :paramtype api_call_delay_sec: float
-    :keyword logger: The logger to use for logging. Defaults to the logger named after the current module.
-    :paramtype logger: logging.Logger
+    :param bots: List of ConversationBot instances participating in the conversation.
+    :type bots: List[ConversationBot]
+    :param session: The session to use for making API calls.
+    :type session: AsyncHttpPipeline
+    :param stopping_criteria: A callable that determines when the conversation should stop.
+    :type stopping_criteria: Callable[[str], bool]
+    :param turn_limit: The maximum number of turns in the conversation. Defaults to 10.
+    :type turn_limit: int
+    :param history_limit: The maximum number of turns to keep in the conversation history. Defaults to 5.
+    :type history_limit: int
+    :param api_call_delay_sec: Delay between API calls in seconds. Defaults to 0.
+    :type api_call_delay_sec: float
+    :param logger: The logger to use for logging. Defaults to the logger named after the current module.
+    :type logger: logging.Logger
     :return: Simulation a conversation between the given bots.
-    :rtype: Tuple[Optional[str], List[ConversationTurn]]
+    :rtype: Tuple
     """
 
     # Read the first prompt.
@@ -110,7 +110,7 @@ async def simulate_conversation(
         turn_number=0,
     )
     if "id" in first_response:
-        conversation_id: Optional[str] = first_response["id"]
+        conversation_id = first_response["id"]
     else:
         conversation_id = None
     first_prompt = first_response["samples"][0]