azure-ai-evaluation 1.0.0b5__py3-none-any.whl → 1.1.0__py3-none-any.whl

azure/ai/evaluation/simulator/_conversation/__init__.py

@@ -9,12 +9,12 @@ import time
 from dataclasses import dataclass
 from typing import Any, Callable, Dict, List, Optional, Tuple, Union, cast
 
+import re
 import jinja2
 
 from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
 from azure.ai.evaluation._http_utils import AsyncHttpPipeline
-
-from .._model_tools import LLMBase, OpenAIChatCompletionsModel
+from .._model_tools import LLMBase, OpenAIChatCompletionsModel, RAIClient
 from .._model_tools._template_handler import TemplateParameters
 from .constants import ConversationRole
 
@@ -128,15 +128,19 @@ class ConversationBot:
         self.conversation_starter: Optional[Union[str, jinja2.Template, Dict]] = None
         if role == ConversationRole.USER:
             if "conversation_starter" in self.persona_template_args:
+                print(self.persona_template_args)
                 conversation_starter_content = self.persona_template_args["conversation_starter"]
                 if isinstance(conversation_starter_content, dict):
                     self.conversation_starter = conversation_starter_content
+                    print(f"Conversation starter content: {conversation_starter_content}")
                 else:
                     try:
                         self.conversation_starter = jinja2.Template(
                             conversation_starter_content, undefined=jinja2.StrictUndefined
                         )
-                    except jinja2.exceptions.TemplateSyntaxError:  # noqa: F841
+                        print("Successfully created a Jinja2 template for the conversation starter.")
+                    except jinja2.exceptions.TemplateSyntaxError as e:  # noqa: F841
+                        print(f"Template syntax error: {e}. Using raw content.")
                         self.conversation_starter = conversation_starter_content
             else:
                 self.logger.info(
@@ -175,6 +179,9 @@ class ConversationBot:
                 samples = [self.conversation_starter.render(**self.persona_template_args)]
             else:
                 samples = [self.conversation_starter]
+            jailbreak_string = self.persona_template_args.get("jailbreak_string", None)
+            if jailbreak_string:
+                samples = [f"{jailbreak_string} {samples[0]}"]
             time_taken = 0
 
             finish_reason = ["stop"]
@@ -271,8 +278,6 @@ class CallbackConversationBot(ConversationBot):
                 "id": None,
                 "template_parameters": {},
             }
-        self.logger.info("Using user provided callback returning response.")
-
         time_taken = end_time - start_time
         try:
             response = {
@@ -290,8 +295,6 @@ class CallbackConversationBot(ConversationBot):
                 blame=ErrorBlame.USER_ERROR,
             ) from exc
 
-        self.logger.info("Parsed callback response")
-
         return response, {}, time_taken, result
 
     # Bug 3354264: template is unused in the method - is this intentional?
@@ -308,9 +311,127 @@ class CallbackConversationBot(ConversationBot):
         }
 
 
+class MultiModalConversationBot(ConversationBot):
+    """MultiModal Conversation bot that uses a user provided callback to generate responses.
+
+    :param callback: The callback function to use to generate responses.
+    :type callback: Callable
+    :param user_template: The template to use for the request.
+    :type user_template: str
+    :param user_template_parameters: The template parameters to use for the request.
+    :type user_template_parameters: Dict
+    :param args: Optional arguments to pass to the parent class.
+    :type args: Any
+    :param kwargs: Optional keyword arguments to pass to the parent class.
+    :type kwargs: Any
+    """
+
+    def __init__(
+        self,
+        callback: Callable,
+        user_template: str,
+        user_template_parameters: TemplateParameters,
+        rai_client: RAIClient,
+        *args,
+        **kwargs,
+    ) -> None:
+        self.callback = callback
+        self.user_template = user_template
+        self.user_template_parameters = user_template_parameters
+        self.rai_client = rai_client
+
+        super().__init__(*args, **kwargs)
+
+    async def generate_response(
+        self,
+        session: AsyncHttpPipeline,
+        conversation_history: List[Any],
+        max_history: int,
+        turn_number: int = 0,
+    ) -> Tuple[dict, dict, float, dict]:
+        previous_prompt = conversation_history[-1]
+        chat_protocol_message = await self._to_chat_protocol(conversation_history, self.user_template_parameters)
+
+        # replace prompt with {image.jpg} tags with image content data.
+        conversation_history.pop()
+        conversation_history.append(
+            ConversationTurn(
+                role=previous_prompt.role,
+                name=previous_prompt.name,
+                message=chat_protocol_message["messages"][0]["content"],
+                full_response=previous_prompt.full_response,
+                request=chat_protocol_message,
+            )
+        )
+        msg_copy = copy.deepcopy(chat_protocol_message)
+        result = {}
+        start_time = time.time()
+        result = await self.callback(msg_copy)
+        end_time = time.time()
+        if not result:
+            result = {
+                "messages": [{"content": "Callback did not return a response.", "role": "assistant"}],
+                "finish_reason": ["stop"],
+                "id": None,
+                "template_parameters": {},
+            }
+
+        time_taken = end_time - start_time
+        try:
+            response = {
+                "samples": [result["messages"][-1]["content"]],
+                "finish_reason": ["stop"],
+                "id": None,
+            }
+        except Exception as exc:
+            msg = "User provided callback does not conform to chat protocol standard."
+            raise EvaluationException(
+                message=msg,
+                internal_message=msg,
+                target=ErrorTarget.CALLBACK_CONVERSATION_BOT,
+                category=ErrorCategory.INVALID_VALUE,
+                blame=ErrorBlame.USER_ERROR,
+            ) from exc
+
+        return response, chat_protocol_message, time_taken, result
+
+    async def _to_chat_protocol(self, conversation_history, template_parameters):  # pylint: disable=unused-argument
+        messages = []
+
+        for _, m in enumerate(conversation_history):
+            if "image:" in m.message:
+                content = await self._to_multi_modal_content(m.message)
+                messages.append({"content": content, "role": m.role.value})
+            else:
+                messages.append({"content": m.message, "role": m.role.value})
+
+        return {
+            "template_parameters": template_parameters,
+            "messages": messages,
+            "$schema": "http://azureml/sdk-2-0/ChatConversation.json",
+        }
+
+    async def _to_multi_modal_content(self, text: str) -> list:
+        split_text = re.findall(r"[^{}]+|\{[^{}]*\}", text)
+        messages = [
+            text.strip("{}").replace("image:", "").strip() if text.startswith("{") else text for text in split_text
+        ]
+        contents = []
+        for msg in messages:
+            if msg.startswith("image_understanding/"):
+                encoded_image = await self.rai_client.get_image_data(msg)
+                contents.append(
+                    {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{encoded_image}"}},
+                )
+            else:
+                contents.append({"type": "text", "text": msg})
+        return contents
+
+
 __all__ = [
     "ConversationRole",
     "ConversationBot",
     "CallbackConversationBot",
+    "MultiModalConversationBot",
     "ConversationTurn",
 ]

azure/ai/evaluation/simulator/_conversation/_conversation.py

@@ -9,7 +9,6 @@ from typing import Callable, Dict, List, Optional, Tuple, Union
 from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
 from azure.ai.evaluation.simulator._constants import SupportedLanguages
 from azure.ai.evaluation.simulator._helpers._language_suffix_mapping import SUPPORTED_LANGUAGES_MAPPING
-
 from ..._http_utils import AsyncHttpPipeline
 from . import ConversationBot, ConversationTurn
 

azure/ai/evaluation/simulator/_direct_attack_simulator.py

@@ -11,6 +11,7 @@ from azure.ai.evaluation._common._experimental import experimental
 from azure.ai.evaluation._common.utils import validate_azure_ai_project
 from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
 from azure.ai.evaluation.simulator import AdversarialScenario
+from azure.ai.evaluation._model_configurations import AzureAIProject
 from azure.core.credentials import TokenCredential
 
 from ._adversarial_simulator import AdversarialSimulator
@@ -30,9 +31,18 @@ class DirectAttackSimulator:
     :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
     :param credential: The credential for connecting to Azure AI project.
     :type credential: ~azure.core.credentials.TokenCredential
+
+    .. admonition:: Example:
+
+        .. literalinclude:: ../samples/evaluation_samples_simulate.py
+            :start-after: [START direct_attack_simulator]
+            :end-before: [END direct_attack_simulator]
+            :language: python
+            :dedent: 8
+            :caption: Run the DirectAttackSimulator to produce 2 results with 3 conversation turns each (6 messages in each result).
     """
 
-    def __init__(self, *, azure_ai_project: dict, credential):
+    def __init__(self, *, azure_ai_project: AzureAIProject, credential: TokenCredential):
         """Constructor."""
 
         try:
@@ -125,7 +135,7 @@ class DirectAttackSimulator:
          - '**$schema**': A string indicating the schema URL for the conversation format.
 
          The 'content' for 'assistant' role messages may includes the messages that your callback returned.
-        :rtype: Dict[str, [List[Dict[str, Any]]]] with two elements
+        :rtype: Dict[str, [List[Dict[str, Any]]]]
 
         **Output format**
 
@@ -178,9 +188,7 @@ class DirectAttackSimulator:
         if not randomization_seed:
             randomization_seed = randint(0, 1000000)
 
-        regular_sim = AdversarialSimulator(
-            azure_ai_project=cast(dict, self.azure_ai_project), credential=self.credential
-        )
+        regular_sim = AdversarialSimulator(azure_ai_project=self.azure_ai_project, credential=self.credential)
         regular_sim_results = await regular_sim(
             scenario=scenario,
             target=target,
@@ -190,10 +198,10 @@ class DirectAttackSimulator:
             api_call_retry_sleep_sec=api_call_retry_sleep_sec,
             api_call_delay_sec=api_call_delay_sec,
             concurrent_async_task=concurrent_async_task,
-            randomize_order=True,
+            randomize_order=False,
             randomization_seed=randomization_seed,
         )
-        jb_sim = AdversarialSimulator(azure_ai_project=cast(dict, self.azure_ai_project), credential=self.credential)
+        jb_sim = AdversarialSimulator(azure_ai_project=self.azure_ai_project, credential=self.credential)
         jb_sim_results = await jb_sim(
             scenario=scenario,
             target=target,
@@ -204,7 +212,7 @@ class DirectAttackSimulator:
             api_call_delay_sec=api_call_delay_sec,
             concurrent_async_task=concurrent_async_task,
             _jailbreak_type="upia",
-            randomize_order=True,
+            randomize_order=False,
             randomization_seed=randomization_seed,
         )
         return {"jailbreak": jb_sim_results, "regular": regular_sim_results}

azure/ai/evaluation/simulator/_indirect_attack_simulator.py

@@ -13,6 +13,7 @@ from azure.ai.evaluation._common.utils import validate_azure_ai_project
 from azure.ai.evaluation._common._experimental import experimental
 from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
 from azure.ai.evaluation.simulator import AdversarialScenarioJailbreak, SupportedLanguages
+from azure.ai.evaluation._model_configurations import AzureAIProject
 from azure.core.credentials import TokenCredential
 
 from ._adversarial_simulator import AdversarialSimulator, JsonLineList
@@ -32,9 +33,18 @@ class IndirectAttackSimulator(AdversarialSimulator):
     :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
     :param credential: The credential for connecting to Azure AI project.
     :type credential: ~azure.core.credentials.TokenCredential
+
+    .. admonition:: Example:
+
+        .. literalinclude:: ../samples/evaluation_samples_simulate.py
+            :start-after: [START indirect_attack_simulator]
+            :end-before: [END indirect_attack_simulator]
+            :language: python
+            :dedent: 8
+            :caption: Run the IndirectAttackSimulator to produce 1 result with 1 conversation turn (2 messages in the result).
     """
 
-    def __init__(self, *, azure_ai_project: dict, credential):
+    def __init__(self, *, azure_ai_project: AzureAIProject, credential: TokenCredential):
         """Constructor."""
 
         try:
@@ -179,6 +189,7 @@ class IndirectAttackSimulator(AdversarialSimulator):
                             api_call_delay_sec=api_call_delay_sec,
                             language=language,
                             semaphore=semaphore,
+                            scenario=scenario,
                         )
                     )
                 )

azure/ai/evaluation/simulator/_model_tools/_identity_manager.py

@@ -14,7 +14,9 @@ from typing import Optional, Union
 from azure.core.credentials import AccessToken, TokenCredential
 from azure.identity import DefaultAzureCredential, ManagedIdentityCredential
 
-AZURE_TOKEN_REFRESH_INTERVAL = 600  # seconds
+AZURE_TOKEN_REFRESH_INTERVAL = int(
+    os.getenv("AZURE_TOKEN_REFRESH_INTERVAL", "600")
+)  # token refresh interval in seconds
 
 
 class TokenScope(Enum):

azure/ai/evaluation/simulator/_model_tools/_rai_client.py

@@ -4,6 +4,7 @@
 import os
 from typing import Any
 from urllib.parse import urljoin, urlparse
+import base64
 
 from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
 from azure.ai.evaluation._http_utils import AsyncHttpPipeline, get_async_http_client, get_http_client
@@ -57,6 +58,7 @@ class RAIClient:  # pylint: disable=client-accepts-api-version-keyword
         # add a "/" at the end of the url
         self.api_url = self.api_url.rstrip("/") + "/"
         self.parameter_json_endpoint = urljoin(self.api_url, "simulation/template/parameters")
+        self.parameter_image_endpoint = urljoin(self.api_url, "simulation/template/parameters/image")
         self.jailbreaks_json_endpoint = urljoin(self.api_url, "simulation/jailbreak")
         self.simulation_submit_endpoint = urljoin(self.api_url, "simulation/chat/completions/submit")
         self.xpia_jailbreaks_json_endpoint = urljoin(self.api_url, "simulation/jailbreak/xpia")
@@ -74,14 +76,18 @@ class RAIClient:  # pylint: disable=client-accepts-api-version-keyword
             timeout=5,
         )
         if response.status_code != 200:
-            msg = "Failed to retrieve the discovery service URL."
+            msg = (
+                f"Failed to connect to your Azure AI project. Please check if the project scope is configured "
+                f"correctly, and make sure you have the necessary access permissions. "
+                f"Status code: {response.status_code}."
+            )
             raise EvaluationException(
                 message=msg,
-                internal_message=msg,
                 target=ErrorTarget.RAI_CLIENT,
-                category=ErrorCategory.SERVICE_UNAVAILABLE,
-                blame=ErrorBlame.UNKNOWN,
+                category=ErrorCategory.PROJECT_ACCESS_ERROR,
+                blame=ErrorBlame.USER_ERROR,
             )
+
         base_url = urlparse(response.json()["properties"]["discoveryUrl"])
         return f"{base_url.scheme}://{base_url.netloc}"
 
@@ -162,3 +168,41 @@ class RAIClient:  # pylint: disable=client-accepts-api-version-keyword
             category=ErrorCategory.UNKNOWN,
             blame=ErrorBlame.USER_ERROR,
         )
+
+    async def get_image_data(self, path: str) -> Any:
+        """Make a GET Image request to the given url
+
+        :param path: The url of the image
+        :type path: str
+        :raises EvaluationException: If the Azure safety evaluation service is not available in the current region
+        :return: The response
+        :rtype: Any
+        """
+        token = self.token_manager.get_token()
+        headers = {
+            "Authorization": f"Bearer {token}",
+            "Content-Type": "application/json",
+            "User-Agent": USER_AGENT,
+        }
+
+        session = self._create_async_client()
+        params = {"path": path}
+        async with session:
+            response = await session.get(
+                url=self.parameter_image_endpoint, params=params, headers=headers
+            )  # pylint: disable=unexpected-keyword-arg
+
+        if response.status_code == 200:
+            return base64.b64encode(response.content).decode("utf-8")
+
+        msg = (
+            "Azure safety evaluation service is not available in your current region, "
+            + "please go to https://aka.ms/azureaistudiosafetyeval to see which regions are supported"
+        )
+        raise EvaluationException(
+            message=msg,
+            internal_message=msg,
+            target=ErrorTarget.RAI_CLIENT,
+            category=ErrorCategory.UNKNOWN,
+            blame=ErrorBlame.USER_ERROR,
+        )

azure/ai/evaluation/simulator/_model_tools/_template_handler.py

@@ -56,6 +56,7 @@ class TemplateParameters(TypedDict):
     category: NotRequired[str]
     target_population: NotRequired[str]
     topic: NotRequired[str]
+    jailbreak_string: NotRequired[str]
 
 
 class _CategorizedParameter(TypedDict):