azure-ai-evaluation 1.0.1__py3-none-any.whl → 1.1.0__py3-none-any.whl

azure/ai/evaluation/_vendor/rouge_score/tokenize.py

@@ -17,10 +17,6 @@
 
 """A library for tokenizing text."""
 
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
 import re
 
 

azure/ai/evaluation/_version.py

@@ -2,4 +2,4 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
 
-VERSION = "1.0.1"
+VERSION = "1.1.0"

azure/ai/evaluation/simulator/_adversarial_scenario.py

@@ -44,3 +44,5 @@ class _UnstableAdversarialScenario(Enum):
     """
 
     ECI = "adv_politics"
+    ADVERSARIAL_IMAGE_GEN = "adv_image_gen"
+    ADVERSARIAL_IMAGE_MULTIMODAL = "adv_image_understanding"

azure/ai/evaluation/simulator/_adversarial_simulator.py

@@ -6,8 +6,7 @@
 import asyncio
 import logging
 import random
-from typing import Any, Callable, Dict, List, Literal, Optional, Union, cast
-from itertools import zip_longest
+from typing import Any, Callable, Dict, List, Optional, Union, cast
 
 from tqdm import tqdm
 
@@ -16,13 +15,19 @@ from azure.ai.evaluation._common.utils import validate_azure_ai_project
 from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
 from azure.ai.evaluation._http_utils import get_async_http_client
 from azure.ai.evaluation._model_configurations import AzureAIProject
-from azure.ai.evaluation.simulator import AdversarialScenario
+from azure.ai.evaluation.simulator import AdversarialScenario, AdversarialScenarioJailbreak
 from azure.ai.evaluation.simulator._adversarial_scenario import _UnstableAdversarialScenario
 from azure.core.credentials import TokenCredential
 from azure.core.pipeline.policies import AsyncRetryPolicy, RetryMode
 
 from ._constants import SupportedLanguages
-from ._conversation import CallbackConversationBot, ConversationBot, ConversationRole, ConversationTurn
+from ._conversation import (
+    CallbackConversationBot,
+    MultiModalConversationBot,
+    ConversationBot,
+    ConversationRole,
+    ConversationTurn,
+)
 from ._conversation._conversation import simulate_conversation
 from ._model_tools import (
     AdversarialTemplateHandler,
@@ -205,7 +210,6 @@ class AdversarialSimulator:
             ncols=100,
             unit="simulations",
         )
-
         if randomize_order:
             # The template parameter lists are persistent across sim runs within a session,
             # So randomize a the selection instead of the parameter list directly,
@@ -214,11 +218,11 @@ class AdversarialSimulator:
                 random.seed(randomization_seed)
             random.shuffle(templates)
         parameter_lists = [t.template_parameters for t in templates]
-        zipped_parameters = list(zip_longest(*parameter_lists))
+        zipped_parameters = list(zip(*parameter_lists))
         for param_group in zipped_parameters:
             for template, parameter in zip(templates, param_group):
                 if _jailbreak_type == "upia":
-                    parameter = self._join_conversation_starter(parameter, random.choice(jailbreak_dataset))
+                    parameter = self._add_jailbreak_parameter(parameter, random.choice(jailbreak_dataset))
                 tasks.append(
                     asyncio.create_task(
                         self._simulate_async(
@@ -231,6 +235,7 @@ class AdversarialSimulator:
                             api_call_delay_sec=api_call_delay_sec,
                             language=language,
                             semaphore=semaphore,
+                            scenario=scenario,
                         )
                     )
                 )
@@ -292,10 +297,13 @@ class AdversarialSimulator:
         api_call_delay_sec: int,
         language: SupportedLanguages,
         semaphore: asyncio.Semaphore,
+        scenario: Union[AdversarialScenario, AdversarialScenarioJailbreak],
     ) -> List[Dict]:
-        user_bot = self._setup_bot(role=ConversationRole.USER, template=template, parameters=parameters)
+        user_bot = self._setup_bot(
+            role=ConversationRole.USER, template=template, parameters=parameters, scenario=scenario
+        )
         system_bot = self._setup_bot(
-            target=target, role=ConversationRole.ASSISTANT, template=template, parameters=parameters
+            target=target, role=ConversationRole.ASSISTANT, template=template, parameters=parameters, scenario=scenario
         )
         bots = [user_bot, system_bot]
         session = get_async_http_client().with_policies(
@@ -341,6 +349,7 @@ class AdversarialSimulator:
         template: AdversarialTemplate,
         parameters: TemplateParameters,
         target: Optional[Callable] = None,
+        scenario: Union[AdversarialScenario, AdversarialScenarioJailbreak],
     ) -> ConversationBot:
         if role is ConversationRole.USER:
             model = self._get_user_proxy_completion_model(
@@ -372,6 +381,21 @@ class AdversarialSimulator:
                 def __call__(self) -> None:
                     pass
 
+            if scenario in [
+                _UnstableAdversarialScenario.ADVERSARIAL_IMAGE_GEN,
+                _UnstableAdversarialScenario.ADVERSARIAL_IMAGE_MULTIMODAL,
+            ]:
+                return MultiModalConversationBot(
+                    callback=target,
+                    role=role,
+                    model=DummyModel(),
+                    user_template=str(template),
+                    user_template_parameters=parameters,
+                    rai_client=self.rai_client,
+                    conversation_template="",
+                    instantiation_parameters={},
+                )
+
             return CallbackConversationBot(
                 callback=target,
                 role=role,
@@ -391,13 +415,8 @@ class AdversarialSimulator:
             blame=ErrorBlame.SYSTEM_ERROR,
         )
 
-    def _join_conversation_starter(self, parameters: TemplateParameters, to_join: str) -> TemplateParameters:
-        key: Literal["conversation_starter"] = "conversation_starter"
-        if key in parameters.keys():
-            parameters[key] = f"{to_join} {parameters[key]}"
-        else:
-            parameters[key] = to_join
-
+    def _add_jailbreak_parameter(self, parameters: TemplateParameters, to_join: str) -> TemplateParameters:
+        parameters["jailbreak_string"] = to_join
         return parameters
 
     def call_sync(

azure/ai/evaluation/simulator/_conversation/__init__.py

@@ -9,12 +9,12 @@ import time
 from dataclasses import dataclass
 from typing import Any, Callable, Dict, List, Optional, Tuple, Union, cast
 
+import re
 import jinja2
 
 from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
 from azure.ai.evaluation._http_utils import AsyncHttpPipeline
-
-from .._model_tools import LLMBase, OpenAIChatCompletionsModel
+from .._model_tools import LLMBase, OpenAIChatCompletionsModel, RAIClient
 from .._model_tools._template_handler import TemplateParameters
 from .constants import ConversationRole
 
@@ -128,15 +128,19 @@ class ConversationBot:
         self.conversation_starter: Optional[Union[str, jinja2.Template, Dict]] = None
         if role == ConversationRole.USER:
             if "conversation_starter" in self.persona_template_args:
+                print(self.persona_template_args)
                 conversation_starter_content = self.persona_template_args["conversation_starter"]
                 if isinstance(conversation_starter_content, dict):
                     self.conversation_starter = conversation_starter_content
+                    print(f"Conversation starter content: {conversation_starter_content}")
                 else:
                     try:
                         self.conversation_starter = jinja2.Template(
                             conversation_starter_content, undefined=jinja2.StrictUndefined
                         )
-                    except jinja2.exceptions.TemplateSyntaxError:  # noqa: F841
+                        print("Successfully created a Jinja2 template for the conversation starter.")
+                    except jinja2.exceptions.TemplateSyntaxError as e:  # noqa: F841
+                        print(f"Template syntax error: {e}. Using raw content.")
                         self.conversation_starter = conversation_starter_content
             else:
                 self.logger.info(
@@ -175,6 +179,9 @@ class ConversationBot:
                 samples = [self.conversation_starter.render(**self.persona_template_args)]
             else:
                 samples = [self.conversation_starter]
+            jailbreak_string = self.persona_template_args.get("jailbreak_string", None)
+            if jailbreak_string:
+                samples = [f"{jailbreak_string} {samples[0]}"]
             time_taken = 0
 
             finish_reason = ["stop"]
@@ -271,8 +278,6 @@ class CallbackConversationBot(ConversationBot):
                 "id": None,
                 "template_parameters": {},
             }
-        self.logger.info("Using user provided callback returning response.")
-
         time_taken = end_time - start_time
         try:
             response = {
@@ -290,8 +295,6 @@ class CallbackConversationBot(ConversationBot):
                 blame=ErrorBlame.USER_ERROR,
             ) from exc
 
-        self.logger.info("Parsed callback response")
-
         return response, {}, time_taken, result
 
     # Bug 3354264: template is unused in the method - is this intentional?
@@ -308,9 +311,127 @@ class CallbackConversationBot(ConversationBot):
         }
 
 
+class MultiModalConversationBot(ConversationBot):
+    """MultiModal Conversation bot that uses a user provided callback to generate responses.
+
+    :param callback: The callback function to use to generate responses.
+    :type callback: Callable
+    :param user_template: The template to use for the request.
+    :type user_template: str
+    :param user_template_parameters: The template parameters to use for the request.
+    :type user_template_parameters: Dict
+    :param args: Optional arguments to pass to the parent class.
+    :type args: Any
+    :param kwargs: Optional keyword arguments to pass to the parent class.
+    :type kwargs: Any
+    """
+
+    def __init__(
+        self,
+        callback: Callable,
+        user_template: str,
+        user_template_parameters: TemplateParameters,
+        rai_client: RAIClient,
+        *args,
+        **kwargs,
+    ) -> None:
+        self.callback = callback
+        self.user_template = user_template
+        self.user_template_parameters = user_template_parameters
+        self.rai_client = rai_client
+
+        super().__init__(*args, **kwargs)
+
+    async def generate_response(
+        self,
+        session: AsyncHttpPipeline,
+        conversation_history: List[Any],
+        max_history: int,
+        turn_number: int = 0,
+    ) -> Tuple[dict, dict, float, dict]:
+        previous_prompt = conversation_history[-1]
+        chat_protocol_message = await self._to_chat_protocol(conversation_history, self.user_template_parameters)
+
+        # replace prompt with {image.jpg} tags with image content data.
+        conversation_history.pop()
+        conversation_history.append(
+            ConversationTurn(
+                role=previous_prompt.role,
+                name=previous_prompt.name,
+                message=chat_protocol_message["messages"][0]["content"],
+                full_response=previous_prompt.full_response,
+                request=chat_protocol_message,
+            )
+        )
+        msg_copy = copy.deepcopy(chat_protocol_message)
+        result = {}
+        start_time = time.time()
+        result = await self.callback(msg_copy)
+        end_time = time.time()
+        if not result:
+            result = {
+                "messages": [{"content": "Callback did not return a response.", "role": "assistant"}],
+                "finish_reason": ["stop"],
+                "id": None,
+                "template_parameters": {},
+            }
+
+        time_taken = end_time - start_time
+        try:
+            response = {
+                "samples": [result["messages"][-1]["content"]],
+                "finish_reason": ["stop"],
+                "id": None,
+            }
+        except Exception as exc:
+            msg = "User provided callback does not conform to chat protocol standard."
+            raise EvaluationException(
+                message=msg,
+                internal_message=msg,
+                target=ErrorTarget.CALLBACK_CONVERSATION_BOT,
+                category=ErrorCategory.INVALID_VALUE,
+                blame=ErrorBlame.USER_ERROR,
+            ) from exc
+
+        return response, chat_protocol_message, time_taken, result
+
+    async def _to_chat_protocol(self, conversation_history, template_parameters):  # pylint: disable=unused-argument
+        messages = []
+
+        for _, m in enumerate(conversation_history):
+            if "image:" in m.message:
+                content = await self._to_multi_modal_content(m.message)
+                messages.append({"content": content, "role": m.role.value})
+            else:
+                messages.append({"content": m.message, "role": m.role.value})
+
+        return {
+            "template_parameters": template_parameters,
+            "messages": messages,
+            "$schema": "http://azureml/sdk-2-0/ChatConversation.json",
+        }
+
+    async def _to_multi_modal_content(self, text: str) -> list:
+        split_text = re.findall(r"[^{}]+|\{[^{}]*\}", text)
+        messages = [
+            text.strip("{}").replace("image:", "").strip() if text.startswith("{") else text for text in split_text
+        ]
+        contents = []
+        for msg in messages:
+            if msg.startswith("image_understanding/"):
+                encoded_image = await self.rai_client.get_image_data(msg)
+                contents.append(
+                    {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{encoded_image}"}},
+                )
+            else:
+                contents.append({"type": "text", "text": msg})
+        return contents
+
+
 __all__ = [
     "ConversationRole",
     "ConversationBot",
     "CallbackConversationBot",
+    "MultiModalConversationBot",
     "ConversationTurn",
 ]

azure/ai/evaluation/simulator/_conversation/_conversation.py

@@ -9,7 +9,6 @@ from typing import Callable, Dict, List, Optional, Tuple, Union
 from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
 from azure.ai.evaluation.simulator._constants import SupportedLanguages
 from azure.ai.evaluation.simulator._helpers._language_suffix_mapping import SUPPORTED_LANGUAGES_MAPPING
-
 from ..._http_utils import AsyncHttpPipeline
 from . import ConversationBot, ConversationTurn
 

azure/ai/evaluation/simulator/_indirect_attack_simulator.py

@@ -189,6 +189,7 @@ class IndirectAttackSimulator(AdversarialSimulator):
                             api_call_delay_sec=api_call_delay_sec,
                             language=language,
                             semaphore=semaphore,
+                            scenario=scenario,
                         )
                     )
                 )

azure/ai/evaluation/simulator/_model_tools/_rai_client.py

@@ -4,6 +4,7 @@
 import os
 from typing import Any
 from urllib.parse import urljoin, urlparse
+import base64
 
 from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
 from azure.ai.evaluation._http_utils import AsyncHttpPipeline, get_async_http_client, get_http_client
@@ -57,6 +58,7 @@ class RAIClient:  # pylint: disable=client-accepts-api-version-keyword
         # add a "/" at the end of the url
         self.api_url = self.api_url.rstrip("/") + "/"
         self.parameter_json_endpoint = urljoin(self.api_url, "simulation/template/parameters")
+        self.parameter_image_endpoint = urljoin(self.api_url, "simulation/template/parameters/image")
         self.jailbreaks_json_endpoint = urljoin(self.api_url, "simulation/jailbreak")
         self.simulation_submit_endpoint = urljoin(self.api_url, "simulation/chat/completions/submit")
         self.xpia_jailbreaks_json_endpoint = urljoin(self.api_url, "simulation/jailbreak/xpia")
@@ -166,3 +168,41 @@ class RAIClient:  # pylint: disable=client-accepts-api-version-keyword
             category=ErrorCategory.UNKNOWN,
             blame=ErrorBlame.USER_ERROR,
         )
+
+    async def get_image_data(self, path: str) -> Any:
+        """Make a GET Image request to the given url
+
+        :param path: The url of the image
+        :type path: str
+        :raises EvaluationException: If the Azure safety evaluation service is not available in the current region
+        :return: The response
+        :rtype: Any
+        """
+        token = self.token_manager.get_token()
+        headers = {
+            "Authorization": f"Bearer {token}",
+            "Content-Type": "application/json",
+            "User-Agent": USER_AGENT,
+        }
+
+        session = self._create_async_client()
+        params = {"path": path}
+        async with session:
+            response = await session.get(
+                url=self.parameter_image_endpoint, params=params, headers=headers
+            )  # pylint: disable=unexpected-keyword-arg
+
+        if response.status_code == 200:
+            return base64.b64encode(response.content).decode("utf-8")
+
+        msg = (
+            "Azure safety evaluation service is not available in your current region, "
+            + "please go to https://aka.ms/azureaistudiosafetyeval to see which regions are supported"
+        )
+        raise EvaluationException(
+            message=msg,
+            internal_message=msg,
+            target=ErrorTarget.RAI_CLIENT,
+            category=ErrorCategory.UNKNOWN,
+            blame=ErrorBlame.USER_ERROR,
+        )

azure/ai/evaluation/simulator/_model_tools/_template_handler.py

@@ -56,6 +56,7 @@ class TemplateParameters(TypedDict):
     category: NotRequired[str]
     target_population: NotRequired[str]
     topic: NotRequired[str]
+    jailbreak_string: NotRequired[str]
 
 
 class _CategorizedParameter(TypedDict):

azure/ai/evaluation/simulator/_simulator.py

@@ -25,6 +25,9 @@ from ._helpers import ConversationHistory, Turn
 from ._utils import JsonLineChatProtocol
 
 
+USER_AGENT += " (type=simulator; subtype=Simulator)"
+
+
 @experimental
 class Simulator:
     """

{azure_ai_evaluation-1.0.1.dist-info → azure_ai_evaluation-1.1.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: azure-ai-evaluation
-Version: 1.0.1
+Version: 1.1.0
 Summary: Microsoft Azure Evaluation Library for Python
 Home-page: https://github.com/Azure/azure-sdk-for-python
 Author: Microsoft Corporation
@@ -28,8 +28,7 @@ Requires-Dist: pyjwt >=2.8.0
 Requires-Dist: azure-identity >=1.16.0
 Requires-Dist: azure-core >=1.30.2
 Requires-Dist: nltk >=3.9.1
-Provides-Extra: remote
-Requires-Dist: promptflow-azure <2.0.0,>=1.15.0 ; extra == 'remote'
+Requires-Dist: azure-storage-blob >=12.10.0
 
 # Azure AI Evaluation client library for Python
 
@@ -359,13 +358,13 @@ This project has adopted the [Microsoft Open Source Code of Conduct][code_of_con
 [evaluate_dataset]: https://learn.microsoft.com/azure/ai-studio/how-to/develop/evaluate-sdk#evaluate-on-test-dataset-using-evaluate
 [evaluators]: https://learn.microsoft.com/python/api/azure-ai-evaluation/azure.ai.evaluation?view=azure-python-preview
 [evaluate_api]: https://learn.microsoft.com/python/api/azure-ai-evaluation/azure.ai.evaluation?view=azure-python-preview#azure-ai-evaluation-evaluate
-[evaluate_app]: https://github.com/Azure-Samples/azureai-samples/tree/main/scenarios/evaluate/evaluate_app
+[evaluate_app]: https://github.com/Azure-Samples/azureai-samples/tree/main/scenarios/evaluate/Supported_Evaluation_Targets/Evaluate_App_Endpoint
 [evaluation_tsg]: https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/evaluation/azure-ai-evaluation/TROUBLESHOOTING.md
 [ai_studio]: https://learn.microsoft.com/azure/ai-studio/what-is-ai-studio
 [ai_project]: https://learn.microsoft.com/azure/ai-studio/how-to/create-projects?tabs=ai-studio
 [azure_openai]: https://learn.microsoft.com/azure/ai-services/openai/
-[evaluate_models]: https://github.com/Azure-Samples/azureai-samples/tree/main/scenarios/evaluate/evaluate_endpoints
-[custom_evaluators]: https://github.com/Azure-Samples/azureai-samples/tree/main/scenarios/evaluate/evaluate_custom
+[evaluate_models]: https://github.com/Azure-Samples/azureai-samples/tree/main/scenarios/evaluate/Supported_Evaluation_Targets/Evaluate_Base_Model_Endpoint
+[custom_evaluators]: https://github.com/Azure-Samples/azureai-samples/tree/main/scenarios/evaluate/Supported_Evaluation_Metrics/Custom_Evaluators
 [evaluate_samples]: https://github.com/Azure-Samples/azureai-samples/tree/main/scenarios/evaluate
 [evaluation_metrics]: https://learn.microsoft.com/azure/ai-studio/concepts/evaluation-metrics-built-in
 [performance_and_quality_evaluators]: https://learn.microsoft.com/azure/ai-studio/how-to/develop/evaluate-sdk#performance-and-quality-evaluators
@@ -373,18 +372,23 @@ This project has adopted the [Microsoft Open Source Code of Conduct][code_of_con
 [composite_evaluators]: https://learn.microsoft.com/azure/ai-studio/how-to/develop/evaluate-sdk#composite-evaluators
 [adversarial_simulation_docs]: https://learn.microsoft.com/azure/ai-studio/how-to/develop/simulator-interaction-data#generate-adversarial-simulations-for-safety-evaluation
 [adversarial_simulation_scenarios]: https://learn.microsoft.com/azure/ai-studio/how-to/develop/simulator-interaction-data#supported-adversarial-simulation-scenarios
-[adversarial_simulation]: https://github.com/Azure-Samples/azureai-samples/tree/main/scenarios/evaluate/simulate_adversarial
-[simulate_with_conversation_starter]: https://github.com/Azure-Samples/azureai-samples/tree/main/scenarios/evaluate/simulate_conversation_starter
+[adversarial_simulation]: https://github.com/Azure-Samples/azureai-samples/tree/main/scenarios/evaluate/Simulators/Simulate_Adversarial_Data
+[simulate_with_conversation_starter]: https://github.com/Azure-Samples/azureai-samples/tree/main/scenarios/evaluate/Simulators/Simulate_Context-Relevant_Data/Simulate_From_Conversation_Starter
 [adversarial_jailbreak]: https://learn.microsoft.com/azure/ai-studio/how-to/develop/simulator-interaction-data#simulating-jailbreak-attacks
 
-
 # Release History
 
+## 1.1.0 (2024-12-12)
+
+### Bugs Fixed
+- Removed `[remote]` extra. This is no longer needed when tracking results in Azure AI Studio.
+- Fixed `AttributeError: 'NoneType' object has no attribute 'get'` while running simulator with 1000+ results
+
 ## 1.0.1 (2024-11-15)
 
 ### Bugs Fixed
-- Fixed `[remote]` extra to be needed only when tracking results in Azure AI Studio.
 - Removing `azure-ai-inference` as dependency.
+- Fixed `AttributeError: 'NoneType' object has no attribute 'get'` while running simulator with 1000+ results
 
 ## 1.0.0 (2024-11-13)
 
@@ -396,6 +400,7 @@ This project has adopted the [Microsoft Open Source Code of Conduct][code_of_con
 - Fixed an issue where the `output_path` parameter in the `evaluate` API did not support relative path.
 - Output of adversarial simulators are of type `JsonLineList` and the helper function `to_eval_qr_json_lines` now outputs context from both user and assistant turns along with `category` if it exists in the conversation
 - Fixed an issue where during long-running simulations, API token expires causing "Forbidden" error. Instead, users can now set an environment variable `AZURE_TOKEN_REFRESH_INTERVAL` to refresh the token more frequently to prevent expiration and ensure continuous operation of the simulation.
+- Fixed an issue with the `ContentSafetyEvaluator` that caused parallel execution of sub-evaluators to fail. Parallel execution is now enabled by default again, but can still be disabled via the '_parallel' boolean keyword argument during class initialization.
 - Fix `evaluate` function not producing aggregated metrics if ANY values to be aggregated were None, NaN, or
 otherwise difficult to process. Such values are ignored fully, so the aggregated metric of `[1, 2, 3, NaN]`
 would be 2, not 1.5.

{azure_ai_evaluation-1.0.1.dist-info → azure_ai_evaluation-1.1.0.dist-info}/RECORD

@@ -1,26 +1,30 @@
 azure/ai/evaluation/__init__.py,sha256=MFxJRoKfSsP_Qlfq0FwynxNf4csNAfTYPQX7jdXc9RU,2757
-azure/ai/evaluation/_constants.py,sha256=kdOdisz3FiWQ6PHg5m0TaFFVRx2m3b_oaUkG3y-bkqA,1984
+azure/ai/evaluation/_constants.py,sha256=d41rQb-w2GmCMHOwiyDD1ieJB1U6JyPPl6APZSJbKzg,2036
 azure/ai/evaluation/_exceptions.py,sha256=MsTbgsPGYPzIxs7MyLKzSeiVKEoCxYkVjONzNfv2tXA,5162
-azure/ai/evaluation/_http_utils.py,sha256=oVbRaxUm41tVFGkYpZdHjT9ss_9va1NzXYuV3DUVr8k,17125
+azure/ai/evaluation/_http_utils.py,sha256=1bGce6pKAL-vmaUGRPxVX7DVO05XVQ8YPIwIQ3q7mfA,17221
 azure/ai/evaluation/_model_configurations.py,sha256=MNN6cQlz7P9vNfHmfEKsUcly3j1FEOEFsA8WV7GPuKQ,4043
 azure/ai/evaluation/_user_agent.py,sha256=O2y-QPBAcw7w7qQ6M2aRPC3Vy3TKd789u5lcs2yuFaI,290
-azure/ai/evaluation/_version.py,sha256=PNwYJcvbJBl8Q8tjRz_IIdkpS8NluC6Ujspj7gJP3CY,199
+azure/ai/evaluation/_version.py,sha256=LzMvSuUB6pmU-LfCPzoYuCoTF0BAqE7ljPjk6r8YaMw,199
 azure/ai/evaluation/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+azure/ai/evaluation/_azure/__init__.py,sha256=Yx1Iq2GNKQ5lYxTotvPwkPL4u0cm6YVxUe-iVbu1clI,180
+azure/ai/evaluation/_azure/_clients.py,sha256=1mFRSxt-Ld5UBn-m3DJkKc-VPP9CbXQHrqLNdLs9RF0,8201
+azure/ai/evaluation/_azure/_models.py,sha256=tKpxjb5Ou476UasjXPCiuvsxTjLTrnoVnSXy5Bfa51M,12483
+azure/ai/evaluation/_azure/_token_manager.py,sha256=1NZHwgEc9BMXWPz5Ear_J5-oYjouD77crLHHqNLldEw,5193
 azure/ai/evaluation/_common/__init__.py,sha256=LHTkf6dMLLxikrGNgbUuREBVQcs4ORHR6Eryo4bm9M8,586
 azure/ai/evaluation/_common/_experimental.py,sha256=GVtSn9r1CeR_yEa578dJVNDJ3P24eqe8WYdH7llbiQY,5694
 azure/ai/evaluation/_common/constants.py,sha256=OsExttFGLnTAyZa26jnY5_PCDTb7uJNFqtE2qsRZ1mg,1957
 azure/ai/evaluation/_common/math.py,sha256=d4bwWe35_RWDIZNcbV1BTBbHNx2QHQ4-I3EofDyyNE0,2863
-azure/ai/evaluation/_common/rai_service.py,sha256=l98dEuNkaXjU4RI9R3Mc6JxRatPlQV3BfwkK7L8Oajs,26023
+azure/ai/evaluation/_common/rai_service.py,sha256=DcakzdOour9qNdMXU-8UFfvLb12oexAoiJXG8XFTRBs,26462
 azure/ai/evaluation/_common/utils.py,sha256=MQIZs95gH5je1L-S3twa_WQi071zRu0Dv54lzCI7ZgU,17642
 azure/ai/evaluation/_evaluate/__init__.py,sha256=Yx1Iq2GNKQ5lYxTotvPwkPL4u0cm6YVxUe-iVbu1clI,180
-azure/ai/evaluation/_evaluate/_eval_run.py,sha256=Jil7ERapJzjr4GIMGT4WgfKFt3AIFgTOo1S1AAP_DB4,23333
-azure/ai/evaluation/_evaluate/_evaluate.py,sha256=mk9hoeISTq9M6rVBcRtlTu7astdCMpN-FtNOSOOmkjY,37279
-azure/ai/evaluation/_evaluate/_utils.py,sha256=IiTkgSBatAUR73oSsq7Mr0W96ZA2cVazw7rKYB-opS0,12280
+azure/ai/evaluation/_evaluate/_eval_run.py,sha256=het3cxjK4J-_hT19dT5a0mC2Cdnk93gM3ONQMJb9bxQ,21923
+azure/ai/evaluation/_evaluate/_evaluate.py,sha256=P5aL70eUBKZT9CVRM9RVSfD0DkuljQyc5ECte37Ycmo,36225
+azure/ai/evaluation/_evaluate/_utils.py,sha256=S4LUUDUBo9JNA41ojSezMC-PZzkWcihhhNdyZwZrpr0,13428
 azure/ai/evaluation/_evaluate/_batch_run/__init__.py,sha256=G8McpeLxAS_gFhNShX52_YWvE-arhJn-bVpAfzjWG3Q,427
 azure/ai/evaluation/_evaluate/_batch_run/code_client.py,sha256=XQLaXfswF6ReHLpQthHLuLLa65Pts8uawGp7kRqmMDs,8260
 azure/ai/evaluation/_evaluate/_batch_run/eval_run_context.py,sha256=p3Bsg_shGs5RXvysOlvo0CQb4Te5herSvX1OP6ylFUQ,3543
 azure/ai/evaluation/_evaluate/_batch_run/proxy_client.py,sha256=T_QRHScDMBM4O6ejkkKdBmHPjH2NOF6owW48aVUYF6k,3775
-azure/ai/evaluation/_evaluate/_batch_run/target_run_context.py,sha256=_e-6QldHyEbPklGFMUOqrQCZHalCUMGHGNiAsVT0wgg,1628
+azure/ai/evaluation/_evaluate/_batch_run/target_run_context.py,sha256=SMos3bVmD73pK6gpIaL4iZZS3-Zda3V4N89Jg0J9sss,1636
 azure/ai/evaluation/_evaluate/_telemetry/__init__.py,sha256=fhLqE41qxdjfBOGi23cpk6QgUe-s1Fw2xhAAUjNESF0,7045
 azure/ai/evaluation/_evaluators/__init__.py,sha256=Yx1Iq2GNKQ5lYxTotvPwkPL4u0cm6YVxUe-iVbu1clI,180
 azure/ai/evaluation/_evaluators/_bleu/__init__.py,sha256=quKKO0kvOSkky5hcoNBvgBuMeeVRFCE9GSv70mAdGP4,260
@@ -29,11 +33,11 @@ azure/ai/evaluation/_evaluators/_coherence/__init__.py,sha256=GRqcSCQse02Spyki0U
 azure/ai/evaluation/_evaluators/_coherence/_coherence.py,sha256=uG9hX2XWkMREKfMAWRoosjicoI4Lg3ptR3UcLEgKd0c,4643
 azure/ai/evaluation/_evaluators/_coherence/coherence.prompty,sha256=ANvh9mDFW7KMejrgdWqBLjj4SIqEO5WW9gg5pE0RLJk,6798
 azure/ai/evaluation/_evaluators/_common/__init__.py,sha256=_hPqTkAla_O6s4ebVtTaBrVLEW3KSdDz66WwxjK50cI,423
-azure/ai/evaluation/_evaluators/_common/_base_eval.py,sha256=_KitrIIOzqhggKP3EL3he0AvpDJv4T3io06PwfAtfg8,15961
-azure/ai/evaluation/_evaluators/_common/_base_prompty_eval.py,sha256=WfCE6KuSK1bNxBvSOl1vPOqh5UEpuVgA5WMN-BOYeQ4,3876
-azure/ai/evaluation/_evaluators/_common/_base_rai_svc_eval.py,sha256=WXGGWf2fsFLeNq0-QL-4s56LXp72CPUhHuTw29H9k-E,5817
+azure/ai/evaluation/_evaluators/_common/_base_eval.py,sha256=n6qldJr8d8H0DnS7IwkQPH9Ep9PdZnVeVtSxQiunADc,19424
+azure/ai/evaluation/_evaluators/_common/_base_prompty_eval.py,sha256=hvJD7jR2ESePkRPN17ytoFhFiS0iTotOfeqmTwG2IMs,4531
+azure/ai/evaluation/_evaluators/_common/_base_rai_svc_eval.py,sha256=czyn1MfaxOmrvvFgdeblf6FaauKgKolgPFsP5f7K29w,7331
 azure/ai/evaluation/_evaluators/_content_safety/__init__.py,sha256=PEYMIybfP64f7byhuTaiq4RiqsYbjqejpW1JsJIG1jA,556
-azure/ai/evaluation/_evaluators/_content_safety/_content_safety.py,sha256=UERxH-cHj1E3mNY7aXMdUz4rAxAkRRNlg8NXqaDdr7M,6332
+azure/ai/evaluation/_evaluators/_content_safety/_content_safety.py,sha256=CIGfBLNOTVXrlF5HIc2UpuDDG5BfzjD7ubJ23CbvobQ,6341
 azure/ai/evaluation/_evaluators/_content_safety/_hate_unfairness.py,sha256=sjw8FfwxC1f0K1J4TkeA8wkfq88aebiNbaKzS-8DWzk,5919
 azure/ai/evaluation/_evaluators/_content_safety/_self_harm.py,sha256=0zaB-JKm8FU6yoxD1nqoYvxp3gvjuZfcQjb-xhSHoQ0,5156
 azure/ai/evaluation/_evaluators/_content_safety/_sexual.py,sha256=q9bEMu6Dp1wxDlH3h2iTayrWv4ux-izLB0kGkxrgEhM,5396
@@ -76,27 +80,27 @@ azure/ai/evaluation/_evaluators/_rouge/_rouge.py,sha256=SV5rESLVARQqh1n0Pf6EMvJo
 azure/ai/evaluation/_evaluators/_service_groundedness/__init__.py,sha256=0DODUGTOgaYyFbO9_zxuwifixDL3SIm3EkwP1sdwn6M,288
 azure/ai/evaluation/_evaluators/_service_groundedness/_service_groundedness.py,sha256=GPvufAgTnoQ2HYs6Xnnpmh23n5E3XxnUV0NGuwjDyU0,6648
 azure/ai/evaluation/_evaluators/_similarity/__init__.py,sha256=V2Mspog99_WBltxTkRHG5NpN5s9XoiTSN4I8POWEkLA,268
-azure/ai/evaluation/_evaluators/_similarity/_similarity.py,sha256=DCoHr8-FN9rM6Kbl2T7yRINabBAmLBuEhHKk7EMz6is,5698
+azure/ai/evaluation/_evaluators/_similarity/_similarity.py,sha256=UVBIa1xIlOIJtPctCu-UCOWvXzE4ysaK_XFdokajCuA,5669
 azure/ai/evaluation/_evaluators/_similarity/similarity.prompty,sha256=eoludASychZoGL625bFCaZai-OY7DIAg90ZLax_o4XE,4594
 azure/ai/evaluation/_evaluators/_xpia/__init__.py,sha256=VMEL8WrpJQeh4sQiOLzP7hRFPnjzsvwfvTzaGCVJPCM,88
 azure/ai/evaluation/_evaluators/_xpia/xpia.py,sha256=Nv14lU7jN0yXKbHgHRXMHEy6pn1rXmesBOYI2Ge9ewk,5849
 azure/ai/evaluation/_vendor/__init__.py,sha256=Yx1Iq2GNKQ5lYxTotvPwkPL4u0cm6YVxUe-iVbu1clI,180
 azure/ai/evaluation/_vendor/rouge_score/__init__.py,sha256=03OkyfS_UmzRnHv6-z9juTaJ6OXJoEJM989hgifIZbc,607
-azure/ai/evaluation/_vendor/rouge_score/rouge_scorer.py,sha256=xDdNtzwtivcdki5RyErEI9BaQ7nksgj4bXYrGz7tLLs,11409
-azure/ai/evaluation/_vendor/rouge_score/scoring.py,sha256=ruwkMrJFJNvs3GWqVLAXudIwDa4EsX_d30pfUPUTf8E,1988
-azure/ai/evaluation/_vendor/rouge_score/tokenize.py,sha256=tdSsUibKxtOMY8fdqGK_3-4sMbeOxZEG6D6L7suDTxQ,1936
+azure/ai/evaluation/_vendor/rouge_score/rouge_scorer.py,sha256=DtNSeshHipzc6vFnvx7kbs5viXe4LNq-ZrgllFvfR4U,11299
+azure/ai/evaluation/_vendor/rouge_score/scoring.py,sha256=0sqdiNE-4R_EmTTqyWL9_DAOgl54250H5004tZDGxEE,1878
+azure/ai/evaluation/_vendor/rouge_score/tokenize.py,sha256=IyHVsWY6IFFZdB23cLiJs8iBZ0DXk1mQlWE1xtdjuuk,1826
 azure/ai/evaluation/_vendor/rouge_score/tokenizers.py,sha256=3_-y1TyvyluHuERhSJ5CdXSwnpcMA7aAKU6PCz9wH_Q,1745
 azure/ai/evaluation/simulator/__init__.py,sha256=JbrPZ8pvTBalyX94SvZ9btHNoovX8rbZV03KmzxxWys,552
-azure/ai/evaluation/simulator/_adversarial_scenario.py,sha256=_hvL719cT7Vgh34KpztJikSlnKhzr16lvNVBXZa6Wwk,1605
-azure/ai/evaluation/simulator/_adversarial_simulator.py,sha256=O-QLbo6-5w-1Qn4-sghCcPECe8uavlenJWg-1x-kc_0,20980
+azure/ai/evaluation/simulator/_adversarial_scenario.py,sha256=9rpAPz594tYjxzM3XMeDq6CZSc2yvf5YaNaGC7nzYhM,1710
+azure/ai/evaluation/simulator/_adversarial_simulator.py,sha256=FPZ3OdpGuwCHDVoOZW-f_j7pyK71PfDN3JPh205tW0c,21706
 azure/ai/evaluation/simulator/_constants.py,sha256=nCL7_1BnYh6k0XvxudxsDVMbiG9MMEvYw5wO9FZHHZ8,857
 azure/ai/evaluation/simulator/_direct_attack_simulator.py,sha256=FTtWf655dHJF5FLJi0xGSBgIlGWNiVWyqaLDJSud9XA,10199
-azure/ai/evaluation/simulator/_indirect_attack_simulator.py,sha256=ktVLlQo7LfzRodVA6wDLc_Dm3YADPa2klX6bPPfrkiw,10179
-azure/ai/evaluation/simulator/_simulator.py,sha256=3wi3hdlao_41sVNvjM6YCXfJ-1A6-tDg_brpkaUat8U,36158
+azure/ai/evaluation/simulator/_indirect_attack_simulator.py,sha256=nweIU_AkUIR50qLQpjmljf_OkpsCPth2Ebf4vusygCA,10226
+azure/ai/evaluation/simulator/_simulator.py,sha256=pWxVfy9ll6gmOyGEk6Ie7Y48X21wJ5DebqY8Re0SIOk,36213
 azure/ai/evaluation/simulator/_tracing.py,sha256=frZ4-usrzINast9F4-ONRzEGGox71y8bYw0UHNufL1Y,3069
 azure/ai/evaluation/simulator/_utils.py,sha256=16NltlywpbMtoFtULwTKqeURguIS1kSKSo3g8uKV8TA,5181
-azure/ai/evaluation/simulator/_conversation/__init__.py,sha256=ulkkJkvRBRROLp_wpAKy1J-HLMJi3Yq6g7Q6VGRuD88,12914
-azure/ai/evaluation/simulator/_conversation/_conversation.py,sha256=vzKdpItmUjZrM5OUSkS2UkTnLnKvIzhak5hZ8xvFwnU,7403
+azure/ai/evaluation/simulator/_conversation/__init__.py,sha256=s8djzJ58_-CiIA8xHB-SbgeZaq1F7ftrc3qJbpUpUdg,17853
+azure/ai/evaluation/simulator/_conversation/_conversation.py,sha256=qdzGMtCPYMxeGpR91NZTEmmz2RtADTvQGj6C-3EUTw4,7402
 azure/ai/evaluation/simulator/_conversation/constants.py,sha256=3v7zkjPwJAPbSpJYIK6VOZZy70bJXMo_QTVqSFGlq9A,984
 azure/ai/evaluation/simulator/_data_sources/__init__.py,sha256=Yx1Iq2GNKQ5lYxTotvPwkPL4u0cm6YVxUe-iVbu1clI,180
 azure/ai/evaluation/simulator/_data_sources/grounding.json,sha256=jqdqHrCgS7hN7K2kXSEcPCmzFjV4cv_qcCSR-Hutwx4,1257075
@@ -106,14 +110,14 @@ azure/ai/evaluation/simulator/_helpers/_simulator_data_classes.py,sha256=BOttMTe
 azure/ai/evaluation/simulator/_model_tools/__init__.py,sha256=aMv5apb7uVjuhMF9ohhA5kQmo652hrGIJlhdl3y2R1I,835
 azure/ai/evaluation/simulator/_model_tools/_identity_manager.py,sha256=-hptp2vpJIcfjvtd0E2c7ry00LVh23LxuYGevsNFfgs,6385
 azure/ai/evaluation/simulator/_model_tools/_proxy_completion_model.py,sha256=Zg_SzqjCGJ3Wt8hktxz6Y1JEJCcV0V5jBC9N06jQP3k,8984
-azure/ai/evaluation/simulator/_model_tools/_rai_client.py,sha256=5WFRbZQbPhp3S8_l1lHE72HHipSgqtlcB-JdRt293aU,7228
-azure/ai/evaluation/simulator/_model_tools/_template_handler.py,sha256=FGKLsWL0FZry47ZxFi53FSem8PZmh0iIy3JN4PBg5Tg,7036
+azure/ai/evaluation/simulator/_model_tools/_rai_client.py,sha256=40MGzIXGv7oVshWH7AbOPLCigI4HlMrqbF2Rq5jFMGo,8755
+azure/ai/evaluation/simulator/_model_tools/_template_handler.py,sha256=NQWqjE7csSzkhb2XdW82AoCA-DxixpTrfBxAnOt2Wlc,7075
 azure/ai/evaluation/simulator/_model_tools/models.py,sha256=bfVm0PV3vfH_8DkdmTMZqYVN-G51hZ6Y0TOO-NiysJY,21811
 azure/ai/evaluation/simulator/_prompty/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 azure/ai/evaluation/simulator/_prompty/task_query_response.prompty,sha256=2BzSqDDYilDushvR56vMRDmqFIaIYAewdUlUZg_elMg,2182
 azure/ai/evaluation/simulator/_prompty/task_simulate.prompty,sha256=NE6lH4bfmibgMn4NgJtm9_l3PMoHSFrfjjosDJEKM0g,939
-azure_ai_evaluation-1.0.1.dist-info/METADATA,sha256=QmfPB60dq4htOHkeAa_YuKh1AywKZVlH0QAl0qqf7CY,28098
-azure_ai_evaluation-1.0.1.dist-info/NOTICE.txt,sha256=4tzi_Yq4-eBGhBvveobWHCgUIVF-ZeouGN0m7hVq5Mk,3592
-azure_ai_evaluation-1.0.1.dist-info/WHEEL,sha256=pL8R0wFFS65tNSRnaOVrsw9EOkOqxLrlUPenUYnJKNo,91
-azure_ai_evaluation-1.0.1.dist-info/top_level.txt,sha256=S7DhWV9m80TBzAhOFjxDUiNbKszzoThbnrSz5MpbHSQ,6
-azure_ai_evaluation-1.0.1.dist-info/RECORD,,
+azure_ai_evaluation-1.1.0.dist-info/METADATA,sha256=zusuZTIcO7487bWQK5V7XZ-Pbqugm7HFzvcCV5yfTmk,28751
+azure_ai_evaluation-1.1.0.dist-info/NOTICE.txt,sha256=4tzi_Yq4-eBGhBvveobWHCgUIVF-ZeouGN0m7hVq5Mk,3592
+azure_ai_evaluation-1.1.0.dist-info/WHEEL,sha256=pL8R0wFFS65tNSRnaOVrsw9EOkOqxLrlUPenUYnJKNo,91
+azure_ai_evaluation-1.1.0.dist-info/top_level.txt,sha256=S7DhWV9m80TBzAhOFjxDUiNbKszzoThbnrSz5MpbHSQ,6
+azure_ai_evaluation-1.1.0.dist-info/RECORD,,