azure-ai-evaluation 1.0.1__py3-none-any.whl → 1.2.0__py3-none-any.whl

azure/ai/evaluation/_evaluators/_similarity/_similarity.py

@@ -2,85 +2,17 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
 
-import math
 import os
-import re
+from typing import Dict
 
-from promptflow._utils.async_utils import async_run_allowing_running_loop
-from promptflow.core import AsyncPrompty
+from typing_extensions import overload, override
 
-from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
+from azure.ai.evaluation._evaluators._common import PromptyEvaluatorBase
 
-from ..._common.utils import construct_prompty_model_config, validate_model_config
 
-try:
-    from ..._user_agent import USER_AGENT
-except ImportError:
-    USER_AGENT = "None"
-
-
-class _AsyncSimilarityEvaluator:
-    # Constants must be defined within eval's directory to be save/loadable
-    _PROMPTY_FILE = "similarity.prompty"
-    _LLM_CALL_TIMEOUT = 600
-    _DEFAULT_OPEN_API_VERSION = "2024-02-15-preview"
-
-    def __init__(self, model_config: dict):
-        prompty_model_config = construct_prompty_model_config(
-            validate_model_config(model_config),
-            self._DEFAULT_OPEN_API_VERSION,
-            USER_AGENT,
-        )
-
-        current_dir = os.path.dirname(__file__)
-        prompty_path = os.path.join(current_dir, self._PROMPTY_FILE)
-        self._flow = AsyncPrompty.load(source=prompty_path, model=prompty_model_config)
-
-    async def __call__(self, *, query: str, response: str, ground_truth: str, **kwargs):
-        """
-        Evaluate similarity.
-
-        :keyword query: The query to be evaluated.
-        :paramtype query: str
-        :keyword response: The response to be evaluated.
-        :paramtype response: str
-        :keyword ground_truth: The ground truth to be evaluated.
-        :paramtype ground_truth: str
-        :return: The similarity score.
-        :rtype: Dict[str, float]
-        """
-        # Validate input parameters
-        query = str(query or "")
-        response = str(response or "")
-        ground_truth = str(ground_truth or "")
-
-        if not (query.strip() and response.strip() and ground_truth.strip()):
-            msg = "'query', 'response' and 'ground_truth' must be non-empty strings."
-            raise EvaluationException(
-                message=msg,
-                internal_message=msg,
-                error_category=ErrorCategory.MISSING_FIELD,
-                error_blame=ErrorBlame.USER_ERROR,
-                error_target=ErrorTarget.SIMILARITY_EVALUATOR,
-            )
-
-        # Run the evaluation flow
-        llm_output = await self._flow(
-            query=query, response=response, ground_truth=ground_truth, timeout=self._LLM_CALL_TIMEOUT, **kwargs
-        )
-
-        score = math.nan
-        if llm_output:
-            match = re.search(r"\d", llm_output)
-            if match:
-                score = float(match.group())
-
-        return {"similarity": float(score), "gpt_similarity": float(score)}
-
-
-class SimilarityEvaluator:
+class SimilarityEvaluator(PromptyEvaluatorBase):
     """
-    Evaluates similarity score for a given query, response, and ground truth or a multi-turn conversation.
+    Evaluates similarity score for a given query, response, and ground truth.
 
     The similarity measure evaluates the likeness between a ground truth sentence (or document) and the
     AI model's generated prediction. This calculation involves creating sentence-level embeddings for both
@@ -113,13 +45,27 @@ class SimilarityEvaluator:
         however, it is recommended to use the new key moving forward as the old key will be deprecated in the future.
     """
 
-    id = "azureml://registries/azureml/models/Similarity-Evaluator/versions/3"
+    # Constants must be defined within eval's directory to be save/loadable
+
+    _PROMPTY_FILE = "similarity.prompty"
+    _RESULT_KEY = "similarity"
+
+    id = "similarity"
     """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
 
+    @override
     def __init__(self, model_config):
-        self._async_evaluator = _AsyncSimilarityEvaluator(model_config)
-
-    def __call__(self, *, query: str, response: str, ground_truth: str, **kwargs):
+        current_dir = os.path.dirname(__file__)
+        prompty_path = os.path.join(current_dir, self._PROMPTY_FILE)
+        super().__init__(model_config=model_config, prompty_file=prompty_path, result_key=self._RESULT_KEY)
+
+    # Ignoring a mypy error about having only 1 overload function.
+    # We want to use the overload style for all evals, even single-inputs. This is both to make
+    # refactoring to multi-input styles easier, stylistic consistency consistency across evals,
+    # and due to the fact that non-overloaded syntax now causes various parsing issues that
+    # we don't want to deal with.
+    @overload  # type: ignore
+    def __call__(self, *, query: str, response: str, ground_truth: str) -> Dict[str, float]:
         """
         Evaluate similarity.
 
@@ -132,9 +78,23 @@ class SimilarityEvaluator:
         :return: The similarity score.
         :rtype: Dict[str, float]
         """
-        return async_run_allowing_running_loop(
-            self._async_evaluator, query=query, response=response, ground_truth=ground_truth, **kwargs
-        )
 
-    def _to_async(self):
-        return self._async_evaluator
+    @override
+    def __call__(  # pylint: disable=docstring-missing-param
+        self,
+        *args,
+        **kwargs,
+    ):
+        """
+        Evaluate similarity.
+
+        :keyword query: The query to be evaluated.
+        :paramtype query: str
+        :keyword response: The response to be evaluated.
+        :paramtype response: str
+        :keyword ground_truth: The ground truth to be evaluated.
+        :paramtype ground_truth: str
+        :return: The similarity score.
+        :rtype: Dict[str, float]
+        """
+        return super().__call__(*args, **kwargs)

azure/ai/evaluation/_http_utils.py

@@ -448,19 +448,21 @@ class AsyncHttpPipeline(AsyncPipeline):
         return cast(Self, await super().__aenter__())
 
 
-def get_http_client() -> HttpPipeline:
+def get_http_client(**kwargs: Any) -> HttpPipeline:
     """Get an HttpPipeline configured with common policies.
 
     :returns: An HttpPipeline with a set of applied policies:
     :rtype: HttpPipeline
     """
-    return HttpPipeline(user_agent_policy=UserAgentPolicy(base_user_agent=USER_AGENT))
+    kwargs.setdefault("user_agent_policy", UserAgentPolicy(base_user_agent=USER_AGENT))
+    return HttpPipeline(**kwargs)
 
 
-def get_async_http_client() -> AsyncHttpPipeline:
+def get_async_http_client(**kwargs: Any) -> AsyncHttpPipeline:
     """Get an AsyncHttpPipeline configured with common policies.
 
     :returns: An AsyncHttpPipeline with a set of applied policies:
     :rtype: AsyncHttpPipeline
     """
-    return AsyncHttpPipeline(user_agent_policy=UserAgentPolicy(base_user_agent=USER_AGENT))
+    kwargs.setdefault("user_agent_policy", UserAgentPolicy(base_user_agent=USER_AGENT))
+    return AsyncHttpPipeline(**kwargs)

azure/ai/evaluation/_vendor/rouge_score/rouge_scorer.py

@@ -32,10 +32,6 @@ ROUGE-1.5.5.pl -m -e data -n 2 -a settings.xml
 In these examples settings.xml lists input files and formats.
 """
 
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
 import collections
 import re
 

azure/ai/evaluation/_vendor/rouge_score/scoring.py

@@ -21,10 +21,6 @@ Aggregation functions use bootstrap resampling to compute confidence intervals
 as per the original ROUGE perl implementation.
 """
 
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
 import abc
 import collections
 from typing import Dict

azure/ai/evaluation/_vendor/rouge_score/tokenize.py

@@ -17,10 +17,6 @@
 
 """A library for tokenizing text."""
 
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
 import re
 
 

azure/ai/evaluation/_version.py

@@ -2,4 +2,4 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
 
-VERSION = "1.0.1"
+VERSION = "1.2.0"

azure/ai/evaluation/simulator/_adversarial_scenario.py

@@ -44,3 +44,5 @@ class _UnstableAdversarialScenario(Enum):
     """
 
     ECI = "adv_politics"
+    ADVERSARIAL_IMAGE_GEN = "adv_image_gen"
+    ADVERSARIAL_IMAGE_MULTIMODAL = "adv_image_understanding"

azure/ai/evaluation/simulator/_adversarial_simulator.py

@@ -6,8 +6,7 @@
 import asyncio
 import logging
 import random
-from typing import Any, Callable, Dict, List, Literal, Optional, Union, cast
-from itertools import zip_longest
+from typing import Any, Callable, Dict, List, Optional, Union, cast
 
 from tqdm import tqdm
 
@@ -16,13 +15,19 @@ from azure.ai.evaluation._common.utils import validate_azure_ai_project
 from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
 from azure.ai.evaluation._http_utils import get_async_http_client
 from azure.ai.evaluation._model_configurations import AzureAIProject
-from azure.ai.evaluation.simulator import AdversarialScenario
+from azure.ai.evaluation.simulator import AdversarialScenario, AdversarialScenarioJailbreak
 from azure.ai.evaluation.simulator._adversarial_scenario import _UnstableAdversarialScenario
 from azure.core.credentials import TokenCredential
 from azure.core.pipeline.policies import AsyncRetryPolicy, RetryMode
 
 from ._constants import SupportedLanguages
-from ._conversation import CallbackConversationBot, ConversationBot, ConversationRole, ConversationTurn
+from ._conversation import (
+    CallbackConversationBot,
+    MultiModalConversationBot,
+    ConversationBot,
+    ConversationRole,
+    ConversationTurn,
+)
 from ._conversation._conversation import simulate_conversation
 from ._model_tools import (
     AdversarialTemplateHandler,
@@ -205,7 +210,6 @@ class AdversarialSimulator:
             ncols=100,
             unit="simulations",
         )
-
         if randomize_order:
             # The template parameter lists are persistent across sim runs within a session,
             # So randomize a the selection instead of the parameter list directly,
@@ -214,11 +218,11 @@ class AdversarialSimulator:
                 random.seed(randomization_seed)
             random.shuffle(templates)
         parameter_lists = [t.template_parameters for t in templates]
-        zipped_parameters = list(zip_longest(*parameter_lists))
+        zipped_parameters = list(zip(*parameter_lists))
         for param_group in zipped_parameters:
             for template, parameter in zip(templates, param_group):
                 if _jailbreak_type == "upia":
-                    parameter = self._join_conversation_starter(parameter, random.choice(jailbreak_dataset))
+                    parameter = self._add_jailbreak_parameter(parameter, random.choice(jailbreak_dataset))
                 tasks.append(
                     asyncio.create_task(
                         self._simulate_async(
@@ -231,6 +235,7 @@ class AdversarialSimulator:
                             api_call_delay_sec=api_call_delay_sec,
                             language=language,
                             semaphore=semaphore,
+                            scenario=scenario,
                         )
                     )
                 )
@@ -292,10 +297,13 @@ class AdversarialSimulator:
         api_call_delay_sec: int,
         language: SupportedLanguages,
         semaphore: asyncio.Semaphore,
+        scenario: Union[AdversarialScenario, AdversarialScenarioJailbreak],
     ) -> List[Dict]:
-        user_bot = self._setup_bot(role=ConversationRole.USER, template=template, parameters=parameters)
+        user_bot = self._setup_bot(
+            role=ConversationRole.USER, template=template, parameters=parameters, scenario=scenario
+        )
         system_bot = self._setup_bot(
-            target=target, role=ConversationRole.ASSISTANT, template=template, parameters=parameters
+            target=target, role=ConversationRole.ASSISTANT, template=template, parameters=parameters, scenario=scenario
         )
         bots = [user_bot, system_bot]
         session = get_async_http_client().with_policies(
@@ -341,6 +349,7 @@ class AdversarialSimulator:
         template: AdversarialTemplate,
         parameters: TemplateParameters,
         target: Optional[Callable] = None,
+        scenario: Union[AdversarialScenario, AdversarialScenarioJailbreak],
     ) -> ConversationBot:
         if role is ConversationRole.USER:
             model = self._get_user_proxy_completion_model(
@@ -372,6 +381,21 @@ class AdversarialSimulator:
                 def __call__(self) -> None:
                     pass
 
+            if scenario in [
+                _UnstableAdversarialScenario.ADVERSARIAL_IMAGE_GEN,
+                _UnstableAdversarialScenario.ADVERSARIAL_IMAGE_MULTIMODAL,
+            ]:
+                return MultiModalConversationBot(
+                    callback=target,
+                    role=role,
+                    model=DummyModel(),
+                    user_template=str(template),
+                    user_template_parameters=parameters,
+                    rai_client=self.rai_client,
+                    conversation_template="",
+                    instantiation_parameters={},
+                )
+
             return CallbackConversationBot(
                 callback=target,
                 role=role,
@@ -391,13 +415,8 @@ class AdversarialSimulator:
             blame=ErrorBlame.SYSTEM_ERROR,
         )
 
-    def _join_conversation_starter(self, parameters: TemplateParameters, to_join: str) -> TemplateParameters:
-        key: Literal["conversation_starter"] = "conversation_starter"
-        if key in parameters.keys():
-            parameters[key] = f"{to_join} {parameters[key]}"
-        else:
-            parameters[key] = to_join
-
+    def _add_jailbreak_parameter(self, parameters: TemplateParameters, to_join: str) -> TemplateParameters:
+        parameters["jailbreak_string"] = to_join
         return parameters
 
     def call_sync(

azure/ai/evaluation/simulator/_conversation/__init__.py

@@ -9,12 +9,12 @@ import time
 from dataclasses import dataclass
 from typing import Any, Callable, Dict, List, Optional, Tuple, Union, cast
 
+import re
 import jinja2
 
 from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
 from azure.ai.evaluation._http_utils import AsyncHttpPipeline
-
-from .._model_tools import LLMBase, OpenAIChatCompletionsModel
+from .._model_tools import LLMBase, OpenAIChatCompletionsModel, RAIClient
 from .._model_tools._template_handler import TemplateParameters
 from .constants import ConversationRole
 
@@ -128,15 +128,19 @@ class ConversationBot:
         self.conversation_starter: Optional[Union[str, jinja2.Template, Dict]] = None
         if role == ConversationRole.USER:
             if "conversation_starter" in self.persona_template_args:
+                print(self.persona_template_args)
                 conversation_starter_content = self.persona_template_args["conversation_starter"]
                 if isinstance(conversation_starter_content, dict):
                     self.conversation_starter = conversation_starter_content
+                    print(f"Conversation starter content: {conversation_starter_content}")
                 else:
                     try:
                         self.conversation_starter = jinja2.Template(
                             conversation_starter_content, undefined=jinja2.StrictUndefined
                         )
-                    except jinja2.exceptions.TemplateSyntaxError:  # noqa: F841
+                        print("Successfully created a Jinja2 template for the conversation starter.")
+                    except jinja2.exceptions.TemplateSyntaxError as e:  # noqa: F841
+                        print(f"Template syntax error: {e}. Using raw content.")
                         self.conversation_starter = conversation_starter_content
             else:
                 self.logger.info(
@@ -175,6 +179,9 @@ class ConversationBot:
                 samples = [self.conversation_starter.render(**self.persona_template_args)]
             else:
                 samples = [self.conversation_starter]
+            jailbreak_string = self.persona_template_args.get("jailbreak_string", None)
+            if jailbreak_string:
+                samples = [f"{jailbreak_string} {samples[0]}"]
             time_taken = 0
 
             finish_reason = ["stop"]
@@ -271,8 +278,6 @@ class CallbackConversationBot(ConversationBot):
                 "id": None,
                 "template_parameters": {},
             }
-        self.logger.info("Using user provided callback returning response.")
-
         time_taken = end_time - start_time
         try:
             response = {
@@ -290,8 +295,6 @@ class CallbackConversationBot(ConversationBot):
                 blame=ErrorBlame.USER_ERROR,
             ) from exc
 
-        self.logger.info("Parsed callback response")
-
         return response, {}, time_taken, result
 
     # Bug 3354264: template is unused in the method - is this intentional?
@@ -308,9 +311,127 @@ class CallbackConversationBot(ConversationBot):
         }
 
 
+class MultiModalConversationBot(ConversationBot):
+    """MultiModal Conversation bot that uses a user provided callback to generate responses.
+
+    :param callback: The callback function to use to generate responses.
+    :type callback: Callable
+    :param user_template: The template to use for the request.
+    :type user_template: str
+    :param user_template_parameters: The template parameters to use for the request.
+    :type user_template_parameters: Dict
+    :param args: Optional arguments to pass to the parent class.
+    :type args: Any
+    :param kwargs: Optional keyword arguments to pass to the parent class.
+    :type kwargs: Any
+    """
+
+    def __init__(
+        self,
+        callback: Callable,
+        user_template: str,
+        user_template_parameters: TemplateParameters,
+        rai_client: RAIClient,
+        *args,
+        **kwargs,
+    ) -> None:
+        self.callback = callback
+        self.user_template = user_template
+        self.user_template_parameters = user_template_parameters
+        self.rai_client = rai_client
+
+        super().__init__(*args, **kwargs)
+
+    async def generate_response(
+        self,
+        session: AsyncHttpPipeline,
+        conversation_history: List[Any],
+        max_history: int,
+        turn_number: int = 0,
+    ) -> Tuple[dict, dict, float, dict]:
+        previous_prompt = conversation_history[-1]
+        chat_protocol_message = await self._to_chat_protocol(conversation_history, self.user_template_parameters)
+
+        # replace prompt with {image.jpg} tags with image content data.
+        conversation_history.pop()
+        conversation_history.append(
+            ConversationTurn(
+                role=previous_prompt.role,
+                name=previous_prompt.name,
+                message=chat_protocol_message["messages"][0]["content"],
+                full_response=previous_prompt.full_response,
+                request=chat_protocol_message,
+            )
+        )
+        msg_copy = copy.deepcopy(chat_protocol_message)
+        result = {}
+        start_time = time.time()
+        result = await self.callback(msg_copy)
+        end_time = time.time()
+        if not result:
+            result = {
+                "messages": [{"content": "Callback did not return a response.", "role": "assistant"}],
+                "finish_reason": ["stop"],
+                "id": None,
+                "template_parameters": {},
+            }
+
+        time_taken = end_time - start_time
+        try:
+            response = {
+                "samples": [result["messages"][-1]["content"]],
+                "finish_reason": ["stop"],
+                "id": None,
+            }
+        except Exception as exc:
+            msg = "User provided callback does not conform to chat protocol standard."
+            raise EvaluationException(
+                message=msg,
+                internal_message=msg,
+                target=ErrorTarget.CALLBACK_CONVERSATION_BOT,
+                category=ErrorCategory.INVALID_VALUE,
+                blame=ErrorBlame.USER_ERROR,
+            ) from exc
+
+        return response, chat_protocol_message, time_taken, result
+
+    async def _to_chat_protocol(self, conversation_history, template_parameters):  # pylint: disable=unused-argument
+        messages = []
+
+        for _, m in enumerate(conversation_history):
+            if "image:" in m.message:
+                content = await self._to_multi_modal_content(m.message)
+                messages.append({"content": content, "role": m.role.value})
+            else:
+                messages.append({"content": m.message, "role": m.role.value})
+
+        return {
+            "template_parameters": template_parameters,
+            "messages": messages,
+            "$schema": "http://azureml/sdk-2-0/ChatConversation.json",
+        }
+
+    async def _to_multi_modal_content(self, text: str) -> list:
+        split_text = re.findall(r"[^{}]+|\{[^{}]*\}", text)
+        messages = [
+            text.strip("{}").replace("image:", "").strip() if text.startswith("{") else text for text in split_text
+        ]
+        contents = []
+        for msg in messages:
+            if msg.startswith("image_understanding/"):
+                encoded_image = await self.rai_client.get_image_data(msg)
+                contents.append(
+                    {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{encoded_image}"}},
+                )
+            else:
+                contents.append({"type": "text", "text": msg})
+        return contents
+
+
 __all__ = [
     "ConversationRole",
     "ConversationBot",
     "CallbackConversationBot",
+    "MultiModalConversationBot",
     "ConversationTurn",
 ]

azure/ai/evaluation/simulator/_conversation/_conversation.py

@@ -9,7 +9,6 @@ from typing import Callable, Dict, List, Optional, Tuple, Union
 from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
 from azure.ai.evaluation.simulator._constants import SupportedLanguages
 from azure.ai.evaluation.simulator._helpers._language_suffix_mapping import SUPPORTED_LANGUAGES_MAPPING
-
 from ..._http_utils import AsyncHttpPipeline
 from . import ConversationBot, ConversationTurn
 

azure/ai/evaluation/simulator/_indirect_attack_simulator.py

@@ -189,6 +189,7 @@ class IndirectAttackSimulator(AdversarialSimulator):
                             api_call_delay_sec=api_call_delay_sec,
                             language=language,
                             semaphore=semaphore,
+                            scenario=scenario,
                         )
                     )
                 )

azure/ai/evaluation/simulator/_model_tools/_rai_client.py

@@ -4,6 +4,7 @@
 import os
 from typing import Any
 from urllib.parse import urljoin, urlparse
+import base64
 
 from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
 from azure.ai.evaluation._http_utils import AsyncHttpPipeline, get_async_http_client, get_http_client
@@ -57,6 +58,7 @@ class RAIClient:  # pylint: disable=client-accepts-api-version-keyword
         # add a "/" at the end of the url
         self.api_url = self.api_url.rstrip("/") + "/"
         self.parameter_json_endpoint = urljoin(self.api_url, "simulation/template/parameters")
+        self.parameter_image_endpoint = urljoin(self.api_url, "simulation/template/parameters/image")
         self.jailbreaks_json_endpoint = urljoin(self.api_url, "simulation/jailbreak")
         self.simulation_submit_endpoint = urljoin(self.api_url, "simulation/chat/completions/submit")
         self.xpia_jailbreaks_json_endpoint = urljoin(self.api_url, "simulation/jailbreak/xpia")
@@ -166,3 +168,41 @@ class RAIClient:  # pylint: disable=client-accepts-api-version-keyword
             category=ErrorCategory.UNKNOWN,
             blame=ErrorBlame.USER_ERROR,
         )
+
+    async def get_image_data(self, path: str) -> Any:
+        """Make a GET Image request to the given url
+
+        :param path: The url of the image
+        :type path: str
+        :raises EvaluationException: If the Azure safety evaluation service is not available in the current region
+        :return: The response
+        :rtype: Any
+        """
+        token = self.token_manager.get_token()
+        headers = {
+            "Authorization": f"Bearer {token}",
+            "Content-Type": "application/json",
+            "User-Agent": USER_AGENT,
+        }
+
+        session = self._create_async_client()
+        params = {"path": path}
+        async with session:
+            response = await session.get(
+                url=self.parameter_image_endpoint, params=params, headers=headers
+            )  # pylint: disable=unexpected-keyword-arg
+
+        if response.status_code == 200:
+            return base64.b64encode(response.content).decode("utf-8")
+
+        msg = (
+            "Azure safety evaluation service is not available in your current region, "
+            + "please go to https://aka.ms/azureaistudiosafetyeval to see which regions are supported"
+        )
+        raise EvaluationException(
+            message=msg,
+            internal_message=msg,
+            target=ErrorTarget.RAI_CLIENT,
+            category=ErrorCategory.UNKNOWN,
+            blame=ErrorBlame.USER_ERROR,
+        )

azure/ai/evaluation/simulator/_model_tools/_template_handler.py

@@ -56,6 +56,7 @@ class TemplateParameters(TypedDict):
     category: NotRequired[str]
     target_population: NotRequired[str]
     topic: NotRequired[str]
+    jailbreak_string: NotRequired[str]
 
 
 class _CategorizedParameter(TypedDict):