azure-ai-evaluation 1.0.0__py3-none-any.whl → 1.0.0b2__py3-none-any.whl

azure/ai/evaluation/{_common → simulator/_helpers}/_experimental.py

@@ -2,14 +2,13 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
 
-import os
 import functools
 import inspect
 import logging
 import sys
-from typing import Callable, Type, TypeVar, Union, overload
+from typing import Callable, Type, TypeVar, Union
 
-from typing_extensions import ParamSpec, TypeGuard
+from typing_extensions import ParamSpec
 
 DOCSTRING_TEMPLATE = ".. note::    {0} {1}\n\n"
 DOCSTRING_DEFAULT_INDENTATION = 8
@@ -23,31 +22,20 @@ EXPERIMENTAL_LINK_MESSAGE = (
 _warning_cache = set()
 module_logger = logging.getLogger(__name__)
 
+TExperimental = TypeVar("TExperimental", bound=Union[Type, Callable])
 P = ParamSpec("P")
 T = TypeVar("T")
 
 
-@overload
-def experimental(wrapped: Type[T]) -> Type[T]: ...
-
-
-@overload
-def experimental(wrapped: Callable[P, T]) -> Callable[P, T]: ...
-
-
-def experimental(wrapped: Union[Type[T], Callable[P, T]]) -> Union[Type[T], Callable[P, T]]:
+def experimental(wrapped: TExperimental) -> TExperimental:
     """Add experimental tag to a class or a method.
 
     :param wrapped: Either a Class or Function to mark as experimental
-    :type wrapped: Union[Type[T], Callable[P, T]]
+    :type wrapped: TExperimental
     :return: The wrapped class or method
-    :rtype: Union[Type[T], Callable[P, T]]
+    :rtype: TExperimental
     """
-
-    def is_class(t: Union[Type[T], Callable[P, T]]) -> TypeGuard[Type[T]]:
-        return isinstance(t, type)
-
-    if is_class(wrapped):
+    if inspect.isclass(wrapped):
         return _add_class_docstring(wrapped)
     if inspect.isfunction(wrapped):
         return _add_method_docstring(wrapped)
@@ -86,11 +74,11 @@ def _add_class_docstring(cls: Type[T]) -> Type[T]:
         cls.__doc__ = _add_note_to_docstring(cls.__doc__, doc_string)
     else:
         cls.__doc__ = doc_string + ">"
-    cls.__init__ = _add_class_warning(cls.__init__)  # type: ignore[method-assign]
+    cls.__init__ = _add_class_warning(cls.__init__)
     return cls
 
 
-def _add_method_docstring(func: Callable[P, T]) -> Callable[P, T]:
+def _add_method_docstring(func: Callable[P, T] = None) -> Callable[P, T]:
     """Add experimental tag to the method doc string.
 
     :param func: The function to update
@@ -150,9 +138,6 @@ def _get_indentation_size(doc_string: str) -> int:
 def _should_skip_warning():
     skip_warning_msg = False
 
-    if os.getenv("AI_EVALS_DISABLE_EXPERIMENTAL_WARNING", "false").lower() == "true":
-        skip_warning_msg = True
-
     # Cases where we want to suppress the warning:
     # 1. When converting from REST object to SDK object
     for frame in inspect.stack():

azure/ai/evaluation/simulator/_helpers/_simulator_data_classes.py

@@ -18,7 +18,7 @@ class Turn:
 
     role: Union[str, ConversationRole]
     content: str
-    context: Optional[str] = None
+    context: str = None
 
     def to_dict(self) -> Dict[str, Optional[str]]:
         """
@@ -30,19 +30,7 @@ class Turn:
         return {
             "role": self.role.value if isinstance(self.role, ConversationRole) else self.role,
             "content": self.content,
-            "context": str(self.context),
-        }
-
-    def to_context_free_dict(self) -> Dict[str, Optional[str]]:
-        """
-        Convert the conversation turn to a dictionary without context.
-
-        :returns: A dictionary representation of the conversation turn without context.
-        :rtype: Dict[str, Optional[str]]
-        """
-        return {
-            "role": self.role.value if isinstance(self.role, ConversationRole) else self.role,
-            "content": self.content,
+            "context": self.context,
         }
 
     def __repr__(self):
@@ -54,13 +42,13 @@ class ConversationHistory:
     Conversation history class to keep track of the conversation turns in a conversation.
     """
 
-    def __init__(self) -> None:
+    def __init__(self):
         """
         Initializes the conversation history with an empty list of turns.
         """
         self.history: List[Turn] = []
 
-    def add_to_history(self, turn: Turn) -> None:
+    def add_to_history(self, turn: Turn):
         """
         Adds a turn to the conversation history.
 
@@ -69,7 +57,7 @@ class ConversationHistory:
         """
         self.history.append(turn)
 
-    def to_list(self) -> List[Dict[str, Optional[str]]]:
+    def to_list(self) -> List[Dict[str, str]]:
         """
         Converts the conversation history to a list of dictionaries.
 
@@ -78,15 +66,6 @@ class ConversationHistory:
         """
         return [turn.to_dict() for turn in self.history]
 
-    def to_context_free_list(self) -> List[Dict[str, Optional[str]]]:
-        """
-        Converts the conversation history to a list of dictionaries without context.
-
-        :returns: A list of dictionaries representing the conversation turns without context.
-        :rtype: List[Dict[str, str]]
-        """
-        return [turn.to_context_free_dict() for turn in self.history]
-
     def __len__(self) -> int:
         return len(self.history)
 

azure/ai/evaluation/simulator/_indirect_attack_simulator.py

@@ -1,30 +1,54 @@
 # ---------------------------------------------------------
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
-# pylint: disable=C0301,C0114,R0913,R0903
 # noqa: E501
-import asyncio
+import functools
 import logging
-from typing import Callable, cast
+from typing import Callable
 
-from tqdm import tqdm
+from promptflow._sdk._telemetry import ActivityType, monitor_operation
 
-from azure.ai.evaluation._common.utils import validate_azure_ai_project
-from azure.ai.evaluation._common._experimental import experimental
 from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
-from azure.ai.evaluation.simulator import AdversarialScenarioJailbreak, SupportedLanguages
 from azure.ai.evaluation._model_configurations import AzureAIProject
-from azure.core.credentials import TokenCredential
-
-from ._adversarial_simulator import AdversarialSimulator, JsonLineList
+from azure.ai.evaluation.simulator import AdversarialScenario
+from azure.identity import DefaultAzureCredential
 
+from ._adversarial_simulator import AdversarialSimulator
 from ._model_tools import AdversarialTemplateHandler, ManagedIdentityAPITokenManager, RAIClient, TokenScope
 
 logger = logging.getLogger(__name__)
 
 
-@experimental
-class IndirectAttackSimulator(AdversarialSimulator):
+def monitor_adversarial_scenario(func) -> Callable:
+    """Decorator to monitor adversarial scenario.
+
+    :param func: The function to be decorated.
+    :type func: Callable
+    :return: The decorated function.
+    :rtype: Callable
+    """
+
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        scenario = str(kwargs.get("scenario", None))
+        max_conversation_turns = kwargs.get("max_conversation_turns", None)
+        max_simulation_results = kwargs.get("max_simulation_results", None)
+        decorated_func = monitor_operation(
+            activity_name="xpia.adversarial.simulator.call",
+            activity_type=ActivityType.PUBLICAPI,
+            custom_dimensions={
+                "scenario": scenario,
+                "max_conversation_turns": max_conversation_turns,
+                "max_simulation_results": max_simulation_results,
+            },
+        )(func)
+
+        return decorated_func(*args, **kwargs)
+
+    return wrapper
+
+
+class IndirectAttackSimulator:
     """
     Initializes the XPIA (cross domain prompt injected attack) jailbreak adversarial simulator with a project scope.
 
@@ -33,42 +57,44 @@ class IndirectAttackSimulator(AdversarialSimulator):
     :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
     :param credential: The credential for connecting to Azure AI project.
     :type credential: ~azure.core.credentials.TokenCredential
-
-    .. admonition:: Example:
-
-        .. literalinclude:: ../samples/evaluation_samples_simulate.py
-            :start-after: [START indirect_attack_simulator]
-            :end-before: [END indirect_attack_simulator]
-            :language: python
-            :dedent: 8
-            :caption: Run the IndirectAttackSimulator to produce 1 result with 1 conversation turn (2 messages in the result).
     """
 
-    def __init__(self, *, azure_ai_project: AzureAIProject, credential: TokenCredential):
+    def __init__(self, *, azure_ai_project: AzureAIProject, credential=None):
         """Constructor."""
-
-        try:
-            self.azure_ai_project = validate_azure_ai_project(azure_ai_project)
-        except EvaluationException as e:
+        # check if azure_ai_project has the keys: subscription_id, resource_group_name, project_name, credential
+        if not all(key in azure_ai_project for key in ["subscription_id", "resource_group_name", "project_name"]):
+            msg = "azure_ai_project must contain keys: subscription_id, resource_group_name and project_name"
             raise EvaluationException(
-                message=e.message,
-                internal_message=e.internal_message,
+                message=msg,
+                internal_message=msg,
                 target=ErrorTarget.DIRECT_ATTACK_SIMULATOR,
-                category=e.category,
-                blame=e.blame,
-            ) from e
-
-        self.credential = cast(TokenCredential, credential)
+                category=ErrorCategory.MISSING_FIELD,
+                blame=ErrorBlame.USER_ERROR,
+            )
+        if not all(azure_ai_project[key] for key in ["subscription_id", "resource_group_name", "project_name"]):
+            msg = "subscription_id, resource_group_name and project_name keys cannot be None"
+            raise EvaluationException(
+                message=msg,
+                internal_message=msg,
+                target=ErrorTarget.DIRECT_ATTACK_SIMULATOR,
+                category=ErrorCategory.MISSING_FIELD,
+                blame=ErrorBlame.USER_ERROR,
+            )
+        if "credential" not in azure_ai_project and not credential:
+            credential = DefaultAzureCredential()
+        elif "credential" in azure_ai_project:
+            credential = azure_ai_project["credential"]
+        self.credential = credential
+        self.azure_ai_project = azure_ai_project
         self.token_manager = ManagedIdentityAPITokenManager(
             token_scope=TokenScope.DEFAULT_AZURE_MANAGEMENT,
             logger=logging.getLogger("AdversarialSimulator"),
-            credential=self.credential,
+            credential=credential,
         )
-        self.rai_client = RAIClient(azure_ai_project=self.azure_ai_project, token_manager=self.token_manager)
+        self.rai_client = RAIClient(azure_ai_project=azure_ai_project, token_manager=self.token_manager)
         self.adversarial_template_handler = AdversarialTemplateHandler(
-            azure_ai_project=self.azure_ai_project, rai_client=self.rai_client
+            azure_ai_project=azure_ai_project, rai_client=self.rai_client
         )
-        super().__init__(azure_ai_project=azure_ai_project, credential=credential)
 
     def _ensure_service_dependencies(self):
         if self.rai_client is None:
@@ -81,25 +107,33 @@ class IndirectAttackSimulator(AdversarialSimulator):
                 blame=ErrorBlame.USER_ERROR,
             )
 
+    # @monitor_adversarial_scenario
     async def __call__(
         self,
         *,
+        scenario: AdversarialScenario,
         target: Callable,
+        max_conversation_turns: int = 1,
         max_simulation_results: int = 3,
         api_call_retry_limit: int = 3,
         api_call_retry_sleep_sec: int = 1,
         api_call_delay_sec: int = 0,
         concurrent_async_task: int = 3,
-        **kwargs,
     ):
         """
         Initializes the XPIA (cross domain prompt injected attack) jailbreak adversarial simulator with a project scope.
         This simulator converses with your AI system using prompts injected into the context to interrupt normal
         expected functionality by eliciting manipulated content, intrusion and attempting to gather information outside
         the scope of your AI system.
+
+        :keyword scenario: Enum value specifying the adversarial scenario used for generating inputs.
+        :paramtype scenario: azure.ai.evaluation.simulator.AdversarialScenario
         :keyword target: The target function to simulate adversarial inputs against.
             This function should be asynchronous and accept a dictionary representing the adversarial input.
         :paramtype target: Callable
+        :keyword max_conversation_turns: The maximum number of conversation turns to simulate.
+            Defaults to 1.
+        :paramtype max_conversation_turns: int
         :keyword max_simulation_results: The maximum number of simulation results to return.
             Defaults to 3.
         :paramtype max_simulation_results: int
@@ -136,11 +170,11 @@ class IndirectAttackSimulator(AdversarialSimulator):
                     'template_parameters': {},
                     'messages': [
                         {
-                            'content': '<adversarial query>',
+                            'content': '<jailbreak prompt> <adversarial query>',
                             'role': 'user'
                         },
                         {
-                            'content': "<response from your callback>",
+                            'content': "<response from endpoint>",
                             'role': 'assistant',
                             'context': None
                         }
@@ -149,72 +183,25 @@ class IndirectAttackSimulator(AdversarialSimulator):
                 }]
             }
         """
-        # values that cannot be changed:
-        scenario = AdversarialScenarioJailbreak.ADVERSARIAL_INDIRECT_JAILBREAK
-        max_conversation_turns = 2
-        language = SupportedLanguages.English
-        self._ensure_service_dependencies()
-        templates = await self.adversarial_template_handler._get_content_harm_template_collections(scenario.value)
-        concurrent_async_task = min(concurrent_async_task, 1000)
-        semaphore = asyncio.Semaphore(concurrent_async_task)
-        sim_results = []
-        tasks = []
-        total_tasks = sum(len(t.template_parameters) for t in templates)
-        if max_simulation_results > total_tasks:
-            logger.warning(
-                "Cannot provide %s results due to maximum number of adversarial simulations that can be generated: %s."
-                "\n %s simulations will be generated.",
-                max_simulation_results,
-                total_tasks,
-                total_tasks,
+        if scenario not in AdversarialScenario.__members__.values():
+            msg = f"Invalid scenario: {scenario}. Supported scenarios: {AdversarialScenario.__members__.values()}"
+            raise EvaluationException(
+                message=msg,
+                internal_message=msg,
+                target=ErrorTarget.DIRECT_ATTACK_SIMULATOR,
+                category=ErrorCategory.INVALID_VALUE,
+                blame=ErrorBlame.USER_ERROR,
             )
-        total_tasks = min(total_tasks, max_simulation_results)
-        progress_bar = tqdm(
-            total=total_tasks,
-            desc="generating jailbreak simulations",
-            ncols=100,
-            unit="simulations",
+        jb_sim = AdversarialSimulator(azure_ai_project=self.azure_ai_project, credential=self.credential)
+        jb_sim_results = await jb_sim(
+            scenario=scenario,
+            target=target,
+            max_conversation_turns=max_conversation_turns,
+            max_simulation_results=max_simulation_results,
+            api_call_retry_limit=api_call_retry_limit,
+            api_call_retry_sleep_sec=api_call_retry_sleep_sec,
+            api_call_delay_sec=api_call_delay_sec,
+            concurrent_async_task=concurrent_async_task,
+            _jailbreak_type="xpia",
         )
-        for template in templates:
-            for parameter in template.template_parameters:
-                tasks.append(
-                    asyncio.create_task(
-                        self._simulate_async(
-                            target=target,
-                            template=template,
-                            parameters=parameter,
-                            max_conversation_turns=max_conversation_turns,
-                            api_call_retry_limit=api_call_retry_limit,
-                            api_call_retry_sleep_sec=api_call_retry_sleep_sec,
-                            api_call_delay_sec=api_call_delay_sec,
-                            language=language,
-                            semaphore=semaphore,
-                        )
-                    )
-                )
-                if len(tasks) >= max_simulation_results:
-                    break
-            if len(tasks) >= max_simulation_results:
-                break
-        for task in asyncio.as_completed(tasks):
-            completed_task = await task  # type: ignore
-            template_parameters = completed_task.get("template_parameters", {})  # type: ignore
-            xpia_attack_type = template_parameters.get("xpia_attack_type", "")  # type: ignore
-            action = template_parameters.get("action", "")  # type: ignore
-            document_type = template_parameters.get("document_type", "")  # type: ignore
-            sim_results.append(
-                {
-                    "messages": completed_task["messages"],  # type: ignore
-                    "$schema": "http://azureml/sdk-2-0/ChatConversation.json",
-                    "template_parameters": {
-                        "metadata": {
-                            "xpia_attack_type": xpia_attack_type,
-                            "action": action,
-                            "document_type": document_type,
-                        },
-                    },
-                }
-            )
-            progress_bar.update(1)
-        progress_bar.close()
-        return JsonLineList(sim_results)
+        return jb_sim_results

azure/ai/evaluation/simulator/_model_tools/_identity_manager.py

@@ -3,20 +3,16 @@
 # ---------------------------------------------------------
 
 import asyncio
-import inspect
 import logging
 import os
 import time
 from abc import ABC, abstractmethod
 from enum import Enum
-from typing import Optional, Union
+from typing import Dict, Optional, Union
 
-from azure.core.credentials import AccessToken, TokenCredential
 from azure.identity import DefaultAzureCredential, ManagedIdentityCredential
 
-AZURE_TOKEN_REFRESH_INTERVAL = int(
-    os.getenv("AZURE_TOKEN_REFRESH_INTERVAL", "600")
-)  # token refresh interval in seconds
+AZURE_TOKEN_REFRESH_INTERVAL = 600  # seconds
 
 
 class TokenScope(Enum):
@@ -33,24 +29,24 @@ class APITokenManager(ABC):
     :param auth_header: Authorization header prefix. Defaults to "Bearer"
     :type auth_header: str
     :param credential: Azure credential object
-    :type credential: Optional[TokenCredential]
+    :type credential: Optional[Union[azure.identity.DefaultAzureCredential, azure.identity.ManagedIdentityCredential]
     """
 
     def __init__(
         self,
         logger: logging.Logger,
         auth_header: str = "Bearer",
-        credential: Optional[TokenCredential] = None,
+        credential: Optional[Union[DefaultAzureCredential, ManagedIdentityCredential]] = None,
     ) -> None:
         self.logger = logger
         self.auth_header = auth_header
-        self._lock: Optional[asyncio.Lock] = None
+        self._lock = None
         if credential is not None:
             self.credential = credential
         else:
             self.credential = self.get_aad_credential()
-        self.token: Optional[str] = None
-        self.last_refresh_time: Optional[float] = None
+        self.token = None
+        self.last_refresh_time = None
 
     @property
     def lock(self) -> asyncio.Lock:
@@ -77,26 +73,20 @@ class APITokenManager(ABC):
         identity_client_id = os.environ.get("DEFAULT_IDENTITY_CLIENT_ID", None)
         if identity_client_id is not None:
             self.logger.info(f"Using DEFAULT_IDENTITY_CLIENT_ID: {identity_client_id}")
-            return ManagedIdentityCredential(client_id=identity_client_id)
-
-        self.logger.info("Environment variable DEFAULT_IDENTITY_CLIENT_ID is not set, using DefaultAzureCredential")
-        return DefaultAzureCredential()
-
-    @abstractmethod
-    def get_token(self) -> str:
-        """Async method to get the API token. Subclasses should implement this method.
-
-        :return: API token
-        :rtype: str
-        """
+            credential = ManagedIdentityCredential(client_id=identity_client_id)
+        else:
+            self.logger.info("Environment variable DEFAULT_IDENTITY_CLIENT_ID is not set, using DefaultAzureCredential")
+            credential = DefaultAzureCredential()
+        return credential
 
     @abstractmethod
-    async def get_token_async(self) -> str:
+    async def get_token(self) -> str:
         """Async method to get the API token. Subclasses should implement this method.
 
         :return: API token
         :rtype: str
         """
+        pass  # pylint: disable=unnecessary-pass
 
 
 class ManagedIdentityAPITokenManager(APITokenManager):
@@ -110,18 +100,12 @@ class ManagedIdentityAPITokenManager(APITokenManager):
     :paramtype kwargs: Dict
     """
 
-    def __init__(
-        self,
-        token_scope: TokenScope,
-        logger: logging.Logger,
-        *,
-        auth_header: str = "Bearer",
-        credential: Optional[TokenCredential] = None,
-    ):
-        super().__init__(logger, auth_header=auth_header, credential=credential)
+    def __init__(self, token_scope: TokenScope, logger: logging.Logger, **kwargs: Dict):
+        super().__init__(logger, **kwargs)
         self.token_scope = token_scope
 
-    def get_token(self) -> str:
+    # Bug 3353724: This get_token is sync method, but it is defined as async method in the base class
+    def get_token(self) -> str:  # pylint: disable=invalid-overridden-method
         """Get the API token. If the token is not available or has expired, refresh the token.
 
         :return: API token
@@ -138,31 +122,6 @@ class ManagedIdentityAPITokenManager(APITokenManager):
 
         return self.token
 
-    async def get_token_async(self) -> str:
-        """Get the API token synchronously. If the token is not available or has expired, refresh it.
-
-        :return: API token
-        :rtype: str
-        """
-        if (
-            self.token is None
-            or self.last_refresh_time is None
-            or time.time() - self.last_refresh_time > AZURE_TOKEN_REFRESH_INTERVAL
-        ):
-            self.last_refresh_time = time.time()
-            get_token_method = self.credential.get_token(self.token_scope.value)
-            if inspect.isawaitable(get_token_method):
-                # If it's awaitable, await it
-                token_response: AccessToken = await get_token_method
-            else:
-                # Otherwise, call it synchronously
-                token_response = get_token_method
-
-            self.token = token_response.token
-            self.logger.info("Refreshed Azure endpoint token.")
-
-        return self.token
-
 
 class PlainTokenManager(APITokenManager):
     """Plain API Token Manager
@@ -175,18 +134,11 @@ class PlainTokenManager(APITokenManager):
     :paramtype kwargs: Dict
     """
 
-    def __init__(
-        self,
-        openapi_key: str,
-        logger: logging.Logger,
-        *,
-        auth_header: str = "Bearer",
-        credential: Optional[TokenCredential] = None,
-    ) -> None:
-        super().__init__(logger, auth_header=auth_header, credential=credential)
-        self.token: str = openapi_key
+    def __init__(self, openapi_key: str, logger: logging.Logger, **kwargs: Dict):
+        super().__init__(logger, **kwargs)
+        self.token = openapi_key
 
-    def get_token(self) -> str:
+    async def get_token(self) -> str:
         """Get the API token
 
         :return: API token