azure-ai-evaluation 1.8.0__py3-none-any.whl → 1.10.0__py3-none-any.whl

azure/ai/evaluation/_safety_evaluation/_safety_evaluation.py

@@ -49,6 +49,7 @@ JAILBREAK_EXT = "_Jailbreak"
 DATA_EXT = "_Data.jsonl"
 RESULTS_EXT = "_Results.jsonl"
 
+
 def _setup_logger():
     """Configure and return a logger instance for the CustomAdversarialSimulator.
 
@@ -115,7 +116,6 @@ class _SafetyEvaluation:
         self.credential = credential
         self.logger = _setup_logger()
 
-
     @staticmethod
     def _validate_model_config(model_config: Any):
         """
@@ -158,7 +158,9 @@ class _SafetyEvaluation:
         max_simulation_results: int = 3,
         conversation_turns: List[List[Union[str, Dict[str, Any]]]] = [],
         tasks: List[str] = [],
-        adversarial_scenario: Optional[Union[AdversarialScenario, AdversarialScenarioJailbreak, _UnstableAdversarialScenario]] = None,
+        adversarial_scenario: Optional[
+            Union[AdversarialScenario, AdversarialScenarioJailbreak, _UnstableAdversarialScenario]
+        ] = None,
         source_text: Optional[str] = None,
         direct_attack: bool = False,
         randomization_seed: Optional[int] = None,
@@ -185,47 +187,53 @@ class _SafetyEvaluation:
         :type direct_attack: bool
         """
 
-        ## Define callback
-        async def callback(
-            messages: List[Dict],
-            stream: bool = False,
-            session_state: Optional[str] = None,
-            context: Optional[Dict] = None,
-        ) -> dict:
-            messages_list = messages["messages"]  # type: ignore
-            latest_message = messages_list[-1]
-            application_input = latest_message["content"]
-            context = latest_message.get("context", None)
-            latest_context = None
-            try:
-                is_async = self._is_async_function(target)
-                if self._check_target_returns_context(target):
-                    if is_async:
-                        response, latest_context = await target(query=application_input)
-                    else:
-                        response, latest_context = target(query=application_input)
-                else:
-                    if is_async:
-                        response = await target(query=application_input)
+        ## Check if target is already a callback-style function
+        if self._check_target_is_callback(target):
+            # Use the target directly as it's already a callback
+            callback = target
+        else:
+            # Define callback wrapper for simple targets
+            async def callback(
+                messages: List[Dict],
+                stream: bool = False,
+                session_state: Optional[str] = None,
+                context: Optional[Dict] = None,
+            ) -> dict:
+                messages_list = messages["messages"]  # type: ignore
+                latest_message = messages_list[-1]
+                application_input = latest_message["content"]
+                context = latest_message.get("context", None)
+                latest_context = None
+                try:
+                    is_async = self._is_async_function(target)
+                    if self._check_target_returns_context(target):
+                        if is_async:
+                            response, latest_context = await target(query=application_input)
+                        else:
+                            response, latest_context = target(query=application_input)
                     else:
-                        response = target(query=application_input)
-            except Exception as e:
-                response = f"Something went wrong {e!s}"
-
-            ## We format the response to follow the openAI chat protocol format
-            formatted_response = {
-                "content": response,
-                "role": "assistant",
-                "context": latest_context if latest_context else context,
-            }
-            ## NOTE: In the future, instead of appending to messages we should just return `formatted_response`
-            messages["messages"].append(formatted_response)  # type: ignore
-            return {
-                "messages": messages_list,
-                "stream": stream,
-                "session_state": session_state,
-                "context": latest_context if latest_context else context,
-            }
+                        if is_async:
+                            response = await target(query=application_input)
+                        else:
+                            response = target(query=application_input)
+                except Exception as e:
+                    response = f"Something went wrong {e!s}"
+
+                ## We format the response to follow the openAI chat protocol
+                formatted_response = {
+                    "content": response,
+                    "role": "assistant",
+                    "context": latest_context if latest_context else context,
+                }
+                ## NOTE: In the future, instead of appending to messages we
+                ## should just return `formatted_response`
+                messages["messages"].append(formatted_response)  # type: ignore
+                return {
+                    "messages": messages_list,
+                    "stream": stream,
+                    "session_state": session_state,
+                    "context": latest_context if latest_context else context,
+                }
 
         ## Run simulator
         simulator = None
@@ -248,7 +256,7 @@ class _SafetyEvaluation:
                 text=source_text,
                 target=callback,
                 randomization_seed=randomization_seed,
-                concurrent_async_task=concurrent_async_tasks
+                concurrent_async_task=concurrent_async_tasks,
             )
 
         # if DirectAttack, run DirectAttackSimulator
@@ -282,6 +290,7 @@ class _SafetyEvaluation:
                 target=callback,
                 text=source_text if source_text else "",
                 concurrent_async_tasks=concurrent_async_tasks,
+                randomization_seed=randomization_seed,
             )
 
         ## Run AdversarialSimulator
@@ -291,14 +300,14 @@ class _SafetyEvaluation:
             )
             simulator = AdversarialSimulator(azure_ai_project=self.azure_ai_project, credential=self.credential)
             simulator_outputs = await simulator(
-                scenario=adversarial_scenario, #type: ignore
+                scenario=adversarial_scenario,  # type: ignore
                 max_conversation_turns=max_conversation_turns,
                 max_simulation_results=max_simulation_results,
                 conversation_turns=conversation_turns,
                 target=callback,
                 text=source_text,
                 randomization_seed=randomization_seed,
-                concurrent_async_task=concurrent_async_tasks
+                concurrent_async_task=concurrent_async_tasks,
             )
 
         ## If no outputs are generated, raise an exception
@@ -312,9 +321,9 @@ class _SafetyEvaluation:
                 category=ErrorCategory.UNKNOWN,
                 blame=ErrorBlame.USER_ERROR,
             )
-        
+
         data_path_base = simulator.__class__.__name__
-        
+
         ## Write outputs to file according to scenario
         if direct_attack and jailbreak_outputs:
             jailbreak_data_path = data_path_base + JAILBREAK_EXT
@@ -360,7 +369,7 @@ class _SafetyEvaluation:
                     ]
                 )
             simulator_data_paths[data_path_base] = data_path_base + DATA_EXT
-            
+
         return simulator_data_paths
 
     def _get_scenario(
@@ -497,7 +506,7 @@ class _SafetyEvaluation:
                     blame=ErrorBlame.USER_ERROR,
                 )
         return evaluators_dict
-    
+
     @staticmethod
     def _check_target_returns_context(target: Callable) -> bool:
         """
@@ -510,7 +519,7 @@ class _SafetyEvaluation:
         ret_type = sig.return_annotation
         if ret_type == inspect.Signature.empty:
             return False
-        
+
         # Check for Coroutine/Awaitable return types for async functions
         origin = getattr(ret_type, "__origin__", None)
         if origin is not None and (origin is Coroutine or origin is Awaitable):
@@ -518,24 +527,24 @@ class _SafetyEvaluation:
             if args and len(args) > 0:
                 # For async functions, check the actual return type inside the Coroutine
                 ret_type = args[-1]
-        
+
         if ret_type is tuple:
             return True
         return False
-    
+
     @staticmethod
     def _check_target_returns_str(target: Callable) -> bool:
-        '''
+        """
         Checks if the target function returns a string.
 
         :param target: The target function to check.
         :type target: Callable
-        '''
+        """
         sig = inspect.signature(target)
         ret_type = sig.return_annotation
         if ret_type == inspect.Signature.empty:
             return False
-        
+
         # Check for Coroutine/Awaitable return types for async functions
         origin = getattr(ret_type, "__origin__", None)
         if origin is not None and (origin is Coroutine or origin is Awaitable):
@@ -543,36 +552,36 @@ class _SafetyEvaluation:
             if args and len(args) > 0:
                 # For async functions, check the actual return type inside the Coroutine
                 ret_type = args[-1]
-                
+
         if ret_type is str:
             return True
         return False
-    
+
     @staticmethod
     def _is_async_function(target: Callable) -> bool:
         """
         Checks if the target function is an async function.
-        
+
         :param target: The target function to check.
         :type target: Callable
         :return: True if the target function is async, False otherwise.
         :rtype: bool
         """
         return asyncio.iscoroutinefunction(target)
-    
+
     @staticmethod
     def _check_target_is_callback(target: Callable) -> bool:
         sig = inspect.signature(target)
         param_names = list(sig.parameters.keys())
-        return 'messages' in param_names and 'stream' in param_names and 'session_state' in param_names and 'context' in param_names
+        return "messages" in param_names and "session_state" in param_names and "context" in param_names
 
     def _validate_inputs(
-            self,
-            evaluators: List[_SafetyEvaluator],
-            target: Union[Callable, AzureOpenAIModelConfiguration, OpenAIModelConfiguration],
-            num_turns: int = 1,
-            scenario: Optional[Union[AdversarialScenario, AdversarialScenarioJailbreak]] = None,
-            source_text: Optional[str] = None,
+        self,
+        evaluators: List[_SafetyEvaluator],
+        target: Union[Callable, AzureOpenAIModelConfiguration, OpenAIModelConfiguration],
+        num_turns: int = 1,
+        scenario: Optional[Union[AdversarialScenario, AdversarialScenarioJailbreak]] = None,
+        source_text: Optional[str] = None,
     ):
         """
         Validates the inputs provided to the __call__ function of the SafetyEvaluation object.
@@ -586,12 +595,28 @@ class _SafetyEvaluation:
         :type scenario: Optional[Union[AdversarialScenario, AdversarialScenarioJailbreak]]
         :param source_text: The source text to use as grounding document in the evaluation.
         :type source_text: Optional[str]
-        """ 
+        """
         if not callable(target):
             self._validate_model_config(target)
-        elif not self._check_target_returns_str(target): 
-            self.logger.error(f"Target function {target} does not return a string.")
-            msg = f"Target function {target} does not return a string."
+        elif not self._check_target_is_callback(target) and not self._check_target_returns_str(target):
+            msg = (
+                f"Invalid target function signature. The target function must be either:\n\n"
+                f"1. A simple function that takes a 'query' parameter and returns a string:\n"
+                f"   def my_target(query: str) -> str:\n"
+                f"       return f'Response to: {{query}}'\n\n"
+                f"2. A callback-style function with these exact parameters:\n"
+                f"   async def my_callback(\n"
+                f"       messages: List[Dict],\n"
+                f"       stream: bool = False,\n"
+                f"       session_state: Any = None,\n"
+                f"       context: Any = None\n"
+                f"   ) -> dict:\n"
+                f"       # Process messages and return dict with 'messages', 'stream', 'session_state', 'context'\n"
+                f"       return {{'messages': messages['messages'], 'stream': stream, 'session_state': session_state, 'context': context}}\n\n"
+                f"Your function '{target.__name__}' does not match either pattern. "
+                f"Please check the function signature and return type."
+            )
+            self.logger.error(msg)
             raise EvaluationException(
                 message=msg,
                 internal_message=msg,
@@ -610,8 +635,8 @@ class _SafetyEvaluation:
                 category=ErrorCategory.MISSING_FIELD,
                 blame=ErrorBlame.USER_ERROR,
             )
-        
-        if scenario and len(evaluators)>0 and not _SafetyEvaluator.CONTENT_SAFETY in evaluators:
+
+        if scenario and len(evaluators) > 0 and not _SafetyEvaluator.CONTENT_SAFETY in evaluators:
             self.logger.error(f"Adversarial scenario {scenario} is not supported without content safety evaluation.")
             msg = f"Adversarial scenario {scenario} is not supported without content safety evaluation."
             raise EvaluationException(
@@ -621,7 +646,7 @@ class _SafetyEvaluation:
                 category=ErrorCategory.INVALID_VALUE,
                 blame=ErrorBlame.USER_ERROR,
             )
-        
+
         if _SafetyEvaluator.CODE_VULNERABILITY in evaluators and num_turns > 1:
             self.logger.error("Code vulnerability evaluation only supports single-turn conversations.")
             msg = "Code vulnerability evaluation only supports single-turn conversations."
@@ -643,9 +668,18 @@ class _SafetyEvaluation:
                 blame=ErrorBlame.USER_ERROR,
             )
 
-        if _SafetyEvaluator.CONTENT_SAFETY in evaluators and scenario and num_turns > 1 and scenario != AdversarialScenario.ADVERSARIAL_CONVERSATION:
-            self.logger.error(f"Adversarial scenario {scenario} is not supported for content safety evaluation with more than 1 turn.")
-            msg = f"Adversarial scenario {scenario} is not supported for content safety evaluation with more than 1 turn."
+        if (
+            _SafetyEvaluator.CONTENT_SAFETY in evaluators
+            and scenario
+            and num_turns > 1
+            and scenario != AdversarialScenario.ADVERSARIAL_CONVERSATION
+        ):
+            self.logger.error(
+                f"Adversarial scenario {scenario} is not supported for content safety evaluation with more than 1 turn."
+            )
+            msg = (
+                f"Adversarial scenario {scenario} is not supported for content safety evaluation with more than 1 turn."
+            )
             raise EvaluationException(
                 message=msg,
                 internal_message=msg,
@@ -681,18 +715,57 @@ class _SafetyEvaluation:
         jailbreak_cols = [key for key in evaluation_result_dict.keys() if JAILBREAK_EXT in key]
         regular_cols = [key for key in evaluation_result_dict.keys() if JAILBREAK_EXT not in key]
         evaluation_result: EvaluationResult = {"metrics": {}, "rows": [], "studio_url": ""}
-        if len(jailbreak_cols)>0 and len(regular_cols)>0:
+        if len(jailbreak_cols) > 0 and len(regular_cols) > 0:
             hate_unfairness_defects = []
             violence_defects = []
             sexual_defects = []
             self_harm_defects = []
-            jailbreak_rows = sorted(evaluation_result_dict[jailbreak_cols[0]]['rows'], key=lambda x: (x.get('inputs.category') is None, x.get('inputs.category', '')))
-            regular_rows = sorted(evaluation_result_dict[regular_cols[0]]['rows'], key=lambda x: (x.get('inputs.category') is None, x.get('inputs.category', '')))
+            jailbreak_rows = sorted(
+                evaluation_result_dict[jailbreak_cols[0]]["rows"],
+                key=lambda x: (x.get("inputs.category") is None, x.get("inputs.category", "")),
+            )
+            regular_rows = sorted(
+                evaluation_result_dict[regular_cols[0]]["rows"],
+                key=lambda x: (x.get("inputs.category") is None, x.get("inputs.category", "")),
+            )
             for i, row in enumerate(zip(jailbreak_rows, regular_rows)):
-                hate_unfairness_defects.append(1 if (row[0]['outputs.content_safety.hate_unfairness_score'] > CONTENT_SAFETY_DEFECT_RATE_THRESHOLD_DEFAULT and not row[1]['outputs.content_safety.hate_unfairness_score'] > CONTENT_SAFETY_DEFECT_RATE_THRESHOLD_DEFAULT) else 0)
-                violence_defects.append(1 if (row[0]['outputs.content_safety.violence_score'] > CONTENT_SAFETY_DEFECT_RATE_THRESHOLD_DEFAULT and not row[1]['outputs.content_safety.violence_score'] > CONTENT_SAFETY_DEFECT_RATE_THRESHOLD_DEFAULT) else 0)
-                sexual_defects.append(1 if (row[0]['outputs.content_safety.sexual_score'] > CONTENT_SAFETY_DEFECT_RATE_THRESHOLD_DEFAULT and not row[1]['outputs.content_safety.sexual_score'] > CONTENT_SAFETY_DEFECT_RATE_THRESHOLD_DEFAULT) else 0)
-                self_harm_defects.append(1 if (row[0]['outputs.content_safety.self_harm_score'] > CONTENT_SAFETY_DEFECT_RATE_THRESHOLD_DEFAULT and not row[1]['outputs.content_safety.self_harm_score'] > CONTENT_SAFETY_DEFECT_RATE_THRESHOLD_DEFAULT) else 0)
+                hate_unfairness_defects.append(
+                    1
+                    if (
+                        row[0]["outputs.content_safety.hate_unfairness_score"]
+                        > CONTENT_SAFETY_DEFECT_RATE_THRESHOLD_DEFAULT
+                        and not row[1]["outputs.content_safety.hate_unfairness_score"]
+                        > CONTENT_SAFETY_DEFECT_RATE_THRESHOLD_DEFAULT
+                    )
+                    else 0
+                )
+                violence_defects.append(
+                    1
+                    if (
+                        row[0]["outputs.content_safety.violence_score"] > CONTENT_SAFETY_DEFECT_RATE_THRESHOLD_DEFAULT
+                        and not row[1]["outputs.content_safety.violence_score"]
+                        > CONTENT_SAFETY_DEFECT_RATE_THRESHOLD_DEFAULT
+                    )
+                    else 0
+                )
+                sexual_defects.append(
+                    1
+                    if (
+                        row[0]["outputs.content_safety.sexual_score"] > CONTENT_SAFETY_DEFECT_RATE_THRESHOLD_DEFAULT
+                        and not row[1]["outputs.content_safety.sexual_score"]
+                        > CONTENT_SAFETY_DEFECT_RATE_THRESHOLD_DEFAULT
+                    )
+                    else 0
+                )
+                self_harm_defects.append(
+                    1
+                    if (
+                        row[0]["outputs.content_safety.self_harm_score"] > CONTENT_SAFETY_DEFECT_RATE_THRESHOLD_DEFAULT
+                        and not row[1]["outputs.content_safety.self_harm_score"]
+                        > CONTENT_SAFETY_DEFECT_RATE_THRESHOLD_DEFAULT
+                    )
+                    else 0
+                )
             hate_unfairness_defect_rate = list_mean_nan_safe(hate_unfairness_defects)
             violence_defect_rate = list_mean_nan_safe(violence_defects)
             sexual_defect_rate = list_mean_nan_safe(sexual_defects)
@@ -708,30 +781,32 @@ class _SafetyEvaluation:
                 "content_safety.self_harm_defect_rate": self_harm_defect_rate,
             }
             evaluation_result["studio_url"] = (
-                evaluation_result_dict[jailbreak_cols[0]]["studio_url"] + "\t" + evaluation_result_dict[regular_cols[0]]["studio_url"]
+                evaluation_result_dict[jailbreak_cols[0]]["studio_url"]
+                + "\t"
+                + evaluation_result_dict[regular_cols[0]]["studio_url"]
             )
         return evaluation_result
-    
+
     async def __call__(
-            self,
-            target: Union[Callable, Awaitable[Any], AzureOpenAIModelConfiguration, OpenAIModelConfiguration],
-            evaluators: List[_SafetyEvaluator] = [],
-            evaluation_name: Optional[str] = None,
-            num_turns : int = 1,
-            num_rows: int = 5,
-            scenario: Optional[Union[AdversarialScenario, AdversarialScenarioJailbreak]] = None,
-            conversation_turns : List[List[Union[str, Dict[str, Any]]]] = [],
-            tasks: List[str] = [],
-            data_only: bool = False,
-            source_text: Optional[str] = None,
-            data_path: Optional[Union[str, os.PathLike]] = None,
-            jailbreak_data_path: Optional[Union[str, os.PathLike]] = None,
-            output_path: Optional[Union[str, os.PathLike]] = None,
-            data_paths: Optional[Union[Dict[str, str], Dict[str, Union[str,os.PathLike]]]] = None,
-            randomization_seed: Optional[int] = None,
-            concurrent_async_tasks: Optional[int] = 5,
-        ) -> Union[Dict[str, EvaluationResult], Dict[str, str], Dict[str, Union[str,os.PathLike]]]:
-        '''
+        self,
+        target: Union[Callable, Awaitable[Any], AzureOpenAIModelConfiguration, OpenAIModelConfiguration],
+        evaluators: List[_SafetyEvaluator] = [],
+        evaluation_name: Optional[str] = None,
+        num_turns: int = 1,
+        num_rows: int = 5,
+        scenario: Optional[Union[AdversarialScenario, AdversarialScenarioJailbreak]] = None,
+        conversation_turns: List[List[Union[str, Dict[str, Any]]]] = [],
+        tasks: List[str] = [],
+        data_only: bool = False,
+        source_text: Optional[str] = None,
+        data_path: Optional[Union[str, os.PathLike]] = None,
+        jailbreak_data_path: Optional[Union[str, os.PathLike]] = None,
+        output_path: Optional[Union[str, os.PathLike]] = None,
+        data_paths: Optional[Union[Dict[str, str], Dict[str, Union[str, os.PathLike]]]] = None,
+        randomization_seed: Optional[int] = None,
+        concurrent_async_tasks: Optional[int] = 5,
+    ) -> Union[Dict[str, EvaluationResult], Dict[str, str], Dict[str, Union[str, os.PathLike]]]:
+        """
         Evaluates the target function based on the provided parameters.
 
         :param target: The target function to call during the evaluation. This can be a synchronous or asynchronous function.
@@ -765,9 +840,11 @@ class _SafetyEvaluation:
         :type randomization_seed: Optional[int]
         :param concurrent_async_tasks: The number of concurrent async tasks to run. If None, the system's default is used.
         :type concurrent_async_tasks: Optional[int]
-        '''
+        """
         ## Log inputs
-        self.logger.info(f"User inputs: evaluators{evaluators}, evaluation_name={evaluation_name}, num_turns={num_turns}, num_rows={num_rows}, scenario={scenario},conversation_turns={conversation_turns}, tasks={tasks}, source_text={source_text}, data_path={data_path}, jailbreak_data_path={jailbreak_data_path}, output_path={output_path}, randomization_seed={randomization_seed}, concurrent_async_tasks={concurrent_async_tasks}")
+        self.logger.info(
+            f"User inputs: evaluators{evaluators}, evaluation_name={evaluation_name}, num_turns={num_turns}, num_rows={num_rows}, scenario={scenario},conversation_turns={conversation_turns}, tasks={tasks}, source_text={source_text}, data_path={data_path}, jailbreak_data_path={jailbreak_data_path}, output_path={output_path}, randomization_seed={randomization_seed}, concurrent_async_tasks={concurrent_async_tasks}"
+        )
 
         ## Validate arguments
         self._validate_inputs(
@@ -798,28 +875,35 @@ class _SafetyEvaluation:
                 source_text=source_text,
                 direct_attack=_SafetyEvaluator.DIRECT_ATTACK in evaluators,
                 randomization_seed=randomization_seed,
+                concurrent_async_tasks=concurrent_async_tasks,
             )
         elif data_path:
             data_paths = {Path(data_path).stem: data_path}
             if jailbreak_data_path:
                 data_paths[Path(jailbreak_data_path).stem + JAILBREAK_EXT] = jailbreak_data_path
 
-        if data_only and data_paths: return data_paths
+        if data_only and data_paths:
+            return data_paths
 
         ## Run evaluation
         evaluation_results = {}
         if data_paths:
             for strategy, data_path in data_paths.items():
-                self.logger.info(f"Running evaluation for data with inputs data_path={data_path}, evaluators={evaluators_dict}, azure_ai_project={self.azure_ai_project}, output_path={output_path}")
-                if evaluation_name: output_prefix = evaluation_name + "_"
-                else: output_prefix = ""
+                self.logger.info(
+                    f"Running evaluation for data with inputs data_path={data_path}, evaluators={evaluators_dict}, azure_ai_project={self.azure_ai_project}, output_path={output_path}"
+                )
+                if evaluation_name:
+                    output_prefix = evaluation_name + "_"
+                else:
+                    output_prefix = ""
                 evaluate_outputs = _evaluate.evaluate(
                     data=data_path,
                     evaluators=evaluators_dict,
                     azure_ai_project=self.azure_ai_project,
                     evaluation_name=evaluation_name,
                     output_path=output_path if output_path else f"{output_prefix}{strategy}{RESULTS_EXT}",
-                    _use_pf_client=False, #TODO: Remove this once eval logic for red team agent is moved to red team agent
+                    _use_pf_client=False,  # TODO: Remove this once eval logic for red team agent is moved to red team agent
+                    _use_run_submitter_client=False,  # TODO: Remove this once eval logic for red team agent is moved to red team agent
                 )
                 evaluation_results[strategy] = evaluate_outputs
             return evaluation_results

azure/ai/evaluation/_user_agent.py

@@ -1,6 +1,37 @@
 # ---------------------------------------------------------
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
+from contextlib import contextmanager
+from typing import Iterator
+
 from azure.ai.evaluation._version import VERSION
 
-USER_AGENT = "{}/{}".format("azure-ai-evaluation", VERSION)
+
+class UserAgentSingleton:
+    __BASE_USER_AGENT: str = "{}/{}".format("azure-ai-evaluation", VERSION)
+
+    @property
+    def value(self):
+        """Get the user-agent"""
+        return self.__BASE_USER_AGENT
+
+    def __str__(self) -> str:
+        return self.value
+
+    @classmethod
+    @contextmanager
+    def add_useragent_product(cls, *product: str) -> Iterator[None]:
+        """Appends a "product" (e.g. `name/version`) to the base user agent
+
+        :param product: User Agent products to append to the base user agent
+
+         ..see-also::
+
+             `User-Agent section of RFC 9110, <https://www.rfc-editor.org/rfc/rfc9110#name-user-agent>`
+        """
+        old_useragent = cls.__BASE_USER_AGENT
+        cls.__BASE_USER_AGENT = f"{old_useragent} {' '.join(product)}"
+
+        yield
+
+        cls.__BASE_USER_AGENT = old_useragent

azure/ai/evaluation/_version.py

@@ -3,4 +3,4 @@
 # ---------------------------------------------------------
 # represents upcoming version
 
-VERSION = "1.8.0"
+VERSION = "1.10.0"

azure/ai/evaluation/red_team/__init__.py

@@ -8,7 +8,9 @@ try:
     from ._attack_objective_generator import RiskCategory
     from ._red_team_result import RedTeamResult
 except ImportError:
-    print("[INFO] Could not import Pyrit. Please install the dependency with `pip install azure-ai-evaluation[redteam]`.")
+    raise ImportError(
+        "Could not import Pyrit. Please install the dependency with `pip install azure-ai-evaluation[redteam]`."
+    )
 
 
 __all__ = [

azure/ai/evaluation/red_team/_agent/__init__.py

@@ -1,3 +1,3 @@
 # ---------------------------------------------------------
 # Copyright (c) Microsoft Corporation. All rights reserved.
-# ---------------------------------------------------------
+# ---------------------------------------------------------