azure-ai-evaluation 1.7.0__py3-none-any.whl → 1.8.0__py3-none-any.whl

azure/ai/evaluation/red_team/_utils/constants.py

@@ -40,7 +40,9 @@ ATTACK_STRATEGY_COMPLEXITY_MAP = {
     str(AttackStrategy.Tense.value): "moderate",
     str(AttackStrategy.MODERATE.value): "moderate",
     str(AttackStrategy.DIFFICULT.value): "difficult",
-    str(AttackStrategy.Jailbreak.value): "easy"
+    str(AttackStrategy.Jailbreak.value): "easy",
+    str(AttackStrategy.MultiTurn.value): "difficult",
+    str(AttackStrategy.Crescendo.value): "difficult",
 }
 
 # Mapping of risk categories to their evaluators
@@ -63,3 +65,5 @@ TASK_STATUS = {
     "TIMEOUT": "timeout",
     "INCOMPLETE": "incomplete",
 }
+
+USER_AGENT = "azure-ai-evaluation-redteam"

azure/ai/evaluation/red_team/_utils/metric_mapping.py

@@ -7,7 +7,7 @@ from azure.ai.evaluation._constants import EvaluationMetrics
 # Mapping of risk categories to their corresponding evaluation metrics
 RISK_CATEGORY_METRIC_MAP = {
     RiskCategory.Violence: EvaluationMetrics.VIOLENCE,
-    RiskCategory.HateUnfairness: EvaluationMetrics.HATE_UNFAIRNESS,
+    RiskCategory.HateUnfairness: EvaluationMetrics.HATE_FAIRNESS,
     RiskCategory.Sexual: EvaluationMetrics.SEXUAL,
     RiskCategory.SelfHarm: EvaluationMetrics.SELF_HARM
 }
@@ -20,4 +20,4 @@ def get_metric_from_risk_category(risk_category: RiskCategory) -> str:
     :return: The corresponding evaluation metric
     :rtype: str
     """
-    return RISK_CATEGORY_METRIC_MAP.get(risk_category, EvaluationMetrics.HATE_UNFAIRNESS)
+    return RISK_CATEGORY_METRIC_MAP.get(risk_category, EvaluationMetrics.HATE_FAIRNESS)

azure/ai/evaluation/red_team/_utils/strategy_utils.py

@@ -66,6 +66,8 @@ def strategy_converter_map() -> Dict[Any, Union[PromptConverter, List[PromptConv
         AttackStrategy.UnicodeSubstitution: UnicodeSubstitutionConverter(),
         AttackStrategy.Url: UrlConverter(),
         AttackStrategy.Jailbreak: None,
+        AttackStrategy.MultiTurn: None,
+        AttackStrategy.Crescendo: None,
     }
 
 

azure/ai/evaluation/simulator/_adversarial_simulator.py

@@ -264,12 +264,19 @@ class AdversarialSimulator:
 
         # Limit to max_simulation_results if needed
         if len(template_parameter_pairs) > max_simulation_results:
-            template_parameter_pairs = template_parameter_pairs[:max_simulation_results]
+            template_parameter_pairs = template_parameter_pairs[:max_simulation_results]        # Create a seeded random instance for jailbreak selection if randomization_seed is provided
+        jailbreak_random = None
+        if _jailbreak_type == "upia" and randomization_seed is not None:
+            jailbreak_random = random.Random(randomization_seed)
 
         # Single task append loop for all scenarios
         for template, parameter in template_parameter_pairs:
             if _jailbreak_type == "upia":
-                parameter = self._add_jailbreak_parameter(parameter, random.choice(jailbreak_dataset))
+                if jailbreak_random is not None:
+                    selected_jailbreak = jailbreak_random.choice(jailbreak_dataset)
+                else:
+                    selected_jailbreak = random.choice(jailbreak_dataset)
+                parameter = self._add_jailbreak_parameter(parameter, selected_jailbreak)
 
             tasks.append(
                 asyncio.create_task(

azure/ai/evaluation/simulator/_model_tools/_generated_rai_client.py

@@ -47,6 +47,7 @@ class GeneratedRAIClient:
             ).rai_svc
         else:
             self._client = AIProjectClient(endpoint=azure_ai_project, credential=token_manager).red_teams
+            self._operations_client = AIProjectClient(endpoint=azure_ai_project, credential=token_manager).evaluations
             self._evaluation_onedp_client = EvaluationServiceOneDPClient(endpoint=azure_ai_project, credential=token_manager)
         
     def _get_service_discovery_url(self):

azure/ai/evaluation/simulator/_model_tools/_proxy_completion_model.py

@@ -43,15 +43,15 @@ class SimulationRequestDTO:
         headers: Dict[str, str],
         payload: Dict[str, Any],
         params: Dict[str, str],
-        template_key: str,
-        template_parameters: Optional[TemplateParameters],
+        templateKey: str,
+        templateParameters: Optional[TemplateParameters],
     ):
         self.url = url
         self.headers = headers
         self.json = json.dumps(payload)
         self.params = params
-        self.template_key = template_key
-        self.templateParameters = template_parameters
+        self.templateKey = templateKey
+        self.templateParameters = templateParameters
 
     def to_dict(self) -> Dict:
         """Convert the DTO to a dictionary.
@@ -186,8 +186,8 @@ class ProxyChatCompletionsModel(OpenAIChatCompletionsModel):
             headers=headers,
             payload=request_data,
             params=params,
-            template_key=self.tkey,
-            template_parameters=self.tparam,
+            templateKey=self.tkey,
+            templateParameters=self.tparam,
         )
 
         time_start = time.time()
@@ -207,7 +207,15 @@ class ProxyChatCompletionsModel(OpenAIChatCompletionsModel):
             request_count = 0
             flag = True
             while flag:
-                response = session.evaluations.operation_results(operation_id, headers=headers)
+                try:
+                    response = session.evaluations.operation_results(operation_id, headers=headers)
+                except Exception as e:
+                    from types import SimpleNamespace  # pylint: disable=forgotten-debug-statement
+                    response = SimpleNamespace(status_code=202, text=str(e), json=lambda: {"error": str(e)})
+                if isinstance(response, dict):
+                    response_data = response
+                    flag = False
+                    break
                 if response.status_code == 200:
                     response_data = cast(List[Dict], response.json())
                     flag = False

{azure_ai_evaluation-1.7.0.dist-info → azure_ai_evaluation-1.8.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: azure-ai-evaluation
-Version: 1.7.0
+Version: 1.8.0
 Summary: Microsoft Azure Evaluation Library for Python
 Home-page: https://github.com/Azure/azure-sdk-for-python
 Author: Microsoft Corporation
@@ -400,6 +400,15 @@ This project has adopted the [Microsoft Open Source Code of Conduct][code_of_con
 
 # Release History
 
+## 1.8.0 (2025-05-29)
+
+### Features Added
+
+- Introduces `AttackStrategy.MultiTurn` and `AttackStrategy.Crescendo` to `RedTeam`. These strategies attack the target of a `RedTeam` scan over the course of multi-turn conversations. 
+
+### Bugs Fixed
+- AdversarialSimulator in `ADVERSARIAL_CONVERSATION` mode was broken. It is now fixed.
+
 ## 1.7.0 (2025-05-12)
 
 ### Bugs Fixed

{azure_ai_evaluation-1.7.0.dist-info → azure_ai_evaluation-1.8.0.dist-info}/RECORD

@@ -5,7 +5,7 @@ azure/ai/evaluation/_exceptions.py,sha256=XVliA6hj3C9Pz1ryBn0aC2izpno953VZwarTEf
 azure/ai/evaluation/_http_utils.py,sha256=1bGce6pKAL-vmaUGRPxVX7DVO05XVQ8YPIwIQ3q7mfA,17221
 azure/ai/evaluation/_model_configurations.py,sha256=MNN6cQlz7P9vNfHmfEKsUcly3j1FEOEFsA8WV7GPuKQ,4043
 azure/ai/evaluation/_user_agent.py,sha256=O2y-QPBAcw7w7qQ6M2aRPC3Vy3TKd789u5lcs2yuFaI,290
-azure/ai/evaluation/_version.py,sha256=tlS1Ht1i2Nh2nXXHGSsfL3WWEGo_hHTCCI0v8jtdvKo,229
+azure/ai/evaluation/_version.py,sha256=s-hB5c2HMI5UlCpMm0EGHnSdbc_xvo8g7tM1oIr9ys4,229
 azure/ai/evaluation/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 azure/ai/evaluation/_aoai/__init__.py,sha256=DE9av37bnDMCbYjgXAY9pcmCol7URCMkJCTIbRn1xHo,264
 azure/ai/evaluation/_aoai/aoai_grader.py,sha256=QyLlEeNy4BE2mdE2lKyArV8sHbiaFLDyI2j0xxlVEZ0,4121
@@ -48,7 +48,7 @@ azure/ai/evaluation/_common/onedp/models/_enums.py,sha256=xWkVAfjZGxnQsaGENmgpfK
 azure/ai/evaluation/_common/onedp/models/_models.py,sha256=VIP7RCP-48LSMfc4m0HPSgt40sCnciKNjlV_C-p5H0E,92197
 azure/ai/evaluation/_common/onedp/models/_patch.py,sha256=xrdxsiXieQiFZyh8B1lHWF80DiQWHmbyjZR5mb8coyM,868
 azure/ai/evaluation/_common/onedp/operations/__init__.py,sha256=2rIhesutTPrcc8BUqSVkRpkQdxTI5lzucEOqj552P0E,1618
-azure/ai/evaluation/_common/onedp/operations/_operations.py,sha256=y1xIio9h_kvw_-CooQpaFKZ1pVU70Qmi22Jaf9YDDuE,240793
+azure/ai/evaluation/_common/onedp/operations/_operations.py,sha256=RirJRnif4ujcsngXNiRXzHtCc0F7M5VNbwG6CBjKU9o,240913
 azure/ai/evaluation/_common/onedp/operations/_patch.py,sha256=xrdxsiXieQiFZyh8B1lHWF80DiQWHmbyjZR5mb8coyM,868
 azure/ai/evaluation/_common/onedp/servicepatterns/__init__.py,sha256=SXUyE9hmQeYBcmIxmXrAZAIeYfMHAA7cy2vPwgRceTo,82
 azure/ai/evaluation/_common/onedp/servicepatterns/aio/__init__.py,sha256=SXUyE9hmQeYBcmIxmXrAZAIeYfMHAA7cy2vPwgRceTo,82
@@ -93,7 +93,7 @@ azure/ai/evaluation/_converters/_ai_services.py,sha256=1ZaCNNpocCcg_fWzwCR4TAgVC
 azure/ai/evaluation/_converters/_models.py,sha256=cd3uEUvXNnxEEEGMYE5_V9XEeIwhtCF_qZOwVIx9JPw,14010
 azure/ai/evaluation/_evaluate/__init__.py,sha256=Yx1Iq2GNKQ5lYxTotvPwkPL4u0cm6YVxUe-iVbu1clI,180
 azure/ai/evaluation/_evaluate/_eval_run.py,sha256=ZlDvTNDiMSZ2VfzVicPmEGx65LL3LmcKcPWft6XjWcA,21991
-azure/ai/evaluation/_evaluate/_evaluate.py,sha256=JwukXczL0rb_1BcvqPtk3IdXweyjJth6wMNxTxJZqdg,50900
+azure/ai/evaluation/_evaluate/_evaluate.py,sha256=YBdCP5gFXUwy-19A83-F4fjgwe7GfJUAcu3s6zjfnoc,50946
 azure/ai/evaluation/_evaluate/_evaluate_aoai.py,sha256=Z7Ku3geqOEWvlODQ7IpYWpT3TXSjKPV7OCtYQBJ10JU,24999
 azure/ai/evaluation/_evaluate/_utils.py,sha256=w-71izKmbqaA4aOLCop6Jj39Z6bWGpB1INy70MpYxws,18899
 azure/ai/evaluation/_evaluate/_batch_run/__init__.py,sha256=cPLi_MJ_pCp8eKBxJbiSoxgTnN3nDLuaP57dMkKuyhg,552
@@ -158,7 +158,7 @@ azure/ai/evaluation/_evaluators/_retrieval/__init__.py,sha256=kMu47ZyTZ7f-4Yh6H3
 azure/ai/evaluation/_evaluators/_retrieval/_retrieval.py,sha256=X4hf_1A1sK2oxU1CfgORIm-QZ_-19U24WzK3w1aOrM4,6292
 azure/ai/evaluation/_evaluators/_retrieval/retrieval.prompty,sha256=_YVoO4Gt_WD42bUcj5n6BDW0dMUqNf0yF3Nj5XMOX2c,16490
 azure/ai/evaluation/_evaluators/_rouge/__init__.py,sha256=kusCDaYcXogDugGefRP8MQSn9xv107oDbrMCqZ6K4GA,291
-azure/ai/evaluation/_evaluators/_rouge/_rouge.py,sha256=P8qyKMeZ70AqSIIpMC_MU-hMyRJnEzFZINeE7j1M4u4,10137
+azure/ai/evaluation/_evaluators/_rouge/_rouge.py,sha256=QrBdcn0YuUy1kKVJ4PftnAKOF9xi97mdFddYfx3HYJY,10137
 azure/ai/evaluation/_evaluators/_service_groundedness/__init__.py,sha256=0DODUGTOgaYyFbO9_zxuwifixDL3SIm3EkwP1sdwn6M,288
 azure/ai/evaluation/_evaluators/_service_groundedness/_service_groundedness.py,sha256=JZAX9o7UiF-vi6WdSJ6dFOXhkg97uDW3dP1nsUFUH_E,8065
 azure/ai/evaluation/_evaluators/_similarity/__init__.py,sha256=V2Mspog99_WBltxTkRHG5NpN5s9XoiTSN4I8POWEkLA,268
@@ -221,20 +221,28 @@ azure/ai/evaluation/_vendor/rouge_score/tokenize.py,sha256=IyHVsWY6IFFZdB23cLiJs
 azure/ai/evaluation/_vendor/rouge_score/tokenizers.py,sha256=3_-y1TyvyluHuERhSJ5CdXSwnpcMA7aAKU6PCz9wH_Q,1745
 azure/ai/evaluation/red_team/__init__.py,sha256=wAjy_85gzdX0YyUlj6kHWrpo01zz2mgOJ1AbH5EkH8s,613
 azure/ai/evaluation/red_team/_attack_objective_generator.py,sha256=yMK5z91_w6GIzbqTQtUXfABuGeH3nfVgF8feOMd37K4,10765
-azure/ai/evaluation/red_team/_attack_strategy.py,sha256=CmMYhp9cm8xTr63LbrFnfFyTxXQ3Xn4PZ5tEfMRFVIQ,1431
+azure/ai/evaluation/red_team/_attack_strategy.py,sha256=aPFb40oKuIGRJA4MTempR6SOfLu2YJPvN_AXljwXk4E,1488
 azure/ai/evaluation/red_team/_callback_chat_target.py,sha256=H58nU6IVEpeDuTZPsxIoYKXAvU3dAWnLRhA8wyaY11s,3071
 azure/ai/evaluation/red_team/_default_converter.py,sha256=K8G775VVdQLsAqff60BBNFrNRj8JijB7St9XOHmGQGA,775
-azure/ai/evaluation/red_team/_red_team.py,sha256=He-iKFL3uqg0f02bJfgLc3XNAc8g1tbjshU46KQbjL8,132526
+azure/ai/evaluation/red_team/_red_team.py,sha256=_TGgPo1UuPAfwrAB09j0F5GAC8A1YM3VJjahdQkJau4,151395
 azure/ai/evaluation/red_team/_red_team_result.py,sha256=BwHf0KBGe9hqyWeTYDOOuoIxJ0rqkY9lHLUOZ02wPJI,17340
+azure/ai/evaluation/red_team/_agent/__init__.py,sha256=UwArlb5cGLN4pRUm8nG7mXsmxhgdIving2OslYhQDAg,179
+azure/ai/evaluation/red_team/_agent/_agent_functions.py,sha256=PjUQdd55jxWTU4RdecuBySTmy0HdtA7ogv-cDR50hjM,10706
+azure/ai/evaluation/red_team/_agent/_agent_tools.py,sha256=tlVGvphmA1YQPXgtv_JCv2qFUCdnRuIkerJKENG4Sek,21846
+azure/ai/evaluation/red_team/_agent/_agent_utils.py,sha256=93qctSkv-7QkdTtNlityVppQnuttDF9TWu7vVi1rIkE,3150
+azure/ai/evaluation/red_team/_agent/_semantic_kernel_plugin.py,sha256=rKwBuqC6_nmQslwe7VQ-RetC_CfhZRIr3F8HIMZRe3M,10961
 azure/ai/evaluation/red_team/_utils/__init__.py,sha256=UwArlb5cGLN4pRUm8nG7mXsmxhgdIving2OslYhQDAg,179
-azure/ai/evaluation/red_team/_utils/constants.py,sha256=ctSsuWiavhaGApRgP6fL3OtNGphnCKuesTQOwR833nk,2289
+azure/ai/evaluation/red_team/_utils/_rai_service_eval_chat_target.py,sha256=uoKzOzCUePTRo65GGbylt_SBwfZLvRzv2kJog6Jn5YY,4934
+azure/ai/evaluation/red_team/_utils/_rai_service_target.py,sha256=Hr90_89MXb7I00cpZukUl1IPFJRI0oK9ax8-0ak9O0U,29966
+azure/ai/evaluation/red_team/_utils/_rai_service_true_false_scorer.py,sha256=Xf-bj_w9VCznhULg3RG-TZFnLLyY1l-Fw1yOC9XeBas,4196
+azure/ai/evaluation/red_team/_utils/constants.py,sha256=1dZiJ_Mnm3qyfjMVL2eccN_jQeIx-etBypIZWJJ1yPs,2441
 azure/ai/evaluation/red_team/_utils/formatting_utils.py,sha256=jztERWOb1P3ClOKEr6-sMmLDQ8subjKUVBKVVxeSFcY,6416
 azure/ai/evaluation/red_team/_utils/logging_utils.py,sha256=jjNq37zlJz0KPo9g8NuuhnDJo56T-06ElJ-7yyeD46M,4838
-azure/ai/evaluation/red_team/_utils/metric_mapping.py,sha256=HhN3YWDNDs41jtGCUJ0HJ7fg-2AuQKIdUsOucOdw9TI,950
-azure/ai/evaluation/red_team/_utils/strategy_utils.py,sha256=wImNcnKNNC-Z8L8TdLCSNfftK8pLTA2RLLoyMXPSRc8,8059
+azure/ai/evaluation/red_team/_utils/metric_mapping.py,sha256=5Q2nhF2utHypWnXqdnvfoiF_x-poYPEOPEjolsEkxAA,946
+azure/ai/evaluation/red_team/_utils/strategy_utils.py,sha256=32hN3KwC36RjdnZvTNr0fzsZldh5LTk6ZPUELd-qprA,8139
 azure/ai/evaluation/simulator/__init__.py,sha256=JbrPZ8pvTBalyX94SvZ9btHNoovX8rbZV03KmzxxWys,552
 azure/ai/evaluation/simulator/_adversarial_scenario.py,sha256=FQ9ZNruqPY4o06uu2uig2MZSbearwAJwieIB26GbPOU,1833
-azure/ai/evaluation/simulator/_adversarial_simulator.py,sha256=-vkPNop_dJDG8skju4v3skCjdx2brl7OomoSP37q-GM,24791
+azure/ai/evaluation/simulator/_adversarial_simulator.py,sha256=nVEuW1N-Eo9Vwo03RTcgOq45H7uE1ViygSnOj4FjxtE,25275
 azure/ai/evaluation/simulator/_constants.py,sha256=jtsj-XzbseurEAxWmB-iSPmXu7B6ZHrM3bl07j-zruM,875
 azure/ai/evaluation/simulator/_direct_attack_simulator.py,sha256=dXF0_ExD7l6WUUCKkO5tmOjGpR00CKAhpSPI-f8804I,11045
 azure/ai/evaluation/simulator/_indirect_attack_simulator.py,sha256=3VQW2Qgk3pdhd960YAJs3SE-Uptc11lSu7XapxQwSPk,11236
@@ -249,17 +257,17 @@ azure/ai/evaluation/simulator/_helpers/__init__.py,sha256=FQwgrJvzq_nv3wF9DBr2py
 azure/ai/evaluation/simulator/_helpers/_language_suffix_mapping.py,sha256=sbKc3O5qsg77LEaSEQfWDfzqEFXVDw612BaDcBo3E7A,1095
 azure/ai/evaluation/simulator/_helpers/_simulator_data_classes.py,sha256=BOttMTec3muMiA4OzwD_iW08GTrhja7PL9XVjRCN3jM,3029
 azure/ai/evaluation/simulator/_model_tools/__init__.py,sha256=soaUHfmd_IK1FINPRKgKens7qKOVw9BZwc7ms7N6zrk,860
-azure/ai/evaluation/simulator/_model_tools/_generated_rai_client.py,sha256=m_Wy8g--p1PzGc8SDs-R4Z0ZuttCBzFEQQweUuvdhxY,6887
+azure/ai/evaluation/simulator/_model_tools/_generated_rai_client.py,sha256=s2HrKI67ee7ObMJQetovwkBcq7_LcJMnw4YbtguOQtg,7006
 azure/ai/evaluation/simulator/_model_tools/_identity_manager.py,sha256=DO9vRSLsp974S1f8KdK3w2fTicdAcv3Y0gntwtXehm4,6433
-azure/ai/evaluation/simulator/_model_tools/_proxy_completion_model.py,sha256=BCKOS3QY1iUn6gLB00fXzKervpzbeufJ8gx__Fb5-8U,10587
+azure/ai/evaluation/simulator/_model_tools/_proxy_completion_model.py,sha256=A3D3joIzN2JhqkQeSX0Dwb1iOHRDWKEUaWu6384J8C4,11003
 azure/ai/evaluation/simulator/_model_tools/_rai_client.py,sha256=nvDS33uaxLpuvZEC4aotu-P9-CHqLiyCInED6Xq9ZQ0,11643
 azure/ai/evaluation/simulator/_model_tools/_template_handler.py,sha256=fCXWP-ZDYNj6vjbxGNKisJalmQ0EcMvCOhcL7YwhCR8,7724
 azure/ai/evaluation/simulator/_model_tools/models.py,sha256=V3Rs9htzt0XtLdLZRaHB_hxMRcJ-z_VR9eXwBOKpi_Y,22373
 azure/ai/evaluation/simulator/_prompty/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 azure/ai/evaluation/simulator/_prompty/task_query_response.prompty,sha256=2BzSqDDYilDushvR56vMRDmqFIaIYAewdUlUZg_elMg,2182
 azure/ai/evaluation/simulator/_prompty/task_simulate.prompty,sha256=NE6lH4bfmibgMn4NgJtm9_l3PMoHSFrfjjosDJEKM0g,939
-azure_ai_evaluation-1.7.0.dist-info/METADATA,sha256=6Temm52FtwSfaOpaa_JWhvidiPF8NVcN412YvQpKFZk,38061
-azure_ai_evaluation-1.7.0.dist-info/NOTICE.txt,sha256=4tzi_Yq4-eBGhBvveobWHCgUIVF-ZeouGN0m7hVq5Mk,3592
-azure_ai_evaluation-1.7.0.dist-info/WHEEL,sha256=iAkIy5fosb7FzIOwONchHf19Qu7_1wCWyFNR5gu9nU0,91
-azure_ai_evaluation-1.7.0.dist-info/top_level.txt,sha256=S7DhWV9m80TBzAhOFjxDUiNbKszzoThbnrSz5MpbHSQ,6
-azure_ai_evaluation-1.7.0.dist-info/RECORD,,
+azure_ai_evaluation-1.8.0.dist-info/METADATA,sha256=wWI1IjTWaeN3tZSv293QkzCeMvdhvEx_fD-GiWiaR78,38394
+azure_ai_evaluation-1.8.0.dist-info/NOTICE.txt,sha256=4tzi_Yq4-eBGhBvveobWHCgUIVF-ZeouGN0m7hVq5Mk,3592
+azure_ai_evaluation-1.8.0.dist-info/WHEEL,sha256=iAkIy5fosb7FzIOwONchHf19Qu7_1wCWyFNR5gu9nU0,91
+azure_ai_evaluation-1.8.0.dist-info/top_level.txt,sha256=S7DhWV9m80TBzAhOFjxDUiNbKszzoThbnrSz5MpbHSQ,6
+azure_ai_evaluation-1.8.0.dist-info/RECORD,,