azure-ai-evaluation 1.2.0__py3-none-any.whl → 1.3.0__py3-none-any.whl

azure/ai/evaluation/simulator/_adversarial_simulator.py

@@ -7,6 +7,7 @@ import asyncio
 import logging
 import random
 from typing import Any, Callable, Dict, List, Optional, Union, cast
+import uuid
 
 from tqdm import tqdm
 
@@ -187,6 +188,8 @@ class AdversarialSimulator:
             )
         self._ensure_service_dependencies()
         templates = await self.adversarial_template_handler._get_content_harm_template_collections(scenario.value)
+        simulation_id = str(uuid.uuid4())
+        logger.warning("Use simulation_id to help debug the issue: %s", str(simulation_id))
         concurrent_async_task = min(concurrent_async_task, 1000)
         semaphore = asyncio.Semaphore(concurrent_async_task)
         sim_results = []
@@ -236,6 +239,7 @@ class AdversarialSimulator:
                             language=language,
                             semaphore=semaphore,
                             scenario=scenario,
+                            simulation_id=simulation_id,
                         )
                     )
                 )
@@ -298,9 +302,10 @@ class AdversarialSimulator:
         language: SupportedLanguages,
         semaphore: asyncio.Semaphore,
         scenario: Union[AdversarialScenario, AdversarialScenarioJailbreak],
+        simulation_id: str = "",
     ) -> List[Dict]:
         user_bot = self._setup_bot(
-            role=ConversationRole.USER, template=template, parameters=parameters, scenario=scenario
+            role=ConversationRole.USER, template=template, parameters=parameters, scenario=scenario, simulation_id=simulation_id
         )
         system_bot = self._setup_bot(
             target=target, role=ConversationRole.ASSISTANT, template=template, parameters=parameters, scenario=scenario
@@ -329,7 +334,7 @@ class AdversarialSimulator:
         )
 
     def _get_user_proxy_completion_model(
-        self, template_key: str, template_parameters: TemplateParameters
+        self, template_key: str, template_parameters: TemplateParameters, simulation_id: str = ""
     ) -> ProxyChatCompletionsModel:
         return ProxyChatCompletionsModel(
             name="raisvc_proxy_model",
@@ -340,6 +345,7 @@ class AdversarialSimulator:
             api_version="2023-07-01-preview",
             max_tokens=1200,
             temperature=0.0,
+            simulation_id=simulation_id,
         )
 
     def _setup_bot(
@@ -350,10 +356,11 @@ class AdversarialSimulator:
         parameters: TemplateParameters,
         target: Optional[Callable] = None,
         scenario: Union[AdversarialScenario, AdversarialScenarioJailbreak],
+        simulation_id: str = "",
     ) -> ConversationBot:
         if role is ConversationRole.USER:
             model = self._get_user_proxy_completion_model(
-                template_key=template.template_name, template_parameters=parameters
+                template_key=template.template_name, template_parameters=parameters, simulation_id=simulation_id,
             )
             return ConversationBot(
                 role=role,

azure/ai/evaluation/simulator/_conversation/__init__.py

@@ -128,19 +128,15 @@ class ConversationBot:
         self.conversation_starter: Optional[Union[str, jinja2.Template, Dict]] = None
         if role == ConversationRole.USER:
             if "conversation_starter" in self.persona_template_args:
-                print(self.persona_template_args)
                 conversation_starter_content = self.persona_template_args["conversation_starter"]
                 if isinstance(conversation_starter_content, dict):
                     self.conversation_starter = conversation_starter_content
-                    print(f"Conversation starter content: {conversation_starter_content}")
                 else:
                     try:
                         self.conversation_starter = jinja2.Template(
                             conversation_starter_content, undefined=jinja2.StrictUndefined
                         )
-                        print("Successfully created a Jinja2 template for the conversation starter.")
                     except jinja2.exceptions.TemplateSyntaxError as e:  # noqa: F841
-                        print(f"Template syntax error: {e}. Using raw content.")
                         self.conversation_starter = conversation_starter_content
             else:
                 self.logger.info(
@@ -153,6 +149,7 @@ class ConversationBot:
         conversation_history: List[ConversationTurn],
         max_history: int,
         turn_number: int = 0,
+        session_state: Optional[Dict[str, Any]] = None,
     ) -> Tuple[dict, dict, float, dict]:
         """
         Prompt the ConversationBot for a response.
@@ -262,6 +259,7 @@ class CallbackConversationBot(ConversationBot):
         conversation_history: List[Any],
         max_history: int,
         turn_number: int = 0,
+        session_state: Optional[Dict[str, Any]] = None,
     ) -> Tuple[dict, dict, float, dict]:
         chat_protocol_message = self._to_chat_protocol(
             self.user_template, conversation_history, self.user_template_parameters
@@ -269,7 +267,7 @@ class CallbackConversationBot(ConversationBot):
         msg_copy = copy.deepcopy(chat_protocol_message)
         result = {}
         start_time = time.time()
-        result = await self.callback(msg_copy)
+        result = await self.callback(msg_copy, session_state=session_state)
         end_time = time.time()
         if not result:
             result = {
@@ -348,6 +346,7 @@ class MultiModalConversationBot(ConversationBot):
         conversation_history: List[Any],
         max_history: int,
         turn_number: int = 0,
+        session_state: Optional[Dict[str, Any]] = None,
     ) -> Tuple[dict, dict, float, dict]:
         previous_prompt = conversation_history[-1]
         chat_protocol_message = await self._to_chat_protocol(conversation_history, self.user_template_parameters)

azure/ai/evaluation/simulator/_conversation/_conversation.py

@@ -101,6 +101,7 @@ async def simulate_conversation(
     :rtype: Tuple[Optional[str], List[ConversationTurn]]
     """
 
+    session_state = {}
     # Read the first prompt.
     (first_response, request, _, full_response) = await bots[0].generate_response(
         session=session,
@@ -149,7 +150,10 @@ async def simulate_conversation(
                 conversation_history=conversation_history,
                 max_history=history_limit,
                 turn_number=current_turn,
+                session_state=session_state,
             )
+            if "session_state" in full_response and full_response["session_state"] is not None:
+                session_state.update(full_response["session_state"])
 
             # check if conversation id is null, which means conversation starter was used. use id from next turn
             if conversation_id is None and "id" in response:

azure/ai/evaluation/simulator/_model_tools/_proxy_completion_model.py

@@ -89,6 +89,7 @@ class ProxyChatCompletionsModel(OpenAIChatCompletionsModel):
         self.tkey = template_key
         self.tparam = template_parameters
         self.result_url: Optional[str] = None
+        self.simulation_id: Optional[str] = kwargs.pop("simulation_id", "")
 
         super().__init__(name=name, **kwargs)
 
@@ -169,6 +170,7 @@ class ProxyChatCompletionsModel(OpenAIChatCompletionsModel):
             "Content-Type": "application/json",
             "X-CV": f"{uuid.uuid4()}",
             "X-ModelType": self.model or "",
+            "x-ms-client-request-id": self.simulation_id,
         }
         # add all additional headers
         headers.update(self.additional_headers)  # type: ignore[arg-type]

{azure_ai_evaluation-1.2.0.dist-info → azure_ai_evaluation-1.3.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: azure-ai-evaluation
-Version: 1.2.0
+Version: 1.3.0
 Summary: Microsoft Azure Evaluation Library for Python
 Home-page: https://github.com/Azure/azure-sdk-for-python
 Author: Microsoft Corporation
@@ -377,6 +377,12 @@ This project has adopted the [Microsoft Open Source Code of Conduct][code_of_con
 
 # Release History
 
+## 1.3.0 (2025-02-28)
+
+### Breaking Changes
+- Multimodal specific evaluators `ContentSafetyMultimodalEvaluator`, `ViolenceMultimodalEvaluator`, `SexualMultimodalEvaluator`, `SelfHarmMultimodalEvaluator`, `HateUnfairnessMultimodalEvaluator` and `ProtectedMaterialMultimodalEvaluator` has been removed. Please use `ContentSafetyEvaluator`, `ViolenceEvaluator`, `SexualEvaluator`, `SelfHarmEvaluator`, `HateUnfairnessEvaluator` and `ProtectedMaterialEvaluator` instead.
+- Metric name in ProtectedMaterialEvaluator's output is changed from `protected_material.fictional_characters_label` to `protected_material.fictional_characters_defect_rate`. It's now consistent with other evaluator's metric names (ending with `_defect_rate`).
+
 ## 1.2.0 (2025-01-27)
 
 ### Features Added

{azure_ai_evaluation-1.2.0.dist-info → azure_ai_evaluation-1.3.0.dist-info}/RECORD

@@ -1,10 +1,10 @@
-azure/ai/evaluation/__init__.py,sha256=MFxJRoKfSsP_Qlfq0FwynxNf4csNAfTYPQX7jdXc9RU,2757
-azure/ai/evaluation/_constants.py,sha256=a7eCgdG6Kid79ebAMu0rPNH7foRF5Aii0K5YQI6cNPc,2765
-azure/ai/evaluation/_exceptions.py,sha256=MsTbgsPGYPzIxs7MyLKzSeiVKEoCxYkVjONzNfv2tXA,5162
+azure/ai/evaluation/__init__.py,sha256=3lV2dhUqPzUz4lBo8DoEr5o4PEF_Lh5yWQtuLDUQZCM,2192
+azure/ai/evaluation/_constants.py,sha256=Avp26R4qPqaze024EJXzAX8A457DExF5rnIKMWcE8X4,2881
+azure/ai/evaluation/_exceptions.py,sha256=o3AjCyIAIBo_rkJAH_TsAT-hEZ4SMKd9Uo0eLmD9VHY,5085
 azure/ai/evaluation/_http_utils.py,sha256=1bGce6pKAL-vmaUGRPxVX7DVO05XVQ8YPIwIQ3q7mfA,17221
 azure/ai/evaluation/_model_configurations.py,sha256=MNN6cQlz7P9vNfHmfEKsUcly3j1FEOEFsA8WV7GPuKQ,4043
 azure/ai/evaluation/_user_agent.py,sha256=O2y-QPBAcw7w7qQ6M2aRPC3Vy3TKd789u5lcs2yuFaI,290
-azure/ai/evaluation/_version.py,sha256=aIrrVLGzX0UDxMjpkbe8HTOCqRr6Y9R8tC8XGAOocbE,199
+azure/ai/evaluation/_version.py,sha256=xsol5X6WDVwo8FFtHWt28TPyzT0is0sJ3N5i9_ikPVI,229
 azure/ai/evaluation/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 azure/ai/evaluation/_azure/__init__.py,sha256=Yx1Iq2GNKQ5lYxTotvPwkPL4u0cm6YVxUe-iVbu1clI,180
 azure/ai/evaluation/_azure/_clients.py,sha256=N1V-LyQkItPuoKl0aieypFPdGSRSld9lQqH1x-n3L7U,9119
@@ -15,10 +15,10 @@ azure/ai/evaluation/_common/_experimental.py,sha256=GVtSn9r1CeR_yEa578dJVNDJ3P24
 azure/ai/evaluation/_common/constants.py,sha256=OsExttFGLnTAyZa26jnY5_PCDTb7uJNFqtE2qsRZ1mg,1957
 azure/ai/evaluation/_common/math.py,sha256=d4bwWe35_RWDIZNcbV1BTBbHNx2QHQ4-I3EofDyyNE0,2863
 azure/ai/evaluation/_common/rai_service.py,sha256=DcakzdOour9qNdMXU-8UFfvLb12oexAoiJXG8XFTRBs,26462
-azure/ai/evaluation/_common/utils.py,sha256=MQIZs95gH5je1L-S3twa_WQi071zRu0Dv54lzCI7ZgU,17642
+azure/ai/evaluation/_common/utils.py,sha256=wssBc9i0JrWwsrvtp45P5_z5ceB7kVGf-KKzpfVQQH4,17594
 azure/ai/evaluation/_evaluate/__init__.py,sha256=Yx1Iq2GNKQ5lYxTotvPwkPL4u0cm6YVxUe-iVbu1clI,180
 azure/ai/evaluation/_evaluate/_eval_run.py,sha256=QBtNBwUxqxsIVmKPU-_H8MDFkF4s_bW7arQYXAniRpo,21965
-azure/ai/evaluation/_evaluate/_evaluate.py,sha256=hj1HG9WCjbvAk8iB0MwnVoV-ceQYKKecfyVTlbc3y4A,38934
+azure/ai/evaluation/_evaluate/_evaluate.py,sha256=pyl0HscBAPllpq385p9lMqR2sqip3RZ5BGfcBa4X334,39068
 azure/ai/evaluation/_evaluate/_utils.py,sha256=sKj_4iN-QjrRlEkiZwA9UNiWozS4LgJcUZ6AWdHrTY4,14231
 azure/ai/evaluation/_evaluate/_batch_run/__init__.py,sha256=Z-TQdSxKTn0bjsF0YosIJMbQFQHDUv_b9zCBu1TeogQ,474
 azure/ai/evaluation/_evaluate/_batch_run/code_client.py,sha256=XQLaXfswF6ReHLpQthHLuLLa65Pts8uawGp7kRqmMDs,8260
@@ -59,14 +59,6 @@ azure/ai/evaluation/_evaluators/_groundedness/groundedness_with_query.prompty,sh
 azure/ai/evaluation/_evaluators/_groundedness/groundedness_without_query.prompty,sha256=8kNShdfxQvkII7GnqjmdqQ5TNelA2B6cjnqWZk8FFe4,5296
 azure/ai/evaluation/_evaluators/_meteor/__init__.py,sha256=209na3pPsdmcuYpYHUYtqQybCpc3yZkc93HnRdicSlI,266
 azure/ai/evaluation/_evaluators/_meteor/_meteor.py,sha256=OpugAjIgcTcNQ6g6Rks_8GVhcRiH524PbmBKH3bTefs,4369
-azure/ai/evaluation/_evaluators/_multimodal/__init__.py,sha256=tPvsY0nv8T3VtiiAwJM6wT5A9FhKP2XXwUlCH994xl4,906
-azure/ai/evaluation/_evaluators/_multimodal/_content_safety_multimodal.py,sha256=x0l6eLQhxVP85jEyGfFCl27C2okMgD0S3aJ_qrgB3Q8,5219
-azure/ai/evaluation/_evaluators/_multimodal/_content_safety_multimodal_base.py,sha256=X2IVw0YvymDD3e4Vx-TfjqgqtYiAKVhUumjBowCpOmA,2441
-azure/ai/evaluation/_evaluators/_multimodal/_hate_unfairness.py,sha256=ral1AAbP5pfsygDe30MtuwajuydiXoXzzCeuLBzIkWc,3779
-azure/ai/evaluation/_evaluators/_multimodal/_protected_material.py,sha256=gMrfyn3KHcV6SoowuEjR7Fon9vVLN7GOPM4rkJRK6xU,4906
-azure/ai/evaluation/_evaluators/_multimodal/_self_harm.py,sha256=QwOCBb618ZXSs-OoVXyNM65N4ZEL7IZt-S1Nqd8xNbY,3703
-azure/ai/evaluation/_evaluators/_multimodal/_sexual.py,sha256=6zz89yzr_SdldqBVv-3wOErz3H5sBO6wYgNh39aHXmY,3668
-azure/ai/evaluation/_evaluators/_multimodal/_violence.py,sha256=t1h3bY6N7SwlSgP_1P-90KGTsq1oWvTYDJpy_uMvzjA,3694
 azure/ai/evaluation/_evaluators/_protected_material/__init__.py,sha256=eRAQIU9diVXfO5bp6aLWxZoYUvOsrDIfy1gnDOeNTiI,109
 azure/ai/evaluation/_evaluators/_protected_material/_protected_material.py,sha256=IABs1YMBZdIi1u57dPi-aQpSiPWIGxEZ4hyt97jvdNA,4604
 azure/ai/evaluation/_evaluators/_qa/__init__.py,sha256=bcXfT--C0hjym2haqd1B2-u9bDciyM0ThOFtU1Q69sk,244
@@ -86,6 +78,8 @@ azure/ai/evaluation/_evaluators/_similarity/_similarity.py,sha256=AeqJ_OJUAsdu9C
 azure/ai/evaluation/_evaluators/_similarity/similarity.prompty,sha256=eoludASychZoGL625bFCaZai-OY7DIAg90ZLax_o4XE,4594
 azure/ai/evaluation/_evaluators/_xpia/__init__.py,sha256=VMEL8WrpJQeh4sQiOLzP7hRFPnjzsvwfvTzaGCVJPCM,88
 azure/ai/evaluation/_evaluators/_xpia/xpia.py,sha256=Nv14lU7jN0yXKbHgHRXMHEy6pn1rXmesBOYI2Ge9ewk,5849
+azure/ai/evaluation/_safety_evaluation/__init__.py,sha256=Yx1Iq2GNKQ5lYxTotvPwkPL4u0cm6YVxUe-iVbu1clI,180
+azure/ai/evaluation/_safety_evaluation/_safety_evaluation.py,sha256=HtT6aKdZ4EP6Dx-JuExdysO_CBERFMwRWmGTWeHQ_NU,33890
 azure/ai/evaluation/_vendor/__init__.py,sha256=Yx1Iq2GNKQ5lYxTotvPwkPL4u0cm6YVxUe-iVbu1clI,180
 azure/ai/evaluation/_vendor/rouge_score/__init__.py,sha256=03OkyfS_UmzRnHv6-z9juTaJ6OXJoEJM989hgifIZbc,607
 azure/ai/evaluation/_vendor/rouge_score/rouge_scorer.py,sha256=DtNSeshHipzc6vFnvx7kbs5viXe4LNq-ZrgllFvfR4U,11299
@@ -94,15 +88,15 @@ azure/ai/evaluation/_vendor/rouge_score/tokenize.py,sha256=IyHVsWY6IFFZdB23cLiJs
 azure/ai/evaluation/_vendor/rouge_score/tokenizers.py,sha256=3_-y1TyvyluHuERhSJ5CdXSwnpcMA7aAKU6PCz9wH_Q,1745
 azure/ai/evaluation/simulator/__init__.py,sha256=JbrPZ8pvTBalyX94SvZ9btHNoovX8rbZV03KmzxxWys,552
 azure/ai/evaluation/simulator/_adversarial_scenario.py,sha256=9rpAPz594tYjxzM3XMeDq6CZSc2yvf5YaNaGC7nzYhM,1710
-azure/ai/evaluation/simulator/_adversarial_simulator.py,sha256=FPZ3OdpGuwCHDVoOZW-f_j7pyK71PfDN3JPh205tW0c,21706
+azure/ai/evaluation/simulator/_adversarial_simulator.py,sha256=2QEt5hHOKnNqMySt3SNsr-LY22DvFyusVnPkvAyur1I,22100
 azure/ai/evaluation/simulator/_constants.py,sha256=nCL7_1BnYh6k0XvxudxsDVMbiG9MMEvYw5wO9FZHHZ8,857
 azure/ai/evaluation/simulator/_direct_attack_simulator.py,sha256=FTtWf655dHJF5FLJi0xGSBgIlGWNiVWyqaLDJSud9XA,10199
 azure/ai/evaluation/simulator/_indirect_attack_simulator.py,sha256=nweIU_AkUIR50qLQpjmljf_OkpsCPth2Ebf4vusygCA,10226
 azure/ai/evaluation/simulator/_simulator.py,sha256=LBzez7qvObpVjTwmlGS_PfhDLo8pRknh5epra2yo9X8,36484
 azure/ai/evaluation/simulator/_tracing.py,sha256=frZ4-usrzINast9F4-ONRzEGGox71y8bYw0UHNufL1Y,3069
 azure/ai/evaluation/simulator/_utils.py,sha256=16NltlywpbMtoFtULwTKqeURguIS1kSKSo3g8uKV8TA,5181
-azure/ai/evaluation/simulator/_conversation/__init__.py,sha256=s8djzJ58_-CiIA8xHB-SbgeZaq1F7ftrc3qJbpUpUdg,17853
-azure/ai/evaluation/simulator/_conversation/_conversation.py,sha256=qdzGMtCPYMxeGpR91NZTEmmz2RtADTvQGj6C-3EUTw4,7402
+azure/ai/evaluation/simulator/_conversation/__init__.py,sha256=LOR5h7vSACrv_cmaS6r7KUqjJcHYg7PAgkDbdvOmd4g,17726
+azure/ai/evaluation/simulator/_conversation/_conversation.py,sha256=h8OHq0sWKiTH821tC5zF44CJ-QgutTgDnEYsFEA7Cw0,7635
 azure/ai/evaluation/simulator/_conversation/constants.py,sha256=3v7zkjPwJAPbSpJYIK6VOZZy70bJXMo_QTVqSFGlq9A,984
 azure/ai/evaluation/simulator/_data_sources/__init__.py,sha256=Yx1Iq2GNKQ5lYxTotvPwkPL4u0cm6YVxUe-iVbu1clI,180
 azure/ai/evaluation/simulator/_data_sources/grounding.json,sha256=jqdqHrCgS7hN7K2kXSEcPCmzFjV4cv_qcCSR-Hutwx4,1257075
@@ -111,15 +105,15 @@ azure/ai/evaluation/simulator/_helpers/_language_suffix_mapping.py,sha256=7BBLH7
 azure/ai/evaluation/simulator/_helpers/_simulator_data_classes.py,sha256=BOttMTec3muMiA4OzwD_iW08GTrhja7PL9XVjRCN3jM,3029
 azure/ai/evaluation/simulator/_model_tools/__init__.py,sha256=aMv5apb7uVjuhMF9ohhA5kQmo652hrGIJlhdl3y2R1I,835
 azure/ai/evaluation/simulator/_model_tools/_identity_manager.py,sha256=-hptp2vpJIcfjvtd0E2c7ry00LVh23LxuYGevsNFfgs,6385
-azure/ai/evaluation/simulator/_model_tools/_proxy_completion_model.py,sha256=Zg_SzqjCGJ3Wt8hktxz6Y1JEJCcV0V5jBC9N06jQP3k,8984
+azure/ai/evaluation/simulator/_model_tools/_proxy_completion_model.py,sha256=D2Am07l2gq_v-vztqDBTX2MxZTUumm5ysaVSL2L0rxQ,9118
 azure/ai/evaluation/simulator/_model_tools/_rai_client.py,sha256=40MGzIXGv7oVshWH7AbOPLCigI4HlMrqbF2Rq5jFMGo,8755
 azure/ai/evaluation/simulator/_model_tools/_template_handler.py,sha256=NQWqjE7csSzkhb2XdW82AoCA-DxixpTrfBxAnOt2Wlc,7075
 azure/ai/evaluation/simulator/_model_tools/models.py,sha256=bfVm0PV3vfH_8DkdmTMZqYVN-G51hZ6Y0TOO-NiysJY,21811
 azure/ai/evaluation/simulator/_prompty/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 azure/ai/evaluation/simulator/_prompty/task_query_response.prompty,sha256=2BzSqDDYilDushvR56vMRDmqFIaIYAewdUlUZg_elMg,2182
 azure/ai/evaluation/simulator/_prompty/task_simulate.prompty,sha256=NE6lH4bfmibgMn4NgJtm9_l3PMoHSFrfjjosDJEKM0g,939
-azure_ai_evaluation-1.2.0.dist-info/METADATA,sha256=NM0mPj138_k-6vWuKICoqUBtDq-TaWGXOeaCLpyT2IU,32156
-azure_ai_evaluation-1.2.0.dist-info/NOTICE.txt,sha256=4tzi_Yq4-eBGhBvveobWHCgUIVF-ZeouGN0m7hVq5Mk,3592
-azure_ai_evaluation-1.2.0.dist-info/WHEEL,sha256=pL8R0wFFS65tNSRnaOVrsw9EOkOqxLrlUPenUYnJKNo,91
-azure_ai_evaluation-1.2.0.dist-info/top_level.txt,sha256=S7DhWV9m80TBzAhOFjxDUiNbKszzoThbnrSz5MpbHSQ,6
-azure_ai_evaluation-1.2.0.dist-info/RECORD,,
+azure_ai_evaluation-1.3.0.dist-info/METADATA,sha256=msxfA0EnNspmcqbsGuYtdEESA3kLPnEfWBpzf7yFHEw,32885
+azure_ai_evaluation-1.3.0.dist-info/NOTICE.txt,sha256=4tzi_Yq4-eBGhBvveobWHCgUIVF-ZeouGN0m7hVq5Mk,3592
+azure_ai_evaluation-1.3.0.dist-info/WHEEL,sha256=pL8R0wFFS65tNSRnaOVrsw9EOkOqxLrlUPenUYnJKNo,91
+azure_ai_evaluation-1.3.0.dist-info/top_level.txt,sha256=S7DhWV9m80TBzAhOFjxDUiNbKszzoThbnrSz5MpbHSQ,6
+azure_ai_evaluation-1.3.0.dist-info/RECORD,,

azure/ai/evaluation/_evaluators/_multimodal/__init__.py

@@ -1,20 +0,0 @@
-# ---------------------------------------------------------
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# ---------------------------------------------------------
-from ._content_safety_multimodal import ContentSafetyMultimodalEvaluator
-from ._content_safety_multimodal_base import ContentSafetyMultimodalEvaluatorBase
-from ._hate_unfairness import HateUnfairnessMultimodalEvaluator
-from ._self_harm import SelfHarmMultimodalEvaluator
-from ._sexual import SexualMultimodalEvaluator
-from ._violence import ViolenceMultimodalEvaluator
-from ._protected_material import ProtectedMaterialMultimodalEvaluator
-
-__all__ = [
-    "ContentSafetyMultimodalEvaluator",
-    "ContentSafetyMultimodalEvaluatorBase",
-    "ViolenceMultimodalEvaluator",
-    "SexualMultimodalEvaluator",
-    "SelfHarmMultimodalEvaluator",
-    "HateUnfairnessMultimodalEvaluator",
-    "ProtectedMaterialMultimodalEvaluator",
-]

azure/ai/evaluation/_evaluators/_multimodal/_content_safety_multimodal.py

@@ -1,132 +0,0 @@
-# ---------------------------------------------------------
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# ---------------------------------------------------------
-import logging
-from concurrent.futures import as_completed
-from typing import Callable, Dict, List, Union
-
-from promptflow.tracing import ThreadPoolExecutorWithContext as ThreadPoolExecutor
-from azure.ai.evaluation._common._experimental import experimental
-from azure.ai.evaluation._common.utils import validate_conversation
-from ._hate_unfairness import HateUnfairnessMultimodalEvaluator
-from ._self_harm import SelfHarmMultimodalEvaluator
-from ._sexual import SexualMultimodalEvaluator
-from ._violence import ViolenceMultimodalEvaluator
-
-
-logger = logging.getLogger(__name__)
-
-
-@experimental
-class ContentSafetyMultimodalEvaluator:
-    """
-    Initialize a content safety multimodal evaluator configured to evaluate content safety metrics
-    in a multimodal scenario.
-
-    :param credential: The credential required for connecting to the Azure AI project.
-    :type credential: ~azure.core.credentials.TokenCredential
-    :param azure_ai_project: The scope of the Azure AI project, containing the subscription ID,
-        resource group, and project name.
-    :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
-    :param kwargs: Additional arguments to pass to the evaluator.
-    :type kwargs: Any
-
-    :return: A function that evaluates multimodal chat messages and generates content safety metrics.
-
-    **Usage Example**
-
-    .. code-block:: python
-
-        azure_ai_project = {
-            "subscription_id": "<subscription_id>",
-            "resource_group_name": "<resource_group_name>",
-            "project_name": "<project_name>",
-        }
-        eval_fn = ContentSafetyMultimodalEvaluator(azure_ai_project)
-        result = eval_fn(conversation=
-            {
-                "messages": [
-                    {
-                        "role": "user",
-                        "content": [
-                            {
-                                "type": "text",
-                                "text": "What's in this image?"
-                            },
-                            {
-                                "type": "image_url",
-                                "image_url": {
-                                    "url": "<image url or base64 encoded image>"
-                                }
-                            }
-                        ]
-                    },
-                    {
-                        "role": "assistant",
-                        "content": "This picture shows an astronaut standing in the desert."
-                    }
-                ]
-            }
-        )
-
-    **Output Format**
-
-    .. code-block:: json
-
-        {
-            "violence": "Medium",
-            "violence_score": 5.0,
-            "violence_reason": "Some reason",
-            "sexual": "Medium",
-            "sexual_score": 5.0,
-            "sexual_reason": "Some reason",
-            "self_harm": "Medium",
-            "self_harm_score": 5.0,
-            "self_harm_reason": "Some reason",
-            "hate_unfairness": "Medium",
-            "hate_unfairness_score": 5.0,
-            "hate_unfairness_reason": "Some reason"
-        }
-
-    """
-
-    id = "content_safety_multimodal"
-    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
-
-    def __init__(self, credential, azure_ai_project, **kwargs):
-        self._parallel = kwargs.pop("_parallel", False)
-        self._evaluators: List[Callable[..., Dict[str, Union[str, float]]]] = [
-            ViolenceMultimodalEvaluator(credential=credential, azure_ai_project=azure_ai_project),
-            SexualMultimodalEvaluator(credential=credential, azure_ai_project=azure_ai_project),
-            SelfHarmMultimodalEvaluator(credential=credential, azure_ai_project=azure_ai_project),
-            HateUnfairnessMultimodalEvaluator(credential=credential, azure_ai_project=azure_ai_project),
-        ]
-
-    def __call__(self, *, conversation, **kwargs):
-        """
-        Evaluates content-safety metrics for list of messages.
-
-        :keyword conversation: The conversation contains list of messages to be evaluated.
-            Each message should have "role" and "content" keys. It supports single turn only.
-        :paramtype conversation: ~azure.ai.evaluation.Conversation
-        :return: The evaluation score based on the Content Safety Metrics.
-        :rtype: Dict[str, Union[float, str]]
-        """
-        # validate inputs
-        validate_conversation(conversation)
-        results: Dict[str, Union[str, float]] = {}
-        if self._parallel:
-            with ThreadPoolExecutor() as executor:
-                futures = {
-                    executor.submit(evaluator, conversation=conversation, **kwargs): evaluator
-                    for evaluator in self._evaluators
-                }
-
-                for future in as_completed(futures):
-                    results.update(future.result())
-        else:
-            for evaluator in self._evaluators:
-                result = evaluator(conversation=conversation, **kwargs)
-                results.update(result)
-
-        return results

azure/ai/evaluation/_evaluators/_multimodal/_content_safety_multimodal_base.py

@@ -1,55 +0,0 @@
-# ---------------------------------------------------------
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# ---------------------------------------------------------
-from abc import ABC
-from typing import Union
-from azure.ai.evaluation._common.rai_service import evaluate_with_rai_service_multimodal
-from azure.ai.evaluation._common.constants import EvaluationMetrics, _InternalEvaluationMetrics
-from azure.ai.evaluation._common.utils import validate_conversation
-from azure.core.credentials import TokenCredential
-
-
-class ContentSafetyMultimodalEvaluatorBase(ABC):
-    """
-    Initialize a evaluator for a specified Evaluation Metric. Base class that is not
-    meant to be instantiated by users.
-
-    :param metric: The metric to be evaluated.
-    :type metric: ~azure.ai.evaluation._evaluators._content_safety.flow.constants.EvaluationMetrics
-    :param credential: The credential for connecting to Azure AI project. Required
-    :type credential: ~azure.core.credentials.TokenCredential
-    :param azure_ai_project: The scope of the Azure AI project.
-        It contains subscription id, resource group, and project name.
-    :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
-    """
-
-    def __init__(
-        self,
-        metric: Union[EvaluationMetrics, _InternalEvaluationMetrics],
-        credential: TokenCredential,
-        azure_ai_project,
-    ):
-        self._metric = metric
-        self._azure_ai_project = azure_ai_project
-        self._credential = credential
-
-    async def __call__(self, *, conversation, **kwargs):
-        """
-        Evaluates content according to this evaluator's metric.
-        :keyword conversation: The conversation contains list of messages to be evaluated.
-            Each message should have "role" and "content" keys.
-        :paramtype conversation: ~azure.ai.evaluation.Conversation
-        :return: The evaluation score computation based on the Content Safety metric (self.metric).
-        :rtype: Dict[str, Union[float, str]]
-        """
-        # validate inputs
-        validate_conversation(conversation)
-        messages = conversation["messages"]
-        # Run score computation based on supplied metric.
-        result = await evaluate_with_rai_service_multimodal(
-            messages=messages,
-            metric_name=self._metric,
-            project_scope=self._azure_ai_project,
-            credential=self._credential,
-        )
-        return result

azure/ai/evaluation/_evaluators/_multimodal/_hate_unfairness.py

@@ -1,100 +0,0 @@
-# ---------------------------------------------------------
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# ---------------------------------------------------------
-from promptflow._utils.async_utils import async_run_allowing_running_loop
-from azure.ai.evaluation._common._experimental import experimental
-from azure.ai.evaluation._common.constants import EvaluationMetrics
-from ._content_safety_multimodal_base import ContentSafetyMultimodalEvaluatorBase
-
-
-@experimental
-class HateUnfairnessMultimodalEvaluator:
-    """
-    Initialize a hate-unfairness multimodal evaluator for hate unfairness score.
-
-    :param credential: The credential for connecting to the Azure AI project. This is required.
-    :type credential: ~azure.core.credentials.TokenCredential
-    :param azure_ai_project: The scope of the Azure AI project,
-        which includes subscription ID, resource group name, and project name.
-    :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
-
-    **Usage Example**
-
-    .. code-block:: python
-
-        azure_ai_project = {
-            "subscription_id": "<subscription_id>",
-            "resource_group_name": "<resource_group_name>",
-            "project_name": "<project_name>",
-        }
-        eval_fn = HateUnfairnessMultimodalEvaluator(azure_ai_project)
-        result = eval_fn(conversation=
-            {
-                "messages": [
-                    {
-                        "role": "user",
-                        "content": [
-                            {
-                                "type": "text",
-                                "text": "What's in this image?"
-                            },
-                            {
-                                "type": "image_url",
-                                "image_url": {
-                                    "url": "<image url or base64 encoded image>"
-                                }
-                            }
-                        ]
-                    },
-                    {
-                        "role": "assistant",
-                        "content": "This picture shows an astronaut standing in the desert."
-                    }
-                ]
-            }
-        )
-
-    **Output Format**
-
-    .. code-block:: json
-
-        {
-            "hate_unfairness": "High",
-            "hate_unfairness_score": 6.5,
-            "hate_unfairness_reason": "Some reason"
-        }
-
-    """
-
-    id = "hate_unfairness_multimodal"
-    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
-
-    def __init__(self, credential, azure_ai_project):
-        self._async_evaluator = _AsyncHateUnfairnessMultimodalEvaluator(credential, azure_ai_project)
-
-    def __call__(self, *, conversation, **kwargs):
-        """
-        Evaluates hate unfairness content.
-
-        :keyword conversation: The conversation contains list of messages to be evaluated.
-            Each message should have "role" and "content" keys. It supports single turn only.
-        :paramtype conversation: ~azure.ai.evaluation.Conversation
-        :return: The hate unfairness score.
-        :rtype: Dict[str, Union[float, str]]
-        """
-        return async_run_allowing_running_loop(self._async_evaluator, conversation=conversation, **kwargs)
-
-    def _to_async(self):
-        return self._async_evaluator
-
-
-class _AsyncHateUnfairnessMultimodalEvaluator(ContentSafetyMultimodalEvaluatorBase):
-    def __init__(self, credential, azure_ai_project):
-        super().__init__(
-            metric=EvaluationMetrics.HATE_FAIRNESS,
-            credential=credential,
-            azure_ai_project=azure_ai_project,
-        )
-
-    async def __call__(self, *, conversation, **kwargs):
-        return await super().__call__(conversation=conversation, **kwargs)