azure-ai-evaluation 1.0.0b2__py3-none-any.whl → 1.0.0b3__py3-none-any.whl

{azure_ai_evaluation-1.0.0b2.dist-info → azure_ai_evaluation-1.0.0b3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: azure-ai-evaluation
-Version: 1.0.0b2
+Version: 1.0.0b3
 Summary: Microsoft Azure Evaluation Library for Python
 Home-page: https://github.com/Azure/azure-sdk-for-python
 Author: Microsoft Corporation
@@ -426,6 +426,64 @@ This project has adopted the [Microsoft Open Source Code of Conduct][code_of_con
 
 # Release History
 
+## 1.0.0b3 (2024-10-01)
+
+### Features Added
+
+- Added `type` field to `AzureOpenAIModelConfiguration` and `OpenAIModelConfiguration`
+- The following evaluators now support `conversation` as an alternative input to their usual single-turn inputs:
+  - `ViolenceEvaluator`
+  - `SexualEvaluator`
+  - `SelfHarmEvaluator`
+  - `HateUnfairnessEvaluator`
+  - `ProtectedMaterialEvaluator`
+  - `IndirectAttackEvaluator`
+  - `CoherenceEvaluator`
+  - `RelevanceEvaluator`
+  - `FluencyEvaluator`
+  - `GroundednessEvaluator`
+- Surfaced `RetrievalScoreEvaluator`, formally an internal part of `ChatEvaluator` as a standalone conversation-only evaluator.
+
+### Breaking Changes
+
+- Removed `ContentSafetyChatEvaluator` and `ChatEvaluator`
+- The `evaluator_config` parameter of `evaluate` now maps in evaluator name to a dictionary `EvaluatorConfig`, which is a `TypedDict`. The
+`column_mapping` between `data` or `target` and evaluator field names should now be specified inside this new dictionary:
+
+Before:
+```python
+evaluate(
+    ...,
+    evaluator_config={
+        "hate_unfairness": {
+            "query": "${data.question}",
+            "response": "${data.answer}",
+        }
+    },
+    ...
+)
+```
+
+After
+```python
+evaluate(
+    ...,
+    evaluator_config={
+        "hate_unfairness": {
+            "column_mapping": {
+                "query": "${data.question}",
+                "response": "${data.answer}",
+             }
+        }
+    },
+    ...
+)
+```
+
+### Bugs Fixed
+
+- Fixed issue where Entra ID authentication was not working with `AzureOpenAIModelConfiguration` 
+
 ## 1.0.0b2 (2024-09-24)
 
 ### Breaking Changes

{azure_ai_evaluation-1.0.0b2.dist-info → azure_ai_evaluation-1.0.0b3.dist-info}/RECORD

@@ -1,18 +1,18 @@
-azure/ai/evaluation/__init__.py,sha256=7ff0POpQe1C1L9eD4yz9P9y9BtFWNr5wHEtSOR2xVA0,1967
-azure/ai/evaluation/_constants.py,sha256=cWnrEjJuPok2CPbibJ40XyX2VSlw4-x_dob3DjFsd5U,1683
-azure/ai/evaluation/_exceptions.py,sha256=HUMfvguDc7ygcbs3MTK14R3PK7UxGNWQQHH3hYXIV3U,4168
+azure/ai/evaluation/__init__.py,sha256=AW8HyrHG5L1NT-0-vzu14o9Em8-ZiKXokPlIAvgF9lI,1977
+azure/ai/evaluation/_constants.py,sha256=RWerL5-uO8xmgZmdyon5TYH2_xPM_31cZqXs7qk28Ms,1743
+azure/ai/evaluation/_exceptions.py,sha256=CH4Waotlr519uHirau38NFExv5cG5JgrjPxjTAHWPGU,4131
 azure/ai/evaluation/_http_utils.py,sha256=kpAuxuoC6ZK-ZHw6qScOC8ePJQXLwf7SJWP5S4OWwUs,13983
-azure/ai/evaluation/_model_configurations.py,sha256=D02AzOdyO6LQCia0k232Msd7ro35-EcwmlQ0tOD_5H0,652
+azure/ai/evaluation/_model_configurations.py,sha256=9yiXXCFw8DiCHB0Rjg6NEHZNSK8AYCfpRdRp_rwd5DU,1869
 azure/ai/evaluation/_user_agent.py,sha256=O2y-QPBAcw7w7qQ6M2aRPC3Vy3TKd789u5lcs2yuFaI,290
-azure/ai/evaluation/_version.py,sha256=Fx-4qD0RW2DqdKCStZHT4KOjqxKVEeT05DbGkqnTSF4,201
+azure/ai/evaluation/_version.py,sha256=O5t2mfbIREHTFuOmpAqCrJ00-kGQ4SRcFh42DDRu2Ac,201
 azure/ai/evaluation/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 azure/ai/evaluation/_common/__init__.py,sha256=LHTkf6dMLLxikrGNgbUuREBVQcs4ORHR6Eryo4bm9M8,586
 azure/ai/evaluation/_common/constants.py,sha256=ZKHGVgGA1Fc6Pvo22X-CeOUX6-m0q_UwpOKOWATTSuI,1639
 azure/ai/evaluation/_common/rai_service.py,sha256=R-1jtWi4Fu4IT4v0j0hKsWyN-D5nwH5HQr1x0AtCp8Q,17539
-azure/ai/evaluation/_common/utils.py,sha256=orrvIGY3L2o8uHEJkTdfNa4jV_8OsscWyL3CzX96U9o,2991
+azure/ai/evaluation/_common/utils.py,sha256=IKCAxHbGcmi5rH6qFZOB76vInXS8A7Oa7dYC56K0BWY,3494
 azure/ai/evaluation/_evaluate/__init__.py,sha256=Yx1Iq2GNKQ5lYxTotvPwkPL4u0cm6YVxUe-iVbu1clI,180
 azure/ai/evaluation/_evaluate/_eval_run.py,sha256=EVCSRjIwOkXfYlIBTv7hosyUqcMTmVqGQ44yvHmm2Eg,20943
-azure/ai/evaluation/_evaluate/_evaluate.py,sha256=50djPOmW-K3f7lbTttPWcgGx96_b-zTeG5iBHuPArRY,29141
+azure/ai/evaluation/_evaluate/_evaluate.py,sha256=XoGXpzO8Z_hYwpPw9amjYarrKqtvNtLB4xLdtE8mmuI,30324
 azure/ai/evaluation/_evaluate/_utils.py,sha256=9I29KAFsYJCp6frlLXb5vwZJzmiNzfzku9CD1eslaHU,9880
 azure/ai/evaluation/_evaluate/_batch_run_client/__init__.py,sha256=BkxhojWca3e2QM3hFwO2xrLiiQ0i-3f8wsMfOx1zchs,361
 azure/ai/evaluation/_evaluate/_batch_run_client/batch_run_context.py,sha256=sn7k9nM7vVZDt5CgNwwQrvQwV--SwfsfLjfjahk23DM,2984
@@ -22,59 +22,58 @@ azure/ai/evaluation/_evaluate/_telemetry/__init__.py,sha256=6kkbiTCsz7BNV5WxOdwC
 azure/ai/evaluation/_evaluators/__init__.py,sha256=Yx1Iq2GNKQ5lYxTotvPwkPL4u0cm6YVxUe-iVbu1clI,180
 azure/ai/evaluation/_evaluators/_bleu/__init__.py,sha256=quKKO0kvOSkky5hcoNBvgBuMeeVRFCE9GSv70mAdGP4,260
 azure/ai/evaluation/_evaluators/_bleu/_bleu.py,sha256=6EJCG9DnL2Y4pU_vhY4o3UOrumvI-6HI92tzEuCoyXk,2413
-azure/ai/evaluation/_evaluators/_chat/__init__.py,sha256=xOsSHYNGJJiZvBMPbmLd_-ZZs8_15Sblvk-OF7iVoIo,250
-azure/ai/evaluation/_evaluators/_chat/_chat.py,sha256=ZCm7L50bRaAvj2-Gw6T_9u4RTHRF6neCYnjDCunRjKw,14787
-azure/ai/evaluation/_evaluators/_chat/retrieval/__init__.py,sha256=DmBjBkwDDlCsSGpBeXfpfMM9ekxIJs62dij4rBXND7k,273
-azure/ai/evaluation/_evaluators/_chat/retrieval/_retrieval.py,sha256=HKkQdvYVD3Mr5kPUzKLBYsct94k1hAcLu5v9MCxZ6lA,5579
-azure/ai/evaluation/_evaluators/_chat/retrieval/retrieval.prompty,sha256=NoHNDf_UE8BGAixqZPhRQ4ecxOUi9w9gO8HfHhJuxaY,1761
 azure/ai/evaluation/_evaluators/_coherence/__init__.py,sha256=GRqcSCQse02Spyki0UsRNWMIXiea2lLtPPXNGvkJzQ0,258
-azure/ai/evaluation/_evaluators/_coherence/_coherence.py,sha256=XMymhai4cw9Gqy9Lfa9IvIcw9s_cnCM1pbyVVnpF53M,3958
-azure/ai/evaluation/_evaluators/_coherence/coherence.prompty,sha256=WVEXxKmh_Gbb11_00N2WCIIJSMgPssFxJ5h2--rMG-w,2725
-azure/ai/evaluation/_evaluators/_content_safety/__init__.py,sha256=mR5CbcMyxV9GQoY71Saoi0bQTpEB74HrYmM8gcVhnAg,746
+azure/ai/evaluation/_evaluators/_coherence/_coherence.py,sha256=ydYo03-XlJDaX6YD432xMfutU8r41ovf1MW7vVuWPw4,2512
+azure/ai/evaluation/_evaluators/_coherence/coherence.prompty,sha256=_GXYhAH04tsl2qntZH5ACx7gFNfUeQ0hZQpOmDoLPNc,2549
+azure/ai/evaluation/_evaluators/_common/__init__.py,sha256=_hPqTkAla_O6s4ebVtTaBrVLEW3KSdDz66WwxjK50cI,423
+azure/ai/evaluation/_evaluators/_common/_base_eval.py,sha256=Ai3jN-HRzNsG2V48pMR9TmK3owT-6YJxKi9gzMloZNE,14072
+azure/ai/evaluation/_evaluators/_common/_base_prompty_eval.py,sha256=bPePVS7-5gvzEtpKxlDBamxNwetBJTqf5nCMt6Wu7ao,3050
+azure/ai/evaluation/_evaluators/_common/_base_rai_svc_eval.py,sha256=O_klXEBsmYJyyqoIGN5gRg0udnvTKtaouQmEeMGpOgM,4331
+azure/ai/evaluation/_evaluators/_content_safety/__init__.py,sha256=PEYMIybfP64f7byhuTaiq4RiqsYbjqejpW1JsJIG1jA,556
 azure/ai/evaluation/_evaluators/_content_safety/_content_safety.py,sha256=uM6BL9jwtv5nJpSchezTYc-E514_VCTN2pACy7oxHuU,3928
-azure/ai/evaluation/_evaluators/_content_safety/_content_safety_base.py,sha256=VlBDKrB2uNXX8ccV7aKsHM74_mK6JLUZ9SAGcvR3Lk8,2733
 azure/ai/evaluation/_evaluators/_content_safety/_content_safety_chat.py,sha256=n5fdL0TPelJY_AuiamkLO7Jiv7P-7gIZqipo5ShyoR8,11958
-azure/ai/evaluation/_evaluators/_content_safety/_hate_unfairness.py,sha256=uA_FecBD9M1C0_hfnbqR9shZNMFbhmBcCAhKgTxv8rI,2807
-azure/ai/evaluation/_evaluators/_content_safety/_self_harm.py,sha256=6_FK2zDAyS3wQTbYe3FmNXxw7qWwYOGqLUKopWUWKvQ,2608
-azure/ai/evaluation/_evaluators/_content_safety/_sexual.py,sha256=d8LZD368nqppZi-Kdimo4Lt8QOTFtc6csPjY66G7SMQ,2576
-azure/ai/evaluation/_evaluators/_content_safety/_violence.py,sha256=e0ZeUcD1YKWPzqfaLBxFdNHmod69K55zIOgLOJAMe_4,2600
+azure/ai/evaluation/_evaluators/_content_safety/_hate_unfairness.py,sha256=YIQLFj6P7WXAyRRHVOflikUePN5sMCanJQmnIpSDeY0,1856
+azure/ai/evaluation/_evaluators/_content_safety/_self_harm.py,sha256=BFRNVDehDe7Qgnt3k1zRzscFQmn_miuHaSpjOytFhds,1810
+azure/ai/evaluation/_evaluators/_content_safety/_sexual.py,sha256=Ap4EvHDPF8YjJ_esKEK83yusSR_xYhJWn6HIn1mkwW0,1788
+azure/ai/evaluation/_evaluators/_content_safety/_violence.py,sha256=HbLsZOqLopr0beDeHW85EmqSMFcTCZIYGUYvxUq_-gM,1804
 azure/ai/evaluation/_evaluators/_eci/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-azure/ai/evaluation/_evaluators/_eci/_eci.py,sha256=Amrch3q5BG2BFAB7GdShnTX19f5IoGY6VF66_M6hbZs,3990
+azure/ai/evaluation/_evaluators/_eci/_eci.py,sha256=EPy_A4BtqHm_10kHApi9xZ2eHYU5CjVGtkfG4zUMRhs,2411
 azure/ai/evaluation/_evaluators/_f1_score/__init__.py,sha256=aEVbO7iMoF20obdpLQKcKm69Yyu3mYnblKELLqu8OGI,260
 azure/ai/evaluation/_evaluators/_f1_score/_f1_score.py,sha256=Cgp-hANXtrNfe1q6SVm-CUG3UEP8Gj1pvH293ay57OI,4554
 azure/ai/evaluation/_evaluators/_fluency/__init__.py,sha256=EEJw39xRa0bOAA1rELTTKXQu2s60n_7CZQRD0Gu2QVw,259
-azure/ai/evaluation/_evaluators/_fluency/_fluency.py,sha256=MhDOoP9oD-rw7uT90pU6O1XP6vqfOP6e608tp1qRy7Y,3919
-azure/ai/evaluation/_evaluators/_fluency/fluency.prompty,sha256=RparSdDZs-xiGbq7lRifz9z7jaD10ldXDU3E7sO0v2s,2579
+azure/ai/evaluation/_evaluators/_fluency/_fluency.py,sha256=k7lq7qRoiI2SaHPqLhW-Frm_STRK-hFHsbAFOejAU7s,2459
+azure/ai/evaluation/_evaluators/_fluency/fluency.prompty,sha256=xdznyssZDQiLELv4ecC-8uUJ4ssM-iij7A6S1aDsxOQ,2403
 azure/ai/evaluation/_evaluators/_gleu/__init__.py,sha256=Ae2EvQ7gqiYAoNO3LwGIhdAAjJPJDfT85rQGKrRrmbA,260
 azure/ai/evaluation/_evaluators/_gleu/_gleu.py,sha256=m02wmIGjdoXjp9dwjnFQAKA8hGOUOTvpppDf2CD4QQo,2326
 azure/ai/evaluation/_evaluators/_groundedness/__init__.py,sha256=UYNJUeRvBwcSVFyZpdsf29un5eyaDzYoo3QvC1gvlLg,274
-azure/ai/evaluation/_evaluators/_groundedness/_groundedness.py,sha256=3HcNMOBxbslIJmmhJWU3itPOL3TOiLqQYiYr_n8wTqU,4138
-azure/ai/evaluation/_evaluators/_groundedness/groundedness.prompty,sha256=dDclp_YowBjeiGhwmCxCnS4A3K9r4v2tzsUm-ccLt-I,3199
+azure/ai/evaluation/_evaluators/_groundedness/_groundedness.py,sha256=FREk-1w_K6oF74eiNii5EdRS4uK_NUxW0dLd5Kzgj6c,2682
+azure/ai/evaluation/_evaluators/_groundedness/groundedness.prompty,sha256=ylgxKa_xipb7wN_QwxSnjrD9AhKcJQCv8pPpWPwFfGg,3023
 azure/ai/evaluation/_evaluators/_meteor/__init__.py,sha256=209na3pPsdmcuYpYHUYtqQybCpc3yZkc93HnRdicSlI,266
 azure/ai/evaluation/_evaluators/_meteor/_meteor.py,sha256=K3EdRuRcuEZYVIlI2jMEp0O9KJYXQB2o6h08q43oKWY,3316
 azure/ai/evaluation/_evaluators/_protected_material/__init__.py,sha256=eRAQIU9diVXfO5bp6aLWxZoYUvOsrDIfy1gnDOeNTiI,109
-azure/ai/evaluation/_evaluators/_protected_material/_protected_material.py,sha256=LnbyS3xVu_7Q1nDuoOxnQnzXhZVrDMSnEzNdu66g6DY,4165
-azure/ai/evaluation/_evaluators/_protected_materials/__init__.py,sha256=A12UsRVIebGvy9FtZLBPsOIAWUskBt8iuhRdILyRcSo,112
-azure/ai/evaluation/_evaluators/_protected_materials/_protected_materials.py,sha256=xF5jHhM29OXh2sHrnXkYtiRYltuVU-fqC7xToiI3WOM,4136
+azure/ai/evaluation/_evaluators/_protected_material/_protected_material.py,sha256=KT8AMLL9nGvI_KVwpUAcZJcPqPN_QNyQMkkmIhmexNE,2117
 azure/ai/evaluation/_evaluators/_qa/__init__.py,sha256=bcXfT--C0hjym2haqd1B2-u9bDciyM0ThOFtU1Q69sk,244
 azure/ai/evaluation/_evaluators/_qa/_qa.py,sha256=w9XJOfDof78mfOpc7tbPF5wec9dGPFmXXAdR5yx2buI,3502
 azure/ai/evaluation/_evaluators/_relevance/__init__.py,sha256=JlxytW32Nl8pbE-fI3GRpfgVuY9EG6zxIAn5VZGSwyc,265
-azure/ai/evaluation/_evaluators/_relevance/_relevance.py,sha256=iMky4Gf1kJB1kvBLaNWwCOBKLSWfhtSSLU0tPTU6ruE,4344
-azure/ai/evaluation/_evaluators/_relevance/relevance.prompty,sha256=AO70ho2nMhBtKcl_q4nKFW4kA1LjYsmSfymNa-Cbcrw,3735
+azure/ai/evaluation/_evaluators/_relevance/_relevance.py,sha256=tHBHzp2wz3szgfA24HQgphP4mF5iJfg-lw6bVqgqkpY,2934
+azure/ai/evaluation/_evaluators/_relevance/relevance.prompty,sha256=QNWlrWxObUPlXFF1hdCDVpfXuw0QDOxHUtWLj1MwrxA,3559
+azure/ai/evaluation/_evaluators/_retrieval/__init__.py,sha256=kMu47ZyTZ7f-4Yh6H3KHxswmxitmPJ8FPSk90qgR0XI,265
+azure/ai/evaluation/_evaluators/_retrieval/_retrieval.py,sha256=u6OqyZ62JpHmYatepRW5aRbtwu1sZByVSCDj_CRZSj8,5160
+azure/ai/evaluation/_evaluators/_retrieval/retrieval.prompty,sha256=HbQu5Gy9Ghw9r8vGCF-4ui441JBD8w45NOU_9ehamd0,1585
 azure/ai/evaluation/_evaluators/_rouge/__init__.py,sha256=kusCDaYcXogDugGefRP8MQSn9xv107oDbrMCqZ6K4GA,291
 azure/ai/evaluation/_evaluators/_rouge/_rouge.py,sha256=28vqjjleeJR5VRsQP5VCCMX_PVUUVxkgh4c3xIvwmXE,3526
 azure/ai/evaluation/_evaluators/_similarity/__init__.py,sha256=V2Mspog99_WBltxTkRHG5NpN5s9XoiTSN4I8POWEkLA,268
-azure/ai/evaluation/_evaluators/_similarity/_similarity.py,sha256=T-aTYausFvIfxCe0u8Lbd3ehKwxRtRLk2gHZ2pJ9YkE,4332
-azure/ai/evaluation/_evaluators/_similarity/similarity.prompty,sha256=p2Tb4IW6QnP2BaGRQsAicW4V0B23Oezhf5l3Hau0nxU,4770
+azure/ai/evaluation/_evaluators/_similarity/_similarity.py,sha256=m4Ub7EcGMHotcOll_WIEYrvUWV1hMjF6K1VGthkEoqk,3883
+azure/ai/evaluation/_evaluators/_similarity/similarity.prompty,sha256=eoludASychZoGL625bFCaZai-OY7DIAg90ZLax_o4XE,4594
 azure/ai/evaluation/_evaluators/_xpia/__init__.py,sha256=VMEL8WrpJQeh4sQiOLzP7hRFPnjzsvwfvTzaGCVJPCM,88
-azure/ai/evaluation/_evaluators/_xpia/xpia.py,sha256=iYXjGt0F_gfc7SF6Q_d3Z_0Bkniqad91wljtBwR6BDM,5750
+azure/ai/evaluation/_evaluators/_xpia/xpia.py,sha256=Mg0nhT00VPgfBqp0Pu-7C4Unf6MEu8yNMFW-Wu7RTXw,2556
 azure/ai/evaluation/simulator/__init__.py,sha256=UtlcXo3SteIQEW_hW2WMhtqLNiDiIGLeW_lIkEUNoMc,486
 azure/ai/evaluation/simulator/_adversarial_scenario.py,sha256=SxpyMw5wmM5-fiUjl1_oJH0GQEnsa7ASso10MAr2Hjw,1030
 azure/ai/evaluation/simulator/_adversarial_simulator.py,sha256=kOL31FcD7vXTpkeFUooASXNaFTe9Vme5st_i0Qa_9sA,20542
 azure/ai/evaluation/simulator/_constants.py,sha256=xM-Or2x7RytfoeBM3N7Vt4JQDJX66UdL3CPz0YN5rvE,485
 azure/ai/evaluation/simulator/_direct_attack_simulator.py,sha256=zFYYdk8Sdg4-_HSd_rumM0LizPevcR57HjqvEdowv8c,11691
 azure/ai/evaluation/simulator/_indirect_attack_simulator.py,sha256=qALFN3LG5o1kSjMjdlLeJInax8GcjD1iPUZCayJp0Kc,9628
-azure/ai/evaluation/simulator/_simulator.py,sha256=MpB8fcCmuge7AvHmoV-K3L9tdEwWHcfpkiU22MJfZ-A,31642
+azure/ai/evaluation/simulator/_simulator.py,sha256=V9xNOwDRTlK9Xf1SyRK4yv8j3pTFd_4D79BYanePoDw,32187
 azure/ai/evaluation/simulator/_tracing.py,sha256=LRPjsVLe9VohmXowFr9aCK_VwD0MHd1CBe8rl9jGQhU,3032
 azure/ai/evaluation/simulator/_utils.py,sha256=aXH5GdzQrwluKvYofWtdT0s_nzgVHS2hP6x4rc5zt-E,4287
 azure/ai/evaluation/simulator/_conversation/__init__.py,sha256=MNfFW4UDsVrk1p2ysIvmYlLzHqjKfxExktQXfSRiBPk,12774
@@ -91,9 +90,9 @@ azure/ai/evaluation/simulator/_model_tools/_rai_client.py,sha256=Bi0tLNlJmz295md
 azure/ai/evaluation/simulator/_model_tools/_template_handler.py,sha256=gGSMvveKWn0LKSQ4FS5AxIwcsxj6iqCbUP53yjvndPw,5471
 azure/ai/evaluation/simulator/_model_tools/models.py,sha256=11O6jcj3Zwo4FZvmF-X0walNp22ux1k3ghi3KFtbdy0,21762
 azure/ai/evaluation/simulator/_prompty/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-azure/ai/evaluation/simulator/_prompty/task_query_response.prompty,sha256=lNSioz2XiGQJb-AXRNYm2JCLMivZKa3JlHfol2Jd7fY,2244
+azure/ai/evaluation/simulator/_prompty/task_query_response.prompty,sha256=wUiDKFL_vnAk3eEW66z33UgNML8Wqd_ReCzqfEBMId8,2350
 azure/ai/evaluation/simulator/_prompty/task_simulate.prompty,sha256=00zLVfNgHZdlbC2XvBedSrwDJOaAhl3B1ohE3LKsGg4,928
-azure_ai_evaluation-1.0.0b2.dist-info/METADATA,sha256=t8N3Aanv8JY6JHI4bElRml_Iy0Bm1wSESBG0K78Ou4Y,15520
-azure_ai_evaluation-1.0.0b2.dist-info/WHEEL,sha256=HiCZjzuy6Dw0hdX5R3LCFPDmFS4BWl8H-8W39XfmgX4,91
-azure_ai_evaluation-1.0.0b2.dist-info/top_level.txt,sha256=S7DhWV9m80TBzAhOFjxDUiNbKszzoThbnrSz5MpbHSQ,6
-azure_ai_evaluation-1.0.0b2.dist-info/RECORD,,
+azure_ai_evaluation-1.0.0b3.dist-info/METADATA,sha256=2jKCZxNNYmFeTFJo87_qO32_nAE9ur1YwSSwnn2Mi9I,17077
+azure_ai_evaluation-1.0.0b3.dist-info/WHEEL,sha256=HiCZjzuy6Dw0hdX5R3LCFPDmFS4BWl8H-8W39XfmgX4,91
+azure_ai_evaluation-1.0.0b3.dist-info/top_level.txt,sha256=S7DhWV9m80TBzAhOFjxDUiNbKszzoThbnrSz5MpbHSQ,6
+azure_ai_evaluation-1.0.0b3.dist-info/RECORD,,

azure/ai/evaluation/_evaluators/_chat/_chat.py

@@ -1,357 +0,0 @@
-# ---------------------------------------------------------
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# ---------------------------------------------------------
-import json
-import logging
-from concurrent.futures import as_completed
-from typing import Dict, List
-
-import numpy as np
-from promptflow.tracing import ThreadPoolExecutorWithContext as ThreadPoolExecutor
-
-from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
-
-from .._coherence import CoherenceEvaluator
-from .._fluency import FluencyEvaluator
-from .._groundedness import GroundednessEvaluator
-from .._relevance import RelevanceEvaluator
-from .retrieval import RetrievalChatEvaluator
-
-logger = logging.getLogger(__name__)
-
-
-class ChatEvaluator:
-    """
-    Initialize a chat evaluator configured for a specific Azure OpenAI model.
-
-    :param model_config: Configuration for the Azure OpenAI model.
-    :type model_config: Union[~azure.ai.evaluation.AzureOpenAIModelConfiguration,
-        ~azure.ai.evaluation.OpenAIModelConfiguration]
-    :param eval_last_turn: Set to True to evaluate only the most recent exchange in the dialogue,
-        focusing on the latest user inquiry and the assistant's corresponding response. Defaults to False
-    :type eval_last_turn: bool
-    :param parallel: If True, use parallel execution for evaluators. Else, use sequential execution.
-        Default is True.
-    :type parallel: bool
-    :return: A function that evaluates and generates metrics for "chat" scenario.
-    :rtype: Callable
-
-    **Usage**
-
-    .. code-block:: python
-
-        chat_eval = ChatEvaluator(model_config)
-        conversation = [
-            {"role": "user", "content": "What is the value of 2 + 2?"},
-            {"role": "assistant", "content": "2 + 2 = 4", "context": {
-                "citations": [
-                        {"id": "math_doc.md", "content": "Information about additions: 1 + 2 = 3, 2 + 2 = 4"}
-                    ]
-                }
-            }
-        ]
-        result = chat_eval(conversation=conversation)
-
-    **Output format**
-
-    .. code-block:: python
-
-        {
-            "evaluation_per_turn": {
-                "gpt_retrieval": [1.0, 2.0],
-                "gpt_groundedness": [5.0, 2.0],
-                "gpt_relevance": [3.0, 5.0],
-                "gpt_coherence": [1.0, 2.0],
-                "gpt_fluency": [3.0, 5.0]
-            }
-            "gpt_retrieval": 1.5,
-            "gpt_groundedness": 3.5,
-            "gpt_relevance": 4.0,
-            "gpt_coherence": 1.5,
-            "gpt_fluency": 4.0
-        }
-    """
-
-    def __init__(
-        self,
-        model_config: dict,
-        eval_last_turn: bool = False,
-        parallel: bool = True,
-    ):
-        self._eval_last_turn = eval_last_turn
-        self._parallel = parallel
-
-        # TODO: Need a built-in evaluator for retrieval. It needs to be added to `self._rag_evaluators` collection
-        self._rag_evaluators = [
-            GroundednessEvaluator(model_config),
-            RelevanceEvaluator(model_config),
-        ]
-        self._non_rag_evaluators = [
-            CoherenceEvaluator(model_config),
-            FluencyEvaluator(model_config),
-        ]
-        # TODO: Temporary workaround to close the gap of missing retrieval score
-        # https://msdata.visualstudio.com/Vienna/_workitems/edit/3186644
-        # For long term, we need to add a built-in evaluator for retrieval after prompt is generalized for QA and Chat
-        self._retrieval_chat_evaluator = RetrievalChatEvaluator(model_config)
-
-    def __call__(self, *, conversation, **kwargs):
-        """
-        Evaluates chat scenario.
-
-        :keyword conversation: The conversation to be evaluated. Each turn should have "role" and "content" keys.
-            "context" key is optional for assistant's turn and should have "citations" key with list of citations.
-        :paramtype conversation: List[Dict]
-        :return: The scores for Chat scenario.
-        :rtype: dict
-        """
-        self._validate_conversation(conversation)
-
-        # Extract queries, responses and contexts from conversation
-        queries = []
-        responses = []
-        contexts = []
-
-        if self._eval_last_turn:
-            # Process only the last two turns if _eval_last_turn is True
-            conversation_slice = conversation[-2:] if len(conversation) >= 2 else conversation
-        else:
-            conversation_slice = conversation
-
-        for each_turn in conversation_slice:
-            role = each_turn["role"]
-            if role == "user":
-                queries.append(each_turn["content"])
-            elif role == "assistant":
-                responses.append(each_turn["content"])
-                if "context" in each_turn and "citations" in each_turn["context"]:
-                    citations = json.dumps(each_turn["context"]["citations"])
-                    contexts.append(citations)
-
-        # Select evaluators to be used for evaluation
-        compute_rag_based_metrics = True
-        if len(responses) != len(contexts):
-            safe_message = (
-                "Skipping rag based metrics as we need citations or "
-                "retrieved_documents in context key of every assistant's turn"
-            )
-            logger.warning(safe_message)
-            compute_rag_based_metrics = False
-
-        selected_evaluators = []
-        selected_evaluators.extend(self._non_rag_evaluators)
-        if compute_rag_based_metrics:
-            selected_evaluators.extend(self._rag_evaluators)
-
-        # Evaluate each turn
-        per_turn_results = []
-        for turn_num in range(len(queries)):
-            current_turn_result = {}
-
-            if self._parallel:
-                # Parallel execution
-                with ThreadPoolExecutor() as executor:
-                    future_to_evaluator = {
-                        executor.submit(
-                            self._evaluate_turn, turn_num, queries, responses, contexts, evaluator
-                        ): evaluator
-                        for evaluator in selected_evaluators
-                    }
-
-                    for future in as_completed(future_to_evaluator):
-                        result = future.result()
-                        current_turn_result.update(result)
-            else:
-                # Sequential execution
-                for evaluator in selected_evaluators:
-                    async_evaluator = evaluator._to_async()
-                    result = self._evaluate_turn(turn_num, queries, responses, contexts, async_evaluator)
-                    current_turn_result.update(result)
-
-            per_turn_results.append(current_turn_result)
-
-        # Aggregate results
-        # Final aggregated results for a conversation will look like:
-        #     "gpt_groundedness": 2.0, # Mean of all groundedness scores
-        #     "evaluation_per_turn": {
-        #         "gpt_groundedness": {
-        #             "score": [1.0, ...],
-        #             "reason": ["reason1", ...],
-        #         },
-        #     },
-        # }
-        aggregated = self._aggregate_results(per_turn_results)
-
-        # Run RetrievalChatEvaluator and merge the results
-        if compute_rag_based_metrics:
-            retrieval_score = self._retrieval_chat_evaluator(conversation=conversation_slice)
-            aggregated["gpt_retrieval"] = retrieval_score["gpt_retrieval"]
-            aggregated["evaluation_per_turn"]["gpt_retrieval"] = retrieval_score["evaluation_per_turn"]["gpt_retrieval"]
-            aggregated = dict(sorted(aggregated.items()))
-
-        return aggregated
-
-    def _evaluate_turn(self, turn_num, queries, responses, contexts, evaluator):
-        try:
-            query = queries[turn_num] if turn_num < len(queries) else ""
-            response = responses[turn_num] if turn_num < len(responses) else ""
-            context = contexts[turn_num] if turn_num < len(contexts) else ""
-
-            score = evaluator(query=query, response=response, context=context)
-
-            return score
-        except Exception as e:  # pylint: disable=broad-exception-caught
-            logger.warning(
-                "Evaluator %s failed for turn %s with exception: %s", evaluator.__class__.__name__, turn_num + 1, e
-            )
-            return {}
-
-    def _aggregate_results(self, per_turn_results: List[Dict]):
-        scores = {}
-        reasons = {}
-
-        for turn in per_turn_results:
-            for metric, value in turn.items():
-                if "reason" in metric:
-                    if metric not in reasons:
-                        reasons[metric] = []
-                    reasons[metric].append(value)
-                else:
-                    if metric not in scores:
-                        scores[metric] = []
-                    scores[metric].append(value)
-
-        aggregated = {}
-        evaluation_per_turn = {}
-
-        for metric, values in scores.items():
-            aggregated[metric] = np.nanmean(values)
-
-            # Prepare per-turn evaluations
-            evaluation_per_turn[metric] = {"score": values}
-            reason_key = f"{metric}_reason"
-            if reason_key in reasons:
-                evaluation_per_turn[metric]["reason"] = reasons[reason_key]
-
-        aggregated["evaluation_per_turn"] = evaluation_per_turn
-
-        return aggregated
-
-    def _validate_conversation(self, conversation: List[Dict]):
-        if conversation is None or not isinstance(conversation, list):
-            msg = "conversation must be a list of dictionaries"
-            raise EvaluationException(
-                message=msg,
-                internal_message=msg,
-                target=ErrorTarget.CHAT_EVALUATOR,
-                category=ErrorCategory.INVALID_VALUE,
-                blame=ErrorBlame.USER_ERROR,
-            )
-
-        expected_role = "user"
-        for turn_num, turn in enumerate(conversation):
-            one_based_turn_num = turn_num + 1
-
-            if not isinstance(turn, dict):
-                msg = f"Each turn in 'conversation' must be a dictionary. Turn number: {one_based_turn_num}"
-                raise EvaluationException(
-                    message=msg,
-                    internal_message=msg,
-                    target=ErrorTarget.CHAT_EVALUATOR,
-                    category=ErrorCategory.INVALID_VALUE,
-                    blame=ErrorBlame.USER_ERROR,
-                )
-
-            if "role" not in turn or "content" not in turn:
-                msg = (
-                    "Each turn in 'conversation' must have 'role' and 'content' keys. "
-                    + f"Turn number: {one_based_turn_num}"
-                )
-                raise EvaluationException(
-                    message=msg,
-                    internal_message=msg,
-                    target=ErrorTarget.CHAT_EVALUATOR,
-                    category=ErrorCategory.INVALID_VALUE,
-                    blame=ErrorBlame.USER_ERROR,
-                )
-
-            if turn["role"] != expected_role:
-                msg = f"Expected role {expected_role} but got {turn['role']}. Turn number: {one_based_turn_num}"
-                raise EvaluationException(
-                    message=msg,
-                    internal_message=msg,
-                    target=ErrorTarget.CHAT_EVALUATOR,
-                    category=ErrorCategory.INVALID_VALUE,
-                    blame=ErrorBlame.USER_ERROR,
-                )
-
-            if not isinstance(turn["content"], str):
-                msg = f"Content in each turn must be a string. Turn number: {one_based_turn_num}"
-                raise EvaluationException(
-                    message=msg,
-                    internal_message=msg,
-                    target=ErrorTarget.CHAT_EVALUATOR,
-                    category=ErrorCategory.INVALID_VALUE,
-                    blame=ErrorBlame.USER_ERROR,
-                )
-
-            if turn["role"] == "assistant" and "context" in turn:
-                if not isinstance(turn["context"], dict):
-                    msg = f"Context in each assistant's turn must be a dictionary. Turn number: {one_based_turn_num}"
-                    raise EvaluationException(
-                        message=msg,
-                        internal_message=msg,
-                        target=ErrorTarget.CHAT_EVALUATOR,
-                        category=ErrorCategory.INVALID_VALUE,
-                        blame=ErrorBlame.USER_ERROR,
-                    )
-
-                if "citations" not in turn["context"]:
-                    msg = (
-                        f"Context in each assistant's turn must have 'citations' key. Turn number: {one_based_turn_num}"
-                    )
-                    raise EvaluationException(
-                        message=msg,
-                        internal_message=msg,
-                        target=ErrorTarget.CHAT_EVALUATOR,
-                        category=ErrorCategory.MISSING_FIELD,
-                        blame=ErrorBlame.USER_ERROR,
-                    )
-
-                if not isinstance(turn["context"]["citations"], list):
-                    msg = f"'citations' in context must be a list. Turn number: {one_based_turn_num}"
-                    raise EvaluationException(
-                        message=msg,
-                        internal_message=msg,
-                        target=ErrorTarget.CHAT_EVALUATOR,
-                        category=ErrorCategory.INVALID_VALUE,
-                        blame=ErrorBlame.USER_ERROR,
-                    )
-
-                for citation_num, citation in enumerate(turn["context"]["citations"]):
-                    if not isinstance(citation, dict):
-                        msg = (
-                            "Each citation in 'citations' must be a dictionary. "
-                            + f"Turn number: {one_based_turn_num}, Citation number: {citation_num + 1}"
-                        )
-                        raise EvaluationException(
-                            message=msg,
-                            internal_message=msg,
-                            target=ErrorTarget.CHAT_EVALUATOR,
-                            category=ErrorCategory.INVALID_VALUE,
-                            blame=ErrorBlame.USER_ERROR,
-                        )
-
-            # Toggle expected role for the next turn
-            expected_role = "user" if expected_role == "assistant" else "assistant"
-
-        # Ensure the conversation ends with an assistant's turn
-        if expected_role != "user":
-            msg = "The conversation must end with an assistant's turn."
-            raise EvaluationException(
-                message=msg,
-                internal_message=msg,
-                target=ErrorTarget.CHAT_EVALUATOR,
-                category=ErrorCategory.INVALID_VALUE,
-                blame=ErrorBlame.USER_ERROR,
-            )

azure/ai/evaluation/_evaluators/_chat/retrieval/__init__.py

@@ -1,9 +0,0 @@
-# ---------------------------------------------------------
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# ---------------------------------------------------------
-
-from ._retrieval import RetrievalChatEvaluator
-
-__all__ = [
-    "RetrievalChatEvaluator",
-]

azure/ai/evaluation/_evaluators/_content_safety/_content_safety_base.py

@@ -1,65 +0,0 @@
-# ---------------------------------------------------------
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# ---------------------------------------------------------
-
-from abc import ABC
-
-from azure.ai.evaluation._common.constants import EvaluationMetrics
-from azure.ai.evaluation._common.rai_service import evaluate_with_rai_service
-from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
-
-
-class ContentSafetyEvaluatorBase(ABC):
-    """
-    Initialize a evaluator for a specified Evaluation Metric. Base class that is not
-    meant to be instantiated by users.
-
-
-    :param metric: The metric to be evaluated.
-    :type metric: ~azure.ai.evaluation._evaluators._content_safety.flow.constants.EvaluationMetrics
-    :param azure_ai_project: The scope of the Azure AI project.
-        It contains subscription id, resource group, and project name.
-    :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
-    :param credential: The credential for connecting to Azure AI project.
-    :type credential: ~azure.core.credentials.TokenCredential
-    """
-
-    def __init__(self, metric: EvaluationMetrics, azure_ai_project: dict, credential=None):
-        self._metric = metric
-        self._azure_ai_project = azure_ai_project
-        self._credential = credential
-
-    async def __call__(self, *, query: str, response: str, **kwargs):
-        """
-        Evaluates content according to this evaluator's metric.
-
-        :keyword query: The query to be evaluated.
-        :paramtype query: str
-        :keyword response: The response to be evaluated.
-        :paramtype response: str
-        :return: The evaluation score computation based on the Content Safety metric (self.metric).
-        :rtype: Any
-        """
-        # Validate inputs
-        # Raises value error if failed, so execution alone signifies success.
-        if not (query and query.strip() and query != "None") or not (
-            response and response.strip() and response != "None"
-        ):
-            msg = "Both 'query' and 'response' must be non-empty strings."
-            raise EvaluationException(
-                message=msg,
-                internal_message=msg,
-                error_category=ErrorCategory.MISSING_FIELD,
-                error_blame=ErrorBlame.USER_ERROR,
-                error_target=ErrorTarget.CONTENT_SAFETY_CHAT_EVALUATOR,
-            )
-
-        # Run score computation based on supplied metric.
-        result = await evaluate_with_rai_service(
-            metric_name=self._metric,
-            query=query,
-            response=response,
-            project_scope=self._azure_ai_project,
-            credential=self._credential,
-        )
-        return result

azure/ai/evaluation/_evaluators/_protected_materials/__init__.py

@@ -1,5 +0,0 @@
-from ._protected_materials import ProtectedMaterialsEvaluator
-
-__all__ = [
-    "ProtectedMaterialsEvaluator",
-]