azure-ai-evaluation 1.0.0b2__py3-none-any.whl → 1.13.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of azure-ai-evaluation might be problematic. Click here for more details.

Files changed (299) hide show
  1. azure/ai/evaluation/__init__.py +100 -5
  2. azure/ai/evaluation/{_evaluators/_chat → _aoai}/__init__.py +3 -2
  3. azure/ai/evaluation/_aoai/aoai_grader.py +140 -0
  4. azure/ai/evaluation/_aoai/label_grader.py +68 -0
  5. azure/ai/evaluation/_aoai/python_grader.py +86 -0
  6. azure/ai/evaluation/_aoai/score_model_grader.py +94 -0
  7. azure/ai/evaluation/_aoai/string_check_grader.py +66 -0
  8. azure/ai/evaluation/_aoai/text_similarity_grader.py +80 -0
  9. azure/ai/evaluation/_azure/__init__.py +3 -0
  10. azure/ai/evaluation/_azure/_clients.py +204 -0
  11. azure/ai/evaluation/_azure/_envs.py +207 -0
  12. azure/ai/evaluation/_azure/_models.py +227 -0
  13. azure/ai/evaluation/_azure/_token_manager.py +129 -0
  14. azure/ai/evaluation/_common/__init__.py +9 -1
  15. azure/ai/evaluation/{simulator/_helpers → _common}/_experimental.py +24 -9
  16. azure/ai/evaluation/_common/constants.py +131 -2
  17. azure/ai/evaluation/_common/evaluation_onedp_client.py +169 -0
  18. azure/ai/evaluation/_common/math.py +89 -0
  19. azure/ai/evaluation/_common/onedp/__init__.py +32 -0
  20. azure/ai/evaluation/_common/onedp/_client.py +166 -0
  21. azure/ai/evaluation/_common/onedp/_configuration.py +72 -0
  22. azure/ai/evaluation/_common/onedp/_model_base.py +1232 -0
  23. azure/ai/evaluation/_common/onedp/_patch.py +21 -0
  24. azure/ai/evaluation/_common/onedp/_serialization.py +2032 -0
  25. azure/ai/evaluation/_common/onedp/_types.py +21 -0
  26. azure/ai/evaluation/_common/onedp/_utils/__init__.py +6 -0
  27. azure/ai/evaluation/_common/onedp/_utils/model_base.py +1232 -0
  28. azure/ai/evaluation/_common/onedp/_utils/serialization.py +2032 -0
  29. azure/ai/evaluation/_common/onedp/_validation.py +66 -0
  30. azure/ai/evaluation/_common/onedp/_vendor.py +50 -0
  31. azure/ai/evaluation/_common/onedp/_version.py +9 -0
  32. azure/ai/evaluation/_common/onedp/aio/__init__.py +29 -0
  33. azure/ai/evaluation/_common/onedp/aio/_client.py +168 -0
  34. azure/ai/evaluation/_common/onedp/aio/_configuration.py +72 -0
  35. azure/ai/evaluation/_common/onedp/aio/_patch.py +21 -0
  36. azure/ai/evaluation/_common/onedp/aio/operations/__init__.py +49 -0
  37. azure/ai/evaluation/_common/onedp/aio/operations/_operations.py +7143 -0
  38. azure/ai/evaluation/_common/onedp/aio/operations/_patch.py +21 -0
  39. azure/ai/evaluation/_common/onedp/models/__init__.py +358 -0
  40. azure/ai/evaluation/_common/onedp/models/_enums.py +447 -0
  41. azure/ai/evaluation/_common/onedp/models/_models.py +5963 -0
  42. azure/ai/evaluation/_common/onedp/models/_patch.py +21 -0
  43. azure/ai/evaluation/_common/onedp/operations/__init__.py +49 -0
  44. azure/ai/evaluation/_common/onedp/operations/_operations.py +8951 -0
  45. azure/ai/evaluation/_common/onedp/operations/_patch.py +21 -0
  46. azure/ai/evaluation/_common/onedp/py.typed +1 -0
  47. azure/ai/evaluation/_common/onedp/servicepatterns/__init__.py +1 -0
  48. azure/ai/evaluation/_common/onedp/servicepatterns/aio/__init__.py +1 -0
  49. azure/ai/evaluation/_common/onedp/servicepatterns/aio/operations/__init__.py +25 -0
  50. azure/ai/evaluation/_common/onedp/servicepatterns/aio/operations/_operations.py +34 -0
  51. azure/ai/evaluation/_common/onedp/servicepatterns/aio/operations/_patch.py +20 -0
  52. azure/ai/evaluation/_common/onedp/servicepatterns/buildingblocks/__init__.py +1 -0
  53. azure/ai/evaluation/_common/onedp/servicepatterns/buildingblocks/aio/__init__.py +1 -0
  54. azure/ai/evaluation/_common/onedp/servicepatterns/buildingblocks/aio/operations/__init__.py +22 -0
  55. azure/ai/evaluation/_common/onedp/servicepatterns/buildingblocks/aio/operations/_operations.py +29 -0
  56. azure/ai/evaluation/_common/onedp/servicepatterns/buildingblocks/aio/operations/_patch.py +20 -0
  57. azure/ai/evaluation/_common/onedp/servicepatterns/buildingblocks/operations/__init__.py +22 -0
  58. azure/ai/evaluation/_common/onedp/servicepatterns/buildingblocks/operations/_operations.py +29 -0
  59. azure/ai/evaluation/_common/onedp/servicepatterns/buildingblocks/operations/_patch.py +20 -0
  60. azure/ai/evaluation/_common/onedp/servicepatterns/operations/__init__.py +25 -0
  61. azure/ai/evaluation/_common/onedp/servicepatterns/operations/_operations.py +34 -0
  62. azure/ai/evaluation/_common/onedp/servicepatterns/operations/_patch.py +20 -0
  63. azure/ai/evaluation/_common/rai_service.py +831 -142
  64. azure/ai/evaluation/_common/raiclient/__init__.py +34 -0
  65. azure/ai/evaluation/_common/raiclient/_client.py +128 -0
  66. azure/ai/evaluation/_common/raiclient/_configuration.py +87 -0
  67. azure/ai/evaluation/_common/raiclient/_model_base.py +1235 -0
  68. azure/ai/evaluation/_common/raiclient/_patch.py +20 -0
  69. azure/ai/evaluation/_common/raiclient/_serialization.py +2050 -0
  70. azure/ai/evaluation/_common/raiclient/_version.py +9 -0
  71. azure/ai/evaluation/_common/raiclient/aio/__init__.py +29 -0
  72. azure/ai/evaluation/_common/raiclient/aio/_client.py +130 -0
  73. azure/ai/evaluation/_common/raiclient/aio/_configuration.py +87 -0
  74. azure/ai/evaluation/_common/raiclient/aio/_patch.py +20 -0
  75. azure/ai/evaluation/_common/raiclient/aio/operations/__init__.py +25 -0
  76. azure/ai/evaluation/_common/raiclient/aio/operations/_operations.py +981 -0
  77. azure/ai/evaluation/_common/raiclient/aio/operations/_patch.py +20 -0
  78. azure/ai/evaluation/_common/raiclient/models/__init__.py +60 -0
  79. azure/ai/evaluation/_common/raiclient/models/_enums.py +18 -0
  80. azure/ai/evaluation/_common/raiclient/models/_models.py +651 -0
  81. azure/ai/evaluation/_common/raiclient/models/_patch.py +20 -0
  82. azure/ai/evaluation/_common/raiclient/operations/__init__.py +25 -0
  83. azure/ai/evaluation/_common/raiclient/operations/_operations.py +1238 -0
  84. azure/ai/evaluation/_common/raiclient/operations/_patch.py +20 -0
  85. azure/ai/evaluation/_common/raiclient/py.typed +1 -0
  86. azure/ai/evaluation/_common/utils.py +870 -34
  87. azure/ai/evaluation/_constants.py +167 -6
  88. azure/ai/evaluation/_converters/__init__.py +3 -0
  89. azure/ai/evaluation/_converters/_ai_services.py +899 -0
  90. azure/ai/evaluation/_converters/_models.py +467 -0
  91. azure/ai/evaluation/_converters/_sk_services.py +495 -0
  92. azure/ai/evaluation/_eval_mapping.py +83 -0
  93. azure/ai/evaluation/_evaluate/_batch_run/__init__.py +17 -0
  94. azure/ai/evaluation/_evaluate/_batch_run/_run_submitter_client.py +176 -0
  95. azure/ai/evaluation/_evaluate/_batch_run/batch_clients.py +82 -0
  96. azure/ai/evaluation/_evaluate/{_batch_run_client → _batch_run}/code_client.py +47 -25
  97. azure/ai/evaluation/_evaluate/{_batch_run_client/batch_run_context.py → _batch_run/eval_run_context.py} +42 -13
  98. azure/ai/evaluation/_evaluate/_batch_run/proxy_client.py +124 -0
  99. azure/ai/evaluation/_evaluate/_batch_run/target_run_context.py +62 -0
  100. azure/ai/evaluation/_evaluate/_eval_run.py +102 -59
  101. azure/ai/evaluation/_evaluate/_evaluate.py +2134 -311
  102. azure/ai/evaluation/_evaluate/_evaluate_aoai.py +992 -0
  103. azure/ai/evaluation/_evaluate/_telemetry/__init__.py +14 -99
  104. azure/ai/evaluation/_evaluate/_utils.py +289 -40
  105. azure/ai/evaluation/_evaluator_definition.py +76 -0
  106. azure/ai/evaluation/_evaluators/_bleu/_bleu.py +93 -42
  107. azure/ai/evaluation/_evaluators/_code_vulnerability/__init__.py +5 -0
  108. azure/ai/evaluation/_evaluators/_code_vulnerability/_code_vulnerability.py +119 -0
  109. azure/ai/evaluation/_evaluators/_coherence/_coherence.py +117 -91
  110. azure/ai/evaluation/_evaluators/_coherence/coherence.prompty +76 -39
  111. azure/ai/evaluation/_evaluators/_common/__init__.py +15 -0
  112. azure/ai/evaluation/_evaluators/_common/_base_eval.py +742 -0
  113. azure/ai/evaluation/_evaluators/_common/_base_multi_eval.py +63 -0
  114. azure/ai/evaluation/_evaluators/_common/_base_prompty_eval.py +345 -0
  115. azure/ai/evaluation/_evaluators/_common/_base_rai_svc_eval.py +198 -0
  116. azure/ai/evaluation/_evaluators/_common/_conversation_aggregators.py +49 -0
  117. azure/ai/evaluation/_evaluators/_content_safety/__init__.py +0 -4
  118. azure/ai/evaluation/_evaluators/_content_safety/_content_safety.py +144 -86
  119. azure/ai/evaluation/_evaluators/_content_safety/_hate_unfairness.py +138 -57
  120. azure/ai/evaluation/_evaluators/_content_safety/_self_harm.py +123 -55
  121. azure/ai/evaluation/_evaluators/_content_safety/_sexual.py +133 -54
  122. azure/ai/evaluation/_evaluators/_content_safety/_violence.py +134 -54
  123. azure/ai/evaluation/_evaluators/_document_retrieval/__init__.py +7 -0
  124. azure/ai/evaluation/_evaluators/_document_retrieval/_document_retrieval.py +442 -0
  125. azure/ai/evaluation/_evaluators/_eci/_eci.py +49 -56
  126. azure/ai/evaluation/_evaluators/_f1_score/_f1_score.py +102 -60
  127. azure/ai/evaluation/_evaluators/_fluency/_fluency.py +115 -92
  128. azure/ai/evaluation/_evaluators/_fluency/fluency.prompty +66 -41
  129. azure/ai/evaluation/_evaluators/_gleu/_gleu.py +90 -37
  130. azure/ai/evaluation/_evaluators/_groundedness/_groundedness.py +318 -82
  131. azure/ai/evaluation/_evaluators/_groundedness/groundedness_with_query.prompty +114 -0
  132. azure/ai/evaluation/_evaluators/_groundedness/groundedness_without_query.prompty +104 -0
  133. azure/ai/evaluation/{_evaluate/_batch_run_client → _evaluators/_intent_resolution}/__init__.py +3 -4
  134. azure/ai/evaluation/_evaluators/_intent_resolution/_intent_resolution.py +196 -0
  135. azure/ai/evaluation/_evaluators/_intent_resolution/intent_resolution.prompty +275 -0
  136. azure/ai/evaluation/_evaluators/_meteor/_meteor.py +107 -61
  137. azure/ai/evaluation/_evaluators/_protected_material/_protected_material.py +104 -77
  138. azure/ai/evaluation/_evaluators/_qa/_qa.py +115 -63
  139. azure/ai/evaluation/_evaluators/_relevance/_relevance.py +182 -98
  140. azure/ai/evaluation/_evaluators/_relevance/relevance.prompty +178 -49
  141. azure/ai/evaluation/_evaluators/_response_completeness/__init__.py +7 -0
  142. azure/ai/evaluation/_evaluators/_response_completeness/_response_completeness.py +202 -0
  143. azure/ai/evaluation/_evaluators/_response_completeness/response_completeness.prompty +84 -0
  144. azure/ai/evaluation/_evaluators/{_chat/retrieval → _retrieval}/__init__.py +2 -2
  145. azure/ai/evaluation/_evaluators/_retrieval/_retrieval.py +148 -0
  146. azure/ai/evaluation/_evaluators/_retrieval/retrieval.prompty +93 -0
  147. azure/ai/evaluation/_evaluators/_rouge/_rouge.py +189 -50
  148. azure/ai/evaluation/_evaluators/_service_groundedness/__init__.py +9 -0
  149. azure/ai/evaluation/_evaluators/_service_groundedness/_service_groundedness.py +179 -0
  150. azure/ai/evaluation/_evaluators/_similarity/_similarity.py +102 -91
  151. azure/ai/evaluation/_evaluators/_similarity/similarity.prompty +0 -5
  152. azure/ai/evaluation/_evaluators/_task_adherence/__init__.py +7 -0
  153. azure/ai/evaluation/_evaluators/_task_adherence/_task_adherence.py +226 -0
  154. azure/ai/evaluation/_evaluators/_task_adherence/task_adherence.prompty +101 -0
  155. azure/ai/evaluation/_evaluators/_task_completion/__init__.py +7 -0
  156. azure/ai/evaluation/_evaluators/_task_completion/_task_completion.py +177 -0
  157. azure/ai/evaluation/_evaluators/_task_completion/task_completion.prompty +220 -0
  158. azure/ai/evaluation/_evaluators/_task_navigation_efficiency/__init__.py +7 -0
  159. azure/ai/evaluation/_evaluators/_task_navigation_efficiency/_task_navigation_efficiency.py +384 -0
  160. azure/ai/evaluation/_evaluators/_tool_call_accuracy/__init__.py +9 -0
  161. azure/ai/evaluation/_evaluators/_tool_call_accuracy/_tool_call_accuracy.py +298 -0
  162. azure/ai/evaluation/_evaluators/_tool_call_accuracy/tool_call_accuracy.prompty +166 -0
  163. azure/ai/evaluation/_evaluators/_tool_input_accuracy/__init__.py +9 -0
  164. azure/ai/evaluation/_evaluators/_tool_input_accuracy/_tool_input_accuracy.py +263 -0
  165. azure/ai/evaluation/_evaluators/_tool_input_accuracy/tool_input_accuracy.prompty +76 -0
  166. azure/ai/evaluation/_evaluators/_tool_output_utilization/__init__.py +7 -0
  167. azure/ai/evaluation/_evaluators/_tool_output_utilization/_tool_output_utilization.py +225 -0
  168. azure/ai/evaluation/_evaluators/_tool_output_utilization/tool_output_utilization.prompty +221 -0
  169. azure/ai/evaluation/_evaluators/_tool_selection/__init__.py +9 -0
  170. azure/ai/evaluation/_evaluators/_tool_selection/_tool_selection.py +266 -0
  171. azure/ai/evaluation/_evaluators/_tool_selection/tool_selection.prompty +104 -0
  172. azure/ai/evaluation/_evaluators/_tool_success/__init__.py +7 -0
  173. azure/ai/evaluation/_evaluators/_tool_success/_tool_success.py +301 -0
  174. azure/ai/evaluation/_evaluators/_tool_success/tool_success.prompty +321 -0
  175. azure/ai/evaluation/_evaluators/_ungrounded_attributes/__init__.py +5 -0
  176. azure/ai/evaluation/_evaluators/_ungrounded_attributes/_ungrounded_attributes.py +102 -0
  177. azure/ai/evaluation/_evaluators/_xpia/xpia.py +109 -107
  178. azure/ai/evaluation/_exceptions.py +51 -7
  179. azure/ai/evaluation/_http_utils.py +210 -137
  180. azure/ai/evaluation/_legacy/__init__.py +3 -0
  181. azure/ai/evaluation/_legacy/_adapters/__init__.py +7 -0
  182. azure/ai/evaluation/_legacy/_adapters/_check.py +17 -0
  183. azure/ai/evaluation/_legacy/_adapters/_configuration.py +45 -0
  184. azure/ai/evaluation/_legacy/_adapters/_constants.py +10 -0
  185. azure/ai/evaluation/_legacy/_adapters/_errors.py +29 -0
  186. azure/ai/evaluation/_legacy/_adapters/_flows.py +28 -0
  187. azure/ai/evaluation/_legacy/_adapters/_service.py +16 -0
  188. azure/ai/evaluation/_legacy/_adapters/client.py +51 -0
  189. azure/ai/evaluation/_legacy/_adapters/entities.py +26 -0
  190. azure/ai/evaluation/_legacy/_adapters/tracing.py +28 -0
  191. azure/ai/evaluation/_legacy/_adapters/types.py +15 -0
  192. azure/ai/evaluation/_legacy/_adapters/utils.py +31 -0
  193. azure/ai/evaluation/_legacy/_batch_engine/__init__.py +9 -0
  194. azure/ai/evaluation/_legacy/_batch_engine/_config.py +48 -0
  195. azure/ai/evaluation/_legacy/_batch_engine/_engine.py +477 -0
  196. azure/ai/evaluation/_legacy/_batch_engine/_exceptions.py +88 -0
  197. azure/ai/evaluation/_legacy/_batch_engine/_openai_injector.py +132 -0
  198. azure/ai/evaluation/_legacy/_batch_engine/_result.py +107 -0
  199. azure/ai/evaluation/_legacy/_batch_engine/_run.py +127 -0
  200. azure/ai/evaluation/_legacy/_batch_engine/_run_storage.py +128 -0
  201. azure/ai/evaluation/_legacy/_batch_engine/_run_submitter.py +262 -0
  202. azure/ai/evaluation/_legacy/_batch_engine/_status.py +25 -0
  203. azure/ai/evaluation/_legacy/_batch_engine/_trace.py +97 -0
  204. azure/ai/evaluation/_legacy/_batch_engine/_utils.py +97 -0
  205. azure/ai/evaluation/_legacy/_batch_engine/_utils_deprecated.py +131 -0
  206. azure/ai/evaluation/_legacy/_common/__init__.py +3 -0
  207. azure/ai/evaluation/_legacy/_common/_async_token_provider.py +117 -0
  208. azure/ai/evaluation/_legacy/_common/_logging.py +292 -0
  209. azure/ai/evaluation/_legacy/_common/_thread_pool_executor_with_context.py +17 -0
  210. azure/ai/evaluation/_legacy/prompty/__init__.py +36 -0
  211. azure/ai/evaluation/_legacy/prompty/_connection.py +119 -0
  212. azure/ai/evaluation/_legacy/prompty/_exceptions.py +139 -0
  213. azure/ai/evaluation/_legacy/prompty/_prompty.py +430 -0
  214. azure/ai/evaluation/_legacy/prompty/_utils.py +663 -0
  215. azure/ai/evaluation/_legacy/prompty/_yaml_utils.py +99 -0
  216. azure/ai/evaluation/_model_configurations.py +130 -8
  217. azure/ai/evaluation/_safety_evaluation/__init__.py +3 -0
  218. azure/ai/evaluation/_safety_evaluation/_generated_rai_client.py +0 -0
  219. azure/ai/evaluation/_safety_evaluation/_safety_evaluation.py +917 -0
  220. azure/ai/evaluation/_user_agent.py +32 -1
  221. azure/ai/evaluation/_vendor/__init__.py +3 -0
  222. azure/ai/evaluation/_vendor/rouge_score/__init__.py +14 -0
  223. azure/ai/evaluation/_vendor/rouge_score/rouge_scorer.py +324 -0
  224. azure/ai/evaluation/_vendor/rouge_score/scoring.py +59 -0
  225. azure/ai/evaluation/_vendor/rouge_score/tokenize.py +59 -0
  226. azure/ai/evaluation/_vendor/rouge_score/tokenizers.py +53 -0
  227. azure/ai/evaluation/_version.py +2 -1
  228. azure/ai/evaluation/red_team/__init__.py +22 -0
  229. azure/ai/evaluation/red_team/_agent/__init__.py +3 -0
  230. azure/ai/evaluation/red_team/_agent/_agent_functions.py +261 -0
  231. azure/ai/evaluation/red_team/_agent/_agent_tools.py +461 -0
  232. azure/ai/evaluation/red_team/_agent/_agent_utils.py +89 -0
  233. azure/ai/evaluation/red_team/_agent/_semantic_kernel_plugin.py +228 -0
  234. azure/ai/evaluation/red_team/_attack_objective_generator.py +268 -0
  235. azure/ai/evaluation/red_team/_attack_strategy.py +49 -0
  236. azure/ai/evaluation/red_team/_callback_chat_target.py +115 -0
  237. azure/ai/evaluation/red_team/_default_converter.py +21 -0
  238. azure/ai/evaluation/red_team/_evaluation_processor.py +505 -0
  239. azure/ai/evaluation/red_team/_mlflow_integration.py +430 -0
  240. azure/ai/evaluation/red_team/_orchestrator_manager.py +803 -0
  241. azure/ai/evaluation/red_team/_red_team.py +1717 -0
  242. azure/ai/evaluation/red_team/_red_team_result.py +661 -0
  243. azure/ai/evaluation/red_team/_result_processor.py +1708 -0
  244. azure/ai/evaluation/red_team/_utils/__init__.py +37 -0
  245. azure/ai/evaluation/red_team/_utils/_rai_service_eval_chat_target.py +128 -0
  246. azure/ai/evaluation/red_team/_utils/_rai_service_target.py +601 -0
  247. azure/ai/evaluation/red_team/_utils/_rai_service_true_false_scorer.py +114 -0
  248. azure/ai/evaluation/red_team/_utils/constants.py +72 -0
  249. azure/ai/evaluation/red_team/_utils/exception_utils.py +345 -0
  250. azure/ai/evaluation/red_team/_utils/file_utils.py +266 -0
  251. azure/ai/evaluation/red_team/_utils/formatting_utils.py +365 -0
  252. azure/ai/evaluation/red_team/_utils/logging_utils.py +139 -0
  253. azure/ai/evaluation/red_team/_utils/metric_mapping.py +73 -0
  254. azure/ai/evaluation/red_team/_utils/objective_utils.py +46 -0
  255. azure/ai/evaluation/red_team/_utils/progress_utils.py +252 -0
  256. azure/ai/evaluation/red_team/_utils/retry_utils.py +218 -0
  257. azure/ai/evaluation/red_team/_utils/strategy_utils.py +218 -0
  258. azure/ai/evaluation/simulator/__init__.py +2 -1
  259. azure/ai/evaluation/simulator/_adversarial_scenario.py +26 -1
  260. azure/ai/evaluation/simulator/_adversarial_simulator.py +270 -144
  261. azure/ai/evaluation/simulator/_constants.py +12 -1
  262. azure/ai/evaluation/simulator/_conversation/__init__.py +151 -23
  263. azure/ai/evaluation/simulator/_conversation/_conversation.py +10 -6
  264. azure/ai/evaluation/simulator/_conversation/constants.py +1 -1
  265. azure/ai/evaluation/simulator/_data_sources/__init__.py +3 -0
  266. azure/ai/evaluation/simulator/_data_sources/grounding.json +1150 -0
  267. azure/ai/evaluation/simulator/_direct_attack_simulator.py +54 -75
  268. azure/ai/evaluation/simulator/_helpers/__init__.py +1 -2
  269. azure/ai/evaluation/simulator/_helpers/_language_suffix_mapping.py +1 -0
  270. azure/ai/evaluation/simulator/_helpers/_simulator_data_classes.py +26 -5
  271. azure/ai/evaluation/simulator/_indirect_attack_simulator.py +145 -104
  272. azure/ai/evaluation/simulator/_model_tools/__init__.py +2 -1
  273. azure/ai/evaluation/simulator/_model_tools/_generated_rai_client.py +225 -0
  274. azure/ai/evaluation/simulator/_model_tools/_identity_manager.py +80 -30
  275. azure/ai/evaluation/simulator/_model_tools/_proxy_completion_model.py +117 -45
  276. azure/ai/evaluation/simulator/_model_tools/_rai_client.py +109 -7
  277. azure/ai/evaluation/simulator/_model_tools/_template_handler.py +97 -33
  278. azure/ai/evaluation/simulator/_model_tools/models.py +30 -27
  279. azure/ai/evaluation/simulator/_prompty/task_query_response.prompty +6 -10
  280. azure/ai/evaluation/simulator/_prompty/task_simulate.prompty +6 -5
  281. azure/ai/evaluation/simulator/_simulator.py +302 -208
  282. azure/ai/evaluation/simulator/_utils.py +31 -13
  283. azure_ai_evaluation-1.13.3.dist-info/METADATA +939 -0
  284. azure_ai_evaluation-1.13.3.dist-info/RECORD +305 -0
  285. {azure_ai_evaluation-1.0.0b2.dist-info → azure_ai_evaluation-1.13.3.dist-info}/WHEEL +1 -1
  286. azure_ai_evaluation-1.13.3.dist-info/licenses/NOTICE.txt +70 -0
  287. azure/ai/evaluation/_evaluate/_batch_run_client/proxy_client.py +0 -71
  288. azure/ai/evaluation/_evaluators/_chat/_chat.py +0 -357
  289. azure/ai/evaluation/_evaluators/_chat/retrieval/_retrieval.py +0 -157
  290. azure/ai/evaluation/_evaluators/_chat/retrieval/retrieval.prompty +0 -48
  291. azure/ai/evaluation/_evaluators/_content_safety/_content_safety_base.py +0 -65
  292. azure/ai/evaluation/_evaluators/_content_safety/_content_safety_chat.py +0 -301
  293. azure/ai/evaluation/_evaluators/_groundedness/groundedness.prompty +0 -54
  294. azure/ai/evaluation/_evaluators/_protected_materials/__init__.py +0 -5
  295. azure/ai/evaluation/_evaluators/_protected_materials/_protected_materials.py +0 -104
  296. azure/ai/evaluation/simulator/_tracing.py +0 -89
  297. azure_ai_evaluation-1.0.0b2.dist-info/METADATA +0 -449
  298. azure_ai_evaluation-1.0.0b2.dist-info/RECORD +0 -99
  299. {azure_ai_evaluation-1.0.0b2.dist-info → azure_ai_evaluation-1.13.3.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,21 @@
1
+ # coding=utf-8
2
+ # --------------------------------------------------------------------------
3
+ # Copyright (c) Microsoft Corporation. All rights reserved.
4
+ # Licensed under the MIT License. See License.txt in the project root for license information.
5
+ # --------------------------------------------------------------------------
6
+ """Customize generated code here.
7
+
8
+ Follow our quickstart for examples: https://aka.ms/azsdk/python/dpcodegen/python/customize
9
+ """
10
+ from typing import List
11
+
12
+ __all__: List[str] = [] # Add all objects you want publicly available to users at this package level
13
+
14
+
15
+ def patch_sdk():
16
+ """Do not remove from this file.
17
+
18
+ `patch_sdk` is a last resort escape hatch that allows you to do customizations
19
+ you can't accomplish using the techniques described in
20
+ https://aka.ms/azsdk/python/dpcodegen/python/customize
21
+ """
@@ -0,0 +1,358 @@
1
+ # coding=utf-8
2
+ # --------------------------------------------------------------------------
3
+ # Copyright (c) Microsoft Corporation. All rights reserved.
4
+ # Licensed under the MIT License. See License.txt in the project root for license information.
5
+ # Code generated by Microsoft (R) Python Code Generator.
6
+ # Changes may cause incorrect behavior and will be lost if the code is regenerated.
7
+ # --------------------------------------------------------------------------
8
+ # pylint: disable=wrong-import-position
9
+
10
+ from typing import TYPE_CHECKING
11
+
12
+ if TYPE_CHECKING:
13
+ from ._patch import * # pylint: disable=unused-wildcard-import
14
+
15
+
16
+ from ._models import ( # type: ignore
17
+ AIContent,
18
+ AgentClusterInsightResult,
19
+ AgentClusterInsightsRequest,
20
+ AgentEvaluation,
21
+ AgentEvaluationRedactionConfiguration,
22
+ AgentEvaluationRequest,
23
+ AgentEvaluationResult,
24
+ AgentEvaluationSamplingConfiguration,
25
+ AgentTaxonomyInput,
26
+ AnnotationDTO,
27
+ ApiKeyCredentials,
28
+ AssetCredentialRequest,
29
+ AssetCredentialResponse,
30
+ AssistantMessage,
31
+ AttackMessage,
32
+ AttackObjective,
33
+ AzureAIAgentTarget,
34
+ AzureAIEvaluator,
35
+ AzureAISearchIndex,
36
+ AzureOpenAIModelConfiguration,
37
+ BaseCredentials,
38
+ BlobReference,
39
+ BlobReferenceForConsumption,
40
+ ChartCoordinate,
41
+ ChatChoice,
42
+ ChatCompletions,
43
+ ChatResponseMessage,
44
+ ClusterInsightResult,
45
+ ClusterTokenUsage,
46
+ CodeBasedEvaluatorDefinition,
47
+ CompletionsUsage,
48
+ CompletionsUsageDetails,
49
+ Connection,
50
+ Content,
51
+ ContinuousEvaluationRuleAction,
52
+ CosmosDBIndex,
53
+ CreateEvalJsonlRunDataSource,
54
+ CronTrigger,
55
+ CustomCredential,
56
+ CustomInference,
57
+ CustomizationParameters,
58
+ DailyRecurrenceSchedule,
59
+ DatasetVersion,
60
+ Deployment,
61
+ DeveloperMessage,
62
+ EmbeddingConfiguration,
63
+ EntraIDCredentials,
64
+ EvalCompareReport,
65
+ EvalJsonlFileContent,
66
+ EvalJsonlFileContentItem,
67
+ EvalJsonlFileContentSource,
68
+ EvalResult,
69
+ EvalRunOutputItem,
70
+ EvalRunResultCompareItem,
71
+ EvalRunResultComparison,
72
+ EvalRunResultSummary,
73
+ Evaluation,
74
+ EvaluationComparisonRequest,
75
+ EvaluationResult,
76
+ EvaluationResultSample,
77
+ EvaluationRule,
78
+ EvaluationRuleAction,
79
+ EvaluationRuleFilter,
80
+ EvaluationRunClusterInsightResult,
81
+ EvaluationRunClusterInsightsRequest,
82
+ EvaluationScheduleTask,
83
+ EvaluationTarget,
84
+ EvaluationTaxonomy,
85
+ EvaluationTaxonomyInput,
86
+ EvaluationUpload,
87
+ EvaluatorConfiguration,
88
+ EvaluatorDefinition,
89
+ EvaluatorMessage,
90
+ EvaluatorMetric,
91
+ EvaluatorVersion,
92
+ FieldMapping,
93
+ FileDatasetVersion,
94
+ FolderDatasetVersion,
95
+ HourlyRecurrenceSchedule,
96
+ HumanEvaluationRuleAction,
97
+ ImageSource,
98
+ ImageUrlContent,
99
+ Index,
100
+ InputData,
101
+ InputDataset,
102
+ Insight,
103
+ InsightCluster,
104
+ InsightModelConfiguration,
105
+ InsightRequest,
106
+ InsightResult,
107
+ InsightSample,
108
+ InsightScheduleTask,
109
+ InsightSummary,
110
+ InsightsMetadata,
111
+ LongRunningResponse,
112
+ ManagedAzureAISearchIndex,
113
+ Message,
114
+ Metadata,
115
+ ModelDeployment,
116
+ ModelResponseGenerationTarget,
117
+ MonthlyRecurrenceSchedule,
118
+ NoAuthenticationCredentials,
119
+ OneTimeTrigger,
120
+ PendingUploadRequest,
121
+ PendingUploadResponse,
122
+ PromptBasedEvaluatorDefinition,
123
+ PromptUsageDetails,
124
+ QueryResponseInlineMessage,
125
+ RecurrenceSchedule,
126
+ RecurrenceTrigger,
127
+ RedTeam,
128
+ RedTeamUpload,
129
+ SASCredentials,
130
+ SasCredential,
131
+ Schedule,
132
+ ScheduleRun,
133
+ ScheduleTask,
134
+ SimulationDTO,
135
+ Sku,
136
+ SyncEvalInput,
137
+ SystemData,
138
+ SystemMessage,
139
+ TargetConfig,
140
+ TargetHarm,
141
+ TaxonomyCategory,
142
+ TaxonomySubCategory,
143
+ TextContent,
144
+ ToolCallContent,
145
+ ToolDescription,
146
+ ToolResultContent,
147
+ Trigger,
148
+ UserMessage,
149
+ WeeklyRecurrenceSchedule,
150
+ )
151
+
152
+ from ._enums import ( # type: ignore
153
+ AttackStrategy,
154
+ ChatRole,
155
+ CompletionsFinishReason,
156
+ ConnectionType,
157
+ CredentialType,
158
+ DatasetType,
159
+ DayOfWeek,
160
+ DeploymentType,
161
+ EvaluationRuleActionType,
162
+ EvaluationRuleEventType,
163
+ EvaluationTargetType,
164
+ EvaluationTaxonomyInputType,
165
+ EvaluatorCategory,
166
+ EvaluatorDefinitionType,
167
+ EvaluatorMetricDirection,
168
+ EvaluatorMetricType,
169
+ EvaluatorType,
170
+ IndexType,
171
+ InsightType,
172
+ ListViewType,
173
+ OperationState,
174
+ PendingUploadType,
175
+ RecurrenceType,
176
+ ResultType,
177
+ RiskCategory,
178
+ SampleType,
179
+ ScheduleProvisioningStatus,
180
+ ScheduleTaskType,
181
+ SimulationType,
182
+ TargetType,
183
+ TreatmentEffectType,
184
+ TriggerType,
185
+ )
186
+ from ._patch import __all__ as _patch_all
187
+ from ._patch import *
188
+ from ._patch import patch_sdk as _patch_sdk
189
+
190
+ __all__ = [
191
+ "AIContent",
192
+ "AgentClusterInsightResult",
193
+ "AgentClusterInsightsRequest",
194
+ "AgentEvaluation",
195
+ "AgentEvaluationRedactionConfiguration",
196
+ "AgentEvaluationRequest",
197
+ "AgentEvaluationResult",
198
+ "AgentEvaluationSamplingConfiguration",
199
+ "AgentTaxonomyInput",
200
+ "AnnotationDTO",
201
+ "ApiKeyCredentials",
202
+ "AssetCredentialRequest",
203
+ "AssetCredentialResponse",
204
+ "AssistantMessage",
205
+ "AttackMessage",
206
+ "AttackObjective",
207
+ "AzureAIAgentTarget",
208
+ "AzureAIEvaluator",
209
+ "AzureAISearchIndex",
210
+ "AzureOpenAIModelConfiguration",
211
+ "BaseCredentials",
212
+ "BlobReference",
213
+ "BlobReferenceForConsumption",
214
+ "ChartCoordinate",
215
+ "ChatChoice",
216
+ "ChatCompletions",
217
+ "ChatResponseMessage",
218
+ "ClusterInsightResult",
219
+ "ClusterTokenUsage",
220
+ "CodeBasedEvaluatorDefinition",
221
+ "CompletionsUsage",
222
+ "CompletionsUsageDetails",
223
+ "Connection",
224
+ "Content",
225
+ "ContinuousEvaluationRuleAction",
226
+ "CosmosDBIndex",
227
+ "CreateEvalJsonlRunDataSource",
228
+ "CronTrigger",
229
+ "CustomCredential",
230
+ "CustomInference",
231
+ "CustomizationParameters",
232
+ "DailyRecurrenceSchedule",
233
+ "DatasetVersion",
234
+ "Deployment",
235
+ "DeveloperMessage",
236
+ "EmbeddingConfiguration",
237
+ "EntraIDCredentials",
238
+ "EvalCompareReport",
239
+ "EvalJsonlFileContent",
240
+ "EvalJsonlFileContentItem",
241
+ "EvalJsonlFileContentSource",
242
+ "EvalResult",
243
+ "EvalRunOutputItem",
244
+ "EvalRunResultCompareItem",
245
+ "EvalRunResultComparison",
246
+ "EvalRunResultSummary",
247
+ "Evaluation",
248
+ "EvaluationComparisonRequest",
249
+ "EvaluationResult",
250
+ "EvaluationResultSample",
251
+ "EvaluationRule",
252
+ "EvaluationRuleAction",
253
+ "EvaluationRuleFilter",
254
+ "EvaluationRunClusterInsightResult",
255
+ "EvaluationRunClusterInsightsRequest",
256
+ "EvaluationScheduleTask",
257
+ "EvaluationTarget",
258
+ "EvaluationTaxonomy",
259
+ "EvaluationTaxonomyInput",
260
+ "EvaluationUpload",
261
+ "EvaluatorConfiguration",
262
+ "EvaluatorDefinition",
263
+ "EvaluatorMessage",
264
+ "EvaluatorMetric",
265
+ "EvaluatorVersion",
266
+ "FieldMapping",
267
+ "FileDatasetVersion",
268
+ "FolderDatasetVersion",
269
+ "HourlyRecurrenceSchedule",
270
+ "HumanEvaluationRuleAction",
271
+ "ImageSource",
272
+ "ImageUrlContent",
273
+ "Index",
274
+ "InputData",
275
+ "InputDataset",
276
+ "Insight",
277
+ "InsightCluster",
278
+ "InsightModelConfiguration",
279
+ "InsightRequest",
280
+ "InsightResult",
281
+ "InsightSample",
282
+ "InsightScheduleTask",
283
+ "InsightSummary",
284
+ "InsightsMetadata",
285
+ "LongRunningResponse",
286
+ "ManagedAzureAISearchIndex",
287
+ "Message",
288
+ "Metadata",
289
+ "ModelDeployment",
290
+ "ModelResponseGenerationTarget",
291
+ "MonthlyRecurrenceSchedule",
292
+ "NoAuthenticationCredentials",
293
+ "OneTimeTrigger",
294
+ "PendingUploadRequest",
295
+ "PendingUploadResponse",
296
+ "PromptBasedEvaluatorDefinition",
297
+ "PromptUsageDetails",
298
+ "QueryResponseInlineMessage",
299
+ "RecurrenceSchedule",
300
+ "RecurrenceTrigger",
301
+ "RedTeam",
302
+ "RedTeamUpload",
303
+ "SASCredentials",
304
+ "SasCredential",
305
+ "Schedule",
306
+ "ScheduleRun",
307
+ "ScheduleTask",
308
+ "SimulationDTO",
309
+ "Sku",
310
+ "SyncEvalInput",
311
+ "SystemData",
312
+ "SystemMessage",
313
+ "TargetConfig",
314
+ "TargetHarm",
315
+ "TaxonomyCategory",
316
+ "TaxonomySubCategory",
317
+ "TextContent",
318
+ "ToolCallContent",
319
+ "ToolDescription",
320
+ "ToolResultContent",
321
+ "Trigger",
322
+ "UserMessage",
323
+ "WeeklyRecurrenceSchedule",
324
+ "AttackStrategy",
325
+ "ChatRole",
326
+ "CompletionsFinishReason",
327
+ "ConnectionType",
328
+ "CredentialType",
329
+ "DatasetType",
330
+ "DayOfWeek",
331
+ "DeploymentType",
332
+ "EvaluationRuleActionType",
333
+ "EvaluationRuleEventType",
334
+ "EvaluationTargetType",
335
+ "EvaluationTaxonomyInputType",
336
+ "EvaluatorCategory",
337
+ "EvaluatorDefinitionType",
338
+ "EvaluatorMetricDirection",
339
+ "EvaluatorMetricType",
340
+ "EvaluatorType",
341
+ "IndexType",
342
+ "InsightType",
343
+ "ListViewType",
344
+ "OperationState",
345
+ "PendingUploadType",
346
+ "RecurrenceType",
347
+ "ResultType",
348
+ "RiskCategory",
349
+ "SampleType",
350
+ "ScheduleProvisioningStatus",
351
+ "ScheduleTaskType",
352
+ "SimulationType",
353
+ "TargetType",
354
+ "TreatmentEffectType",
355
+ "TriggerType",
356
+ ]
357
+ __all__.extend([p for p in _patch_all if p not in __all__]) # pyright: ignore
358
+ _patch_sdk()