azure-ai-evaluation 1.0.0b2__py3-none-any.whl → 1.13.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of azure-ai-evaluation might be problematic. Click here for more details.

Files changed (299) hide show
  1. azure/ai/evaluation/__init__.py +100 -5
  2. azure/ai/evaluation/{_evaluators/_chat → _aoai}/__init__.py +3 -2
  3. azure/ai/evaluation/_aoai/aoai_grader.py +140 -0
  4. azure/ai/evaluation/_aoai/label_grader.py +68 -0
  5. azure/ai/evaluation/_aoai/python_grader.py +86 -0
  6. azure/ai/evaluation/_aoai/score_model_grader.py +94 -0
  7. azure/ai/evaluation/_aoai/string_check_grader.py +66 -0
  8. azure/ai/evaluation/_aoai/text_similarity_grader.py +80 -0
  9. azure/ai/evaluation/_azure/__init__.py +3 -0
  10. azure/ai/evaluation/_azure/_clients.py +204 -0
  11. azure/ai/evaluation/_azure/_envs.py +207 -0
  12. azure/ai/evaluation/_azure/_models.py +227 -0
  13. azure/ai/evaluation/_azure/_token_manager.py +129 -0
  14. azure/ai/evaluation/_common/__init__.py +9 -1
  15. azure/ai/evaluation/{simulator/_helpers → _common}/_experimental.py +24 -9
  16. azure/ai/evaluation/_common/constants.py +131 -2
  17. azure/ai/evaluation/_common/evaluation_onedp_client.py +169 -0
  18. azure/ai/evaluation/_common/math.py +89 -0
  19. azure/ai/evaluation/_common/onedp/__init__.py +32 -0
  20. azure/ai/evaluation/_common/onedp/_client.py +166 -0
  21. azure/ai/evaluation/_common/onedp/_configuration.py +72 -0
  22. azure/ai/evaluation/_common/onedp/_model_base.py +1232 -0
  23. azure/ai/evaluation/_common/onedp/_patch.py +21 -0
  24. azure/ai/evaluation/_common/onedp/_serialization.py +2032 -0
  25. azure/ai/evaluation/_common/onedp/_types.py +21 -0
  26. azure/ai/evaluation/_common/onedp/_utils/__init__.py +6 -0
  27. azure/ai/evaluation/_common/onedp/_utils/model_base.py +1232 -0
  28. azure/ai/evaluation/_common/onedp/_utils/serialization.py +2032 -0
  29. azure/ai/evaluation/_common/onedp/_validation.py +66 -0
  30. azure/ai/evaluation/_common/onedp/_vendor.py +50 -0
  31. azure/ai/evaluation/_common/onedp/_version.py +9 -0
  32. azure/ai/evaluation/_common/onedp/aio/__init__.py +29 -0
  33. azure/ai/evaluation/_common/onedp/aio/_client.py +168 -0
  34. azure/ai/evaluation/_common/onedp/aio/_configuration.py +72 -0
  35. azure/ai/evaluation/_common/onedp/aio/_patch.py +21 -0
  36. azure/ai/evaluation/_common/onedp/aio/operations/__init__.py +49 -0
  37. azure/ai/evaluation/_common/onedp/aio/operations/_operations.py +7143 -0
  38. azure/ai/evaluation/_common/onedp/aio/operations/_patch.py +21 -0
  39. azure/ai/evaluation/_common/onedp/models/__init__.py +358 -0
  40. azure/ai/evaluation/_common/onedp/models/_enums.py +447 -0
  41. azure/ai/evaluation/_common/onedp/models/_models.py +5963 -0
  42. azure/ai/evaluation/_common/onedp/models/_patch.py +21 -0
  43. azure/ai/evaluation/_common/onedp/operations/__init__.py +49 -0
  44. azure/ai/evaluation/_common/onedp/operations/_operations.py +8951 -0
  45. azure/ai/evaluation/_common/onedp/operations/_patch.py +21 -0
  46. azure/ai/evaluation/_common/onedp/py.typed +1 -0
  47. azure/ai/evaluation/_common/onedp/servicepatterns/__init__.py +1 -0
  48. azure/ai/evaluation/_common/onedp/servicepatterns/aio/__init__.py +1 -0
  49. azure/ai/evaluation/_common/onedp/servicepatterns/aio/operations/__init__.py +25 -0
  50. azure/ai/evaluation/_common/onedp/servicepatterns/aio/operations/_operations.py +34 -0
  51. azure/ai/evaluation/_common/onedp/servicepatterns/aio/operations/_patch.py +20 -0
  52. azure/ai/evaluation/_common/onedp/servicepatterns/buildingblocks/__init__.py +1 -0
  53. azure/ai/evaluation/_common/onedp/servicepatterns/buildingblocks/aio/__init__.py +1 -0
  54. azure/ai/evaluation/_common/onedp/servicepatterns/buildingblocks/aio/operations/__init__.py +22 -0
  55. azure/ai/evaluation/_common/onedp/servicepatterns/buildingblocks/aio/operations/_operations.py +29 -0
  56. azure/ai/evaluation/_common/onedp/servicepatterns/buildingblocks/aio/operations/_patch.py +20 -0
  57. azure/ai/evaluation/_common/onedp/servicepatterns/buildingblocks/operations/__init__.py +22 -0
  58. azure/ai/evaluation/_common/onedp/servicepatterns/buildingblocks/operations/_operations.py +29 -0
  59. azure/ai/evaluation/_common/onedp/servicepatterns/buildingblocks/operations/_patch.py +20 -0
  60. azure/ai/evaluation/_common/onedp/servicepatterns/operations/__init__.py +25 -0
  61. azure/ai/evaluation/_common/onedp/servicepatterns/operations/_operations.py +34 -0
  62. azure/ai/evaluation/_common/onedp/servicepatterns/operations/_patch.py +20 -0
  63. azure/ai/evaluation/_common/rai_service.py +831 -142
  64. azure/ai/evaluation/_common/raiclient/__init__.py +34 -0
  65. azure/ai/evaluation/_common/raiclient/_client.py +128 -0
  66. azure/ai/evaluation/_common/raiclient/_configuration.py +87 -0
  67. azure/ai/evaluation/_common/raiclient/_model_base.py +1235 -0
  68. azure/ai/evaluation/_common/raiclient/_patch.py +20 -0
  69. azure/ai/evaluation/_common/raiclient/_serialization.py +2050 -0
  70. azure/ai/evaluation/_common/raiclient/_version.py +9 -0
  71. azure/ai/evaluation/_common/raiclient/aio/__init__.py +29 -0
  72. azure/ai/evaluation/_common/raiclient/aio/_client.py +130 -0
  73. azure/ai/evaluation/_common/raiclient/aio/_configuration.py +87 -0
  74. azure/ai/evaluation/_common/raiclient/aio/_patch.py +20 -0
  75. azure/ai/evaluation/_common/raiclient/aio/operations/__init__.py +25 -0
  76. azure/ai/evaluation/_common/raiclient/aio/operations/_operations.py +981 -0
  77. azure/ai/evaluation/_common/raiclient/aio/operations/_patch.py +20 -0
  78. azure/ai/evaluation/_common/raiclient/models/__init__.py +60 -0
  79. azure/ai/evaluation/_common/raiclient/models/_enums.py +18 -0
  80. azure/ai/evaluation/_common/raiclient/models/_models.py +651 -0
  81. azure/ai/evaluation/_common/raiclient/models/_patch.py +20 -0
  82. azure/ai/evaluation/_common/raiclient/operations/__init__.py +25 -0
  83. azure/ai/evaluation/_common/raiclient/operations/_operations.py +1238 -0
  84. azure/ai/evaluation/_common/raiclient/operations/_patch.py +20 -0
  85. azure/ai/evaluation/_common/raiclient/py.typed +1 -0
  86. azure/ai/evaluation/_common/utils.py +870 -34
  87. azure/ai/evaluation/_constants.py +167 -6
  88. azure/ai/evaluation/_converters/__init__.py +3 -0
  89. azure/ai/evaluation/_converters/_ai_services.py +899 -0
  90. azure/ai/evaluation/_converters/_models.py +467 -0
  91. azure/ai/evaluation/_converters/_sk_services.py +495 -0
  92. azure/ai/evaluation/_eval_mapping.py +83 -0
  93. azure/ai/evaluation/_evaluate/_batch_run/__init__.py +17 -0
  94. azure/ai/evaluation/_evaluate/_batch_run/_run_submitter_client.py +176 -0
  95. azure/ai/evaluation/_evaluate/_batch_run/batch_clients.py +82 -0
  96. azure/ai/evaluation/_evaluate/{_batch_run_client → _batch_run}/code_client.py +47 -25
  97. azure/ai/evaluation/_evaluate/{_batch_run_client/batch_run_context.py → _batch_run/eval_run_context.py} +42 -13
  98. azure/ai/evaluation/_evaluate/_batch_run/proxy_client.py +124 -0
  99. azure/ai/evaluation/_evaluate/_batch_run/target_run_context.py +62 -0
  100. azure/ai/evaluation/_evaluate/_eval_run.py +102 -59
  101. azure/ai/evaluation/_evaluate/_evaluate.py +2134 -311
  102. azure/ai/evaluation/_evaluate/_evaluate_aoai.py +992 -0
  103. azure/ai/evaluation/_evaluate/_telemetry/__init__.py +14 -99
  104. azure/ai/evaluation/_evaluate/_utils.py +289 -40
  105. azure/ai/evaluation/_evaluator_definition.py +76 -0
  106. azure/ai/evaluation/_evaluators/_bleu/_bleu.py +93 -42
  107. azure/ai/evaluation/_evaluators/_code_vulnerability/__init__.py +5 -0
  108. azure/ai/evaluation/_evaluators/_code_vulnerability/_code_vulnerability.py +119 -0
  109. azure/ai/evaluation/_evaluators/_coherence/_coherence.py +117 -91
  110. azure/ai/evaluation/_evaluators/_coherence/coherence.prompty +76 -39
  111. azure/ai/evaluation/_evaluators/_common/__init__.py +15 -0
  112. azure/ai/evaluation/_evaluators/_common/_base_eval.py +742 -0
  113. azure/ai/evaluation/_evaluators/_common/_base_multi_eval.py +63 -0
  114. azure/ai/evaluation/_evaluators/_common/_base_prompty_eval.py +345 -0
  115. azure/ai/evaluation/_evaluators/_common/_base_rai_svc_eval.py +198 -0
  116. azure/ai/evaluation/_evaluators/_common/_conversation_aggregators.py +49 -0
  117. azure/ai/evaluation/_evaluators/_content_safety/__init__.py +0 -4
  118. azure/ai/evaluation/_evaluators/_content_safety/_content_safety.py +144 -86
  119. azure/ai/evaluation/_evaluators/_content_safety/_hate_unfairness.py +138 -57
  120. azure/ai/evaluation/_evaluators/_content_safety/_self_harm.py +123 -55
  121. azure/ai/evaluation/_evaluators/_content_safety/_sexual.py +133 -54
  122. azure/ai/evaluation/_evaluators/_content_safety/_violence.py +134 -54
  123. azure/ai/evaluation/_evaluators/_document_retrieval/__init__.py +7 -0
  124. azure/ai/evaluation/_evaluators/_document_retrieval/_document_retrieval.py +442 -0
  125. azure/ai/evaluation/_evaluators/_eci/_eci.py +49 -56
  126. azure/ai/evaluation/_evaluators/_f1_score/_f1_score.py +102 -60
  127. azure/ai/evaluation/_evaluators/_fluency/_fluency.py +115 -92
  128. azure/ai/evaluation/_evaluators/_fluency/fluency.prompty +66 -41
  129. azure/ai/evaluation/_evaluators/_gleu/_gleu.py +90 -37
  130. azure/ai/evaluation/_evaluators/_groundedness/_groundedness.py +318 -82
  131. azure/ai/evaluation/_evaluators/_groundedness/groundedness_with_query.prompty +114 -0
  132. azure/ai/evaluation/_evaluators/_groundedness/groundedness_without_query.prompty +104 -0
  133. azure/ai/evaluation/{_evaluate/_batch_run_client → _evaluators/_intent_resolution}/__init__.py +3 -4
  134. azure/ai/evaluation/_evaluators/_intent_resolution/_intent_resolution.py +196 -0
  135. azure/ai/evaluation/_evaluators/_intent_resolution/intent_resolution.prompty +275 -0
  136. azure/ai/evaluation/_evaluators/_meteor/_meteor.py +107 -61
  137. azure/ai/evaluation/_evaluators/_protected_material/_protected_material.py +104 -77
  138. azure/ai/evaluation/_evaluators/_qa/_qa.py +115 -63
  139. azure/ai/evaluation/_evaluators/_relevance/_relevance.py +182 -98
  140. azure/ai/evaluation/_evaluators/_relevance/relevance.prompty +178 -49
  141. azure/ai/evaluation/_evaluators/_response_completeness/__init__.py +7 -0
  142. azure/ai/evaluation/_evaluators/_response_completeness/_response_completeness.py +202 -0
  143. azure/ai/evaluation/_evaluators/_response_completeness/response_completeness.prompty +84 -0
  144. azure/ai/evaluation/_evaluators/{_chat/retrieval → _retrieval}/__init__.py +2 -2
  145. azure/ai/evaluation/_evaluators/_retrieval/_retrieval.py +148 -0
  146. azure/ai/evaluation/_evaluators/_retrieval/retrieval.prompty +93 -0
  147. azure/ai/evaluation/_evaluators/_rouge/_rouge.py +189 -50
  148. azure/ai/evaluation/_evaluators/_service_groundedness/__init__.py +9 -0
  149. azure/ai/evaluation/_evaluators/_service_groundedness/_service_groundedness.py +179 -0
  150. azure/ai/evaluation/_evaluators/_similarity/_similarity.py +102 -91
  151. azure/ai/evaluation/_evaluators/_similarity/similarity.prompty +0 -5
  152. azure/ai/evaluation/_evaluators/_task_adherence/__init__.py +7 -0
  153. azure/ai/evaluation/_evaluators/_task_adherence/_task_adherence.py +226 -0
  154. azure/ai/evaluation/_evaluators/_task_adherence/task_adherence.prompty +101 -0
  155. azure/ai/evaluation/_evaluators/_task_completion/__init__.py +7 -0
  156. azure/ai/evaluation/_evaluators/_task_completion/_task_completion.py +177 -0
  157. azure/ai/evaluation/_evaluators/_task_completion/task_completion.prompty +220 -0
  158. azure/ai/evaluation/_evaluators/_task_navigation_efficiency/__init__.py +7 -0
  159. azure/ai/evaluation/_evaluators/_task_navigation_efficiency/_task_navigation_efficiency.py +384 -0
  160. azure/ai/evaluation/_evaluators/_tool_call_accuracy/__init__.py +9 -0
  161. azure/ai/evaluation/_evaluators/_tool_call_accuracy/_tool_call_accuracy.py +298 -0
  162. azure/ai/evaluation/_evaluators/_tool_call_accuracy/tool_call_accuracy.prompty +166 -0
  163. azure/ai/evaluation/_evaluators/_tool_input_accuracy/__init__.py +9 -0
  164. azure/ai/evaluation/_evaluators/_tool_input_accuracy/_tool_input_accuracy.py +263 -0
  165. azure/ai/evaluation/_evaluators/_tool_input_accuracy/tool_input_accuracy.prompty +76 -0
  166. azure/ai/evaluation/_evaluators/_tool_output_utilization/__init__.py +7 -0
  167. azure/ai/evaluation/_evaluators/_tool_output_utilization/_tool_output_utilization.py +225 -0
  168. azure/ai/evaluation/_evaluators/_tool_output_utilization/tool_output_utilization.prompty +221 -0
  169. azure/ai/evaluation/_evaluators/_tool_selection/__init__.py +9 -0
  170. azure/ai/evaluation/_evaluators/_tool_selection/_tool_selection.py +266 -0
  171. azure/ai/evaluation/_evaluators/_tool_selection/tool_selection.prompty +104 -0
  172. azure/ai/evaluation/_evaluators/_tool_success/__init__.py +7 -0
  173. azure/ai/evaluation/_evaluators/_tool_success/_tool_success.py +301 -0
  174. azure/ai/evaluation/_evaluators/_tool_success/tool_success.prompty +321 -0
  175. azure/ai/evaluation/_evaluators/_ungrounded_attributes/__init__.py +5 -0
  176. azure/ai/evaluation/_evaluators/_ungrounded_attributes/_ungrounded_attributes.py +102 -0
  177. azure/ai/evaluation/_evaluators/_xpia/xpia.py +109 -107
  178. azure/ai/evaluation/_exceptions.py +51 -7
  179. azure/ai/evaluation/_http_utils.py +210 -137
  180. azure/ai/evaluation/_legacy/__init__.py +3 -0
  181. azure/ai/evaluation/_legacy/_adapters/__init__.py +7 -0
  182. azure/ai/evaluation/_legacy/_adapters/_check.py +17 -0
  183. azure/ai/evaluation/_legacy/_adapters/_configuration.py +45 -0
  184. azure/ai/evaluation/_legacy/_adapters/_constants.py +10 -0
  185. azure/ai/evaluation/_legacy/_adapters/_errors.py +29 -0
  186. azure/ai/evaluation/_legacy/_adapters/_flows.py +28 -0
  187. azure/ai/evaluation/_legacy/_adapters/_service.py +16 -0
  188. azure/ai/evaluation/_legacy/_adapters/client.py +51 -0
  189. azure/ai/evaluation/_legacy/_adapters/entities.py +26 -0
  190. azure/ai/evaluation/_legacy/_adapters/tracing.py +28 -0
  191. azure/ai/evaluation/_legacy/_adapters/types.py +15 -0
  192. azure/ai/evaluation/_legacy/_adapters/utils.py +31 -0
  193. azure/ai/evaluation/_legacy/_batch_engine/__init__.py +9 -0
  194. azure/ai/evaluation/_legacy/_batch_engine/_config.py +48 -0
  195. azure/ai/evaluation/_legacy/_batch_engine/_engine.py +477 -0
  196. azure/ai/evaluation/_legacy/_batch_engine/_exceptions.py +88 -0
  197. azure/ai/evaluation/_legacy/_batch_engine/_openai_injector.py +132 -0
  198. azure/ai/evaluation/_legacy/_batch_engine/_result.py +107 -0
  199. azure/ai/evaluation/_legacy/_batch_engine/_run.py +127 -0
  200. azure/ai/evaluation/_legacy/_batch_engine/_run_storage.py +128 -0
  201. azure/ai/evaluation/_legacy/_batch_engine/_run_submitter.py +262 -0
  202. azure/ai/evaluation/_legacy/_batch_engine/_status.py +25 -0
  203. azure/ai/evaluation/_legacy/_batch_engine/_trace.py +97 -0
  204. azure/ai/evaluation/_legacy/_batch_engine/_utils.py +97 -0
  205. azure/ai/evaluation/_legacy/_batch_engine/_utils_deprecated.py +131 -0
  206. azure/ai/evaluation/_legacy/_common/__init__.py +3 -0
  207. azure/ai/evaluation/_legacy/_common/_async_token_provider.py +117 -0
  208. azure/ai/evaluation/_legacy/_common/_logging.py +292 -0
  209. azure/ai/evaluation/_legacy/_common/_thread_pool_executor_with_context.py +17 -0
  210. azure/ai/evaluation/_legacy/prompty/__init__.py +36 -0
  211. azure/ai/evaluation/_legacy/prompty/_connection.py +119 -0
  212. azure/ai/evaluation/_legacy/prompty/_exceptions.py +139 -0
  213. azure/ai/evaluation/_legacy/prompty/_prompty.py +430 -0
  214. azure/ai/evaluation/_legacy/prompty/_utils.py +663 -0
  215. azure/ai/evaluation/_legacy/prompty/_yaml_utils.py +99 -0
  216. azure/ai/evaluation/_model_configurations.py +130 -8
  217. azure/ai/evaluation/_safety_evaluation/__init__.py +3 -0
  218. azure/ai/evaluation/_safety_evaluation/_generated_rai_client.py +0 -0
  219. azure/ai/evaluation/_safety_evaluation/_safety_evaluation.py +917 -0
  220. azure/ai/evaluation/_user_agent.py +32 -1
  221. azure/ai/evaluation/_vendor/__init__.py +3 -0
  222. azure/ai/evaluation/_vendor/rouge_score/__init__.py +14 -0
  223. azure/ai/evaluation/_vendor/rouge_score/rouge_scorer.py +324 -0
  224. azure/ai/evaluation/_vendor/rouge_score/scoring.py +59 -0
  225. azure/ai/evaluation/_vendor/rouge_score/tokenize.py +59 -0
  226. azure/ai/evaluation/_vendor/rouge_score/tokenizers.py +53 -0
  227. azure/ai/evaluation/_version.py +2 -1
  228. azure/ai/evaluation/red_team/__init__.py +22 -0
  229. azure/ai/evaluation/red_team/_agent/__init__.py +3 -0
  230. azure/ai/evaluation/red_team/_agent/_agent_functions.py +261 -0
  231. azure/ai/evaluation/red_team/_agent/_agent_tools.py +461 -0
  232. azure/ai/evaluation/red_team/_agent/_agent_utils.py +89 -0
  233. azure/ai/evaluation/red_team/_agent/_semantic_kernel_plugin.py +228 -0
  234. azure/ai/evaluation/red_team/_attack_objective_generator.py +268 -0
  235. azure/ai/evaluation/red_team/_attack_strategy.py +49 -0
  236. azure/ai/evaluation/red_team/_callback_chat_target.py +115 -0
  237. azure/ai/evaluation/red_team/_default_converter.py +21 -0
  238. azure/ai/evaluation/red_team/_evaluation_processor.py +505 -0
  239. azure/ai/evaluation/red_team/_mlflow_integration.py +430 -0
  240. azure/ai/evaluation/red_team/_orchestrator_manager.py +803 -0
  241. azure/ai/evaluation/red_team/_red_team.py +1717 -0
  242. azure/ai/evaluation/red_team/_red_team_result.py +661 -0
  243. azure/ai/evaluation/red_team/_result_processor.py +1708 -0
  244. azure/ai/evaluation/red_team/_utils/__init__.py +37 -0
  245. azure/ai/evaluation/red_team/_utils/_rai_service_eval_chat_target.py +128 -0
  246. azure/ai/evaluation/red_team/_utils/_rai_service_target.py +601 -0
  247. azure/ai/evaluation/red_team/_utils/_rai_service_true_false_scorer.py +114 -0
  248. azure/ai/evaluation/red_team/_utils/constants.py +72 -0
  249. azure/ai/evaluation/red_team/_utils/exception_utils.py +345 -0
  250. azure/ai/evaluation/red_team/_utils/file_utils.py +266 -0
  251. azure/ai/evaluation/red_team/_utils/formatting_utils.py +365 -0
  252. azure/ai/evaluation/red_team/_utils/logging_utils.py +139 -0
  253. azure/ai/evaluation/red_team/_utils/metric_mapping.py +73 -0
  254. azure/ai/evaluation/red_team/_utils/objective_utils.py +46 -0
  255. azure/ai/evaluation/red_team/_utils/progress_utils.py +252 -0
  256. azure/ai/evaluation/red_team/_utils/retry_utils.py +218 -0
  257. azure/ai/evaluation/red_team/_utils/strategy_utils.py +218 -0
  258. azure/ai/evaluation/simulator/__init__.py +2 -1
  259. azure/ai/evaluation/simulator/_adversarial_scenario.py +26 -1
  260. azure/ai/evaluation/simulator/_adversarial_simulator.py +270 -144
  261. azure/ai/evaluation/simulator/_constants.py +12 -1
  262. azure/ai/evaluation/simulator/_conversation/__init__.py +151 -23
  263. azure/ai/evaluation/simulator/_conversation/_conversation.py +10 -6
  264. azure/ai/evaluation/simulator/_conversation/constants.py +1 -1
  265. azure/ai/evaluation/simulator/_data_sources/__init__.py +3 -0
  266. azure/ai/evaluation/simulator/_data_sources/grounding.json +1150 -0
  267. azure/ai/evaluation/simulator/_direct_attack_simulator.py +54 -75
  268. azure/ai/evaluation/simulator/_helpers/__init__.py +1 -2
  269. azure/ai/evaluation/simulator/_helpers/_language_suffix_mapping.py +1 -0
  270. azure/ai/evaluation/simulator/_helpers/_simulator_data_classes.py +26 -5
  271. azure/ai/evaluation/simulator/_indirect_attack_simulator.py +145 -104
  272. azure/ai/evaluation/simulator/_model_tools/__init__.py +2 -1
  273. azure/ai/evaluation/simulator/_model_tools/_generated_rai_client.py +225 -0
  274. azure/ai/evaluation/simulator/_model_tools/_identity_manager.py +80 -30
  275. azure/ai/evaluation/simulator/_model_tools/_proxy_completion_model.py +117 -45
  276. azure/ai/evaluation/simulator/_model_tools/_rai_client.py +109 -7
  277. azure/ai/evaluation/simulator/_model_tools/_template_handler.py +97 -33
  278. azure/ai/evaluation/simulator/_model_tools/models.py +30 -27
  279. azure/ai/evaluation/simulator/_prompty/task_query_response.prompty +6 -10
  280. azure/ai/evaluation/simulator/_prompty/task_simulate.prompty +6 -5
  281. azure/ai/evaluation/simulator/_simulator.py +302 -208
  282. azure/ai/evaluation/simulator/_utils.py +31 -13
  283. azure_ai_evaluation-1.13.3.dist-info/METADATA +939 -0
  284. azure_ai_evaluation-1.13.3.dist-info/RECORD +305 -0
  285. {azure_ai_evaluation-1.0.0b2.dist-info → azure_ai_evaluation-1.13.3.dist-info}/WHEEL +1 -1
  286. azure_ai_evaluation-1.13.3.dist-info/licenses/NOTICE.txt +70 -0
  287. azure/ai/evaluation/_evaluate/_batch_run_client/proxy_client.py +0 -71
  288. azure/ai/evaluation/_evaluators/_chat/_chat.py +0 -357
  289. azure/ai/evaluation/_evaluators/_chat/retrieval/_retrieval.py +0 -157
  290. azure/ai/evaluation/_evaluators/_chat/retrieval/retrieval.prompty +0 -48
  291. azure/ai/evaluation/_evaluators/_content_safety/_content_safety_base.py +0 -65
  292. azure/ai/evaluation/_evaluators/_content_safety/_content_safety_chat.py +0 -301
  293. azure/ai/evaluation/_evaluators/_groundedness/groundedness.prompty +0 -54
  294. azure/ai/evaluation/_evaluators/_protected_materials/__init__.py +0 -5
  295. azure/ai/evaluation/_evaluators/_protected_materials/_protected_materials.py +0 -104
  296. azure/ai/evaluation/simulator/_tracing.py +0 -89
  297. azure_ai_evaluation-1.0.0b2.dist-info/METADATA +0 -449
  298. azure_ai_evaluation-1.0.0b2.dist-info/RECORD +0 -99
  299. {azure_ai_evaluation-1.0.0b2.dist-info → azure_ai_evaluation-1.13.3.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,1238 @@
1
+ # pylint: disable=too-many-lines
2
+ # coding=utf-8
3
+ # --------------------------------------------------------------------------
4
+ # Copyright (c) Microsoft Corporation. All rights reserved.
5
+ # Licensed under the MIT License. See License.txt in the project root for license information.
6
+ # Code generated by Microsoft (R) Python Code Generator.
7
+ # Changes may cause incorrect behavior and will be lost if the code is regenerated.
8
+ # --------------------------------------------------------------------------
9
+ from io import IOBase
10
+ import json
11
+ import sys
12
+ from typing import Any, Callable, Dict, IO, List, Optional, TypeVar, Union, overload
13
+
14
+ from azure.core import PipelineClient
15
+ from azure.core.exceptions import (
16
+ ClientAuthenticationError,
17
+ HttpResponseError,
18
+ ResourceExistsError,
19
+ ResourceNotFoundError,
20
+ ResourceNotModifiedError,
21
+ StreamClosedError,
22
+ StreamConsumedError,
23
+ map_error,
24
+ )
25
+ from azure.core.pipeline import PipelineResponse
26
+ from azure.core.rest import HttpRequest, HttpResponse
27
+ from azure.core.tracing.decorator import distributed_trace
28
+ from azure.core.utils import case_insensitive_dict
29
+
30
+ from .. import models as _models
31
+ from .._configuration import MachineLearningServicesClientConfiguration
32
+ from .._model_base import SdkJSONEncoder, _deserialize
33
+ from .._serialization import Deserializer, Serializer
34
+
35
+ if sys.version_info >= (3, 9):
36
+ from collections.abc import MutableMapping
37
+ else:
38
+ from typing import MutableMapping # type: ignore
39
+ JSON = MutableMapping[str, Any] # pylint: disable=unsubscriptable-object
40
+ T = TypeVar("T")
41
+ ClsType = Optional[Callable[[PipelineResponse[HttpRequest, HttpResponse], T, Dict[str, Any]], Any]]
42
+
43
+ _SERIALIZER = Serializer()
44
+ _SERIALIZER.client_side_validation = False
45
+
46
+
47
+ def build_rai_svc_get_annotation_request(**kwargs: Any) -> HttpRequest:
48
+ _headers = case_insensitive_dict(kwargs.pop("headers", {}) or {})
49
+ _params = case_insensitive_dict(kwargs.pop("params", {}) or {})
50
+
51
+ api_version: str = kwargs.pop("api_version", _params.pop("api-version", "2022-11-01-preview"))
52
+ accept = _headers.pop("Accept", "application/json")
53
+
54
+ # Construct URL
55
+ _url = "/checkannotation"
56
+
57
+ # Construct parameters
58
+ _params["api-version"] = _SERIALIZER.query("api_version", api_version, "str")
59
+
60
+ # Construct headers
61
+ _headers["Accept"] = _SERIALIZER.header("accept", accept, "str")
62
+
63
+ return HttpRequest(method="GET", url=_url, params=_params, headers=_headers, **kwargs)
64
+
65
+
66
+ def build_rai_svc_submit_annotation_request(**kwargs: Any) -> HttpRequest:
67
+ _headers = case_insensitive_dict(kwargs.pop("headers", {}) or {})
68
+ _params = case_insensitive_dict(kwargs.pop("params", {}) or {})
69
+
70
+ content_type: Optional[str] = kwargs.pop("content_type", _headers.pop("Content-Type", None))
71
+ api_version: str = kwargs.pop("api_version", _params.pop("api-version", "2022-11-01-preview"))
72
+ accept = _headers.pop("Accept", "application/json")
73
+
74
+ # Construct URL
75
+ _url = "/submitannotation"
76
+
77
+ # Construct parameters
78
+ _params["api-version"] = _SERIALIZER.query("api_version", api_version, "str")
79
+
80
+ # Construct headers
81
+ if content_type is not None:
82
+ _headers["Content-Type"] = _SERIALIZER.header("content_type", content_type, "str")
83
+ _headers["Accept"] = _SERIALIZER.header("accept", accept, "str")
84
+
85
+ return HttpRequest(method="POST", url=_url, params=_params, headers=_headers, **kwargs)
86
+
87
+
88
+ def build_rai_svc_get_jail_break_dataset_with_type_request( # pylint: disable=name-too-long
89
+ type: str, **kwargs: Any
90
+ ) -> HttpRequest:
91
+ _headers = case_insensitive_dict(kwargs.pop("headers", {}) or {})
92
+ _params = case_insensitive_dict(kwargs.pop("params", {}) or {})
93
+
94
+ api_version: str = kwargs.pop("api_version", _params.pop("api-version", "2022-11-01-preview"))
95
+ accept = _headers.pop("Accept", "application/json")
96
+
97
+ # Construct URL
98
+ _url = "/simulation/jailbreak/{type}"
99
+ path_format_arguments = {
100
+ "type": _SERIALIZER.url("type", type, "str"),
101
+ }
102
+
103
+ _url: str = _url.format(**path_format_arguments) # type: ignore
104
+
105
+ # Construct parameters
106
+ _params["api-version"] = _SERIALIZER.query("api_version", api_version, "str")
107
+
108
+ # Construct headers
109
+ _headers["Accept"] = _SERIALIZER.header("accept", accept, "str")
110
+
111
+ return HttpRequest(method="GET", url=_url, params=_params, headers=_headers, **kwargs)
112
+
113
+
114
+ def build_rai_svc_get_attack_objectives_request( # pylint: disable=name-too-long
115
+ *,
116
+ risk_types: Optional[List[str]] = None,
117
+ risk_categories: Optional[List[str]] = None,
118
+ lang: Optional[str] = None,
119
+ strategy: Optional[str] = None,
120
+ **kwargs: Any
121
+ ) -> HttpRequest:
122
+ _headers = case_insensitive_dict(kwargs.pop("headers", {}) or {})
123
+ _params = case_insensitive_dict(kwargs.pop("params", {}) or {})
124
+
125
+ api_version: str = kwargs.pop("api_version", _params.pop("api-version", "2022-11-01-preview"))
126
+ accept = _headers.pop("Accept", "application/json")
127
+
128
+ # Construct URL
129
+ _url = "/simulation/attackobjectives"
130
+
131
+ # Construct parameters
132
+ _params["api-version"] = _SERIALIZER.query("api_version", api_version, "str")
133
+ if risk_types is not None:
134
+ _params["riskTypes"] = [_SERIALIZER.query("risk_types", q, "str") if q is not None else "" for q in risk_types]
135
+ if risk_categories is not None:
136
+ _params["riskCategory"] = [
137
+ _SERIALIZER.query("risk_categories", q, "str") if q is not None else "" for q in risk_categories
138
+ ]
139
+ if lang is not None:
140
+ _params["lang"] = _SERIALIZER.query("lang", lang, "str")
141
+ if strategy is not None:
142
+ _params["strategy"] = _SERIALIZER.query("strategy", strategy, "str")
143
+
144
+ # Construct headers
145
+ _headers["Accept"] = _SERIALIZER.header("accept", accept, "str")
146
+
147
+ return HttpRequest(method="GET", url=_url, params=_params, headers=_headers, **kwargs)
148
+
149
+
150
+ def build_rai_svc_get_jail_break_dataset_request(**kwargs: Any) -> HttpRequest: # pylint: disable=name-too-long
151
+ _headers = case_insensitive_dict(kwargs.pop("headers", {}) or {})
152
+ _params = case_insensitive_dict(kwargs.pop("params", {}) or {})
153
+
154
+ api_version: str = kwargs.pop("api_version", _params.pop("api-version", "2022-11-01-preview"))
155
+ accept = _headers.pop("Accept", "application/json")
156
+
157
+ # Construct URL
158
+ _url = "/simulation/jailbreak/"
159
+
160
+ # Construct parameters
161
+ _params["api-version"] = _SERIALIZER.query("api_version", api_version, "str")
162
+
163
+ # Construct headers
164
+ _headers["Accept"] = _SERIALIZER.header("accept", accept, "str")
165
+
166
+ return HttpRequest(method="GET", url=_url, params=_params, headers=_headers, **kwargs)
167
+
168
+
169
+ def build_rai_svc_get_template_parameters_with_type_request( # pylint: disable=name-too-long
170
+ type: str, **kwargs: Any
171
+ ) -> HttpRequest:
172
+ _headers = case_insensitive_dict(kwargs.pop("headers", {}) or {})
173
+ _params = case_insensitive_dict(kwargs.pop("params", {}) or {})
174
+
175
+ api_version: str = kwargs.pop("api_version", _params.pop("api-version", "2022-11-01-preview"))
176
+ accept = _headers.pop("Accept", "application/json")
177
+
178
+ # Construct URL
179
+ _url = "/simulation/template/parameters/{type}"
180
+ path_format_arguments = {
181
+ "type": _SERIALIZER.url("type", type, "str"),
182
+ }
183
+
184
+ _url: str = _url.format(**path_format_arguments) # type: ignore
185
+
186
+ # Construct parameters
187
+ _params["api-version"] = _SERIALIZER.query("api_version", api_version, "str")
188
+
189
+ # Construct headers
190
+ _headers["Accept"] = _SERIALIZER.header("accept", accept, "str")
191
+
192
+ return HttpRequest(method="GET", url=_url, params=_params, headers=_headers, **kwargs)
193
+
194
+
195
+ def build_rai_svc_get_template_parameters_request(**kwargs: Any) -> HttpRequest: # pylint: disable=name-too-long
196
+ _headers = case_insensitive_dict(kwargs.pop("headers", {}) or {})
197
+ _params = case_insensitive_dict(kwargs.pop("params", {}) or {})
198
+
199
+ api_version: str = kwargs.pop("api_version", _params.pop("api-version", "2022-11-01-preview"))
200
+ accept = _headers.pop("Accept", "application/json")
201
+
202
+ # Construct URL
203
+ _url = "/simulation/template/parameters/"
204
+
205
+ # Construct parameters
206
+ _params["api-version"] = _SERIALIZER.query("api_version", api_version, "str")
207
+
208
+ # Construct headers
209
+ _headers["Accept"] = _SERIALIZER.header("accept", accept, "str")
210
+
211
+ return HttpRequest(method="GET", url=_url, params=_params, headers=_headers, **kwargs)
212
+
213
+
214
+ def build_rai_svc_get_template_parameters_image_request( # pylint: disable=name-too-long
215
+ *, path: str, **kwargs: Any
216
+ ) -> HttpRequest:
217
+ _headers = case_insensitive_dict(kwargs.pop("headers", {}) or {})
218
+ _params = case_insensitive_dict(kwargs.pop("params", {}) or {})
219
+
220
+ api_version: str = kwargs.pop("api_version", _params.pop("api-version", "2022-11-01-preview"))
221
+ accept = _headers.pop("Accept", "application/json")
222
+
223
+ # Construct URL
224
+ _url = "/simulation/template/parameters/image"
225
+
226
+ # Construct parameters
227
+ _params["api-version"] = _SERIALIZER.query("api_version", api_version, "str")
228
+ _params["path"] = _SERIALIZER.query("path", path, "str")
229
+
230
+ # Construct headers
231
+ _headers["Accept"] = _SERIALIZER.header("accept", accept, "str")
232
+
233
+ return HttpRequest(method="GET", url=_url, params=_params, headers=_headers, **kwargs)
234
+
235
+
236
+ def build_rai_svc_submit_simulation_request(**kwargs: Any) -> HttpRequest:
237
+ _headers = case_insensitive_dict(kwargs.pop("headers", {}) or {})
238
+ _params = case_insensitive_dict(kwargs.pop("params", {}) or {})
239
+
240
+ content_type: Optional[str] = kwargs.pop("content_type", _headers.pop("Content-Type", None))
241
+ api_version: str = kwargs.pop("api_version", _params.pop("api-version", "2022-11-01-preview"))
242
+ accept = _headers.pop("Accept", "application/json")
243
+
244
+ # Construct URL
245
+ _url = "/simulation/chat/completions/submit"
246
+
247
+ # Construct parameters
248
+ _params["api-version"] = _SERIALIZER.query("api_version", api_version, "str")
249
+
250
+ # Construct headers
251
+ if content_type is not None:
252
+ _headers["Content-Type"] = _SERIALIZER.header("content_type", content_type, "str")
253
+ _headers["Accept"] = _SERIALIZER.header("accept", accept, "str")
254
+
255
+ return HttpRequest(method="POST", url=_url, params=_params, headers=_headers, **kwargs)
256
+
257
+
258
+ def build_rai_svc_submit_aoai_evaluation_request(**kwargs: Any) -> HttpRequest: # pylint: disable=name-too-long
259
+ _headers = case_insensitive_dict(kwargs.pop("headers", {}) or {})
260
+ _params = case_insensitive_dict(kwargs.pop("params", {}) or {})
261
+
262
+ content_type: Optional[str] = kwargs.pop("content_type", _headers.pop("Content-Type", None))
263
+ api_version: str = kwargs.pop("api_version", _params.pop("api-version", "2022-11-01-preview"))
264
+ accept = _headers.pop("Accept", "application/json")
265
+
266
+ # Construct URL
267
+ _url = "/submitaoaievaluation"
268
+
269
+ # Construct parameters
270
+ _params["api-version"] = _SERIALIZER.query("api_version", api_version, "str")
271
+
272
+ # Construct headers
273
+ if content_type is not None:
274
+ _headers["Content-Type"] = _SERIALIZER.header("content_type", content_type, "str")
275
+ _headers["Accept"] = _SERIALIZER.header("accept", accept, "str")
276
+
277
+ return HttpRequest(method="POST", url=_url, params=_params, headers=_headers, **kwargs)
278
+
279
+
280
+ def build_rai_svc_get_operation_result_request( # pylint: disable=name-too-long
281
+ operation_id: str, *, api_key: Optional[str] = None, model_endpoint: Optional[str] = None, **kwargs: Any
282
+ ) -> HttpRequest:
283
+ _headers = case_insensitive_dict(kwargs.pop("headers", {}) or {})
284
+ _params = case_insensitive_dict(kwargs.pop("params", {}) or {})
285
+
286
+ api_version: str = kwargs.pop("api_version", _params.pop("api-version", "2022-11-01-preview"))
287
+ accept = _headers.pop("Accept", "application/json")
288
+
289
+ # Construct URL
290
+ _url = "/operations/{operationId}"
291
+ path_format_arguments = {
292
+ "operationId": _SERIALIZER.url("operation_id", operation_id, "str"),
293
+ }
294
+
295
+ _url: str = _url.format(**path_format_arguments) # type: ignore
296
+
297
+ # Construct parameters
298
+ _params["api-version"] = _SERIALIZER.query("api_version", api_version, "str")
299
+
300
+ # Construct headers
301
+ if api_key is not None:
302
+ _headers["api-key"] = _SERIALIZER.header("api_key", api_key, "str")
303
+ if model_endpoint is not None:
304
+ _headers["model-endpoint"] = _SERIALIZER.header("model_endpoint", model_endpoint, "str")
305
+ _headers["Accept"] = _SERIALIZER.header("accept", accept, "str")
306
+
307
+ return HttpRequest(method="GET", url=_url, params=_params, headers=_headers, **kwargs)
308
+
309
+
310
+ class RAISvcOperations:
311
+ """
312
+ .. warning::
313
+ **DO NOT** instantiate this class directly.
314
+
315
+ Instead, you should access the following operations through
316
+ :class:`~raiclient.MachineLearningServicesClient`'s
317
+ :attr:`rai_svc` attribute.
318
+ """
319
+
320
+ def __init__(self, *args, **kwargs):
321
+ input_args = list(args)
322
+ self._client: PipelineClient = input_args.pop(0) if input_args else kwargs.pop("client")
323
+ self._config: MachineLearningServicesClientConfiguration = (
324
+ input_args.pop(0) if input_args else kwargs.pop("config")
325
+ )
326
+ self._serialize: Serializer = input_args.pop(0) if input_args else kwargs.pop("serializer")
327
+ self._deserialize: Deserializer = input_args.pop(0) if input_args else kwargs.pop("deserializer")
328
+
329
+ @distributed_trace
330
+ def get_annotation(self, **kwargs: Any) -> List[str]:
331
+ """Get the supported annotation tasks.
332
+
333
+ :return: list of str
334
+ :rtype: list[str]
335
+ :raises ~azure.core.exceptions.HttpResponseError:
336
+ """
337
+ error_map: MutableMapping = {
338
+ 401: ClientAuthenticationError,
339
+ 404: ResourceNotFoundError,
340
+ 409: ResourceExistsError,
341
+ 304: ResourceNotModifiedError,
342
+ }
343
+ error_map.update(kwargs.pop("error_map", {}) or {})
344
+
345
+ _headers = kwargs.pop("headers", {}) or {}
346
+ _params = kwargs.pop("params", {}) or {}
347
+
348
+ cls: ClsType[List[str]] = kwargs.pop("cls", None)
349
+
350
+ _request = build_rai_svc_get_annotation_request(
351
+ api_version=self._config.api_version,
352
+ headers=_headers,
353
+ params=_params,
354
+ )
355
+ path_format_arguments = {
356
+ "endpoint": self._serialize.url("self._config.endpoint", self._config.endpoint, "str", skip_quote=True),
357
+ "subscriptionId": self._serialize.url("self._config.subscription_id", self._config.subscription_id, "str"),
358
+ "resourceGroupName": self._serialize.url(
359
+ "self._config.resource_group_name", self._config.resource_group_name, "str"
360
+ ),
361
+ "workspaceName": self._serialize.url("self._config.workspace_name", self._config.workspace_name, "str"),
362
+ }
363
+ _request.url = self._client.format_url(_request.url, **path_format_arguments)
364
+
365
+ _stream = kwargs.pop("stream", False)
366
+ pipeline_response: PipelineResponse = self._client._pipeline.run( # pylint: disable=protected-access
367
+ _request, stream=_stream, **kwargs
368
+ )
369
+
370
+ response = pipeline_response.http_response
371
+
372
+ if response.status_code not in [200]:
373
+ if _stream:
374
+ try:
375
+ response.read() # Load the body in memory and close the socket
376
+ except (StreamConsumedError, StreamClosedError):
377
+ pass
378
+ map_error(status_code=response.status_code, response=response, error_map=error_map)
379
+ raise HttpResponseError(response=response)
380
+
381
+ if _stream:
382
+ deserialized = response.iter_bytes()
383
+ else:
384
+ deserialized = _deserialize(List[str], response.json())
385
+
386
+ if cls:
387
+ return cls(pipeline_response, deserialized, {}) # type: ignore
388
+
389
+ return deserialized # type: ignore
390
+
391
+ @overload
392
+ def submit_annotation(
393
+ self, body: _models.AnnotationDTO, *, content_type: str = "application/json", **kwargs: Any
394
+ ) -> _models.LongRunningResponse:
395
+ """Submit a request for annotation.
396
+
397
+ :param body: Properties of a Prompt Version. Required.
398
+ :type body: ~raiclient.models.AnnotationDTO
399
+ :keyword content_type: Body Parameter content-type. Content type parameter for JSON body.
400
+ Default value is "application/json".
401
+ :paramtype content_type: str
402
+ :return: LongRunningResponse. The LongRunningResponse is compatible with MutableMapping
403
+ :rtype: ~raiclient.models.LongRunningResponse
404
+ :raises ~azure.core.exceptions.HttpResponseError:
405
+ """
406
+
407
+ @overload
408
+ def submit_annotation(
409
+ self, body: JSON, *, content_type: str = "application/json", **kwargs: Any
410
+ ) -> _models.LongRunningResponse:
411
+ """Submit a request for annotation.
412
+
413
+ :param body: Properties of a Prompt Version. Required.
414
+ :type body: JSON
415
+ :keyword content_type: Body Parameter content-type. Content type parameter for JSON body.
416
+ Default value is "application/json".
417
+ :paramtype content_type: str
418
+ :return: LongRunningResponse. The LongRunningResponse is compatible with MutableMapping
419
+ :rtype: ~raiclient.models.LongRunningResponse
420
+ :raises ~azure.core.exceptions.HttpResponseError:
421
+ """
422
+
423
+ @overload
424
+ def submit_annotation(
425
+ self, body: IO[bytes], *, content_type: str = "application/json", **kwargs: Any
426
+ ) -> _models.LongRunningResponse:
427
+ """Submit a request for annotation.
428
+
429
+ :param body: Properties of a Prompt Version. Required.
430
+ :type body: IO[bytes]
431
+ :keyword content_type: Body Parameter content-type. Content type parameter for binary body.
432
+ Default value is "application/json".
433
+ :paramtype content_type: str
434
+ :return: LongRunningResponse. The LongRunningResponse is compatible with MutableMapping
435
+ :rtype: ~raiclient.models.LongRunningResponse
436
+ :raises ~azure.core.exceptions.HttpResponseError:
437
+ """
438
+
439
+ @distributed_trace
440
+ def submit_annotation(
441
+ self, body: Union[_models.AnnotationDTO, JSON, IO[bytes]], **kwargs: Any
442
+ ) -> _models.LongRunningResponse:
443
+ """Submit a request for annotation.
444
+
445
+ :param body: Properties of a Prompt Version. Is one of the following types: AnnotationDTO,
446
+ JSON, IO[bytes] Required.
447
+ :type body: ~raiclient.models.AnnotationDTO or JSON or IO[bytes]
448
+ :return: LongRunningResponse. The LongRunningResponse is compatible with MutableMapping
449
+ :rtype: ~raiclient.models.LongRunningResponse
450
+ :raises ~azure.core.exceptions.HttpResponseError:
451
+ """
452
+ error_map: MutableMapping = {
453
+ 401: ClientAuthenticationError,
454
+ 404: ResourceNotFoundError,
455
+ 409: ResourceExistsError,
456
+ 304: ResourceNotModifiedError,
457
+ }
458
+ error_map.update(kwargs.pop("error_map", {}) or {})
459
+
460
+ _headers = case_insensitive_dict(kwargs.pop("headers", {}) or {})
461
+ _params = kwargs.pop("params", {}) or {}
462
+
463
+ content_type: Optional[str] = kwargs.pop("content_type", _headers.pop("Content-Type", None))
464
+ cls: ClsType[_models.LongRunningResponse] = kwargs.pop("cls", None)
465
+
466
+ content_type = content_type or "application/json"
467
+ _content = None
468
+ if isinstance(body, (IOBase, bytes)):
469
+ _content = body
470
+ else:
471
+ _content = json.dumps(body, cls=SdkJSONEncoder, exclude_readonly=True) # type: ignore
472
+
473
+ _request = build_rai_svc_submit_annotation_request(
474
+ content_type=content_type,
475
+ api_version=self._config.api_version,
476
+ content=_content,
477
+ headers=_headers,
478
+ params=_params,
479
+ )
480
+ path_format_arguments = {
481
+ "endpoint": self._serialize.url("self._config.endpoint", self._config.endpoint, "str", skip_quote=True),
482
+ "subscriptionId": self._serialize.url("self._config.subscription_id", self._config.subscription_id, "str"),
483
+ "resourceGroupName": self._serialize.url(
484
+ "self._config.resource_group_name", self._config.resource_group_name, "str"
485
+ ),
486
+ "workspaceName": self._serialize.url("self._config.workspace_name", self._config.workspace_name, "str"),
487
+ }
488
+ _request.url = self._client.format_url(_request.url, **path_format_arguments)
489
+
490
+ _stream = kwargs.pop("stream", False)
491
+ pipeline_response: PipelineResponse = self._client._pipeline.run( # pylint: disable=protected-access
492
+ _request, stream=_stream, **kwargs
493
+ )
494
+
495
+ response = pipeline_response.http_response
496
+
497
+ if response.status_code not in [202]:
498
+ if _stream:
499
+ try:
500
+ response.read() # Load the body in memory and close the socket
501
+ except (StreamConsumedError, StreamClosedError):
502
+ pass
503
+ map_error(status_code=response.status_code, response=response, error_map=error_map)
504
+ raise HttpResponseError(response=response)
505
+
506
+ if _stream:
507
+ deserialized = response.iter_bytes()
508
+ else:
509
+ deserialized = _deserialize(_models.LongRunningResponse, response.json())
510
+
511
+ if cls:
512
+ return cls(pipeline_response, deserialized, {}) # type: ignore
513
+
514
+ return deserialized # type: ignore
515
+
516
+ @distributed_trace
517
+ def get_jail_break_dataset_with_type(self, type: str, **kwargs: Any) -> List[str]:
518
+ """Get the jailbreak dataset with type.
519
+
520
+ :param type: Type of jailbreak dataset. Required.
521
+ :type type: str
522
+ :return: list of str
523
+ :rtype: list[str]
524
+ :raises ~azure.core.exceptions.HttpResponseError:
525
+ """
526
+ error_map: MutableMapping = {
527
+ 401: ClientAuthenticationError,
528
+ 404: ResourceNotFoundError,
529
+ 409: ResourceExistsError,
530
+ 304: ResourceNotModifiedError,
531
+ }
532
+ error_map.update(kwargs.pop("error_map", {}) or {})
533
+
534
+ _headers = kwargs.pop("headers", {}) or {}
535
+ _params = kwargs.pop("params", {}) or {}
536
+
537
+ cls: ClsType[List[str]] = kwargs.pop("cls", None)
538
+
539
+ _request = build_rai_svc_get_jail_break_dataset_with_type_request(
540
+ type=type,
541
+ api_version=self._config.api_version,
542
+ headers=_headers,
543
+ params=_params,
544
+ )
545
+ path_format_arguments = {
546
+ "endpoint": self._serialize.url("self._config.endpoint", self._config.endpoint, "str", skip_quote=True),
547
+ "subscriptionId": self._serialize.url("self._config.subscription_id", self._config.subscription_id, "str"),
548
+ "resourceGroupName": self._serialize.url(
549
+ "self._config.resource_group_name", self._config.resource_group_name, "str"
550
+ ),
551
+ "workspaceName": self._serialize.url("self._config.workspace_name", self._config.workspace_name, "str"),
552
+ }
553
+ _request.url = self._client.format_url(_request.url, **path_format_arguments)
554
+
555
+ _stream = kwargs.pop("stream", False)
556
+ pipeline_response: PipelineResponse = self._client._pipeline.run( # pylint: disable=protected-access
557
+ _request, stream=_stream, **kwargs
558
+ )
559
+
560
+ response = pipeline_response.http_response
561
+
562
+ if response.status_code not in [200]:
563
+ if _stream:
564
+ try:
565
+ response.read() # Load the body in memory and close the socket
566
+ except (StreamConsumedError, StreamClosedError):
567
+ pass
568
+ map_error(status_code=response.status_code, response=response, error_map=error_map)
569
+ raise HttpResponseError(response=response)
570
+
571
+ if _stream:
572
+ deserialized = response.iter_bytes()
573
+ else:
574
+ deserialized = _deserialize(List[str], response.json())
575
+
576
+ if cls:
577
+ return cls(pipeline_response, deserialized, {}) # type: ignore
578
+
579
+ return deserialized # type: ignore
580
+
581
+ @distributed_trace
582
+ def get_attack_objectives(
583
+ self,
584
+ *,
585
+ risk_category: str,
586
+ risk_types: Optional[List[str]] = None,
587
+ lang: Optional[str] = None,
588
+ strategy: Optional[str] = None,
589
+ **kwargs: Any
590
+ ) -> List[_models.AttackObjective]:
591
+ """Get the attack objectives.
592
+
593
+ :keyword risk_category: Risk category for the attack objectives. Required.
594
+ :paramtype risk_category: str
595
+ :keyword risk_types: Risk types for the attack objectives dataset. Default value is None.
596
+ :paramtype risk_types: list[str]
597
+ :keyword lang: The language for the attack objectives dataset, defaults to 'en'. Default value
598
+ is None.
599
+ :paramtype lang: str
600
+ :keyword strategy: The strategy. Default value is None.
601
+ :paramtype strategy: str
602
+ :return: list of AttackObjective
603
+ :rtype: list[~raiclient.models.AttackObjective]
604
+ :raises ~azure.core.exceptions.HttpResponseError:
605
+ """
606
+ error_map: MutableMapping = {
607
+ 401: ClientAuthenticationError,
608
+ 404: ResourceNotFoundError,
609
+ 409: ResourceExistsError,
610
+ 304: ResourceNotModifiedError,
611
+ }
612
+ error_map.update(kwargs.pop("error_map", {}) or {})
613
+
614
+ _headers = kwargs.pop("headers", {}) or {}
615
+ _params = kwargs.pop("params", {}) or {}
616
+
617
+ cls: ClsType[List[_models.AttackObjective]] = kwargs.pop("cls", None)
618
+
619
+ _request = build_rai_svc_get_attack_objectives_request(
620
+ risk_categories=[risk_category],
621
+ risk_types=risk_types,
622
+ lang=lang,
623
+ strategy=strategy,
624
+ api_version=self._config.api_version,
625
+ headers=_headers,
626
+ params=_params,
627
+ )
628
+ path_format_arguments = {
629
+ "endpoint": self._serialize.url("self._config.endpoint", self._config.endpoint, "str", skip_quote=True),
630
+ "subscriptionId": self._serialize.url("self._config.subscription_id", self._config.subscription_id, "str"),
631
+ "resourceGroupName": self._serialize.url(
632
+ "self._config.resource_group_name", self._config.resource_group_name, "str"
633
+ ),
634
+ "workspaceName": self._serialize.url("self._config.workspace_name", self._config.workspace_name, "str"),
635
+ }
636
+ _request.url = self._client.format_url(_request.url, **path_format_arguments)
637
+ _stream = kwargs.pop("stream", False)
638
+ pipeline_response: PipelineResponse = self._client._pipeline.run( # pylint: disable=protected-access
639
+ _request, stream=_stream, **kwargs
640
+ )
641
+
642
+ response = pipeline_response.http_response
643
+
644
+ if response.status_code not in [200]:
645
+ if _stream:
646
+ try:
647
+ response.read() # Load the body in memory and close the socket
648
+ except (StreamConsumedError, StreamClosedError):
649
+ pass
650
+ map_error(status_code=response.status_code, response=response, error_map=error_map)
651
+ raise HttpResponseError(response=response)
652
+
653
+ if _stream:
654
+ deserialized = response.iter_bytes()
655
+ else:
656
+ deserialized = _deserialize(List[_models.AttackObjective], response.json())
657
+
658
+ if cls:
659
+ return cls(pipeline_response, deserialized, {}) # type: ignore
660
+
661
+ return deserialized # type: ignore
662
+
663
+ @distributed_trace
664
+ def get_jail_break_dataset(self, **kwargs: Any) -> List[str]:
665
+ """Get the jailbreak dataset.
666
+
667
+ :return: list of str
668
+ :rtype: list[str]
669
+ :raises ~azure.core.exceptions.HttpResponseError:
670
+ """
671
+ error_map: MutableMapping = {
672
+ 401: ClientAuthenticationError,
673
+ 404: ResourceNotFoundError,
674
+ 409: ResourceExistsError,
675
+ 304: ResourceNotModifiedError,
676
+ }
677
+ error_map.update(kwargs.pop("error_map", {}) or {})
678
+
679
+ _headers = kwargs.pop("headers", {}) or {}
680
+ _params = kwargs.pop("params", {}) or {}
681
+
682
+ cls: ClsType[List[str]] = kwargs.pop("cls", None)
683
+
684
+ _request = build_rai_svc_get_jail_break_dataset_request(
685
+ api_version=self._config.api_version,
686
+ headers=_headers,
687
+ params=_params,
688
+ )
689
+ path_format_arguments = {
690
+ "endpoint": self._serialize.url("self._config.endpoint", self._config.endpoint, "str", skip_quote=True),
691
+ "subscriptionId": self._serialize.url("self._config.subscription_id", self._config.subscription_id, "str"),
692
+ "resourceGroupName": self._serialize.url(
693
+ "self._config.resource_group_name", self._config.resource_group_name, "str"
694
+ ),
695
+ "workspaceName": self._serialize.url("self._config.workspace_name", self._config.workspace_name, "str"),
696
+ }
697
+ _request.url = self._client.format_url(_request.url, **path_format_arguments)
698
+
699
+ _stream = kwargs.pop("stream", False)
700
+ pipeline_response: PipelineResponse = self._client._pipeline.run( # pylint: disable=protected-access
701
+ _request, stream=_stream, **kwargs
702
+ )
703
+
704
+ response = pipeline_response.http_response
705
+
706
+ if response.status_code not in [200]:
707
+ if _stream:
708
+ try:
709
+ response.read() # Load the body in memory and close the socket
710
+ except (StreamConsumedError, StreamClosedError):
711
+ pass
712
+ map_error(status_code=response.status_code, response=response, error_map=error_map)
713
+ raise HttpResponseError(response=response)
714
+
715
+ if _stream:
716
+ deserialized = response.iter_bytes()
717
+ else:
718
+ deserialized = _deserialize(List[str], response.json())
719
+
720
+ if cls:
721
+ return cls(pipeline_response, deserialized, {}) # type: ignore
722
+
723
+ return deserialized # type: ignore
724
+
725
+ @distributed_trace
726
+ def get_template_parameters_with_type(self, type: str, **kwargs: Any) -> str:
727
+ """Get template parameters with type.
728
+
729
+ :param type: Type for the template parameters. Required.
730
+ :type type: str
731
+ :return: str
732
+ :rtype: str
733
+ :raises ~azure.core.exceptions.HttpResponseError:
734
+ """
735
+ error_map: MutableMapping = {
736
+ 401: ClientAuthenticationError,
737
+ 404: ResourceNotFoundError,
738
+ 409: ResourceExistsError,
739
+ 304: ResourceNotModifiedError,
740
+ }
741
+ error_map.update(kwargs.pop("error_map", {}) or {})
742
+
743
+ _headers = kwargs.pop("headers", {}) or {}
744
+ _params = kwargs.pop("params", {}) or {}
745
+
746
+ cls: ClsType[str] = kwargs.pop("cls", None)
747
+
748
+ _request = build_rai_svc_get_template_parameters_with_type_request(
749
+ type=type,
750
+ api_version=self._config.api_version,
751
+ headers=_headers,
752
+ params=_params,
753
+ )
754
+ path_format_arguments = {
755
+ "endpoint": self._serialize.url("self._config.endpoint", self._config.endpoint, "str", skip_quote=True),
756
+ "subscriptionId": self._serialize.url("self._config.subscription_id", self._config.subscription_id, "str"),
757
+ "resourceGroupName": self._serialize.url(
758
+ "self._config.resource_group_name", self._config.resource_group_name, "str"
759
+ ),
760
+ "workspaceName": self._serialize.url("self._config.workspace_name", self._config.workspace_name, "str"),
761
+ }
762
+ _request.url = self._client.format_url(_request.url, **path_format_arguments)
763
+
764
+ _stream = kwargs.pop("stream", False)
765
+ pipeline_response: PipelineResponse = self._client._pipeline.run( # pylint: disable=protected-access
766
+ _request, stream=_stream, **kwargs
767
+ )
768
+
769
+ response = pipeline_response.http_response
770
+
771
+ if response.status_code not in [200]:
772
+ if _stream:
773
+ try:
774
+ response.read() # Load the body in memory and close the socket
775
+ except (StreamConsumedError, StreamClosedError):
776
+ pass
777
+ map_error(status_code=response.status_code, response=response, error_map=error_map)
778
+ raise HttpResponseError(response=response)
779
+
780
+ if _stream:
781
+ deserialized = response.iter_bytes()
782
+ else:
783
+ deserialized = _deserialize(str, response.json())
784
+
785
+ if cls:
786
+ return cls(pipeline_response, deserialized, {}) # type: ignore
787
+
788
+ return deserialized # type: ignore
789
+
790
+ @distributed_trace
791
+ def get_template_parameters(self, **kwargs: Any) -> str:
792
+ """Get template parameters.
793
+
794
+ :return: str
795
+ :rtype: str
796
+ :raises ~azure.core.exceptions.HttpResponseError:
797
+ """
798
+ error_map: MutableMapping = {
799
+ 401: ClientAuthenticationError,
800
+ 404: ResourceNotFoundError,
801
+ 409: ResourceExistsError,
802
+ 304: ResourceNotModifiedError,
803
+ }
804
+ error_map.update(kwargs.pop("error_map", {}) or {})
805
+
806
+ _headers = kwargs.pop("headers", {}) or {}
807
+ _params = kwargs.pop("params", {}) or {}
808
+
809
+ cls: ClsType[str] = kwargs.pop("cls", None)
810
+
811
+ _request = build_rai_svc_get_template_parameters_request(
812
+ api_version=self._config.api_version,
813
+ headers=_headers,
814
+ params=_params,
815
+ )
816
+ path_format_arguments = {
817
+ "endpoint": self._serialize.url("self._config.endpoint", self._config.endpoint, "str", skip_quote=True),
818
+ "subscriptionId": self._serialize.url("self._config.subscription_id", self._config.subscription_id, "str"),
819
+ "resourceGroupName": self._serialize.url(
820
+ "self._config.resource_group_name", self._config.resource_group_name, "str"
821
+ ),
822
+ "workspaceName": self._serialize.url("self._config.workspace_name", self._config.workspace_name, "str"),
823
+ }
824
+ _request.url = self._client.format_url(_request.url, **path_format_arguments)
825
+
826
+ _stream = kwargs.pop("stream", False)
827
+ pipeline_response: PipelineResponse = self._client._pipeline.run( # pylint: disable=protected-access
828
+ _request, stream=_stream, **kwargs
829
+ )
830
+
831
+ response = pipeline_response.http_response
832
+
833
+ if response.status_code not in [200]:
834
+ if _stream:
835
+ try:
836
+ response.read() # Load the body in memory and close the socket
837
+ except (StreamConsumedError, StreamClosedError):
838
+ pass
839
+ map_error(status_code=response.status_code, response=response, error_map=error_map)
840
+ raise HttpResponseError(response=response)
841
+
842
+ if _stream:
843
+ deserialized = response.iter_bytes()
844
+ else:
845
+ deserialized = _deserialize(str, response.json())
846
+
847
+ if cls:
848
+ return cls(pipeline_response, deserialized, {}) # type: ignore
849
+
850
+ return deserialized # type: ignore
851
+
852
+ @distributed_trace
853
+ def get_template_parameters_image(self, *, path: str, **kwargs: Any) -> str:
854
+ """Get the template parameters image.
855
+
856
+ :keyword path: Image path. Required.
857
+ :paramtype path: str
858
+ :return: str
859
+ :rtype: str
860
+ :raises ~azure.core.exceptions.HttpResponseError:
861
+ """
862
+ error_map: MutableMapping = {
863
+ 401: ClientAuthenticationError,
864
+ 404: ResourceNotFoundError,
865
+ 409: ResourceExistsError,
866
+ 304: ResourceNotModifiedError,
867
+ }
868
+ error_map.update(kwargs.pop("error_map", {}) or {})
869
+
870
+ _headers = kwargs.pop("headers", {}) or {}
871
+ _params = kwargs.pop("params", {}) or {}
872
+
873
+ cls: ClsType[str] = kwargs.pop("cls", None)
874
+
875
+ _request = build_rai_svc_get_template_parameters_image_request(
876
+ path=path,
877
+ api_version=self._config.api_version,
878
+ headers=_headers,
879
+ params=_params,
880
+ )
881
+ path_format_arguments = {
882
+ "endpoint": self._serialize.url("self._config.endpoint", self._config.endpoint, "str", skip_quote=True),
883
+ "subscriptionId": self._serialize.url("self._config.subscription_id", self._config.subscription_id, "str"),
884
+ "resourceGroupName": self._serialize.url(
885
+ "self._config.resource_group_name", self._config.resource_group_name, "str"
886
+ ),
887
+ "workspaceName": self._serialize.url("self._config.workspace_name", self._config.workspace_name, "str"),
888
+ }
889
+ _request.url = self._client.format_url(_request.url, **path_format_arguments)
890
+
891
+ _stream = kwargs.pop("stream", False)
892
+ pipeline_response: PipelineResponse = self._client._pipeline.run( # pylint: disable=protected-access
893
+ _request, stream=_stream, **kwargs
894
+ )
895
+
896
+ response = pipeline_response.http_response
897
+
898
+ if response.status_code not in [200]:
899
+ if _stream:
900
+ try:
901
+ response.read() # Load the body in memory and close the socket
902
+ except (StreamConsumedError, StreamClosedError):
903
+ pass
904
+ map_error(status_code=response.status_code, response=response, error_map=error_map)
905
+ raise HttpResponseError(response=response)
906
+
907
+ if _stream:
908
+ deserialized = response.iter_bytes()
909
+ else:
910
+ deserialized = _deserialize(str, response.json())
911
+
912
+ if cls:
913
+ return cls(pipeline_response, deserialized, {}) # type: ignore
914
+
915
+ return deserialized # type: ignore
916
+
917
+ @overload
918
+ def submit_simulation(
919
+ self, body: _models.SimulationDTO, *, content_type: str = "application/json", **kwargs: Any
920
+ ) -> _models.LongRunningResponse:
921
+ """Submit a request for simulation.
922
+
923
+ :param body: Properties of a Prompt Version. Required.
924
+ :type body: ~raiclient.models.SimulationDTO
925
+ :keyword content_type: Body Parameter content-type. Content type parameter for JSON body.
926
+ Default value is "application/json".
927
+ :paramtype content_type: str
928
+ :return: LongRunningResponse. The LongRunningResponse is compatible with MutableMapping
929
+ :rtype: ~raiclient.models.LongRunningResponse
930
+ :raises ~azure.core.exceptions.HttpResponseError:
931
+ """
932
+
933
+ @overload
934
+ def submit_simulation(
935
+ self, body: JSON, *, content_type: str = "application/json", **kwargs: Any
936
+ ) -> _models.LongRunningResponse:
937
+ """Submit a request for simulation.
938
+
939
+ :param body: Properties of a Prompt Version. Required.
940
+ :type body: JSON
941
+ :keyword content_type: Body Parameter content-type. Content type parameter for JSON body.
942
+ Default value is "application/json".
943
+ :paramtype content_type: str
944
+ :return: LongRunningResponse. The LongRunningResponse is compatible with MutableMapping
945
+ :rtype: ~raiclient.models.LongRunningResponse
946
+ :raises ~azure.core.exceptions.HttpResponseError:
947
+ """
948
+
949
+ @overload
950
+ def submit_simulation(
951
+ self, body: IO[bytes], *, content_type: str = "application/json", **kwargs: Any
952
+ ) -> _models.LongRunningResponse:
953
+ """Submit a request for simulation.
954
+
955
+ :param body: Properties of a Prompt Version. Required.
956
+ :type body: IO[bytes]
957
+ :keyword content_type: Body Parameter content-type. Content type parameter for binary body.
958
+ Default value is "application/json".
959
+ :paramtype content_type: str
960
+ :return: LongRunningResponse. The LongRunningResponse is compatible with MutableMapping
961
+ :rtype: ~raiclient.models.LongRunningResponse
962
+ :raises ~azure.core.exceptions.HttpResponseError:
963
+ """
964
+
965
+ @distributed_trace
966
+ def submit_simulation(
967
+ self, body: Union[_models.SimulationDTO, JSON, IO[bytes]], **kwargs: Any
968
+ ) -> _models.LongRunningResponse:
969
+ """Submit a request for simulation.
970
+
971
+ :param body: Properties of a Prompt Version. Is one of the following types: SimulationDTO,
972
+ JSON, IO[bytes] Required.
973
+ :type body: ~raiclient.models.SimulationDTO or JSON or IO[bytes]
974
+ :return: LongRunningResponse. The LongRunningResponse is compatible with MutableMapping
975
+ :rtype: ~raiclient.models.LongRunningResponse
976
+ :raises ~azure.core.exceptions.HttpResponseError:
977
+ """
978
+ error_map: MutableMapping = {
979
+ 401: ClientAuthenticationError,
980
+ 404: ResourceNotFoundError,
981
+ 409: ResourceExistsError,
982
+ 304: ResourceNotModifiedError,
983
+ }
984
+ error_map.update(kwargs.pop("error_map", {}) or {})
985
+
986
+ _headers = case_insensitive_dict(kwargs.pop("headers", {}) or {})
987
+ _params = kwargs.pop("params", {}) or {}
988
+
989
+ content_type: Optional[str] = kwargs.pop("content_type", _headers.pop("Content-Type", None))
990
+ cls: ClsType[_models.LongRunningResponse] = kwargs.pop("cls", None)
991
+
992
+ content_type = content_type or "application/json"
993
+ _content = None
994
+ if isinstance(body, (IOBase, bytes)):
995
+ _content = body
996
+ else:
997
+ _content = json.dumps(body, cls=SdkJSONEncoder, exclude_readonly=True) # type: ignore
998
+
999
+ _request = build_rai_svc_submit_simulation_request(
1000
+ content_type=content_type,
1001
+ api_version=self._config.api_version,
1002
+ content=_content,
1003
+ headers=_headers,
1004
+ params=_params,
1005
+ )
1006
+ path_format_arguments = {
1007
+ "endpoint": self._serialize.url("self._config.endpoint", self._config.endpoint, "str", skip_quote=True),
1008
+ "subscriptionId": self._serialize.url("self._config.subscription_id", self._config.subscription_id, "str"),
1009
+ "resourceGroupName": self._serialize.url(
1010
+ "self._config.resource_group_name", self._config.resource_group_name, "str"
1011
+ ),
1012
+ "workspaceName": self._serialize.url("self._config.workspace_name", self._config.workspace_name, "str"),
1013
+ }
1014
+ _request.url = self._client.format_url(_request.url, **path_format_arguments)
1015
+
1016
+ _stream = kwargs.pop("stream", False)
1017
+ pipeline_response: PipelineResponse = self._client._pipeline.run( # pylint: disable=protected-access
1018
+ _request, stream=_stream, **kwargs
1019
+ )
1020
+
1021
+ response = pipeline_response.http_response
1022
+
1023
+ if response.status_code not in [202]:
1024
+ if _stream:
1025
+ try:
1026
+ response.read() # Load the body in memory and close the socket
1027
+ except (StreamConsumedError, StreamClosedError):
1028
+ pass
1029
+ map_error(status_code=response.status_code, response=response, error_map=error_map)
1030
+ raise HttpResponseError(response=response)
1031
+
1032
+ if _stream:
1033
+ deserialized = response.iter_bytes()
1034
+ else:
1035
+ deserialized = _deserialize(_models.LongRunningResponse, response.json())
1036
+
1037
+ if cls:
1038
+ return cls(pipeline_response, deserialized, {}) # type: ignore
1039
+
1040
+ return deserialized # type: ignore
1041
+
1042
+ @overload
1043
+ def submit_aoai_evaluation(
1044
+ self, body: _models.GradersDTO, *, content_type: str = "application/json", **kwargs: Any
1045
+ ) -> _models.LongRunningResponse:
1046
+ """Submit a request for graders.
1047
+
1048
+ :param body: Properties of a Prompt Version. Required.
1049
+ :type body: ~raiclient.models.GradersDTO
1050
+ :keyword content_type: Body Parameter content-type. Content type parameter for JSON body.
1051
+ Default value is "application/json".
1052
+ :paramtype content_type: str
1053
+ :return: LongRunningResponse. The LongRunningResponse is compatible with MutableMapping
1054
+ :rtype: ~raiclient.models.LongRunningResponse
1055
+ :raises ~azure.core.exceptions.HttpResponseError:
1056
+ """
1057
+
1058
+ @overload
1059
+ def submit_aoai_evaluation(
1060
+ self, body: JSON, *, content_type: str = "application/json", **kwargs: Any
1061
+ ) -> _models.LongRunningResponse:
1062
+ """Submit a request for graders.
1063
+
1064
+ :param body: Properties of a Prompt Version. Required.
1065
+ :type body: JSON
1066
+ :keyword content_type: Body Parameter content-type. Content type parameter for JSON body.
1067
+ Default value is "application/json".
1068
+ :paramtype content_type: str
1069
+ :return: LongRunningResponse. The LongRunningResponse is compatible with MutableMapping
1070
+ :rtype: ~raiclient.models.LongRunningResponse
1071
+ :raises ~azure.core.exceptions.HttpResponseError:
1072
+ """
1073
+
1074
+ @overload
1075
+ def submit_aoai_evaluation(
1076
+ self, body: IO[bytes], *, content_type: str = "application/json", **kwargs: Any
1077
+ ) -> _models.LongRunningResponse:
1078
+ """Submit a request for graders.
1079
+
1080
+ :param body: Properties of a Prompt Version. Required.
1081
+ :type body: IO[bytes]
1082
+ :keyword content_type: Body Parameter content-type. Content type parameter for binary body.
1083
+ Default value is "application/json".
1084
+ :paramtype content_type: str
1085
+ :return: LongRunningResponse. The LongRunningResponse is compatible with MutableMapping
1086
+ :rtype: ~raiclient.models.LongRunningResponse
1087
+ :raises ~azure.core.exceptions.HttpResponseError:
1088
+ """
1089
+
1090
+ @distributed_trace
1091
+ def submit_aoai_evaluation(
1092
+ self, body: Union[_models.GradersDTO, JSON, IO[bytes]], **kwargs: Any
1093
+ ) -> _models.LongRunningResponse:
1094
+ """Submit a request for graders.
1095
+
1096
+ :param body: Properties of a Prompt Version. Is one of the following types: GradersDTO, JSON,
1097
+ IO[bytes] Required.
1098
+ :type body: ~raiclient.models.GradersDTO or JSON or IO[bytes]
1099
+ :return: LongRunningResponse. The LongRunningResponse is compatible with MutableMapping
1100
+ :rtype: ~raiclient.models.LongRunningResponse
1101
+ :raises ~azure.core.exceptions.HttpResponseError:
1102
+ """
1103
+ error_map: MutableMapping = {
1104
+ 401: ClientAuthenticationError,
1105
+ 404: ResourceNotFoundError,
1106
+ 409: ResourceExistsError,
1107
+ 304: ResourceNotModifiedError,
1108
+ }
1109
+ error_map.update(kwargs.pop("error_map", {}) or {})
1110
+
1111
+ _headers = case_insensitive_dict(kwargs.pop("headers", {}) or {})
1112
+ _params = kwargs.pop("params", {}) or {}
1113
+
1114
+ content_type: Optional[str] = kwargs.pop("content_type", _headers.pop("Content-Type", None))
1115
+ cls: ClsType[_models.LongRunningResponse] = kwargs.pop("cls", None)
1116
+
1117
+ content_type = content_type or "application/json"
1118
+ _content = None
1119
+ if isinstance(body, (IOBase, bytes)):
1120
+ _content = body
1121
+ else:
1122
+ _content = json.dumps(body, cls=SdkJSONEncoder, exclude_readonly=True) # type: ignore
1123
+
1124
+ _request = build_rai_svc_submit_aoai_evaluation_request(
1125
+ content_type=content_type,
1126
+ api_version=self._config.api_version,
1127
+ content=_content,
1128
+ headers=_headers,
1129
+ params=_params,
1130
+ )
1131
+ path_format_arguments = {
1132
+ "endpoint": self._serialize.url("self._config.endpoint", self._config.endpoint, "str", skip_quote=True),
1133
+ "subscriptionId": self._serialize.url("self._config.subscription_id", self._config.subscription_id, "str"),
1134
+ "resourceGroupName": self._serialize.url(
1135
+ "self._config.resource_group_name", self._config.resource_group_name, "str"
1136
+ ),
1137
+ "workspaceName": self._serialize.url("self._config.workspace_name", self._config.workspace_name, "str"),
1138
+ }
1139
+ _request.url = self._client.format_url(_request.url, **path_format_arguments)
1140
+
1141
+ _stream = kwargs.pop("stream", False)
1142
+ pipeline_response: PipelineResponse = self._client._pipeline.run( # pylint: disable=protected-access
1143
+ _request, stream=_stream, **kwargs
1144
+ )
1145
+
1146
+ response = pipeline_response.http_response
1147
+
1148
+ if response.status_code not in [202]:
1149
+ if _stream:
1150
+ try:
1151
+ response.read() # Load the body in memory and close the socket
1152
+ except (StreamConsumedError, StreamClosedError):
1153
+ pass
1154
+ map_error(status_code=response.status_code, response=response, error_map=error_map)
1155
+ raise HttpResponseError(response=response)
1156
+
1157
+ if _stream:
1158
+ deserialized = response.iter_bytes()
1159
+ else:
1160
+ deserialized = _deserialize(_models.LongRunningResponse, response.json())
1161
+
1162
+ if cls:
1163
+ return cls(pipeline_response, deserialized, {}) # type: ignore
1164
+
1165
+ return deserialized # type: ignore
1166
+
1167
+ @distributed_trace
1168
+ def get_operation_result(
1169
+ self, operation_id: str, *, api_key: Optional[str] = None, model_endpoint: Optional[str] = None, **kwargs: Any
1170
+ ) -> str:
1171
+ """Get the operation result.
1172
+
1173
+ :param operation_id: Operation id. Required.
1174
+ :type operation_id: str
1175
+ :keyword api_key: Api key. Default value is None.
1176
+ :paramtype api_key: str
1177
+ :keyword model_endpoint: Model Endpoint. Default value is None.
1178
+ :paramtype model_endpoint: str
1179
+ :return: str
1180
+ :rtype: str
1181
+ :raises ~azure.core.exceptions.HttpResponseError:
1182
+ """
1183
+ error_map: MutableMapping = {
1184
+ 401: ClientAuthenticationError,
1185
+ 404: ResourceNotFoundError,
1186
+ 409: ResourceExistsError,
1187
+ 304: ResourceNotModifiedError,
1188
+ }
1189
+ error_map.update(kwargs.pop("error_map", {}) or {})
1190
+
1191
+ _headers = kwargs.pop("headers", {}) or {}
1192
+ _params = kwargs.pop("params", {}) or {}
1193
+
1194
+ cls: ClsType[str] = kwargs.pop("cls", None)
1195
+
1196
+ _request = build_rai_svc_get_operation_result_request(
1197
+ operation_id=operation_id,
1198
+ api_key=api_key,
1199
+ model_endpoint=model_endpoint,
1200
+ api_version=self._config.api_version,
1201
+ headers=_headers,
1202
+ params=_params,
1203
+ )
1204
+ path_format_arguments = {
1205
+ "endpoint": self._serialize.url("self._config.endpoint", self._config.endpoint, "str", skip_quote=True),
1206
+ "subscriptionId": self._serialize.url("self._config.subscription_id", self._config.subscription_id, "str"),
1207
+ "resourceGroupName": self._serialize.url(
1208
+ "self._config.resource_group_name", self._config.resource_group_name, "str"
1209
+ ),
1210
+ "workspaceName": self._serialize.url("self._config.workspace_name", self._config.workspace_name, "str"),
1211
+ }
1212
+ _request.url = self._client.format_url(_request.url, **path_format_arguments)
1213
+
1214
+ _stream = kwargs.pop("stream", False)
1215
+ pipeline_response: PipelineResponse = self._client._pipeline.run( # pylint: disable=protected-access
1216
+ _request, stream=_stream, **kwargs
1217
+ )
1218
+
1219
+ response = pipeline_response.http_response
1220
+
1221
+ if response.status_code not in [200]:
1222
+ if _stream:
1223
+ try:
1224
+ response.read() # Load the body in memory and close the socket
1225
+ except (StreamConsumedError, StreamClosedError):
1226
+ pass
1227
+ map_error(status_code=response.status_code, response=response, error_map=error_map)
1228
+ raise HttpResponseError(response=response)
1229
+
1230
+ if _stream:
1231
+ deserialized = response.iter_bytes()
1232
+ else:
1233
+ deserialized = _deserialize(str, response.json())
1234
+
1235
+ if cls:
1236
+ return cls(pipeline_response, deserialized, {}) # type: ignore
1237
+
1238
+ return deserialized # type: ignore