azure-ai-evaluation 1.0.0b2__py3-none-any.whl → 1.13.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of azure-ai-evaluation might be problematic. Click here for more details.

Files changed (299) hide show
  1. azure/ai/evaluation/__init__.py +100 -5
  2. azure/ai/evaluation/{_evaluators/_chat → _aoai}/__init__.py +3 -2
  3. azure/ai/evaluation/_aoai/aoai_grader.py +140 -0
  4. azure/ai/evaluation/_aoai/label_grader.py +68 -0
  5. azure/ai/evaluation/_aoai/python_grader.py +86 -0
  6. azure/ai/evaluation/_aoai/score_model_grader.py +94 -0
  7. azure/ai/evaluation/_aoai/string_check_grader.py +66 -0
  8. azure/ai/evaluation/_aoai/text_similarity_grader.py +80 -0
  9. azure/ai/evaluation/_azure/__init__.py +3 -0
  10. azure/ai/evaluation/_azure/_clients.py +204 -0
  11. azure/ai/evaluation/_azure/_envs.py +207 -0
  12. azure/ai/evaluation/_azure/_models.py +227 -0
  13. azure/ai/evaluation/_azure/_token_manager.py +129 -0
  14. azure/ai/evaluation/_common/__init__.py +9 -1
  15. azure/ai/evaluation/{simulator/_helpers → _common}/_experimental.py +24 -9
  16. azure/ai/evaluation/_common/constants.py +131 -2
  17. azure/ai/evaluation/_common/evaluation_onedp_client.py +169 -0
  18. azure/ai/evaluation/_common/math.py +89 -0
  19. azure/ai/evaluation/_common/onedp/__init__.py +32 -0
  20. azure/ai/evaluation/_common/onedp/_client.py +166 -0
  21. azure/ai/evaluation/_common/onedp/_configuration.py +72 -0
  22. azure/ai/evaluation/_common/onedp/_model_base.py +1232 -0
  23. azure/ai/evaluation/_common/onedp/_patch.py +21 -0
  24. azure/ai/evaluation/_common/onedp/_serialization.py +2032 -0
  25. azure/ai/evaluation/_common/onedp/_types.py +21 -0
  26. azure/ai/evaluation/_common/onedp/_utils/__init__.py +6 -0
  27. azure/ai/evaluation/_common/onedp/_utils/model_base.py +1232 -0
  28. azure/ai/evaluation/_common/onedp/_utils/serialization.py +2032 -0
  29. azure/ai/evaluation/_common/onedp/_validation.py +66 -0
  30. azure/ai/evaluation/_common/onedp/_vendor.py +50 -0
  31. azure/ai/evaluation/_common/onedp/_version.py +9 -0
  32. azure/ai/evaluation/_common/onedp/aio/__init__.py +29 -0
  33. azure/ai/evaluation/_common/onedp/aio/_client.py +168 -0
  34. azure/ai/evaluation/_common/onedp/aio/_configuration.py +72 -0
  35. azure/ai/evaluation/_common/onedp/aio/_patch.py +21 -0
  36. azure/ai/evaluation/_common/onedp/aio/operations/__init__.py +49 -0
  37. azure/ai/evaluation/_common/onedp/aio/operations/_operations.py +7143 -0
  38. azure/ai/evaluation/_common/onedp/aio/operations/_patch.py +21 -0
  39. azure/ai/evaluation/_common/onedp/models/__init__.py +358 -0
  40. azure/ai/evaluation/_common/onedp/models/_enums.py +447 -0
  41. azure/ai/evaluation/_common/onedp/models/_models.py +5963 -0
  42. azure/ai/evaluation/_common/onedp/models/_patch.py +21 -0
  43. azure/ai/evaluation/_common/onedp/operations/__init__.py +49 -0
  44. azure/ai/evaluation/_common/onedp/operations/_operations.py +8951 -0
  45. azure/ai/evaluation/_common/onedp/operations/_patch.py +21 -0
  46. azure/ai/evaluation/_common/onedp/py.typed +1 -0
  47. azure/ai/evaluation/_common/onedp/servicepatterns/__init__.py +1 -0
  48. azure/ai/evaluation/_common/onedp/servicepatterns/aio/__init__.py +1 -0
  49. azure/ai/evaluation/_common/onedp/servicepatterns/aio/operations/__init__.py +25 -0
  50. azure/ai/evaluation/_common/onedp/servicepatterns/aio/operations/_operations.py +34 -0
  51. azure/ai/evaluation/_common/onedp/servicepatterns/aio/operations/_patch.py +20 -0
  52. azure/ai/evaluation/_common/onedp/servicepatterns/buildingblocks/__init__.py +1 -0
  53. azure/ai/evaluation/_common/onedp/servicepatterns/buildingblocks/aio/__init__.py +1 -0
  54. azure/ai/evaluation/_common/onedp/servicepatterns/buildingblocks/aio/operations/__init__.py +22 -0
  55. azure/ai/evaluation/_common/onedp/servicepatterns/buildingblocks/aio/operations/_operations.py +29 -0
  56. azure/ai/evaluation/_common/onedp/servicepatterns/buildingblocks/aio/operations/_patch.py +20 -0
  57. azure/ai/evaluation/_common/onedp/servicepatterns/buildingblocks/operations/__init__.py +22 -0
  58. azure/ai/evaluation/_common/onedp/servicepatterns/buildingblocks/operations/_operations.py +29 -0
  59. azure/ai/evaluation/_common/onedp/servicepatterns/buildingblocks/operations/_patch.py +20 -0
  60. azure/ai/evaluation/_common/onedp/servicepatterns/operations/__init__.py +25 -0
  61. azure/ai/evaluation/_common/onedp/servicepatterns/operations/_operations.py +34 -0
  62. azure/ai/evaluation/_common/onedp/servicepatterns/operations/_patch.py +20 -0
  63. azure/ai/evaluation/_common/rai_service.py +831 -142
  64. azure/ai/evaluation/_common/raiclient/__init__.py +34 -0
  65. azure/ai/evaluation/_common/raiclient/_client.py +128 -0
  66. azure/ai/evaluation/_common/raiclient/_configuration.py +87 -0
  67. azure/ai/evaluation/_common/raiclient/_model_base.py +1235 -0
  68. azure/ai/evaluation/_common/raiclient/_patch.py +20 -0
  69. azure/ai/evaluation/_common/raiclient/_serialization.py +2050 -0
  70. azure/ai/evaluation/_common/raiclient/_version.py +9 -0
  71. azure/ai/evaluation/_common/raiclient/aio/__init__.py +29 -0
  72. azure/ai/evaluation/_common/raiclient/aio/_client.py +130 -0
  73. azure/ai/evaluation/_common/raiclient/aio/_configuration.py +87 -0
  74. azure/ai/evaluation/_common/raiclient/aio/_patch.py +20 -0
  75. azure/ai/evaluation/_common/raiclient/aio/operations/__init__.py +25 -0
  76. azure/ai/evaluation/_common/raiclient/aio/operations/_operations.py +981 -0
  77. azure/ai/evaluation/_common/raiclient/aio/operations/_patch.py +20 -0
  78. azure/ai/evaluation/_common/raiclient/models/__init__.py +60 -0
  79. azure/ai/evaluation/_common/raiclient/models/_enums.py +18 -0
  80. azure/ai/evaluation/_common/raiclient/models/_models.py +651 -0
  81. azure/ai/evaluation/_common/raiclient/models/_patch.py +20 -0
  82. azure/ai/evaluation/_common/raiclient/operations/__init__.py +25 -0
  83. azure/ai/evaluation/_common/raiclient/operations/_operations.py +1238 -0
  84. azure/ai/evaluation/_common/raiclient/operations/_patch.py +20 -0
  85. azure/ai/evaluation/_common/raiclient/py.typed +1 -0
  86. azure/ai/evaluation/_common/utils.py +870 -34
  87. azure/ai/evaluation/_constants.py +167 -6
  88. azure/ai/evaluation/_converters/__init__.py +3 -0
  89. azure/ai/evaluation/_converters/_ai_services.py +899 -0
  90. azure/ai/evaluation/_converters/_models.py +467 -0
  91. azure/ai/evaluation/_converters/_sk_services.py +495 -0
  92. azure/ai/evaluation/_eval_mapping.py +83 -0
  93. azure/ai/evaluation/_evaluate/_batch_run/__init__.py +17 -0
  94. azure/ai/evaluation/_evaluate/_batch_run/_run_submitter_client.py +176 -0
  95. azure/ai/evaluation/_evaluate/_batch_run/batch_clients.py +82 -0
  96. azure/ai/evaluation/_evaluate/{_batch_run_client → _batch_run}/code_client.py +47 -25
  97. azure/ai/evaluation/_evaluate/{_batch_run_client/batch_run_context.py → _batch_run/eval_run_context.py} +42 -13
  98. azure/ai/evaluation/_evaluate/_batch_run/proxy_client.py +124 -0
  99. azure/ai/evaluation/_evaluate/_batch_run/target_run_context.py +62 -0
  100. azure/ai/evaluation/_evaluate/_eval_run.py +102 -59
  101. azure/ai/evaluation/_evaluate/_evaluate.py +2134 -311
  102. azure/ai/evaluation/_evaluate/_evaluate_aoai.py +992 -0
  103. azure/ai/evaluation/_evaluate/_telemetry/__init__.py +14 -99
  104. azure/ai/evaluation/_evaluate/_utils.py +289 -40
  105. azure/ai/evaluation/_evaluator_definition.py +76 -0
  106. azure/ai/evaluation/_evaluators/_bleu/_bleu.py +93 -42
  107. azure/ai/evaluation/_evaluators/_code_vulnerability/__init__.py +5 -0
  108. azure/ai/evaluation/_evaluators/_code_vulnerability/_code_vulnerability.py +119 -0
  109. azure/ai/evaluation/_evaluators/_coherence/_coherence.py +117 -91
  110. azure/ai/evaluation/_evaluators/_coherence/coherence.prompty +76 -39
  111. azure/ai/evaluation/_evaluators/_common/__init__.py +15 -0
  112. azure/ai/evaluation/_evaluators/_common/_base_eval.py +742 -0
  113. azure/ai/evaluation/_evaluators/_common/_base_multi_eval.py +63 -0
  114. azure/ai/evaluation/_evaluators/_common/_base_prompty_eval.py +345 -0
  115. azure/ai/evaluation/_evaluators/_common/_base_rai_svc_eval.py +198 -0
  116. azure/ai/evaluation/_evaluators/_common/_conversation_aggregators.py +49 -0
  117. azure/ai/evaluation/_evaluators/_content_safety/__init__.py +0 -4
  118. azure/ai/evaluation/_evaluators/_content_safety/_content_safety.py +144 -86
  119. azure/ai/evaluation/_evaluators/_content_safety/_hate_unfairness.py +138 -57
  120. azure/ai/evaluation/_evaluators/_content_safety/_self_harm.py +123 -55
  121. azure/ai/evaluation/_evaluators/_content_safety/_sexual.py +133 -54
  122. azure/ai/evaluation/_evaluators/_content_safety/_violence.py +134 -54
  123. azure/ai/evaluation/_evaluators/_document_retrieval/__init__.py +7 -0
  124. azure/ai/evaluation/_evaluators/_document_retrieval/_document_retrieval.py +442 -0
  125. azure/ai/evaluation/_evaluators/_eci/_eci.py +49 -56
  126. azure/ai/evaluation/_evaluators/_f1_score/_f1_score.py +102 -60
  127. azure/ai/evaluation/_evaluators/_fluency/_fluency.py +115 -92
  128. azure/ai/evaluation/_evaluators/_fluency/fluency.prompty +66 -41
  129. azure/ai/evaluation/_evaluators/_gleu/_gleu.py +90 -37
  130. azure/ai/evaluation/_evaluators/_groundedness/_groundedness.py +318 -82
  131. azure/ai/evaluation/_evaluators/_groundedness/groundedness_with_query.prompty +114 -0
  132. azure/ai/evaluation/_evaluators/_groundedness/groundedness_without_query.prompty +104 -0
  133. azure/ai/evaluation/{_evaluate/_batch_run_client → _evaluators/_intent_resolution}/__init__.py +3 -4
  134. azure/ai/evaluation/_evaluators/_intent_resolution/_intent_resolution.py +196 -0
  135. azure/ai/evaluation/_evaluators/_intent_resolution/intent_resolution.prompty +275 -0
  136. azure/ai/evaluation/_evaluators/_meteor/_meteor.py +107 -61
  137. azure/ai/evaluation/_evaluators/_protected_material/_protected_material.py +104 -77
  138. azure/ai/evaluation/_evaluators/_qa/_qa.py +115 -63
  139. azure/ai/evaluation/_evaluators/_relevance/_relevance.py +182 -98
  140. azure/ai/evaluation/_evaluators/_relevance/relevance.prompty +178 -49
  141. azure/ai/evaluation/_evaluators/_response_completeness/__init__.py +7 -0
  142. azure/ai/evaluation/_evaluators/_response_completeness/_response_completeness.py +202 -0
  143. azure/ai/evaluation/_evaluators/_response_completeness/response_completeness.prompty +84 -0
  144. azure/ai/evaluation/_evaluators/{_chat/retrieval → _retrieval}/__init__.py +2 -2
  145. azure/ai/evaluation/_evaluators/_retrieval/_retrieval.py +148 -0
  146. azure/ai/evaluation/_evaluators/_retrieval/retrieval.prompty +93 -0
  147. azure/ai/evaluation/_evaluators/_rouge/_rouge.py +189 -50
  148. azure/ai/evaluation/_evaluators/_service_groundedness/__init__.py +9 -0
  149. azure/ai/evaluation/_evaluators/_service_groundedness/_service_groundedness.py +179 -0
  150. azure/ai/evaluation/_evaluators/_similarity/_similarity.py +102 -91
  151. azure/ai/evaluation/_evaluators/_similarity/similarity.prompty +0 -5
  152. azure/ai/evaluation/_evaluators/_task_adherence/__init__.py +7 -0
  153. azure/ai/evaluation/_evaluators/_task_adherence/_task_adherence.py +226 -0
  154. azure/ai/evaluation/_evaluators/_task_adherence/task_adherence.prompty +101 -0
  155. azure/ai/evaluation/_evaluators/_task_completion/__init__.py +7 -0
  156. azure/ai/evaluation/_evaluators/_task_completion/_task_completion.py +177 -0
  157. azure/ai/evaluation/_evaluators/_task_completion/task_completion.prompty +220 -0
  158. azure/ai/evaluation/_evaluators/_task_navigation_efficiency/__init__.py +7 -0
  159. azure/ai/evaluation/_evaluators/_task_navigation_efficiency/_task_navigation_efficiency.py +384 -0
  160. azure/ai/evaluation/_evaluators/_tool_call_accuracy/__init__.py +9 -0
  161. azure/ai/evaluation/_evaluators/_tool_call_accuracy/_tool_call_accuracy.py +298 -0
  162. azure/ai/evaluation/_evaluators/_tool_call_accuracy/tool_call_accuracy.prompty +166 -0
  163. azure/ai/evaluation/_evaluators/_tool_input_accuracy/__init__.py +9 -0
  164. azure/ai/evaluation/_evaluators/_tool_input_accuracy/_tool_input_accuracy.py +263 -0
  165. azure/ai/evaluation/_evaluators/_tool_input_accuracy/tool_input_accuracy.prompty +76 -0
  166. azure/ai/evaluation/_evaluators/_tool_output_utilization/__init__.py +7 -0
  167. azure/ai/evaluation/_evaluators/_tool_output_utilization/_tool_output_utilization.py +225 -0
  168. azure/ai/evaluation/_evaluators/_tool_output_utilization/tool_output_utilization.prompty +221 -0
  169. azure/ai/evaluation/_evaluators/_tool_selection/__init__.py +9 -0
  170. azure/ai/evaluation/_evaluators/_tool_selection/_tool_selection.py +266 -0
  171. azure/ai/evaluation/_evaluators/_tool_selection/tool_selection.prompty +104 -0
  172. azure/ai/evaluation/_evaluators/_tool_success/__init__.py +7 -0
  173. azure/ai/evaluation/_evaluators/_tool_success/_tool_success.py +301 -0
  174. azure/ai/evaluation/_evaluators/_tool_success/tool_success.prompty +321 -0
  175. azure/ai/evaluation/_evaluators/_ungrounded_attributes/__init__.py +5 -0
  176. azure/ai/evaluation/_evaluators/_ungrounded_attributes/_ungrounded_attributes.py +102 -0
  177. azure/ai/evaluation/_evaluators/_xpia/xpia.py +109 -107
  178. azure/ai/evaluation/_exceptions.py +51 -7
  179. azure/ai/evaluation/_http_utils.py +210 -137
  180. azure/ai/evaluation/_legacy/__init__.py +3 -0
  181. azure/ai/evaluation/_legacy/_adapters/__init__.py +7 -0
  182. azure/ai/evaluation/_legacy/_adapters/_check.py +17 -0
  183. azure/ai/evaluation/_legacy/_adapters/_configuration.py +45 -0
  184. azure/ai/evaluation/_legacy/_adapters/_constants.py +10 -0
  185. azure/ai/evaluation/_legacy/_adapters/_errors.py +29 -0
  186. azure/ai/evaluation/_legacy/_adapters/_flows.py +28 -0
  187. azure/ai/evaluation/_legacy/_adapters/_service.py +16 -0
  188. azure/ai/evaluation/_legacy/_adapters/client.py +51 -0
  189. azure/ai/evaluation/_legacy/_adapters/entities.py +26 -0
  190. azure/ai/evaluation/_legacy/_adapters/tracing.py +28 -0
  191. azure/ai/evaluation/_legacy/_adapters/types.py +15 -0
  192. azure/ai/evaluation/_legacy/_adapters/utils.py +31 -0
  193. azure/ai/evaluation/_legacy/_batch_engine/__init__.py +9 -0
  194. azure/ai/evaluation/_legacy/_batch_engine/_config.py +48 -0
  195. azure/ai/evaluation/_legacy/_batch_engine/_engine.py +477 -0
  196. azure/ai/evaluation/_legacy/_batch_engine/_exceptions.py +88 -0
  197. azure/ai/evaluation/_legacy/_batch_engine/_openai_injector.py +132 -0
  198. azure/ai/evaluation/_legacy/_batch_engine/_result.py +107 -0
  199. azure/ai/evaluation/_legacy/_batch_engine/_run.py +127 -0
  200. azure/ai/evaluation/_legacy/_batch_engine/_run_storage.py +128 -0
  201. azure/ai/evaluation/_legacy/_batch_engine/_run_submitter.py +262 -0
  202. azure/ai/evaluation/_legacy/_batch_engine/_status.py +25 -0
  203. azure/ai/evaluation/_legacy/_batch_engine/_trace.py +97 -0
  204. azure/ai/evaluation/_legacy/_batch_engine/_utils.py +97 -0
  205. azure/ai/evaluation/_legacy/_batch_engine/_utils_deprecated.py +131 -0
  206. azure/ai/evaluation/_legacy/_common/__init__.py +3 -0
  207. azure/ai/evaluation/_legacy/_common/_async_token_provider.py +117 -0
  208. azure/ai/evaluation/_legacy/_common/_logging.py +292 -0
  209. azure/ai/evaluation/_legacy/_common/_thread_pool_executor_with_context.py +17 -0
  210. azure/ai/evaluation/_legacy/prompty/__init__.py +36 -0
  211. azure/ai/evaluation/_legacy/prompty/_connection.py +119 -0
  212. azure/ai/evaluation/_legacy/prompty/_exceptions.py +139 -0
  213. azure/ai/evaluation/_legacy/prompty/_prompty.py +430 -0
  214. azure/ai/evaluation/_legacy/prompty/_utils.py +663 -0
  215. azure/ai/evaluation/_legacy/prompty/_yaml_utils.py +99 -0
  216. azure/ai/evaluation/_model_configurations.py +130 -8
  217. azure/ai/evaluation/_safety_evaluation/__init__.py +3 -0
  218. azure/ai/evaluation/_safety_evaluation/_generated_rai_client.py +0 -0
  219. azure/ai/evaluation/_safety_evaluation/_safety_evaluation.py +917 -0
  220. azure/ai/evaluation/_user_agent.py +32 -1
  221. azure/ai/evaluation/_vendor/__init__.py +3 -0
  222. azure/ai/evaluation/_vendor/rouge_score/__init__.py +14 -0
  223. azure/ai/evaluation/_vendor/rouge_score/rouge_scorer.py +324 -0
  224. azure/ai/evaluation/_vendor/rouge_score/scoring.py +59 -0
  225. azure/ai/evaluation/_vendor/rouge_score/tokenize.py +59 -0
  226. azure/ai/evaluation/_vendor/rouge_score/tokenizers.py +53 -0
  227. azure/ai/evaluation/_version.py +2 -1
  228. azure/ai/evaluation/red_team/__init__.py +22 -0
  229. azure/ai/evaluation/red_team/_agent/__init__.py +3 -0
  230. azure/ai/evaluation/red_team/_agent/_agent_functions.py +261 -0
  231. azure/ai/evaluation/red_team/_agent/_agent_tools.py +461 -0
  232. azure/ai/evaluation/red_team/_agent/_agent_utils.py +89 -0
  233. azure/ai/evaluation/red_team/_agent/_semantic_kernel_plugin.py +228 -0
  234. azure/ai/evaluation/red_team/_attack_objective_generator.py +268 -0
  235. azure/ai/evaluation/red_team/_attack_strategy.py +49 -0
  236. azure/ai/evaluation/red_team/_callback_chat_target.py +115 -0
  237. azure/ai/evaluation/red_team/_default_converter.py +21 -0
  238. azure/ai/evaluation/red_team/_evaluation_processor.py +505 -0
  239. azure/ai/evaluation/red_team/_mlflow_integration.py +430 -0
  240. azure/ai/evaluation/red_team/_orchestrator_manager.py +803 -0
  241. azure/ai/evaluation/red_team/_red_team.py +1717 -0
  242. azure/ai/evaluation/red_team/_red_team_result.py +661 -0
  243. azure/ai/evaluation/red_team/_result_processor.py +1708 -0
  244. azure/ai/evaluation/red_team/_utils/__init__.py +37 -0
  245. azure/ai/evaluation/red_team/_utils/_rai_service_eval_chat_target.py +128 -0
  246. azure/ai/evaluation/red_team/_utils/_rai_service_target.py +601 -0
  247. azure/ai/evaluation/red_team/_utils/_rai_service_true_false_scorer.py +114 -0
  248. azure/ai/evaluation/red_team/_utils/constants.py +72 -0
  249. azure/ai/evaluation/red_team/_utils/exception_utils.py +345 -0
  250. azure/ai/evaluation/red_team/_utils/file_utils.py +266 -0
  251. azure/ai/evaluation/red_team/_utils/formatting_utils.py +365 -0
  252. azure/ai/evaluation/red_team/_utils/logging_utils.py +139 -0
  253. azure/ai/evaluation/red_team/_utils/metric_mapping.py +73 -0
  254. azure/ai/evaluation/red_team/_utils/objective_utils.py +46 -0
  255. azure/ai/evaluation/red_team/_utils/progress_utils.py +252 -0
  256. azure/ai/evaluation/red_team/_utils/retry_utils.py +218 -0
  257. azure/ai/evaluation/red_team/_utils/strategy_utils.py +218 -0
  258. azure/ai/evaluation/simulator/__init__.py +2 -1
  259. azure/ai/evaluation/simulator/_adversarial_scenario.py +26 -1
  260. azure/ai/evaluation/simulator/_adversarial_simulator.py +270 -144
  261. azure/ai/evaluation/simulator/_constants.py +12 -1
  262. azure/ai/evaluation/simulator/_conversation/__init__.py +151 -23
  263. azure/ai/evaluation/simulator/_conversation/_conversation.py +10 -6
  264. azure/ai/evaluation/simulator/_conversation/constants.py +1 -1
  265. azure/ai/evaluation/simulator/_data_sources/__init__.py +3 -0
  266. azure/ai/evaluation/simulator/_data_sources/grounding.json +1150 -0
  267. azure/ai/evaluation/simulator/_direct_attack_simulator.py +54 -75
  268. azure/ai/evaluation/simulator/_helpers/__init__.py +1 -2
  269. azure/ai/evaluation/simulator/_helpers/_language_suffix_mapping.py +1 -0
  270. azure/ai/evaluation/simulator/_helpers/_simulator_data_classes.py +26 -5
  271. azure/ai/evaluation/simulator/_indirect_attack_simulator.py +145 -104
  272. azure/ai/evaluation/simulator/_model_tools/__init__.py +2 -1
  273. azure/ai/evaluation/simulator/_model_tools/_generated_rai_client.py +225 -0
  274. azure/ai/evaluation/simulator/_model_tools/_identity_manager.py +80 -30
  275. azure/ai/evaluation/simulator/_model_tools/_proxy_completion_model.py +117 -45
  276. azure/ai/evaluation/simulator/_model_tools/_rai_client.py +109 -7
  277. azure/ai/evaluation/simulator/_model_tools/_template_handler.py +97 -33
  278. azure/ai/evaluation/simulator/_model_tools/models.py +30 -27
  279. azure/ai/evaluation/simulator/_prompty/task_query_response.prompty +6 -10
  280. azure/ai/evaluation/simulator/_prompty/task_simulate.prompty +6 -5
  281. azure/ai/evaluation/simulator/_simulator.py +302 -208
  282. azure/ai/evaluation/simulator/_utils.py +31 -13
  283. azure_ai_evaluation-1.13.3.dist-info/METADATA +939 -0
  284. azure_ai_evaluation-1.13.3.dist-info/RECORD +305 -0
  285. {azure_ai_evaluation-1.0.0b2.dist-info → azure_ai_evaluation-1.13.3.dist-info}/WHEEL +1 -1
  286. azure_ai_evaluation-1.13.3.dist-info/licenses/NOTICE.txt +70 -0
  287. azure/ai/evaluation/_evaluate/_batch_run_client/proxy_client.py +0 -71
  288. azure/ai/evaluation/_evaluators/_chat/_chat.py +0 -357
  289. azure/ai/evaluation/_evaluators/_chat/retrieval/_retrieval.py +0 -157
  290. azure/ai/evaluation/_evaluators/_chat/retrieval/retrieval.prompty +0 -48
  291. azure/ai/evaluation/_evaluators/_content_safety/_content_safety_base.py +0 -65
  292. azure/ai/evaluation/_evaluators/_content_safety/_content_safety_chat.py +0 -301
  293. azure/ai/evaluation/_evaluators/_groundedness/groundedness.prompty +0 -54
  294. azure/ai/evaluation/_evaluators/_protected_materials/__init__.py +0 -5
  295. azure/ai/evaluation/_evaluators/_protected_materials/_protected_materials.py +0 -104
  296. azure/ai/evaluation/simulator/_tracing.py +0 -89
  297. azure_ai_evaluation-1.0.0b2.dist-info/METADATA +0 -449
  298. azure_ai_evaluation-1.0.0b2.dist-info/RECORD +0 -99
  299. {azure_ai_evaluation-1.0.0b2.dist-info → azure_ai_evaluation-1.13.3.dist-info}/top_level.txt +0 -0
@@ -3,12 +3,11 @@
3
3
  # ---------------------------------------------------------
4
4
 
5
5
 
6
- from functools import wraps
7
- from typing import Any, Awaitable, Callable, Dict, MutableMapping, Optional
6
+ from typing import Any, Dict, MutableMapping, Optional, TypedDict, cast
8
7
 
9
- from typing_extensions import Self
8
+ from typing_extensions import Self, Unpack
10
9
 
11
- from azure.ai.evaluation._user_agent import USER_AGENT
10
+ from azure.ai.evaluation._user_agent import UserAgentSingleton
12
11
  from azure.core.configuration import Configuration
13
12
  from azure.core.pipeline import AsyncPipeline, Pipeline
14
13
  from azure.core.pipeline.policies import (
@@ -33,78 +32,25 @@ from azure.core.rest import AsyncHttpResponse, HttpRequest, HttpResponse
33
32
  from azure.core.rest._rest_py3 import ContentType, FilesType, ParamsType
34
33
 
35
34
 
36
- def _request_fn(f: Callable[["HttpPipeline"], None]):
37
- """Decorator to generate convenience methods for HTTP method.
35
+ class RequestKwargs(TypedDict, total=False):
36
+ """Keyword arguments for request-style http request functions
38
37
 
39
- :param Callable[["HttpPipeline"],None] f: A HttpPipeline classmethod to wrap.
40
- The f.__name__ is the HTTP method used
41
- :return: A wrapped callable that sends a `f.__name__` request
42
- :rtype: Callable
43
- """
38
+ .. note::
44
39
 
45
- @wraps(f)
46
- def request_fn(
47
- self: "HttpPipeline",
48
- url: str,
49
- *,
50
- params: Optional[ParamsType] = None,
51
- headers: Optional[MutableMapping[str, str]] = None,
52
- json: Any = None,
53
- content: Optional[ContentType] = None,
54
- data: Optional[Dict[str, Any]] = None,
55
- files: Optional[FilesType] = None,
56
- **kwargs,
57
- ) -> HttpResponse:
58
- return self.request(
59
- f.__name__.upper(),
60
- url,
61
- params=params,
62
- headers=headers,
63
- json=json,
64
- content=content,
65
- data=data,
66
- files=files,
67
- **kwargs,
68
- )
40
+ Ideally, we'd be able to express that these are the known subset of kwargs, but it's possible to provide
41
+ others. But that currently isn't possible; there's no way currently to express a TypedDict that expects
42
+ a known set of keys and an unknown set of keys.
69
43
 
70
- return request_fn
71
-
72
-
73
- def _async_request_fn(f: Callable[["AsyncHttpPipeline"], Awaitable[None]]):
74
- """Decorator to generate convenience methods for HTTP method.
75
-
76
- :param Callable[["HttpPipeline"],None] f: A HttpPipeline classmethod to wrap.
77
- The f.__name__ is the HTTP method used
78
- :return: A wrapped callable that sends a `f.__name__` request
79
- :rtype: Callable
44
+ PEP 728 - TypedDict with Typed Extra Items (https://peps.python.org/pep-0728/) would rectify this but it's
45
+ still in Draft status.
80
46
  """
81
47
 
82
- @wraps(f)
83
- async def request_fn(
84
- self: "AsyncHttpPipeline",
85
- url: str,
86
- *,
87
- params: Optional[ParamsType] = None,
88
- headers: Optional[MutableMapping[str, str]] = None,
89
- json: Any = None,
90
- content: Optional[ContentType] = None,
91
- data: Optional[Dict[str, Any]] = None,
92
- files: Optional[FilesType] = None,
93
- **kwargs,
94
- ) -> AsyncHttpResponse:
95
- return await self.request(
96
- f.__name__.upper(),
97
- url,
98
- params=params,
99
- headers=headers,
100
- json=json,
101
- content=content,
102
- data=data,
103
- files=files,
104
- **kwargs,
105
- )
106
-
107
- return request_fn
48
+ params: ParamsType
49
+ headers: MutableMapping[str, str]
50
+ json: Any
51
+ content: ContentType
52
+ data: Dict[str, Any]
53
+ files: FilesType
108
54
 
109
55
 
110
56
  class HttpPipeline(Pipeline):
@@ -145,14 +91,32 @@ class HttpPipeline(Pipeline):
145
91
  :param RedirectPolicy redirect_policy:
146
92
  """
147
93
  config = config or Configuration()
148
- config.headers_policy = headers_policy or config.headers_policy or HeadersPolicy(**kwargs)
149
- config.proxy_policy = proxy_policy or config.proxy_policy or ProxyPolicy(**kwargs)
150
- config.redirect_policy = redirect_policy or config.redirect_policy or RedirectPolicy(**kwargs)
151
- config.retry_policy = retry_policy or config.retry_policy or RetryPolicy(**kwargs)
152
- config.custom_hook_policy = custom_hook_policy or config.custom_hook_policy or CustomHookPolicy(**kwargs)
153
- config.logging_policy = logging_policy or config.logging_policy or NetworkTraceLoggingPolicy(**kwargs)
154
- config.http_logging_policy = http_logging_policy or config.http_logging_policy or HttpLoggingPolicy(**kwargs)
155
- config.user_agent_policy = user_agent_policy or config.user_agent_policy or UserAgentPolicy(**kwargs)
94
+ config.headers_policy = (
95
+ headers_policy or cast(Optional[HeadersPolicy], config.headers_policy) or HeadersPolicy(**kwargs)
96
+ )
97
+ config.proxy_policy = proxy_policy or cast(Optional[ProxyPolicy], config.proxy_policy) or ProxyPolicy(**kwargs)
98
+ config.redirect_policy = (
99
+ redirect_policy or cast(Optional[RedirectPolicy], config.redirect_policy) or RedirectPolicy(**kwargs)
100
+ )
101
+ config.retry_policy = retry_policy or cast(Optional[RetryPolicy], config.retry_policy) or RetryPolicy(**kwargs)
102
+ config.custom_hook_policy = (
103
+ custom_hook_policy
104
+ or cast(Optional[CustomHookPolicy], config.custom_hook_policy)
105
+ or CustomHookPolicy(**kwargs)
106
+ )
107
+ config.logging_policy = (
108
+ logging_policy
109
+ or cast(Optional[NetworkTraceLoggingPolicy], config.logging_policy)
110
+ or NetworkTraceLoggingPolicy(**kwargs)
111
+ )
112
+ config.http_logging_policy = (
113
+ http_logging_policy
114
+ or cast(Optional[HttpLoggingPolicy], config.http_logging_policy)
115
+ or HttpLoggingPolicy(**kwargs)
116
+ )
117
+ config.user_agent_policy = (
118
+ user_agent_policy or cast(Optional[UserAgentPolicy], config.user_agent_policy) or UserAgentPolicy(**kwargs)
119
+ )
156
120
  config.polling_interval = kwargs.get("polling_interval", 30)
157
121
 
158
122
  super().__init__(
@@ -166,7 +130,6 @@ class HttpPipeline(Pipeline):
166
130
  config.proxy_policy,
167
131
  config.redirect_policy,
168
132
  config.retry_policy,
169
- config.authentication_policy,
170
133
  config.custom_hook_policy,
171
134
  config.logging_policy,
172
135
  ],
@@ -199,7 +162,6 @@ class HttpPipeline(Pipeline):
199
162
  files: Optional[FilesType] = None,
200
163
  **kwargs,
201
164
  ) -> HttpResponse:
202
-
203
165
  request = HttpRequest(
204
166
  method,
205
167
  url,
@@ -213,33 +175,78 @@ class HttpPipeline(Pipeline):
213
175
 
214
176
  return self.run(request, **kwargs).http_response
215
177
 
216
- @_request_fn
217
- def delete(self) -> None:
218
- """Send a DELETE request."""
178
+ def delete(self: "HttpPipeline", url: str, **kwargs: Unpack[RequestKwargs]) -> HttpResponse:
179
+ """Send a DELETE request.
219
180
 
220
- @_request_fn
221
- def put(self) -> None:
222
- """Send a PUT request."""
181
+ :param str url: The request url
182
+ :returns: The request response
183
+ :rtype: HttpResponse
184
+ """
223
185
 
224
- @_request_fn
225
- def get(self) -> None:
226
- """Send a GET request."""
186
+ return self.request(self.delete.__name__.upper(), url, **kwargs)
227
187
 
228
- @_request_fn
229
- def post(self) -> None:
230
- """Send a POST request."""
188
+ def put(self: "HttpPipeline", url: str, **kwargs: Unpack[RequestKwargs]) -> HttpResponse:
189
+ """Send a PUT request.
231
190
 
232
- @_request_fn
233
- def head(self) -> None:
234
- """Send a HEAD request."""
191
+ :param str url: The request url
192
+ :returns: The request response
193
+ :rtype: HttpResponse
194
+ """
235
195
 
236
- @_request_fn
237
- def options(self) -> None:
238
- """Send a OPTIONS request."""
196
+ return self.request(self.put.__name__.upper(), url, **kwargs)
239
197
 
240
- @_request_fn
241
- def patch(self) -> None:
242
- """Send a PATCH request."""
198
+ def get(self: "HttpPipeline", url: str, **kwargs: Unpack[RequestKwargs]) -> HttpResponse:
199
+ """Send a GET request.
200
+
201
+ :param str url: The request url
202
+ :returns: The request response
203
+ :rtype: HttpResponse
204
+ """
205
+
206
+ return self.request(self.get.__name__.upper(), url, **kwargs)
207
+
208
+ def post(self: "HttpPipeline", url: str, **kwargs: Unpack[RequestKwargs]) -> HttpResponse:
209
+ """Send a POST request.
210
+
211
+ :param str url: The request url
212
+ :returns: The request response
213
+ :rtype: HttpResponse
214
+ """
215
+
216
+ return self.request(self.post.__name__.upper(), url, **kwargs)
217
+
218
+ def head(self: "HttpPipeline", url: str, **kwargs: Unpack[RequestKwargs]) -> HttpResponse:
219
+ """Send a HEAD request.
220
+
221
+ :param str url: The request url
222
+ :returns: The request response
223
+ :rtype: HttpResponse
224
+ """
225
+
226
+ return self.request(self.head.__name__.upper(), url, **kwargs)
227
+
228
+ def options(self: "HttpPipeline", url: str, **kwargs: Unpack[RequestKwargs]) -> HttpResponse:
229
+ """Send a OPTIONS request.
230
+
231
+ :param str url: The request url
232
+ :returns: The request response
233
+ :rtype: HttpResponse
234
+ """
235
+
236
+ return self.request(self.options.__name__.upper(), url, **kwargs)
237
+
238
+ def patch(self: "HttpPipeline", url: str, **kwargs: Unpack[RequestKwargs]) -> HttpResponse:
239
+ """Send a PATCH request.
240
+
241
+ :param str url: The request url
242
+ :returns: The request response
243
+ :rtype: HttpResponse
244
+ """
245
+
246
+ return self.request(self.patch.__name__.upper(), url, **kwargs)
247
+
248
+ def __enter__(self) -> Self:
249
+ return cast(Self, super().__enter__())
243
250
 
244
251
 
245
252
  class AsyncHttpPipeline(AsyncPipeline):
@@ -280,14 +287,36 @@ class AsyncHttpPipeline(AsyncPipeline):
280
287
  :param AsyncRedirectPolicy redirect_policy:
281
288
  """
282
289
  config = config or Configuration()
283
- config.headers_policy = headers_policy or config.headers_policy or HeadersPolicy(**kwargs)
284
- config.proxy_policy = proxy_policy or config.proxy_policy or ProxyPolicy(**kwargs)
285
- config.redirect_policy = redirect_policy or config.redirect_policy or AsyncRedirectPolicy(**kwargs)
286
- config.retry_policy = retry_policy or config.retry_policy or AsyncRetryPolicy(**kwargs)
287
- config.custom_hook_policy = custom_hook_policy or config.custom_hook_policy or CustomHookPolicy(**kwargs)
288
- config.logging_policy = logging_policy or config.logging_policy or NetworkTraceLoggingPolicy(**kwargs)
289
- config.http_logging_policy = http_logging_policy or config.http_logging_policy or HttpLoggingPolicy(**kwargs)
290
- config.user_agent_policy = user_agent_policy or config.user_agent_policy or UserAgentPolicy(**kwargs)
290
+ config.headers_policy = (
291
+ headers_policy or cast(Optional[HeadersPolicy], config.headers_policy) or HeadersPolicy(**kwargs)
292
+ )
293
+ config.proxy_policy = proxy_policy or cast(Optional[ProxyPolicy], config.proxy_policy) or ProxyPolicy(**kwargs)
294
+ config.redirect_policy = (
295
+ redirect_policy
296
+ or cast(Optional[AsyncRedirectPolicy], config.redirect_policy)
297
+ or AsyncRedirectPolicy(**kwargs)
298
+ )
299
+ config.retry_policy = (
300
+ retry_policy or cast(Optional[AsyncRetryPolicy], config.retry_policy) or AsyncRetryPolicy(**kwargs)
301
+ )
302
+ config.custom_hook_policy = (
303
+ custom_hook_policy
304
+ or cast(Optional[CustomHookPolicy], config.custom_hook_policy)
305
+ or CustomHookPolicy(**kwargs)
306
+ )
307
+ config.logging_policy = (
308
+ logging_policy
309
+ or cast(Optional[NetworkTraceLoggingPolicy], config.logging_policy)
310
+ or NetworkTraceLoggingPolicy(**kwargs)
311
+ )
312
+ config.http_logging_policy = (
313
+ http_logging_policy
314
+ or cast(Optional[HttpLoggingPolicy], config.http_logging_policy)
315
+ or HttpLoggingPolicy(**kwargs)
316
+ )
317
+ config.user_agent_policy = (
318
+ user_agent_policy or cast(Optional[UserAgentPolicy], config.user_agent_policy) or UserAgentPolicy(**kwargs)
319
+ )
291
320
  config.polling_interval = kwargs.get("polling_interval", 30)
292
321
 
293
322
  super().__init__(
@@ -301,7 +330,6 @@ class AsyncHttpPipeline(AsyncPipeline):
301
330
  config.proxy_policy,
302
331
  config.redirect_policy,
303
332
  config.retry_policy,
304
- config.authentication_policy,
305
333
  config.custom_hook_policy,
306
334
  config.logging_policy,
307
335
  ],
@@ -334,7 +362,6 @@ class AsyncHttpPipeline(AsyncPipeline):
334
362
  files: Optional[FilesType] = None,
335
363
  **kwargs,
336
364
  ) -> AsyncHttpResponse:
337
-
338
365
  request = HttpRequest(
339
366
  method,
340
367
  url,
@@ -348,48 +375,94 @@ class AsyncHttpPipeline(AsyncPipeline):
348
375
 
349
376
  return (await self.run(request, **kwargs)).http_response
350
377
 
351
- @_async_request_fn
352
- async def delete(self) -> None:
353
- """Send a DELETE request."""
378
+ async def delete(self: "AsyncHttpPipeline", url: str, **kwargs: Unpack[RequestKwargs]) -> AsyncHttpResponse:
379
+ """Send a DELETE request.
380
+
381
+ :param str url: The request url
382
+ :returns: The request response
383
+ :rtype: AsyncHttpResponse
384
+ """
385
+ return await self.request(self.delete.__name__.upper(), url, **kwargs)
386
+
387
+ async def put(self: "AsyncHttpPipeline", url: str, **kwargs: Unpack[RequestKwargs]) -> AsyncHttpResponse:
388
+ """Send a PUT request.
389
+
390
+ :param str url: The request url
391
+ :returns: The request response
392
+ :rtype: AsyncHttpResponse
393
+ """
394
+
395
+ return await self.request(self.put.__name__.upper(), url, **kwargs)
396
+
397
+ async def get(self: "AsyncHttpPipeline", url: str, **kwargs: Unpack[RequestKwargs]) -> AsyncHttpResponse:
398
+ """Send a GET request.
399
+
400
+ :param str url: The request url
401
+ :returns: The request response
402
+ :rtype: AsyncHttpResponse
403
+ """
404
+
405
+ return await self.request(self.get.__name__.upper(), url, **kwargs)
406
+
407
+ async def post(self: "AsyncHttpPipeline", url: str, **kwargs: Unpack[RequestKwargs]) -> AsyncHttpResponse:
408
+ """Send a POST request.
409
+
410
+ :param str url: The request url
411
+ :returns: The request response
412
+ :rtype: AsyncHttpResponse
413
+ """
414
+
415
+ return await self.request(self.post.__name__.upper(), url, **kwargs)
416
+
417
+ async def head(self: "AsyncHttpPipeline", url: str, **kwargs: Unpack[RequestKwargs]) -> AsyncHttpResponse:
418
+ """Send a HEAD request.
354
419
 
355
- @_async_request_fn
356
- async def put(self) -> None:
357
- """Send a PUT request."""
420
+ :param str url: The request url
421
+ :returns: The request response
422
+ :rtype: AsyncHttpResponse
423
+ """
424
+
425
+ return await self.request(self.head.__name__.upper(), url, **kwargs)
426
+
427
+ async def options(self: "AsyncHttpPipeline", url: str, **kwargs: Unpack[RequestKwargs]) -> AsyncHttpResponse:
428
+ """Send a OPTIONS request.
358
429
 
359
- @_async_request_fn
360
- async def get(self) -> None:
361
- """Send a GET request."""
430
+ :param str url: The request url
431
+ :returns: The request response
432
+ :rtype: AsyncHttpResponse
433
+ """
434
+
435
+ return await self.request(self.options.__name__.upper(), url, **kwargs)
362
436
 
363
- @_async_request_fn
364
- async def post(self) -> None:
365
- """Send a POST request."""
437
+ async def patch(self: "AsyncHttpPipeline", url: str, **kwargs: Unpack[RequestKwargs]) -> AsyncHttpResponse:
438
+ """Send a PATCH request.
366
439
 
367
- @_async_request_fn
368
- async def head(self) -> None:
369
- """Send a HEAD request."""
440
+ :param str url: The request url
441
+ :returns: The request response
442
+ :rtype: AsyncHttpResponse
443
+ """
370
444
 
371
- @_async_request_fn
372
- async def options(self) -> None:
373
- """Send a OPTIONS request."""
445
+ return await self.request(self.patch.__name__.upper(), url, **kwargs)
374
446
 
375
- @_async_request_fn
376
- async def patch(self) -> None:
377
- """Send a PATCH request."""
447
+ async def __aenter__(self) -> Self:
448
+ return cast(Self, await super().__aenter__())
378
449
 
379
450
 
380
- def get_http_client() -> HttpPipeline:
451
+ def get_http_client(**kwargs: Any) -> HttpPipeline:
381
452
  """Get an HttpPipeline configured with common policies.
382
453
 
383
454
  :returns: An HttpPipeline with a set of applied policies:
384
455
  :rtype: HttpPipeline
385
456
  """
386
- return HttpPipeline(user_agent_policy=UserAgentPolicy(base_user_agent=USER_AGENT))
457
+ kwargs.setdefault("user_agent_policy", UserAgentPolicy(base_user_agent=UserAgentSingleton().value))
458
+ return HttpPipeline(**kwargs)
387
459
 
388
460
 
389
- def get_async_http_client() -> AsyncHttpPipeline:
461
+ def get_async_http_client(**kwargs: Any) -> AsyncHttpPipeline:
390
462
  """Get an AsyncHttpPipeline configured with common policies.
391
463
 
392
464
  :returns: An AsyncHttpPipeline with a set of applied policies:
393
465
  :rtype: AsyncHttpPipeline
394
466
  """
395
- return AsyncHttpPipeline(user_agent_policy=UserAgentPolicy(base_user_agent=USER_AGENT))
467
+ kwargs.setdefault("user_agent_policy", UserAgentPolicy(base_user_agent=UserAgentSingleton().value))
468
+ return AsyncHttpPipeline(**kwargs)
@@ -0,0 +1,3 @@
1
+ # ---------------------------------------------------------
2
+ # Copyright (c) Microsoft Corporation. All rights reserved.
3
+ # ---------------------------------------------------------
@@ -0,0 +1,7 @@
1
+ # ---------------------------------------------------------
2
+ # Copyright (c) Microsoft Corporation. All rights reserved.
3
+ # ---------------------------------------------------------
4
+
5
+ # NOTE: This contains adapters that make the Promptflow dependency optional. In the first phase,
6
+ # Promptflow will still be installed as part of the azure-ai-evaluation dependencies. This
7
+ # will be removed in the future once the code migration is complete.
@@ -0,0 +1,17 @@
1
+ # ---------------------------------------------------------
2
+ # Copyright (c) Microsoft Corporation. All rights reserved.
3
+ # ---------------------------------------------------------
4
+
5
+ from typing import Final
6
+
7
+
8
+ _has_legacy = False
9
+ try:
10
+ from promptflow._constants import FlowType
11
+
12
+ _has_legacy = True
13
+ except ImportError:
14
+ pass
15
+
16
+ HAS_LEGACY_SDK: Final[bool] = _has_legacy
17
+ MISSING_LEGACY_SDK: Final[bool] = not _has_legacy
@@ -0,0 +1,45 @@
1
+ # ---------------------------------------------------------
2
+ # Copyright (c) Microsoft Corporation. All rights reserved.
3
+ # ---------------------------------------------------------
4
+
5
+ from pathlib import Path
6
+ from typing import Any, Dict, Final, Optional
7
+ from typing_extensions import TypeAlias
8
+
9
+
10
+ try:
11
+ from promptflow._sdk._configuration import Configuration as _Configuration
12
+ except ImportError:
13
+ _global_config: Final[Dict[str, Any]] = {}
14
+
15
+ class _Configuration:
16
+ TRACE_DESTINATION: Final[str] = "trace.destination"
17
+ _instance = None
18
+
19
+ def __init__(self, *, override_config: Optional[Dict[str, Any]] = None) -> None:
20
+ self._config = override_config or {}
21
+
22
+ @classmethod
23
+ def get_instance(cls) -> "_Configuration":
24
+ """Use this to get instance to avoid multiple copies of same global config."""
25
+ if cls._instance is None:
26
+ cls._instance = Configuration(override_config=_global_config)
27
+ return cls._instance
28
+
29
+ def set_config(self, key: str, value: Any) -> None:
30
+ # Simulated config storage
31
+ self._config[key] = value
32
+
33
+ def get_config(self, key: str) -> Any:
34
+ # Simulated config storage
35
+ if key in self._config:
36
+ return self._config[key]
37
+ return _global_config.get(key, None)
38
+
39
+ def get_trace_destination(self, path: Optional[Path] = None) -> Optional[str]:
40
+ if path:
41
+ raise NotImplementedError("Setting trace destination with a path is not supported.")
42
+ return self._config.get("trace.destination", None)
43
+
44
+
45
+ Configuration: TypeAlias = _Configuration
@@ -0,0 +1,10 @@
1
+ # ---------------------------------------------------------
2
+ # Copyright (c) Microsoft Corporation. All rights reserved.
3
+ # ---------------------------------------------------------
4
+
5
+ from typing import Final
6
+
7
+
8
+ PF_FLOW_ENTRY_IN_TMP: Final[str] = "PF_FLOW_ENTRY_IN_TMP"
9
+ PF_FLOW_META_LOAD_IN_SUBPROCESS: Final[str] = "PF_FLOW_META_LOAD_IN_SUBPROCESS"
10
+ LINE_NUMBER: Final[str] = "line_number"
@@ -0,0 +1,29 @@
1
+ # ---------------------------------------------------------
2
+ # Copyright (c) Microsoft Corporation. All rights reserved.
3
+ # ---------------------------------------------------------
4
+
5
+ from typing import Any
6
+ from typing_extensions import TypeAlias
7
+
8
+
9
+ try:
10
+ from promptflow.core._errors import MissingRequiredPackage as _MissingRequiredPackage
11
+ except ImportError:
12
+ from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
13
+
14
+ class _MissingRequiredPackage(EvaluationException):
15
+ """Raised when a required package is missing.
16
+
17
+ :param message: A message describing the error. This is the error message the user will see.
18
+ :type message: str
19
+ """
20
+
21
+ def __init__(self, message: str, **kwargs: Any):
22
+ kwargs.setdefault("category", ErrorCategory.MISSING_PACKAGE)
23
+ kwargs.setdefault("blame", ErrorBlame.SYSTEM_ERROR)
24
+ kwargs.setdefault("target", ErrorTarget.EVALUATE)
25
+ kwargs.setdefault("internal_message", "Missing required package.")
26
+ super().__init__(message=message, **kwargs)
27
+
28
+
29
+ MissingRequiredPackage: TypeAlias = _MissingRequiredPackage
@@ -0,0 +1,28 @@
1
+ # ---------------------------------------------------------
2
+ # Copyright (c) Microsoft Corporation. All rights reserved.
3
+ # ---------------------------------------------------------
4
+
5
+ from typing_extensions import TypeAlias
6
+
7
+
8
+ try:
9
+ from promptflow.core._flow import AsyncPrompty as _AsyncPrompty
10
+ from promptflow._sdk.entities._flows import FlexFlow as _FlexFlow
11
+ from promptflow._sdk.entities._flows.dag import Flow as _Flow
12
+ except ImportError:
13
+ from azure.ai.evaluation._legacy.prompty import AsyncPrompty as _AsyncPrompty
14
+
15
+ class _FlexFlow:
16
+ pass
17
+
18
+ _FlexFlow.__name__ = "FlexFlow"
19
+
20
+ class _Flow:
21
+ name: str
22
+
23
+ _Flow.__name__ = "Flow"
24
+
25
+
26
+ AsyncPrompty: TypeAlias = _AsyncPrompty
27
+ FlexFlow: TypeAlias = _FlexFlow
28
+ Flow: TypeAlias = _Flow
@@ -0,0 +1,16 @@
1
+ # ---------------------------------------------------------
2
+ # Copyright (c) Microsoft Corporation. All rights reserved.
3
+ # ---------------------------------------------------------
4
+
5
+ from typing import Callable, Final
6
+
7
+
8
+ try:
9
+ from promptflow._cli._pf._service import stop_service as _stop_service
10
+ except ImportError:
11
+
12
+ def _stop_service() -> None:
13
+ pass
14
+
15
+
16
+ stop_service: Final[Callable[[], None]] = _stop_service
@@ -0,0 +1,51 @@
1
+ # ---------------------------------------------------------
2
+ # Copyright (c) Microsoft Corporation. All rights reserved.
3
+ # ---------------------------------------------------------
4
+
5
+ from os import PathLike
6
+ from typing import Any, Callable, Dict, Optional, Union
7
+ from typing_extensions import TypeAlias
8
+
9
+ import pandas as pd
10
+
11
+ from ._errors import MissingRequiredPackage
12
+ from ._configuration import Configuration
13
+ from .entities import Run
14
+
15
+
16
+ try:
17
+ from promptflow.client import PFClient as _PFClient
18
+ except ImportError:
19
+
20
+ class _PFClient:
21
+ def __init__(self, **kwargs):
22
+ self._config = Configuration(override_config=kwargs.pop("config", None))
23
+
24
+ def run(
25
+ self,
26
+ flow: Union[str, PathLike, Callable],
27
+ *,
28
+ data: Union[str, PathLike],
29
+ run: Optional[Union[str, Run]] = None,
30
+ column_mapping: Optional[dict] = None,
31
+ variant: Optional[str] = None,
32
+ connections: Optional[dict] = None,
33
+ environment_variables: Optional[dict] = None,
34
+ name: Optional[str] = None,
35
+ display_name: Optional[str] = None,
36
+ tags: Optional[Dict[str, str]] = None,
37
+ resume_from: Optional[Union[str, Run]] = None,
38
+ code: Optional[Union[str, PathLike]] = None,
39
+ init: Optional[dict] = None,
40
+ **kwargs,
41
+ ) -> Run:
42
+ raise MissingRequiredPackage("Please install 'promptflow' package to use PFClient")
43
+
44
+ def get_details(self, run: Union[str, Run], max_results: int = 100, all_results: bool = False) -> pd.DataFrame:
45
+ return pd.DataFrame()
46
+
47
+ def get_metrics(self, run: Union[str, Run]) -> Dict[str, Any]:
48
+ return {}
49
+
50
+
51
+ PFClient: TypeAlias = _PFClient