azure-ai-evaluation 1.4.0__py3-none-any.whl → 1.6.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.

This version of azure-ai-evaluation might be problematic. Click here for more details.

Files changed (150) hide show
  1. azure/ai/evaluation/__init__.py +9 -16
  2. azure/ai/evaluation/_aoai/__init__.py +10 -0
  3. azure/ai/evaluation/_aoai/aoai_grader.py +89 -0
  4. azure/ai/evaluation/_aoai/label_grader.py +66 -0
  5. azure/ai/evaluation/_aoai/string_check_grader.py +65 -0
  6. azure/ai/evaluation/_aoai/text_similarity_grader.py +88 -0
  7. azure/ai/evaluation/_azure/_clients.py +4 -4
  8. azure/ai/evaluation/_azure/_envs.py +208 -0
  9. azure/ai/evaluation/_azure/_token_manager.py +12 -7
  10. azure/ai/evaluation/_common/__init__.py +5 -0
  11. azure/ai/evaluation/_common/evaluation_onedp_client.py +118 -0
  12. azure/ai/evaluation/_common/onedp/__init__.py +32 -0
  13. azure/ai/evaluation/_common/onedp/_client.py +139 -0
  14. azure/ai/evaluation/_common/onedp/_configuration.py +73 -0
  15. azure/ai/evaluation/_common/onedp/_model_base.py +1232 -0
  16. azure/ai/evaluation/_common/onedp/_patch.py +21 -0
  17. azure/ai/evaluation/_common/onedp/_serialization.py +2032 -0
  18. azure/ai/evaluation/_common/onedp/_types.py +21 -0
  19. azure/ai/evaluation/_common/onedp/_validation.py +50 -0
  20. azure/ai/evaluation/_common/onedp/_vendor.py +50 -0
  21. azure/ai/evaluation/_common/onedp/_version.py +9 -0
  22. azure/ai/evaluation/_common/onedp/aio/__init__.py +29 -0
  23. azure/ai/evaluation/_common/onedp/aio/_client.py +143 -0
  24. azure/ai/evaluation/_common/onedp/aio/_configuration.py +75 -0
  25. azure/ai/evaluation/_common/onedp/aio/_patch.py +21 -0
  26. azure/ai/evaluation/_common/onedp/aio/_vendor.py +40 -0
  27. azure/ai/evaluation/_common/onedp/aio/operations/__init__.py +39 -0
  28. azure/ai/evaluation/_common/onedp/aio/operations/_operations.py +4494 -0
  29. azure/ai/evaluation/_common/onedp/aio/operations/_patch.py +21 -0
  30. azure/ai/evaluation/_common/onedp/models/__init__.py +142 -0
  31. azure/ai/evaluation/_common/onedp/models/_enums.py +162 -0
  32. azure/ai/evaluation/_common/onedp/models/_models.py +2228 -0
  33. azure/ai/evaluation/_common/onedp/models/_patch.py +21 -0
  34. azure/ai/evaluation/_common/onedp/operations/__init__.py +39 -0
  35. azure/ai/evaluation/_common/onedp/operations/_operations.py +5655 -0
  36. azure/ai/evaluation/_common/onedp/operations/_patch.py +21 -0
  37. azure/ai/evaluation/_common/onedp/py.typed +1 -0
  38. azure/ai/evaluation/_common/onedp/servicepatterns/__init__.py +1 -0
  39. azure/ai/evaluation/_common/onedp/servicepatterns/aio/__init__.py +1 -0
  40. azure/ai/evaluation/_common/onedp/servicepatterns/aio/operations/__init__.py +25 -0
  41. azure/ai/evaluation/_common/onedp/servicepatterns/aio/operations/_operations.py +34 -0
  42. azure/ai/evaluation/_common/onedp/servicepatterns/aio/operations/_patch.py +20 -0
  43. azure/ai/evaluation/_common/onedp/servicepatterns/buildingblocks/__init__.py +1 -0
  44. azure/ai/evaluation/_common/onedp/servicepatterns/buildingblocks/aio/__init__.py +1 -0
  45. azure/ai/evaluation/_common/onedp/servicepatterns/buildingblocks/aio/operations/__init__.py +22 -0
  46. azure/ai/evaluation/_common/onedp/servicepatterns/buildingblocks/aio/operations/_operations.py +29 -0
  47. azure/ai/evaluation/_common/onedp/servicepatterns/buildingblocks/aio/operations/_patch.py +20 -0
  48. azure/ai/evaluation/_common/onedp/servicepatterns/buildingblocks/operations/__init__.py +22 -0
  49. azure/ai/evaluation/_common/onedp/servicepatterns/buildingblocks/operations/_operations.py +29 -0
  50. azure/ai/evaluation/_common/onedp/servicepatterns/buildingblocks/operations/_patch.py +20 -0
  51. azure/ai/evaluation/_common/onedp/servicepatterns/operations/__init__.py +25 -0
  52. azure/ai/evaluation/_common/onedp/servicepatterns/operations/_operations.py +34 -0
  53. azure/ai/evaluation/_common/onedp/servicepatterns/operations/_patch.py +20 -0
  54. azure/ai/evaluation/_common/rai_service.py +159 -29
  55. azure/ai/evaluation/_common/raiclient/_version.py +1 -1
  56. azure/ai/evaluation/_common/utils.py +80 -2
  57. azure/ai/evaluation/_constants.py +16 -0
  58. azure/ai/evaluation/_converters/__init__.py +1 -1
  59. azure/ai/evaluation/_converters/_ai_services.py +4 -4
  60. azure/ai/evaluation/_eval_mapping.py +71 -0
  61. azure/ai/evaluation/_evaluate/_batch_run/_run_submitter_client.py +30 -16
  62. azure/ai/evaluation/_evaluate/_batch_run/code_client.py +18 -12
  63. azure/ai/evaluation/_evaluate/_batch_run/eval_run_context.py +17 -4
  64. azure/ai/evaluation/_evaluate/_batch_run/proxy_client.py +47 -22
  65. azure/ai/evaluation/_evaluate/_batch_run/target_run_context.py +18 -2
  66. azure/ai/evaluation/_evaluate/_eval_run.py +2 -2
  67. azure/ai/evaluation/_evaluate/_evaluate.py +372 -105
  68. azure/ai/evaluation/_evaluate/_evaluate_aoai.py +534 -0
  69. azure/ai/evaluation/_evaluate/_telemetry/__init__.py +5 -89
  70. azure/ai/evaluation/_evaluate/_utils.py +120 -7
  71. azure/ai/evaluation/_evaluators/_common/_base_eval.py +9 -4
  72. azure/ai/evaluation/_evaluators/_common/_base_multi_eval.py +1 -1
  73. azure/ai/evaluation/_evaluators/_common/_base_prompty_eval.py +12 -3
  74. azure/ai/evaluation/_evaluators/_common/_base_rai_svc_eval.py +2 -2
  75. azure/ai/evaluation/_evaluators/_document_retrieval/__init__.py +11 -0
  76. azure/ai/evaluation/_evaluators/_document_retrieval/_document_retrieval.py +467 -0
  77. azure/ai/evaluation/_evaluators/_fluency/_fluency.py +1 -1
  78. azure/ai/evaluation/_evaluators/_groundedness/_groundedness.py +2 -2
  79. azure/ai/evaluation/_evaluators/_intent_resolution/_intent_resolution.py +6 -2
  80. azure/ai/evaluation/_evaluators/_relevance/_relevance.py +1 -1
  81. azure/ai/evaluation/_evaluators/_response_completeness/_response_completeness.py +8 -2
  82. azure/ai/evaluation/_evaluators/_response_completeness/response_completeness.prompty +31 -46
  83. azure/ai/evaluation/_evaluators/_similarity/_similarity.py +1 -1
  84. azure/ai/evaluation/_evaluators/_task_adherence/_task_adherence.py +5 -2
  85. azure/ai/evaluation/_evaluators/_tool_call_accuracy/_tool_call_accuracy.py +6 -2
  86. azure/ai/evaluation/_exceptions.py +2 -0
  87. azure/ai/evaluation/_legacy/_adapters/__init__.py +7 -0
  88. azure/ai/evaluation/_legacy/_adapters/_check.py +17 -0
  89. azure/ai/evaluation/_legacy/_adapters/_configuration.py +45 -0
  90. azure/ai/evaluation/_legacy/_adapters/_constants.py +10 -0
  91. azure/ai/evaluation/_legacy/_adapters/_errors.py +29 -0
  92. azure/ai/evaluation/_legacy/_adapters/_flows.py +28 -0
  93. azure/ai/evaluation/_legacy/_adapters/_service.py +16 -0
  94. azure/ai/evaluation/_legacy/_adapters/client.py +51 -0
  95. azure/ai/evaluation/_legacy/_adapters/entities.py +26 -0
  96. azure/ai/evaluation/_legacy/_adapters/tracing.py +28 -0
  97. azure/ai/evaluation/_legacy/_adapters/types.py +15 -0
  98. azure/ai/evaluation/_legacy/_adapters/utils.py +31 -0
  99. azure/ai/evaluation/_legacy/_batch_engine/_engine.py +51 -32
  100. azure/ai/evaluation/_legacy/_batch_engine/_openai_injector.py +114 -8
  101. azure/ai/evaluation/_legacy/_batch_engine/_result.py +7 -1
  102. azure/ai/evaluation/_legacy/_batch_engine/_run.py +6 -0
  103. azure/ai/evaluation/_legacy/_batch_engine/_run_submitter.py +69 -29
  104. azure/ai/evaluation/_legacy/_batch_engine/_status.py +1 -1
  105. azure/ai/evaluation/_legacy/_batch_engine/_trace.py +54 -62
  106. azure/ai/evaluation/_legacy/_batch_engine/_utils.py +19 -1
  107. azure/ai/evaluation/{_red_team/_utils → _legacy/_common}/__init__.py +1 -1
  108. azure/ai/evaluation/_legacy/_common/_async_token_provider.py +124 -0
  109. azure/ai/evaluation/_legacy/_common/_thread_pool_executor_with_context.py +15 -0
  110. azure/ai/evaluation/_legacy/prompty/_connection.py +11 -74
  111. azure/ai/evaluation/_legacy/prompty/_exceptions.py +80 -0
  112. azure/ai/evaluation/_legacy/prompty/_prompty.py +119 -9
  113. azure/ai/evaluation/_legacy/prompty/_utils.py +72 -2
  114. azure/ai/evaluation/_safety_evaluation/_safety_evaluation.py +90 -17
  115. azure/ai/evaluation/_version.py +1 -1
  116. azure/ai/evaluation/red_team/__init__.py +19 -0
  117. azure/ai/evaluation/{_red_team → red_team}/_attack_objective_generator.py +3 -0
  118. azure/ai/evaluation/{_red_team → red_team}/_attack_strategy.py +4 -1
  119. azure/ai/evaluation/{_red_team → red_team}/_red_team.py +885 -481
  120. azure/ai/evaluation/red_team/_red_team_result.py +382 -0
  121. azure/ai/evaluation/{_red_team → red_team}/_utils/constants.py +2 -1
  122. azure/ai/evaluation/{_red_team → red_team}/_utils/formatting_utils.py +23 -22
  123. azure/ai/evaluation/{_red_team → red_team}/_utils/logging_utils.py +1 -1
  124. azure/ai/evaluation/red_team/_utils/metric_mapping.py +23 -0
  125. azure/ai/evaluation/{_red_team → red_team}/_utils/strategy_utils.py +9 -5
  126. azure/ai/evaluation/simulator/_adversarial_simulator.py +63 -39
  127. azure/ai/evaluation/simulator/_constants.py +1 -0
  128. azure/ai/evaluation/simulator/_conversation/__init__.py +13 -6
  129. azure/ai/evaluation/simulator/_conversation/_conversation.py +2 -1
  130. azure/ai/evaluation/simulator/_direct_attack_simulator.py +35 -22
  131. azure/ai/evaluation/simulator/_helpers/_language_suffix_mapping.py +1 -0
  132. azure/ai/evaluation/simulator/_indirect_attack_simulator.py +40 -25
  133. azure/ai/evaluation/simulator/_model_tools/__init__.py +2 -1
  134. azure/ai/evaluation/simulator/_model_tools/_generated_rai_client.py +24 -18
  135. azure/ai/evaluation/simulator/_model_tools/_identity_manager.py +5 -10
  136. azure/ai/evaluation/simulator/_model_tools/_proxy_completion_model.py +65 -41
  137. azure/ai/evaluation/simulator/_model_tools/_template_handler.py +9 -5
  138. azure/ai/evaluation/simulator/_model_tools/models.py +20 -17
  139. azure/ai/evaluation/simulator/_simulator.py +1 -1
  140. {azure_ai_evaluation-1.4.0.dist-info → azure_ai_evaluation-1.6.0.dist-info}/METADATA +36 -2
  141. {azure_ai_evaluation-1.4.0.dist-info → azure_ai_evaluation-1.6.0.dist-info}/RECORD +148 -80
  142. azure/ai/evaluation/_red_team/_red_team_result.py +0 -246
  143. azure/ai/evaluation/simulator/_tracing.py +0 -89
  144. /azure/ai/evaluation/_legacy/{_batch_engine → _common}/_logging.py +0 -0
  145. /azure/ai/evaluation/{_red_team → red_team}/_callback_chat_target.py +0 -0
  146. /azure/ai/evaluation/{_red_team → red_team}/_default_converter.py +0 -0
  147. /azure/ai/evaluation/{_red_team → red_team/_utils}/__init__.py +0 -0
  148. {azure_ai_evaluation-1.4.0.dist-info → azure_ai_evaluation-1.6.0.dist-info}/NOTICE.txt +0 -0
  149. {azure_ai_evaluation-1.4.0.dist-info → azure_ai_evaluation-1.6.0.dist-info}/WHEEL +0 -0
  150. {azure_ai_evaluation-1.4.0.dist-info → azure_ai_evaluation-1.6.0.dist-info}/top_level.txt +0 -0
azure/ai/evaluation/_common/onedp/_patch.py ADDED
@@ -0,0 +1,21 @@
1
+ # coding=utf-8
2
+ # --------------------------------------------------------------------------
3
+ # Copyright (c) Microsoft Corporation. All rights reserved.
4
+ # Licensed under the MIT License. See License.txt in the project root for license information.
5
+ # --------------------------------------------------------------------------
6
+ """Customize generated code here.
7
+
8
+ Follow our quickstart for examples: https://aka.ms/azsdk/python/dpcodegen/python/customize
9
+ """
10
+ from typing import List
11
+
12
+ __all__: List[str] = [] # Add all objects you want publicly available to users at this package level
13
+
14
+
15
+ def patch_sdk():
16
+ """Do not remove from this file.
17
+
18
+ `patch_sdk` is a last resort escape hatch that allows you to do customizations
19
+ you can't accomplish using the techniques described in
20
+ https://aka.ms/azsdk/python/dpcodegen/python/customize
21
+ """