azure-ai-evaluation 1.0.0b2__py3-none-any.whl → 1.13.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of azure-ai-evaluation might be problematic. Click here for more details.
- azure/ai/evaluation/__init__.py +100 -5
- azure/ai/evaluation/{_evaluators/_chat → _aoai}/__init__.py +3 -2
- azure/ai/evaluation/_aoai/aoai_grader.py +140 -0
- azure/ai/evaluation/_aoai/label_grader.py +68 -0
- azure/ai/evaluation/_aoai/python_grader.py +86 -0
- azure/ai/evaluation/_aoai/score_model_grader.py +94 -0
- azure/ai/evaluation/_aoai/string_check_grader.py +66 -0
- azure/ai/evaluation/_aoai/text_similarity_grader.py +80 -0
- azure/ai/evaluation/_azure/__init__.py +3 -0
- azure/ai/evaluation/_azure/_clients.py +204 -0
- azure/ai/evaluation/_azure/_envs.py +207 -0
- azure/ai/evaluation/_azure/_models.py +227 -0
- azure/ai/evaluation/_azure/_token_manager.py +129 -0
- azure/ai/evaluation/_common/__init__.py +9 -1
- azure/ai/evaluation/{simulator/_helpers → _common}/_experimental.py +24 -9
- azure/ai/evaluation/_common/constants.py +131 -2
- azure/ai/evaluation/_common/evaluation_onedp_client.py +169 -0
- azure/ai/evaluation/_common/math.py +89 -0
- azure/ai/evaluation/_common/onedp/__init__.py +32 -0
- azure/ai/evaluation/_common/onedp/_client.py +166 -0
- azure/ai/evaluation/_common/onedp/_configuration.py +72 -0
- azure/ai/evaluation/_common/onedp/_model_base.py +1232 -0
- azure/ai/evaluation/_common/onedp/_patch.py +21 -0
- azure/ai/evaluation/_common/onedp/_serialization.py +2032 -0
- azure/ai/evaluation/_common/onedp/_types.py +21 -0
- azure/ai/evaluation/_common/onedp/_utils/__init__.py +6 -0
- azure/ai/evaluation/_common/onedp/_utils/model_base.py +1232 -0
- azure/ai/evaluation/_common/onedp/_utils/serialization.py +2032 -0
- azure/ai/evaluation/_common/onedp/_validation.py +66 -0
- azure/ai/evaluation/_common/onedp/_vendor.py +50 -0
- azure/ai/evaluation/_common/onedp/_version.py +9 -0
- azure/ai/evaluation/_common/onedp/aio/__init__.py +29 -0
- azure/ai/evaluation/_common/onedp/aio/_client.py +168 -0
- azure/ai/evaluation/_common/onedp/aio/_configuration.py +72 -0
- azure/ai/evaluation/_common/onedp/aio/_patch.py +21 -0
- azure/ai/evaluation/_common/onedp/aio/operations/__init__.py +49 -0
- azure/ai/evaluation/_common/onedp/aio/operations/_operations.py +7143 -0
- azure/ai/evaluation/_common/onedp/aio/operations/_patch.py +21 -0
- azure/ai/evaluation/_common/onedp/models/__init__.py +358 -0
- azure/ai/evaluation/_common/onedp/models/_enums.py +447 -0
- azure/ai/evaluation/_common/onedp/models/_models.py +5963 -0
- azure/ai/evaluation/_common/onedp/models/_patch.py +21 -0
- azure/ai/evaluation/_common/onedp/operations/__init__.py +49 -0
- azure/ai/evaluation/_common/onedp/operations/_operations.py +8951 -0
- azure/ai/evaluation/_common/onedp/operations/_patch.py +21 -0
- azure/ai/evaluation/_common/onedp/py.typed +1 -0
- azure/ai/evaluation/_common/onedp/servicepatterns/__init__.py +1 -0
- azure/ai/evaluation/_common/onedp/servicepatterns/aio/__init__.py +1 -0
- azure/ai/evaluation/_common/onedp/servicepatterns/aio/operations/__init__.py +25 -0
- azure/ai/evaluation/_common/onedp/servicepatterns/aio/operations/_operations.py +34 -0
- azure/ai/evaluation/_common/onedp/servicepatterns/aio/operations/_patch.py +20 -0
- azure/ai/evaluation/_common/onedp/servicepatterns/buildingblocks/__init__.py +1 -0
- azure/ai/evaluation/_common/onedp/servicepatterns/buildingblocks/aio/__init__.py +1 -0
- azure/ai/evaluation/_common/onedp/servicepatterns/buildingblocks/aio/operations/__init__.py +22 -0
- azure/ai/evaluation/_common/onedp/servicepatterns/buildingblocks/aio/operations/_operations.py +29 -0
- azure/ai/evaluation/_common/onedp/servicepatterns/buildingblocks/aio/operations/_patch.py +20 -0
- azure/ai/evaluation/_common/onedp/servicepatterns/buildingblocks/operations/__init__.py +22 -0
- azure/ai/evaluation/_common/onedp/servicepatterns/buildingblocks/operations/_operations.py +29 -0
- azure/ai/evaluation/_common/onedp/servicepatterns/buildingblocks/operations/_patch.py +20 -0
- azure/ai/evaluation/_common/onedp/servicepatterns/operations/__init__.py +25 -0
- azure/ai/evaluation/_common/onedp/servicepatterns/operations/_operations.py +34 -0
- azure/ai/evaluation/_common/onedp/servicepatterns/operations/_patch.py +20 -0
- azure/ai/evaluation/_common/rai_service.py +831 -142
- azure/ai/evaluation/_common/raiclient/__init__.py +34 -0
- azure/ai/evaluation/_common/raiclient/_client.py +128 -0
- azure/ai/evaluation/_common/raiclient/_configuration.py +87 -0
- azure/ai/evaluation/_common/raiclient/_model_base.py +1235 -0
- azure/ai/evaluation/_common/raiclient/_patch.py +20 -0
- azure/ai/evaluation/_common/raiclient/_serialization.py +2050 -0
- azure/ai/evaluation/_common/raiclient/_version.py +9 -0
- azure/ai/evaluation/_common/raiclient/aio/__init__.py +29 -0
- azure/ai/evaluation/_common/raiclient/aio/_client.py +130 -0
- azure/ai/evaluation/_common/raiclient/aio/_configuration.py +87 -0
- azure/ai/evaluation/_common/raiclient/aio/_patch.py +20 -0
- azure/ai/evaluation/_common/raiclient/aio/operations/__init__.py +25 -0
- azure/ai/evaluation/_common/raiclient/aio/operations/_operations.py +981 -0
- azure/ai/evaluation/_common/raiclient/aio/operations/_patch.py +20 -0
- azure/ai/evaluation/_common/raiclient/models/__init__.py +60 -0
- azure/ai/evaluation/_common/raiclient/models/_enums.py +18 -0
- azure/ai/evaluation/_common/raiclient/models/_models.py +651 -0
- azure/ai/evaluation/_common/raiclient/models/_patch.py +20 -0
- azure/ai/evaluation/_common/raiclient/operations/__init__.py +25 -0
- azure/ai/evaluation/_common/raiclient/operations/_operations.py +1238 -0
- azure/ai/evaluation/_common/raiclient/operations/_patch.py +20 -0
- azure/ai/evaluation/_common/raiclient/py.typed +1 -0
- azure/ai/evaluation/_common/utils.py +870 -34
- azure/ai/evaluation/_constants.py +167 -6
- azure/ai/evaluation/_converters/__init__.py +3 -0
- azure/ai/evaluation/_converters/_ai_services.py +899 -0
- azure/ai/evaluation/_converters/_models.py +467 -0
- azure/ai/evaluation/_converters/_sk_services.py +495 -0
- azure/ai/evaluation/_eval_mapping.py +83 -0
- azure/ai/evaluation/_evaluate/_batch_run/__init__.py +17 -0
- azure/ai/evaluation/_evaluate/_batch_run/_run_submitter_client.py +176 -0
- azure/ai/evaluation/_evaluate/_batch_run/batch_clients.py +82 -0
- azure/ai/evaluation/_evaluate/{_batch_run_client → _batch_run}/code_client.py +47 -25
- azure/ai/evaluation/_evaluate/{_batch_run_client/batch_run_context.py → _batch_run/eval_run_context.py} +42 -13
- azure/ai/evaluation/_evaluate/_batch_run/proxy_client.py +124 -0
- azure/ai/evaluation/_evaluate/_batch_run/target_run_context.py +62 -0
- azure/ai/evaluation/_evaluate/_eval_run.py +102 -59
- azure/ai/evaluation/_evaluate/_evaluate.py +2134 -311
- azure/ai/evaluation/_evaluate/_evaluate_aoai.py +992 -0
- azure/ai/evaluation/_evaluate/_telemetry/__init__.py +14 -99
- azure/ai/evaluation/_evaluate/_utils.py +289 -40
- azure/ai/evaluation/_evaluator_definition.py +76 -0
- azure/ai/evaluation/_evaluators/_bleu/_bleu.py +93 -42
- azure/ai/evaluation/_evaluators/_code_vulnerability/__init__.py +5 -0
- azure/ai/evaluation/_evaluators/_code_vulnerability/_code_vulnerability.py +119 -0
- azure/ai/evaluation/_evaluators/_coherence/_coherence.py +117 -91
- azure/ai/evaluation/_evaluators/_coherence/coherence.prompty +76 -39
- azure/ai/evaluation/_evaluators/_common/__init__.py +15 -0
- azure/ai/evaluation/_evaluators/_common/_base_eval.py +742 -0
- azure/ai/evaluation/_evaluators/_common/_base_multi_eval.py +63 -0
- azure/ai/evaluation/_evaluators/_common/_base_prompty_eval.py +345 -0
- azure/ai/evaluation/_evaluators/_common/_base_rai_svc_eval.py +198 -0
- azure/ai/evaluation/_evaluators/_common/_conversation_aggregators.py +49 -0
- azure/ai/evaluation/_evaluators/_content_safety/__init__.py +0 -4
- azure/ai/evaluation/_evaluators/_content_safety/_content_safety.py +144 -86
- azure/ai/evaluation/_evaluators/_content_safety/_hate_unfairness.py +138 -57
- azure/ai/evaluation/_evaluators/_content_safety/_self_harm.py +123 -55
- azure/ai/evaluation/_evaluators/_content_safety/_sexual.py +133 -54
- azure/ai/evaluation/_evaluators/_content_safety/_violence.py +134 -54
- azure/ai/evaluation/_evaluators/_document_retrieval/__init__.py +7 -0
- azure/ai/evaluation/_evaluators/_document_retrieval/_document_retrieval.py +442 -0
- azure/ai/evaluation/_evaluators/_eci/_eci.py +49 -56
- azure/ai/evaluation/_evaluators/_f1_score/_f1_score.py +102 -60
- azure/ai/evaluation/_evaluators/_fluency/_fluency.py +115 -92
- azure/ai/evaluation/_evaluators/_fluency/fluency.prompty +66 -41
- azure/ai/evaluation/_evaluators/_gleu/_gleu.py +90 -37
- azure/ai/evaluation/_evaluators/_groundedness/_groundedness.py +318 -82
- azure/ai/evaluation/_evaluators/_groundedness/groundedness_with_query.prompty +114 -0
- azure/ai/evaluation/_evaluators/_groundedness/groundedness_without_query.prompty +104 -0
- azure/ai/evaluation/{_evaluate/_batch_run_client → _evaluators/_intent_resolution}/__init__.py +3 -4
- azure/ai/evaluation/_evaluators/_intent_resolution/_intent_resolution.py +196 -0
- azure/ai/evaluation/_evaluators/_intent_resolution/intent_resolution.prompty +275 -0
- azure/ai/evaluation/_evaluators/_meteor/_meteor.py +107 -61
- azure/ai/evaluation/_evaluators/_protected_material/_protected_material.py +104 -77
- azure/ai/evaluation/_evaluators/_qa/_qa.py +115 -63
- azure/ai/evaluation/_evaluators/_relevance/_relevance.py +182 -98
- azure/ai/evaluation/_evaluators/_relevance/relevance.prompty +178 -49
- azure/ai/evaluation/_evaluators/_response_completeness/__init__.py +7 -0
- azure/ai/evaluation/_evaluators/_response_completeness/_response_completeness.py +202 -0
- azure/ai/evaluation/_evaluators/_response_completeness/response_completeness.prompty +84 -0
- azure/ai/evaluation/_evaluators/{_chat/retrieval → _retrieval}/__init__.py +2 -2
- azure/ai/evaluation/_evaluators/_retrieval/_retrieval.py +148 -0
- azure/ai/evaluation/_evaluators/_retrieval/retrieval.prompty +93 -0
- azure/ai/evaluation/_evaluators/_rouge/_rouge.py +189 -50
- azure/ai/evaluation/_evaluators/_service_groundedness/__init__.py +9 -0
- azure/ai/evaluation/_evaluators/_service_groundedness/_service_groundedness.py +179 -0
- azure/ai/evaluation/_evaluators/_similarity/_similarity.py +102 -91
- azure/ai/evaluation/_evaluators/_similarity/similarity.prompty +0 -5
- azure/ai/evaluation/_evaluators/_task_adherence/__init__.py +7 -0
- azure/ai/evaluation/_evaluators/_task_adherence/_task_adherence.py +226 -0
- azure/ai/evaluation/_evaluators/_task_adherence/task_adherence.prompty +101 -0
- azure/ai/evaluation/_evaluators/_task_completion/__init__.py +7 -0
- azure/ai/evaluation/_evaluators/_task_completion/_task_completion.py +177 -0
- azure/ai/evaluation/_evaluators/_task_completion/task_completion.prompty +220 -0
- azure/ai/evaluation/_evaluators/_task_navigation_efficiency/__init__.py +7 -0
- azure/ai/evaluation/_evaluators/_task_navigation_efficiency/_task_navigation_efficiency.py +384 -0
- azure/ai/evaluation/_evaluators/_tool_call_accuracy/__init__.py +9 -0
- azure/ai/evaluation/_evaluators/_tool_call_accuracy/_tool_call_accuracy.py +298 -0
- azure/ai/evaluation/_evaluators/_tool_call_accuracy/tool_call_accuracy.prompty +166 -0
- azure/ai/evaluation/_evaluators/_tool_input_accuracy/__init__.py +9 -0
- azure/ai/evaluation/_evaluators/_tool_input_accuracy/_tool_input_accuracy.py +263 -0
- azure/ai/evaluation/_evaluators/_tool_input_accuracy/tool_input_accuracy.prompty +76 -0
- azure/ai/evaluation/_evaluators/_tool_output_utilization/__init__.py +7 -0
- azure/ai/evaluation/_evaluators/_tool_output_utilization/_tool_output_utilization.py +225 -0
- azure/ai/evaluation/_evaluators/_tool_output_utilization/tool_output_utilization.prompty +221 -0
- azure/ai/evaluation/_evaluators/_tool_selection/__init__.py +9 -0
- azure/ai/evaluation/_evaluators/_tool_selection/_tool_selection.py +266 -0
- azure/ai/evaluation/_evaluators/_tool_selection/tool_selection.prompty +104 -0
- azure/ai/evaluation/_evaluators/_tool_success/__init__.py +7 -0
- azure/ai/evaluation/_evaluators/_tool_success/_tool_success.py +301 -0
- azure/ai/evaluation/_evaluators/_tool_success/tool_success.prompty +321 -0
- azure/ai/evaluation/_evaluators/_ungrounded_attributes/__init__.py +5 -0
- azure/ai/evaluation/_evaluators/_ungrounded_attributes/_ungrounded_attributes.py +102 -0
- azure/ai/evaluation/_evaluators/_xpia/xpia.py +109 -107
- azure/ai/evaluation/_exceptions.py +51 -7
- azure/ai/evaluation/_http_utils.py +210 -137
- azure/ai/evaluation/_legacy/__init__.py +3 -0
- azure/ai/evaluation/_legacy/_adapters/__init__.py +7 -0
- azure/ai/evaluation/_legacy/_adapters/_check.py +17 -0
- azure/ai/evaluation/_legacy/_adapters/_configuration.py +45 -0
- azure/ai/evaluation/_legacy/_adapters/_constants.py +10 -0
- azure/ai/evaluation/_legacy/_adapters/_errors.py +29 -0
- azure/ai/evaluation/_legacy/_adapters/_flows.py +28 -0
- azure/ai/evaluation/_legacy/_adapters/_service.py +16 -0
- azure/ai/evaluation/_legacy/_adapters/client.py +51 -0
- azure/ai/evaluation/_legacy/_adapters/entities.py +26 -0
- azure/ai/evaluation/_legacy/_adapters/tracing.py +28 -0
- azure/ai/evaluation/_legacy/_adapters/types.py +15 -0
- azure/ai/evaluation/_legacy/_adapters/utils.py +31 -0
- azure/ai/evaluation/_legacy/_batch_engine/__init__.py +9 -0
- azure/ai/evaluation/_legacy/_batch_engine/_config.py +48 -0
- azure/ai/evaluation/_legacy/_batch_engine/_engine.py +477 -0
- azure/ai/evaluation/_legacy/_batch_engine/_exceptions.py +88 -0
- azure/ai/evaluation/_legacy/_batch_engine/_openai_injector.py +132 -0
- azure/ai/evaluation/_legacy/_batch_engine/_result.py +107 -0
- azure/ai/evaluation/_legacy/_batch_engine/_run.py +127 -0
- azure/ai/evaluation/_legacy/_batch_engine/_run_storage.py +128 -0
- azure/ai/evaluation/_legacy/_batch_engine/_run_submitter.py +262 -0
- azure/ai/evaluation/_legacy/_batch_engine/_status.py +25 -0
- azure/ai/evaluation/_legacy/_batch_engine/_trace.py +97 -0
- azure/ai/evaluation/_legacy/_batch_engine/_utils.py +97 -0
- azure/ai/evaluation/_legacy/_batch_engine/_utils_deprecated.py +131 -0
- azure/ai/evaluation/_legacy/_common/__init__.py +3 -0
- azure/ai/evaluation/_legacy/_common/_async_token_provider.py +117 -0
- azure/ai/evaluation/_legacy/_common/_logging.py +292 -0
- azure/ai/evaluation/_legacy/_common/_thread_pool_executor_with_context.py +17 -0
- azure/ai/evaluation/_legacy/prompty/__init__.py +36 -0
- azure/ai/evaluation/_legacy/prompty/_connection.py +119 -0
- azure/ai/evaluation/_legacy/prompty/_exceptions.py +139 -0
- azure/ai/evaluation/_legacy/prompty/_prompty.py +430 -0
- azure/ai/evaluation/_legacy/prompty/_utils.py +663 -0
- azure/ai/evaluation/_legacy/prompty/_yaml_utils.py +99 -0
- azure/ai/evaluation/_model_configurations.py +130 -8
- azure/ai/evaluation/_safety_evaluation/__init__.py +3 -0
- azure/ai/evaluation/_safety_evaluation/_generated_rai_client.py +0 -0
- azure/ai/evaluation/_safety_evaluation/_safety_evaluation.py +917 -0
- azure/ai/evaluation/_user_agent.py +32 -1
- azure/ai/evaluation/_vendor/__init__.py +3 -0
- azure/ai/evaluation/_vendor/rouge_score/__init__.py +14 -0
- azure/ai/evaluation/_vendor/rouge_score/rouge_scorer.py +324 -0
- azure/ai/evaluation/_vendor/rouge_score/scoring.py +59 -0
- azure/ai/evaluation/_vendor/rouge_score/tokenize.py +59 -0
- azure/ai/evaluation/_vendor/rouge_score/tokenizers.py +53 -0
- azure/ai/evaluation/_version.py +2 -1
- azure/ai/evaluation/red_team/__init__.py +22 -0
- azure/ai/evaluation/red_team/_agent/__init__.py +3 -0
- azure/ai/evaluation/red_team/_agent/_agent_functions.py +261 -0
- azure/ai/evaluation/red_team/_agent/_agent_tools.py +461 -0
- azure/ai/evaluation/red_team/_agent/_agent_utils.py +89 -0
- azure/ai/evaluation/red_team/_agent/_semantic_kernel_plugin.py +228 -0
- azure/ai/evaluation/red_team/_attack_objective_generator.py +268 -0
- azure/ai/evaluation/red_team/_attack_strategy.py +49 -0
- azure/ai/evaluation/red_team/_callback_chat_target.py +115 -0
- azure/ai/evaluation/red_team/_default_converter.py +21 -0
- azure/ai/evaluation/red_team/_evaluation_processor.py +505 -0
- azure/ai/evaluation/red_team/_mlflow_integration.py +430 -0
- azure/ai/evaluation/red_team/_orchestrator_manager.py +803 -0
- azure/ai/evaluation/red_team/_red_team.py +1717 -0
- azure/ai/evaluation/red_team/_red_team_result.py +661 -0
- azure/ai/evaluation/red_team/_result_processor.py +1708 -0
- azure/ai/evaluation/red_team/_utils/__init__.py +37 -0
- azure/ai/evaluation/red_team/_utils/_rai_service_eval_chat_target.py +128 -0
- azure/ai/evaluation/red_team/_utils/_rai_service_target.py +601 -0
- azure/ai/evaluation/red_team/_utils/_rai_service_true_false_scorer.py +114 -0
- azure/ai/evaluation/red_team/_utils/constants.py +72 -0
- azure/ai/evaluation/red_team/_utils/exception_utils.py +345 -0
- azure/ai/evaluation/red_team/_utils/file_utils.py +266 -0
- azure/ai/evaluation/red_team/_utils/formatting_utils.py +365 -0
- azure/ai/evaluation/red_team/_utils/logging_utils.py +139 -0
- azure/ai/evaluation/red_team/_utils/metric_mapping.py +73 -0
- azure/ai/evaluation/red_team/_utils/objective_utils.py +46 -0
- azure/ai/evaluation/red_team/_utils/progress_utils.py +252 -0
- azure/ai/evaluation/red_team/_utils/retry_utils.py +218 -0
- azure/ai/evaluation/red_team/_utils/strategy_utils.py +218 -0
- azure/ai/evaluation/simulator/__init__.py +2 -1
- azure/ai/evaluation/simulator/_adversarial_scenario.py +26 -1
- azure/ai/evaluation/simulator/_adversarial_simulator.py +270 -144
- azure/ai/evaluation/simulator/_constants.py +12 -1
- azure/ai/evaluation/simulator/_conversation/__init__.py +151 -23
- azure/ai/evaluation/simulator/_conversation/_conversation.py +10 -6
- azure/ai/evaluation/simulator/_conversation/constants.py +1 -1
- azure/ai/evaluation/simulator/_data_sources/__init__.py +3 -0
- azure/ai/evaluation/simulator/_data_sources/grounding.json +1150 -0
- azure/ai/evaluation/simulator/_direct_attack_simulator.py +54 -75
- azure/ai/evaluation/simulator/_helpers/__init__.py +1 -2
- azure/ai/evaluation/simulator/_helpers/_language_suffix_mapping.py +1 -0
- azure/ai/evaluation/simulator/_helpers/_simulator_data_classes.py +26 -5
- azure/ai/evaluation/simulator/_indirect_attack_simulator.py +145 -104
- azure/ai/evaluation/simulator/_model_tools/__init__.py +2 -1
- azure/ai/evaluation/simulator/_model_tools/_generated_rai_client.py +225 -0
- azure/ai/evaluation/simulator/_model_tools/_identity_manager.py +80 -30
- azure/ai/evaluation/simulator/_model_tools/_proxy_completion_model.py +117 -45
- azure/ai/evaluation/simulator/_model_tools/_rai_client.py +109 -7
- azure/ai/evaluation/simulator/_model_tools/_template_handler.py +97 -33
- azure/ai/evaluation/simulator/_model_tools/models.py +30 -27
- azure/ai/evaluation/simulator/_prompty/task_query_response.prompty +6 -10
- azure/ai/evaluation/simulator/_prompty/task_simulate.prompty +6 -5
- azure/ai/evaluation/simulator/_simulator.py +302 -208
- azure/ai/evaluation/simulator/_utils.py +31 -13
- azure_ai_evaluation-1.13.3.dist-info/METADATA +939 -0
- azure_ai_evaluation-1.13.3.dist-info/RECORD +305 -0
- {azure_ai_evaluation-1.0.0b2.dist-info → azure_ai_evaluation-1.13.3.dist-info}/WHEEL +1 -1
- azure_ai_evaluation-1.13.3.dist-info/licenses/NOTICE.txt +70 -0
- azure/ai/evaluation/_evaluate/_batch_run_client/proxy_client.py +0 -71
- azure/ai/evaluation/_evaluators/_chat/_chat.py +0 -357
- azure/ai/evaluation/_evaluators/_chat/retrieval/_retrieval.py +0 -157
- azure/ai/evaluation/_evaluators/_chat/retrieval/retrieval.prompty +0 -48
- azure/ai/evaluation/_evaluators/_content_safety/_content_safety_base.py +0 -65
- azure/ai/evaluation/_evaluators/_content_safety/_content_safety_chat.py +0 -301
- azure/ai/evaluation/_evaluators/_groundedness/groundedness.prompty +0 -54
- azure/ai/evaluation/_evaluators/_protected_materials/__init__.py +0 -5
- azure/ai/evaluation/_evaluators/_protected_materials/_protected_materials.py +0 -104
- azure/ai/evaluation/simulator/_tracing.py +0 -89
- azure_ai_evaluation-1.0.0b2.dist-info/METADATA +0 -449
- azure_ai_evaluation-1.0.0b2.dist-info/RECORD +0 -99
- {azure_ai_evaluation-1.0.0b2.dist-info → azure_ai_evaluation-1.13.3.dist-info}/top_level.txt +0 -0
|
@@ -3,12 +3,11 @@
|
|
|
3
3
|
# ---------------------------------------------------------
|
|
4
4
|
|
|
5
5
|
|
|
6
|
-
from
|
|
7
|
-
from typing import Any, Awaitable, Callable, Dict, MutableMapping, Optional
|
|
6
|
+
from typing import Any, Dict, MutableMapping, Optional, TypedDict, cast
|
|
8
7
|
|
|
9
|
-
from typing_extensions import Self
|
|
8
|
+
from typing_extensions import Self, Unpack
|
|
10
9
|
|
|
11
|
-
from azure.ai.evaluation._user_agent import
|
|
10
|
+
from azure.ai.evaluation._user_agent import UserAgentSingleton
|
|
12
11
|
from azure.core.configuration import Configuration
|
|
13
12
|
from azure.core.pipeline import AsyncPipeline, Pipeline
|
|
14
13
|
from azure.core.pipeline.policies import (
|
|
@@ -33,78 +32,25 @@ from azure.core.rest import AsyncHttpResponse, HttpRequest, HttpResponse
|
|
|
33
32
|
from azure.core.rest._rest_py3 import ContentType, FilesType, ParamsType
|
|
34
33
|
|
|
35
34
|
|
|
36
|
-
|
|
37
|
-
"""
|
|
35
|
+
class RequestKwargs(TypedDict, total=False):
|
|
36
|
+
"""Keyword arguments for request-style http request functions
|
|
38
37
|
|
|
39
|
-
|
|
40
|
-
The f.__name__ is the HTTP method used
|
|
41
|
-
:return: A wrapped callable that sends a `f.__name__` request
|
|
42
|
-
:rtype: Callable
|
|
43
|
-
"""
|
|
38
|
+
.. note::
|
|
44
39
|
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
url: str,
|
|
49
|
-
*,
|
|
50
|
-
params: Optional[ParamsType] = None,
|
|
51
|
-
headers: Optional[MutableMapping[str, str]] = None,
|
|
52
|
-
json: Any = None,
|
|
53
|
-
content: Optional[ContentType] = None,
|
|
54
|
-
data: Optional[Dict[str, Any]] = None,
|
|
55
|
-
files: Optional[FilesType] = None,
|
|
56
|
-
**kwargs,
|
|
57
|
-
) -> HttpResponse:
|
|
58
|
-
return self.request(
|
|
59
|
-
f.__name__.upper(),
|
|
60
|
-
url,
|
|
61
|
-
params=params,
|
|
62
|
-
headers=headers,
|
|
63
|
-
json=json,
|
|
64
|
-
content=content,
|
|
65
|
-
data=data,
|
|
66
|
-
files=files,
|
|
67
|
-
**kwargs,
|
|
68
|
-
)
|
|
40
|
+
Ideally, we'd be able to express that these are the known subset of kwargs, but it's possible to provide
|
|
41
|
+
others. But that currently isn't possible; there's no way currently to express a TypedDict that expects
|
|
42
|
+
a known set of keys and an unknown set of keys.
|
|
69
43
|
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
def _async_request_fn(f: Callable[["AsyncHttpPipeline"], Awaitable[None]]):
|
|
74
|
-
"""Decorator to generate convenience methods for HTTP method.
|
|
75
|
-
|
|
76
|
-
:param Callable[["HttpPipeline"],None] f: A HttpPipeline classmethod to wrap.
|
|
77
|
-
The f.__name__ is the HTTP method used
|
|
78
|
-
:return: A wrapped callable that sends a `f.__name__` request
|
|
79
|
-
:rtype: Callable
|
|
44
|
+
PEP 728 - TypedDict with Typed Extra Items (https://peps.python.org/pep-0728/) would rectify this but it's
|
|
45
|
+
still in Draft status.
|
|
80
46
|
"""
|
|
81
47
|
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
headers: Optional[MutableMapping[str, str]] = None,
|
|
89
|
-
json: Any = None,
|
|
90
|
-
content: Optional[ContentType] = None,
|
|
91
|
-
data: Optional[Dict[str, Any]] = None,
|
|
92
|
-
files: Optional[FilesType] = None,
|
|
93
|
-
**kwargs,
|
|
94
|
-
) -> AsyncHttpResponse:
|
|
95
|
-
return await self.request(
|
|
96
|
-
f.__name__.upper(),
|
|
97
|
-
url,
|
|
98
|
-
params=params,
|
|
99
|
-
headers=headers,
|
|
100
|
-
json=json,
|
|
101
|
-
content=content,
|
|
102
|
-
data=data,
|
|
103
|
-
files=files,
|
|
104
|
-
**kwargs,
|
|
105
|
-
)
|
|
106
|
-
|
|
107
|
-
return request_fn
|
|
48
|
+
params: ParamsType
|
|
49
|
+
headers: MutableMapping[str, str]
|
|
50
|
+
json: Any
|
|
51
|
+
content: ContentType
|
|
52
|
+
data: Dict[str, Any]
|
|
53
|
+
files: FilesType
|
|
108
54
|
|
|
109
55
|
|
|
110
56
|
class HttpPipeline(Pipeline):
|
|
@@ -145,14 +91,32 @@ class HttpPipeline(Pipeline):
|
|
|
145
91
|
:param RedirectPolicy redirect_policy:
|
|
146
92
|
"""
|
|
147
93
|
config = config or Configuration()
|
|
148
|
-
config.headers_policy =
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
config.
|
|
152
|
-
config.
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
config.
|
|
94
|
+
config.headers_policy = (
|
|
95
|
+
headers_policy or cast(Optional[HeadersPolicy], config.headers_policy) or HeadersPolicy(**kwargs)
|
|
96
|
+
)
|
|
97
|
+
config.proxy_policy = proxy_policy or cast(Optional[ProxyPolicy], config.proxy_policy) or ProxyPolicy(**kwargs)
|
|
98
|
+
config.redirect_policy = (
|
|
99
|
+
redirect_policy or cast(Optional[RedirectPolicy], config.redirect_policy) or RedirectPolicy(**kwargs)
|
|
100
|
+
)
|
|
101
|
+
config.retry_policy = retry_policy or cast(Optional[RetryPolicy], config.retry_policy) or RetryPolicy(**kwargs)
|
|
102
|
+
config.custom_hook_policy = (
|
|
103
|
+
custom_hook_policy
|
|
104
|
+
or cast(Optional[CustomHookPolicy], config.custom_hook_policy)
|
|
105
|
+
or CustomHookPolicy(**kwargs)
|
|
106
|
+
)
|
|
107
|
+
config.logging_policy = (
|
|
108
|
+
logging_policy
|
|
109
|
+
or cast(Optional[NetworkTraceLoggingPolicy], config.logging_policy)
|
|
110
|
+
or NetworkTraceLoggingPolicy(**kwargs)
|
|
111
|
+
)
|
|
112
|
+
config.http_logging_policy = (
|
|
113
|
+
http_logging_policy
|
|
114
|
+
or cast(Optional[HttpLoggingPolicy], config.http_logging_policy)
|
|
115
|
+
or HttpLoggingPolicy(**kwargs)
|
|
116
|
+
)
|
|
117
|
+
config.user_agent_policy = (
|
|
118
|
+
user_agent_policy or cast(Optional[UserAgentPolicy], config.user_agent_policy) or UserAgentPolicy(**kwargs)
|
|
119
|
+
)
|
|
156
120
|
config.polling_interval = kwargs.get("polling_interval", 30)
|
|
157
121
|
|
|
158
122
|
super().__init__(
|
|
@@ -166,7 +130,6 @@ class HttpPipeline(Pipeline):
|
|
|
166
130
|
config.proxy_policy,
|
|
167
131
|
config.redirect_policy,
|
|
168
132
|
config.retry_policy,
|
|
169
|
-
config.authentication_policy,
|
|
170
133
|
config.custom_hook_policy,
|
|
171
134
|
config.logging_policy,
|
|
172
135
|
],
|
|
@@ -199,7 +162,6 @@ class HttpPipeline(Pipeline):
|
|
|
199
162
|
files: Optional[FilesType] = None,
|
|
200
163
|
**kwargs,
|
|
201
164
|
) -> HttpResponse:
|
|
202
|
-
|
|
203
165
|
request = HttpRequest(
|
|
204
166
|
method,
|
|
205
167
|
url,
|
|
@@ -213,33 +175,78 @@ class HttpPipeline(Pipeline):
|
|
|
213
175
|
|
|
214
176
|
return self.run(request, **kwargs).http_response
|
|
215
177
|
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
"""Send a DELETE request."""
|
|
178
|
+
def delete(self: "HttpPipeline", url: str, **kwargs: Unpack[RequestKwargs]) -> HttpResponse:
|
|
179
|
+
"""Send a DELETE request.
|
|
219
180
|
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
181
|
+
:param str url: The request url
|
|
182
|
+
:returns: The request response
|
|
183
|
+
:rtype: HttpResponse
|
|
184
|
+
"""
|
|
223
185
|
|
|
224
|
-
|
|
225
|
-
def get(self) -> None:
|
|
226
|
-
"""Send a GET request."""
|
|
186
|
+
return self.request(self.delete.__name__.upper(), url, **kwargs)
|
|
227
187
|
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
"""Send a POST request."""
|
|
188
|
+
def put(self: "HttpPipeline", url: str, **kwargs: Unpack[RequestKwargs]) -> HttpResponse:
|
|
189
|
+
"""Send a PUT request.
|
|
231
190
|
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
191
|
+
:param str url: The request url
|
|
192
|
+
:returns: The request response
|
|
193
|
+
:rtype: HttpResponse
|
|
194
|
+
"""
|
|
235
195
|
|
|
236
|
-
|
|
237
|
-
def options(self) -> None:
|
|
238
|
-
"""Send a OPTIONS request."""
|
|
196
|
+
return self.request(self.put.__name__.upper(), url, **kwargs)
|
|
239
197
|
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
198
|
+
def get(self: "HttpPipeline", url: str, **kwargs: Unpack[RequestKwargs]) -> HttpResponse:
|
|
199
|
+
"""Send a GET request.
|
|
200
|
+
|
|
201
|
+
:param str url: The request url
|
|
202
|
+
:returns: The request response
|
|
203
|
+
:rtype: HttpResponse
|
|
204
|
+
"""
|
|
205
|
+
|
|
206
|
+
return self.request(self.get.__name__.upper(), url, **kwargs)
|
|
207
|
+
|
|
208
|
+
def post(self: "HttpPipeline", url: str, **kwargs: Unpack[RequestKwargs]) -> HttpResponse:
|
|
209
|
+
"""Send a POST request.
|
|
210
|
+
|
|
211
|
+
:param str url: The request url
|
|
212
|
+
:returns: The request response
|
|
213
|
+
:rtype: HttpResponse
|
|
214
|
+
"""
|
|
215
|
+
|
|
216
|
+
return self.request(self.post.__name__.upper(), url, **kwargs)
|
|
217
|
+
|
|
218
|
+
def head(self: "HttpPipeline", url: str, **kwargs: Unpack[RequestKwargs]) -> HttpResponse:
|
|
219
|
+
"""Send a HEAD request.
|
|
220
|
+
|
|
221
|
+
:param str url: The request url
|
|
222
|
+
:returns: The request response
|
|
223
|
+
:rtype: HttpResponse
|
|
224
|
+
"""
|
|
225
|
+
|
|
226
|
+
return self.request(self.head.__name__.upper(), url, **kwargs)
|
|
227
|
+
|
|
228
|
+
def options(self: "HttpPipeline", url: str, **kwargs: Unpack[RequestKwargs]) -> HttpResponse:
|
|
229
|
+
"""Send a OPTIONS request.
|
|
230
|
+
|
|
231
|
+
:param str url: The request url
|
|
232
|
+
:returns: The request response
|
|
233
|
+
:rtype: HttpResponse
|
|
234
|
+
"""
|
|
235
|
+
|
|
236
|
+
return self.request(self.options.__name__.upper(), url, **kwargs)
|
|
237
|
+
|
|
238
|
+
def patch(self: "HttpPipeline", url: str, **kwargs: Unpack[RequestKwargs]) -> HttpResponse:
|
|
239
|
+
"""Send a PATCH request.
|
|
240
|
+
|
|
241
|
+
:param str url: The request url
|
|
242
|
+
:returns: The request response
|
|
243
|
+
:rtype: HttpResponse
|
|
244
|
+
"""
|
|
245
|
+
|
|
246
|
+
return self.request(self.patch.__name__.upper(), url, **kwargs)
|
|
247
|
+
|
|
248
|
+
def __enter__(self) -> Self:
|
|
249
|
+
return cast(Self, super().__enter__())
|
|
243
250
|
|
|
244
251
|
|
|
245
252
|
class AsyncHttpPipeline(AsyncPipeline):
|
|
@@ -280,14 +287,36 @@ class AsyncHttpPipeline(AsyncPipeline):
|
|
|
280
287
|
:param AsyncRedirectPolicy redirect_policy:
|
|
281
288
|
"""
|
|
282
289
|
config = config or Configuration()
|
|
283
|
-
config.headers_policy =
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
config.
|
|
287
|
-
config.
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
290
|
+
config.headers_policy = (
|
|
291
|
+
headers_policy or cast(Optional[HeadersPolicy], config.headers_policy) or HeadersPolicy(**kwargs)
|
|
292
|
+
)
|
|
293
|
+
config.proxy_policy = proxy_policy or cast(Optional[ProxyPolicy], config.proxy_policy) or ProxyPolicy(**kwargs)
|
|
294
|
+
config.redirect_policy = (
|
|
295
|
+
redirect_policy
|
|
296
|
+
or cast(Optional[AsyncRedirectPolicy], config.redirect_policy)
|
|
297
|
+
or AsyncRedirectPolicy(**kwargs)
|
|
298
|
+
)
|
|
299
|
+
config.retry_policy = (
|
|
300
|
+
retry_policy or cast(Optional[AsyncRetryPolicy], config.retry_policy) or AsyncRetryPolicy(**kwargs)
|
|
301
|
+
)
|
|
302
|
+
config.custom_hook_policy = (
|
|
303
|
+
custom_hook_policy
|
|
304
|
+
or cast(Optional[CustomHookPolicy], config.custom_hook_policy)
|
|
305
|
+
or CustomHookPolicy(**kwargs)
|
|
306
|
+
)
|
|
307
|
+
config.logging_policy = (
|
|
308
|
+
logging_policy
|
|
309
|
+
or cast(Optional[NetworkTraceLoggingPolicy], config.logging_policy)
|
|
310
|
+
or NetworkTraceLoggingPolicy(**kwargs)
|
|
311
|
+
)
|
|
312
|
+
config.http_logging_policy = (
|
|
313
|
+
http_logging_policy
|
|
314
|
+
or cast(Optional[HttpLoggingPolicy], config.http_logging_policy)
|
|
315
|
+
or HttpLoggingPolicy(**kwargs)
|
|
316
|
+
)
|
|
317
|
+
config.user_agent_policy = (
|
|
318
|
+
user_agent_policy or cast(Optional[UserAgentPolicy], config.user_agent_policy) or UserAgentPolicy(**kwargs)
|
|
319
|
+
)
|
|
291
320
|
config.polling_interval = kwargs.get("polling_interval", 30)
|
|
292
321
|
|
|
293
322
|
super().__init__(
|
|
@@ -301,7 +330,6 @@ class AsyncHttpPipeline(AsyncPipeline):
|
|
|
301
330
|
config.proxy_policy,
|
|
302
331
|
config.redirect_policy,
|
|
303
332
|
config.retry_policy,
|
|
304
|
-
config.authentication_policy,
|
|
305
333
|
config.custom_hook_policy,
|
|
306
334
|
config.logging_policy,
|
|
307
335
|
],
|
|
@@ -334,7 +362,6 @@ class AsyncHttpPipeline(AsyncPipeline):
|
|
|
334
362
|
files: Optional[FilesType] = None,
|
|
335
363
|
**kwargs,
|
|
336
364
|
) -> AsyncHttpResponse:
|
|
337
|
-
|
|
338
365
|
request = HttpRequest(
|
|
339
366
|
method,
|
|
340
367
|
url,
|
|
@@ -348,48 +375,94 @@ class AsyncHttpPipeline(AsyncPipeline):
|
|
|
348
375
|
|
|
349
376
|
return (await self.run(request, **kwargs)).http_response
|
|
350
377
|
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
378
|
+
async def delete(self: "AsyncHttpPipeline", url: str, **kwargs: Unpack[RequestKwargs]) -> AsyncHttpResponse:
|
|
379
|
+
"""Send a DELETE request.
|
|
380
|
+
|
|
381
|
+
:param str url: The request url
|
|
382
|
+
:returns: The request response
|
|
383
|
+
:rtype: AsyncHttpResponse
|
|
384
|
+
"""
|
|
385
|
+
return await self.request(self.delete.__name__.upper(), url, **kwargs)
|
|
386
|
+
|
|
387
|
+
async def put(self: "AsyncHttpPipeline", url: str, **kwargs: Unpack[RequestKwargs]) -> AsyncHttpResponse:
|
|
388
|
+
"""Send a PUT request.
|
|
389
|
+
|
|
390
|
+
:param str url: The request url
|
|
391
|
+
:returns: The request response
|
|
392
|
+
:rtype: AsyncHttpResponse
|
|
393
|
+
"""
|
|
394
|
+
|
|
395
|
+
return await self.request(self.put.__name__.upper(), url, **kwargs)
|
|
396
|
+
|
|
397
|
+
async def get(self: "AsyncHttpPipeline", url: str, **kwargs: Unpack[RequestKwargs]) -> AsyncHttpResponse:
|
|
398
|
+
"""Send a GET request.
|
|
399
|
+
|
|
400
|
+
:param str url: The request url
|
|
401
|
+
:returns: The request response
|
|
402
|
+
:rtype: AsyncHttpResponse
|
|
403
|
+
"""
|
|
404
|
+
|
|
405
|
+
return await self.request(self.get.__name__.upper(), url, **kwargs)
|
|
406
|
+
|
|
407
|
+
async def post(self: "AsyncHttpPipeline", url: str, **kwargs: Unpack[RequestKwargs]) -> AsyncHttpResponse:
|
|
408
|
+
"""Send a POST request.
|
|
409
|
+
|
|
410
|
+
:param str url: The request url
|
|
411
|
+
:returns: The request response
|
|
412
|
+
:rtype: AsyncHttpResponse
|
|
413
|
+
"""
|
|
414
|
+
|
|
415
|
+
return await self.request(self.post.__name__.upper(), url, **kwargs)
|
|
416
|
+
|
|
417
|
+
async def head(self: "AsyncHttpPipeline", url: str, **kwargs: Unpack[RequestKwargs]) -> AsyncHttpResponse:
|
|
418
|
+
"""Send a HEAD request.
|
|
354
419
|
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
420
|
+
:param str url: The request url
|
|
421
|
+
:returns: The request response
|
|
422
|
+
:rtype: AsyncHttpResponse
|
|
423
|
+
"""
|
|
424
|
+
|
|
425
|
+
return await self.request(self.head.__name__.upper(), url, **kwargs)
|
|
426
|
+
|
|
427
|
+
async def options(self: "AsyncHttpPipeline", url: str, **kwargs: Unpack[RequestKwargs]) -> AsyncHttpResponse:
|
|
428
|
+
"""Send a OPTIONS request.
|
|
358
429
|
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
430
|
+
:param str url: The request url
|
|
431
|
+
:returns: The request response
|
|
432
|
+
:rtype: AsyncHttpResponse
|
|
433
|
+
"""
|
|
434
|
+
|
|
435
|
+
return await self.request(self.options.__name__.upper(), url, **kwargs)
|
|
362
436
|
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
"""Send a POST request."""
|
|
437
|
+
async def patch(self: "AsyncHttpPipeline", url: str, **kwargs: Unpack[RequestKwargs]) -> AsyncHttpResponse:
|
|
438
|
+
"""Send a PATCH request.
|
|
366
439
|
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
440
|
+
:param str url: The request url
|
|
441
|
+
:returns: The request response
|
|
442
|
+
:rtype: AsyncHttpResponse
|
|
443
|
+
"""
|
|
370
444
|
|
|
371
|
-
|
|
372
|
-
async def options(self) -> None:
|
|
373
|
-
"""Send a OPTIONS request."""
|
|
445
|
+
return await self.request(self.patch.__name__.upper(), url, **kwargs)
|
|
374
446
|
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
"""Send a PATCH request."""
|
|
447
|
+
async def __aenter__(self) -> Self:
|
|
448
|
+
return cast(Self, await super().__aenter__())
|
|
378
449
|
|
|
379
450
|
|
|
380
|
-
def get_http_client() -> HttpPipeline:
|
|
451
|
+
def get_http_client(**kwargs: Any) -> HttpPipeline:
|
|
381
452
|
"""Get an HttpPipeline configured with common policies.
|
|
382
453
|
|
|
383
454
|
:returns: An HttpPipeline with a set of applied policies:
|
|
384
455
|
:rtype: HttpPipeline
|
|
385
456
|
"""
|
|
386
|
-
|
|
457
|
+
kwargs.setdefault("user_agent_policy", UserAgentPolicy(base_user_agent=UserAgentSingleton().value))
|
|
458
|
+
return HttpPipeline(**kwargs)
|
|
387
459
|
|
|
388
460
|
|
|
389
|
-
def get_async_http_client() -> AsyncHttpPipeline:
|
|
461
|
+
def get_async_http_client(**kwargs: Any) -> AsyncHttpPipeline:
|
|
390
462
|
"""Get an AsyncHttpPipeline configured with common policies.
|
|
391
463
|
|
|
392
464
|
:returns: An AsyncHttpPipeline with a set of applied policies:
|
|
393
465
|
:rtype: AsyncHttpPipeline
|
|
394
466
|
"""
|
|
395
|
-
|
|
467
|
+
kwargs.setdefault("user_agent_policy", UserAgentPolicy(base_user_agent=UserAgentSingleton().value))
|
|
468
|
+
return AsyncHttpPipeline(**kwargs)
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
# ---------------------------------------------------------
|
|
2
|
+
# Copyright (c) Microsoft Corporation. All rights reserved.
|
|
3
|
+
# ---------------------------------------------------------
|
|
4
|
+
|
|
5
|
+
# NOTE: This contains adapters that make the Promptflow dependency optional. In the first phase,
|
|
6
|
+
# Promptflow will still be installed as part of the azure-ai-evaluation dependencies. This
|
|
7
|
+
# will be removed in the future once the code migration is complete.
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# ---------------------------------------------------------
|
|
2
|
+
# Copyright (c) Microsoft Corporation. All rights reserved.
|
|
3
|
+
# ---------------------------------------------------------
|
|
4
|
+
|
|
5
|
+
from typing import Final
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
_has_legacy = False
|
|
9
|
+
try:
|
|
10
|
+
from promptflow._constants import FlowType
|
|
11
|
+
|
|
12
|
+
_has_legacy = True
|
|
13
|
+
except ImportError:
|
|
14
|
+
pass
|
|
15
|
+
|
|
16
|
+
HAS_LEGACY_SDK: Final[bool] = _has_legacy
|
|
17
|
+
MISSING_LEGACY_SDK: Final[bool] = not _has_legacy
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# ---------------------------------------------------------
|
|
2
|
+
# Copyright (c) Microsoft Corporation. All rights reserved.
|
|
3
|
+
# ---------------------------------------------------------
|
|
4
|
+
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import Any, Dict, Final, Optional
|
|
7
|
+
from typing_extensions import TypeAlias
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
try:
|
|
11
|
+
from promptflow._sdk._configuration import Configuration as _Configuration
|
|
12
|
+
except ImportError:
|
|
13
|
+
_global_config: Final[Dict[str, Any]] = {}
|
|
14
|
+
|
|
15
|
+
class _Configuration:
|
|
16
|
+
TRACE_DESTINATION: Final[str] = "trace.destination"
|
|
17
|
+
_instance = None
|
|
18
|
+
|
|
19
|
+
def __init__(self, *, override_config: Optional[Dict[str, Any]] = None) -> None:
|
|
20
|
+
self._config = override_config or {}
|
|
21
|
+
|
|
22
|
+
@classmethod
|
|
23
|
+
def get_instance(cls) -> "_Configuration":
|
|
24
|
+
"""Use this to get instance to avoid multiple copies of same global config."""
|
|
25
|
+
if cls._instance is None:
|
|
26
|
+
cls._instance = Configuration(override_config=_global_config)
|
|
27
|
+
return cls._instance
|
|
28
|
+
|
|
29
|
+
def set_config(self, key: str, value: Any) -> None:
|
|
30
|
+
# Simulated config storage
|
|
31
|
+
self._config[key] = value
|
|
32
|
+
|
|
33
|
+
def get_config(self, key: str) -> Any:
|
|
34
|
+
# Simulated config storage
|
|
35
|
+
if key in self._config:
|
|
36
|
+
return self._config[key]
|
|
37
|
+
return _global_config.get(key, None)
|
|
38
|
+
|
|
39
|
+
def get_trace_destination(self, path: Optional[Path] = None) -> Optional[str]:
|
|
40
|
+
if path:
|
|
41
|
+
raise NotImplementedError("Setting trace destination with a path is not supported.")
|
|
42
|
+
return self._config.get("trace.destination", None)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
Configuration: TypeAlias = _Configuration
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
# ---------------------------------------------------------
|
|
2
|
+
# Copyright (c) Microsoft Corporation. All rights reserved.
|
|
3
|
+
# ---------------------------------------------------------
|
|
4
|
+
|
|
5
|
+
from typing import Final
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
PF_FLOW_ENTRY_IN_TMP: Final[str] = "PF_FLOW_ENTRY_IN_TMP"
|
|
9
|
+
PF_FLOW_META_LOAD_IN_SUBPROCESS: Final[str] = "PF_FLOW_META_LOAD_IN_SUBPROCESS"
|
|
10
|
+
LINE_NUMBER: Final[str] = "line_number"
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# ---------------------------------------------------------
|
|
2
|
+
# Copyright (c) Microsoft Corporation. All rights reserved.
|
|
3
|
+
# ---------------------------------------------------------
|
|
4
|
+
|
|
5
|
+
from typing import Any
|
|
6
|
+
from typing_extensions import TypeAlias
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
try:
|
|
10
|
+
from promptflow.core._errors import MissingRequiredPackage as _MissingRequiredPackage
|
|
11
|
+
except ImportError:
|
|
12
|
+
from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
|
|
13
|
+
|
|
14
|
+
class _MissingRequiredPackage(EvaluationException):
|
|
15
|
+
"""Raised when a required package is missing.
|
|
16
|
+
|
|
17
|
+
:param message: A message describing the error. This is the error message the user will see.
|
|
18
|
+
:type message: str
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
def __init__(self, message: str, **kwargs: Any):
|
|
22
|
+
kwargs.setdefault("category", ErrorCategory.MISSING_PACKAGE)
|
|
23
|
+
kwargs.setdefault("blame", ErrorBlame.SYSTEM_ERROR)
|
|
24
|
+
kwargs.setdefault("target", ErrorTarget.EVALUATE)
|
|
25
|
+
kwargs.setdefault("internal_message", "Missing required package.")
|
|
26
|
+
super().__init__(message=message, **kwargs)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
MissingRequiredPackage: TypeAlias = _MissingRequiredPackage
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
# ---------------------------------------------------------
|
|
2
|
+
# Copyright (c) Microsoft Corporation. All rights reserved.
|
|
3
|
+
# ---------------------------------------------------------
|
|
4
|
+
|
|
5
|
+
from typing_extensions import TypeAlias
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
try:
|
|
9
|
+
from promptflow.core._flow import AsyncPrompty as _AsyncPrompty
|
|
10
|
+
from promptflow._sdk.entities._flows import FlexFlow as _FlexFlow
|
|
11
|
+
from promptflow._sdk.entities._flows.dag import Flow as _Flow
|
|
12
|
+
except ImportError:
|
|
13
|
+
from azure.ai.evaluation._legacy.prompty import AsyncPrompty as _AsyncPrompty
|
|
14
|
+
|
|
15
|
+
class _FlexFlow:
|
|
16
|
+
pass
|
|
17
|
+
|
|
18
|
+
_FlexFlow.__name__ = "FlexFlow"
|
|
19
|
+
|
|
20
|
+
class _Flow:
|
|
21
|
+
name: str
|
|
22
|
+
|
|
23
|
+
_Flow.__name__ = "Flow"
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
AsyncPrompty: TypeAlias = _AsyncPrompty
|
|
27
|
+
FlexFlow: TypeAlias = _FlexFlow
|
|
28
|
+
Flow: TypeAlias = _Flow
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
# ---------------------------------------------------------
|
|
2
|
+
# Copyright (c) Microsoft Corporation. All rights reserved.
|
|
3
|
+
# ---------------------------------------------------------
|
|
4
|
+
|
|
5
|
+
from typing import Callable, Final
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
try:
|
|
9
|
+
from promptflow._cli._pf._service import stop_service as _stop_service
|
|
10
|
+
except ImportError:
|
|
11
|
+
|
|
12
|
+
def _stop_service() -> None:
|
|
13
|
+
pass
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
stop_service: Final[Callable[[], None]] = _stop_service
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
# ---------------------------------------------------------
|
|
2
|
+
# Copyright (c) Microsoft Corporation. All rights reserved.
|
|
3
|
+
# ---------------------------------------------------------
|
|
4
|
+
|
|
5
|
+
from os import PathLike
|
|
6
|
+
from typing import Any, Callable, Dict, Optional, Union
|
|
7
|
+
from typing_extensions import TypeAlias
|
|
8
|
+
|
|
9
|
+
import pandas as pd
|
|
10
|
+
|
|
11
|
+
from ._errors import MissingRequiredPackage
|
|
12
|
+
from ._configuration import Configuration
|
|
13
|
+
from .entities import Run
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
try:
|
|
17
|
+
from promptflow.client import PFClient as _PFClient
|
|
18
|
+
except ImportError:
|
|
19
|
+
|
|
20
|
+
class _PFClient:
|
|
21
|
+
def __init__(self, **kwargs):
|
|
22
|
+
self._config = Configuration(override_config=kwargs.pop("config", None))
|
|
23
|
+
|
|
24
|
+
def run(
|
|
25
|
+
self,
|
|
26
|
+
flow: Union[str, PathLike, Callable],
|
|
27
|
+
*,
|
|
28
|
+
data: Union[str, PathLike],
|
|
29
|
+
run: Optional[Union[str, Run]] = None,
|
|
30
|
+
column_mapping: Optional[dict] = None,
|
|
31
|
+
variant: Optional[str] = None,
|
|
32
|
+
connections: Optional[dict] = None,
|
|
33
|
+
environment_variables: Optional[dict] = None,
|
|
34
|
+
name: Optional[str] = None,
|
|
35
|
+
display_name: Optional[str] = None,
|
|
36
|
+
tags: Optional[Dict[str, str]] = None,
|
|
37
|
+
resume_from: Optional[Union[str, Run]] = None,
|
|
38
|
+
code: Optional[Union[str, PathLike]] = None,
|
|
39
|
+
init: Optional[dict] = None,
|
|
40
|
+
**kwargs,
|
|
41
|
+
) -> Run:
|
|
42
|
+
raise MissingRequiredPackage("Please install 'promptflow' package to use PFClient")
|
|
43
|
+
|
|
44
|
+
def get_details(self, run: Union[str, Run], max_results: int = 100, all_results: bool = False) -> pd.DataFrame:
|
|
45
|
+
return pd.DataFrame()
|
|
46
|
+
|
|
47
|
+
def get_metrics(self, run: Union[str, Run]) -> Dict[str, Any]:
|
|
48
|
+
return {}
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
PFClient: TypeAlias = _PFClient
|