azure-ai-evaluation 1.0.0b3__py3-none-any.whl → 1.0.0b5__py3-none-any.whl

azure/ai/evaluation/_common/utils.py

@@ -2,21 +2,30 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
 
+import re
+import math
 import threading
-from typing import List, Union
+from typing import Any, List, Literal, Mapping, Type, TypeVar, Tuple, Union, cast, get_args, get_origin
 
 import nltk
-import numpy as np
-
-from azure.ai.evaluation._model_configurations import AzureOpenAIModelConfiguration, OpenAIModelConfiguration
+from typing_extensions import NotRequired, Required, TypeGuard
+from promptflow.core._errors import MissingRequiredPackage
 from azure.ai.evaluation._constants import AZURE_OPENAI_TYPE, OPENAI_TYPE
+from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
+from azure.ai.evaluation._model_configurations import (
+    AzureAIProject,
+    AzureOpenAIModelConfiguration,
+    OpenAIModelConfiguration,
+)
 
 from . import constants
 
 _nltk_data_download_lock = threading.Lock()
 
+T_TypedDict = TypeVar("T_TypedDict", bound=Mapping[Any, Any])
+
 
-def get_harm_severity_level(harm_score: int) -> str:
+def get_harm_severity_level(harm_score: Union[float, int]) -> Union[str, float]:
     """Generate harm severity level based on harm score.
 
     :param harm_score: The harm score to be evaluated.
@@ -30,23 +39,29 @@ def get_harm_severity_level(harm_score: int) -> str:
         constants.HarmSeverityLevel.Medium: [4, 5],
         constants.HarmSeverityLevel.High: [6, 7],
     }
-    if harm_score == np.nan or harm_score is None:
-        return np.nan
+    if math.isnan(harm_score) or harm_score is None:
+        return math.nan
     for harm_level, harm_score_range in HARM_SEVERITY_LEVEL_MAPPING.items():
         if harm_score_range[0] <= harm_score <= harm_score_range[1]:
             return harm_level.value
-    return np.nan
+    return math.nan
 
 
 def ensure_nltk_data_downloaded():
     """Download NLTK data packages if not already downloaded."""
+    nltk_data = [
+        ("wordnet", "corpora/wordnet.zip"),
+        ("perluniprops", "misc/perluniprops.zip"),
+        ("punkt", "tokenizers/punkt.zip"),
+        ("punkt_tab", "tokenizers/punkt_tab.zip"),
+    ]
+
     with _nltk_data_download_lock:
-        try:
-            from nltk.tokenize.nist import NISTTokenizer  # pylint: disable=unused-import
-        except LookupError:
-            nltk.download("perluniprops")
-            nltk.download("punkt")
-            nltk.download("punkt_tab")
+        for _id, resource_name in nltk_data:
+            try:
+                nltk.find(resource_name)
+            except LookupError:
+                nltk.download(_id)
 
 
 def nltk_tokenize(text: str) -> List[str]:
@@ -71,12 +86,20 @@ def nltk_tokenize(text: str) -> List[str]:
     return list(tokens)
 
 
+def _is_aoi_model_config(val: object) -> TypeGuard[AzureOpenAIModelConfiguration]:
+    return isinstance(val, dict) and all(isinstance(val.get(k), str) for k in ("azure_endpoint", "azure_deployment"))
+
+
+def _is_openai_model_config(val: object) -> TypeGuard[OpenAIModelConfiguration]:
+    return isinstance(val, dict) and all(isinstance(val.get(k), str) for k in ("model"))
+
+
 def parse_model_config_type(
     model_config: Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration],
 ) -> None:
-    if "azure_endpoint" in model_config or "azure_deployment" in model_config:
+    if _is_aoi_model_config(model_config):
         model_config["type"] = AZURE_OPENAI_TYPE
-    else:
+    elif _is_openai_model_config(model_config):
         model_config["type"] = OPENAI_TYPE
 
 
@@ -87,16 +110,302 @@ def construct_prompty_model_config(
 ) -> dict:
     parse_model_config_type(model_config)
 
-    if model_config["type"] == AZURE_OPENAI_TYPE:
+    if _is_aoi_model_config(model_config):
         model_config["api_version"] = model_config.get("api_version", default_api_version)
 
-    prompty_model_config = {"configuration": model_config, "parameters": {"extra_headers": {}}}
+    prompty_model_config: dict = {"configuration": model_config, "parameters": {"extra_headers": {}}}
 
     # Handle "RuntimeError: Event loop is closed" from httpx AsyncClient
     # https://github.com/encode/httpx/discussions/2959
     prompty_model_config["parameters"]["extra_headers"].update({"Connection": "close"})
 
-    if model_config["type"] == AZURE_OPENAI_TYPE and user_agent:
+    if _is_aoi_model_config(model_config) and user_agent:
         prompty_model_config["parameters"]["extra_headers"].update({"x-ms-useragent": user_agent})
 
     return prompty_model_config
+
+
+def validate_azure_ai_project(o: object) -> AzureAIProject:
+    fields = {"subscription_id": str, "resource_group_name": str, "project_name": str}
+
+    if not isinstance(o, dict):
+        msg = "The 'azure_ai_project' parameter must be a dictionary."
+        raise EvaluationException(
+            message=msg,
+            category=ErrorCategory.INVALID_VALUE,
+            blame=ErrorBlame.USER_ERROR,
+        )
+
+    missing_fields = set(fields.keys()) - o.keys()
+
+    if missing_fields:
+        msg = (
+            "The 'azure_ai_project' dictionary is missing the following required "
+            f"field(s): {', '.join(f'{field}' for field in missing_fields)}."
+        )
+        raise EvaluationException(
+            message=msg,
+            category=ErrorCategory.INVALID_VALUE,
+            blame=ErrorBlame.USER_ERROR,
+        )
+
+    for field_name, expected_type in fields.items():
+        if isinstance(o[field_name], expected_type):
+            continue
+
+        msg = f"Invalid type for field '{field_name}'. Expected {expected_type}, but got {type(o[field_name])}."
+        raise EvaluationException(
+            message=msg,
+            category=ErrorCategory.INVALID_VALUE,
+            blame=ErrorBlame.USER_ERROR,
+        )
+
+    return cast(AzureAIProject, o)
+
+
+def validate_model_config(config: dict) -> Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration]:
+    try:
+        return _validate_typed_dict(config, AzureOpenAIModelConfiguration)
+    except TypeError:
+        try:
+            return _validate_typed_dict(config, OpenAIModelConfiguration)
+        except TypeError as e:
+            msg = "Model config validation failed."
+            raise EvaluationException(
+                message=msg, internal_message=msg, category=ErrorCategory.MISSING_FIELD, blame=ErrorBlame.USER_ERROR
+            ) from e
+
+
+def _validate_typed_dict(o: object, t: Type[T_TypedDict]) -> T_TypedDict:
+    """Do very basic runtime validation that an object is a typed dict
+
+    .. warning::
+
+        This validation is very basic, robust enough to cover some very simple TypedDicts.
+        Ideally, validation of this kind should be delegated to something more robust.
+
+        You will very quickly run into limitations trying to apply this function more broadly:
+           * Doesn't support stringized annotations at all
+           * Very limited support for generics, and "special form" (NoReturn, NotRequired, Required, etc...) types.
+           * Error messages are poor, especially if there is any nesting.
+
+    :param object o: The object to check
+    :param Type[T_TypedDict] t: The TypedDict to validate against
+    :raises NotImplementedError: Several forms of validation are unsupported
+        * Checking against stringized annotations
+        * Checking a generic that is not one of a few basic forms
+    :raises TypeError: If a value does not match the specified annotation
+    :raises ValueError: If t's annotation is not a string, type of a special form (e.g. NotRequired, Required, etc...)
+    :returns: The object passed in
+    :rtype: T_TypedDict
+    """
+    if not isinstance(o, dict):
+        raise TypeError(f"Expected type 'dict', got type '{type(object)}'.")
+
+    annotations = t.__annotations__
+    is_total = getattr(t, "__total__", False)
+    unknown_keys = set(o.keys()) - annotations.keys()
+
+    if unknown_keys:
+        raise TypeError(f"dict contains unknown keys: {list(unknown_keys)!r}")
+
+    required_keys = {
+        k
+        for k in annotations
+        if (is_total and get_origin(annotations[k]) is not NotRequired)
+        or (not is_total and get_origin(annotations[k]) is Required)
+    }
+
+    missing_keys = required_keys - o.keys()
+
+    if missing_keys:
+        raise TypeError(f"Missing required keys: {list(missing_keys)!r}.")
+
+    def validate_annotation(v: object, annotation: Union[str, type, object]) -> bool:
+        if isinstance(annotation, str):
+            raise NotImplementedError("Missing support for validating against stringized annotations.")
+
+        if (origin := get_origin(annotation)) is not None:
+            if origin is tuple:
+                validate_annotation(v, tuple)
+                tuple_args = get_args(annotation)
+                if len(cast(tuple, v)) != len(tuple_args):
+                    raise TypeError(f"Expected a {len(tuple_args)}-tuple, got a {len(cast(tuple, v))}-tuple.")
+                for tuple_val, tuple_args in zip(cast(tuple, v), tuple_args):
+                    validate_annotation(tuple_val, tuple_args)
+            elif origin is dict:
+                validate_annotation(v, dict)
+                dict_key_ann, dict_val_ann = get_args(annotation)
+                for dict_key, dict_val in cast(dict, v).items():
+                    validate_annotation(dict_val, dict_val_ann)
+                    validate_annotation(dict_key, dict_key_ann)
+            elif origin is list:
+                validate_annotation(v, list)
+                list_val_ann = get_args(annotation)[0]
+                for list_val in cast(list, v):
+                    validate_annotation(list_val, list_val_ann)
+            elif origin is Union:
+                for generic_arg in get_args(annotation):
+                    try:
+                        validate_annotation(v, generic_arg)
+                        return True
+                    except TypeError:
+                        pass
+                    raise TypeError(f"Expected value to have type {annotation}. Received type {type(v)}")
+            elif origin is Literal:
+                literal_args = get_args(annotation)
+                if not any(type(literal) is type(v) and literal == v for literal in literal_args):
+                    raise TypeError(f"Expected value to be one of {list(literal_args)!r}. Received type {type(v)}")
+            elif any(origin is g for g in (NotRequired, Required)):
+                validate_annotation(v, get_args(annotation)[0])
+            else:
+                raise NotImplementedError(f"Validation not implemented for generic {origin}.")
+            return True
+
+        if isinstance(annotation, type):
+            if not isinstance(v, annotation):
+                raise TypeError(f"Expected value to have type {annotation}. Received type {type(v)}.")
+            return True
+
+        raise ValueError("Annotation to validate against should be a str, type, or generic.")
+
+    for k, v in o.items():
+        validate_annotation(v, annotations[k])
+
+    return cast(T_TypedDict, o)
+
+
+def parse_quality_evaluator_reason_score(llm_output: str) -> Tuple[float, str]:
+    """Parse the output of prompt-based quality evaluators that return a score and reason.
+
+    Current supported evaluators:
+        - Fluency
+        - Relevance
+        - Retrieval
+        - Groundedness
+        - Coherence
+
+    :param llm_output: The output of the prompt-based quality evaluator.
+    :type llm_output: str
+    :return: The score and reason.
+    :rtype: Tuple[float, str]
+    """
+    score = math.nan
+    reason = ""
+    if llm_output:
+        score_pattern = r"<S2>(.*?)</S2>"
+        reason_pattern = r"<S1>(.*?)</S1>"
+        score_match = re.findall(score_pattern, llm_output, re.DOTALL)
+        reason_match = re.findall(reason_pattern, llm_output, re.DOTALL)
+        if score_match:
+            score = float(score_match[0].strip())
+        if reason_match:
+            reason = reason_match[0].strip()
+
+    return score, reason
+
+
+def remove_optional_singletons(eval_class, singletons):
+    required_singletons = singletons.copy()
+    if hasattr(eval_class, "_OPTIONAL_PARAMS"):  # pylint: disable=protected-access
+        for param in eval_class._OPTIONAL_PARAMS:  # pylint: disable=protected-access
+            if param in singletons:
+                del required_singletons[param]
+    return required_singletons
+
+
+def retrieve_content_type(assistant_messages: List, metric: str) -> str:
+    """Get the content type for service payload.
+
+    :param assistant_messages: The list of messages to be annotated by evaluation service
+    :type assistant_messages: list
+    :param metric: A string representing the metric type
+    :type metric: str
+    :return: A text representing the content type. Example: 'text', or 'image'
+    :rtype: str
+    """
+    # Check if metric is "protected_material"
+    if metric == "protected_material":
+        return "image"
+
+    # Iterate through each message
+    for item in assistant_messages:
+        # Ensure "content" exists in the message and is iterable
+        content = item.get("content", [])
+        for message in content:
+            if message.get("type", "") == "image_url":
+                return "image"
+    # Default return if no image was found
+    return "text"
+
+
+def validate_conversation(conversation):
+    def raise_exception(msg, target):
+        raise EvaluationException(
+            message=msg,
+            internal_message=msg,
+            target=target,
+            category=ErrorCategory.INVALID_VALUE,
+            blame=ErrorBlame.USER_ERROR,
+        )
+
+    if not conversation or "messages" not in conversation:
+        raise_exception(
+            "Attribute 'messages' is missing in the request",
+            ErrorTarget.CONTENT_SAFETY_CHAT_EVALUATOR,
+        )
+    messages = conversation["messages"]
+    if not isinstance(messages, list):
+        raise_exception(
+            "'messages' parameter must be a JSON-compatible list of chat messages",
+            ErrorTarget.CONTENT_SAFETY_MULTIMODAL_EVALUATOR,
+        )
+    expected_roles = {"user", "assistant", "system"}
+    image_found = False
+    for num, message in enumerate(messages, 1):
+        if not isinstance(message, dict):
+            try:
+                from azure.ai.inference.models import (
+                    ChatRequestMessage,
+                    UserMessage,
+                    AssistantMessage,
+                    SystemMessage,
+                    ImageContentItem,
+                )
+            except ImportError as ex:
+                raise MissingRequiredPackage(
+                    message="Please install 'azure-ai-inference' package to use SystemMessage, AssistantMessage"
+                ) from ex
+
+            if isinstance(messages[0], ChatRequestMessage) and not isinstance(
+                message, (UserMessage, AssistantMessage, SystemMessage)
+            ):
+                raise_exception(
+                    f"Messages must be a strongly typed class of ChatRequestMessage. Message number: {num}",
+                    ErrorTarget.CONTENT_SAFETY_MULTIMODAL_EVALUATOR,
+                )
+
+            if isinstance(message.content, list) and any(
+                isinstance(item, ImageContentItem) for item in message.content
+            ):
+                image_found = True
+            continue
+        if message.get("role") not in expected_roles:
+            raise_exception(
+                f"Invalid role provided: {message.get('role')}. Message number: {num}",
+                ErrorTarget.CONTENT_SAFETY_MULTIMODAL_EVALUATOR,
+            )
+        content = message.get("content")
+        if not isinstance(content, (str, list)):
+            raise_exception(
+                f"Content in each turn must be a string or array. Message number: {num}",
+                ErrorTarget.CONTENT_SAFETY_MULTIMODAL_EVALUATOR,
+            )
+        if isinstance(content, list):
+            if any(item.get("type") == "image_url" and "url" in item.get("image_url", {}) for item in content):
+                image_found = True
+    if not image_found:
+        raise_exception(
+            "Message needs to have multi-modal input like images.",
+            ErrorTarget.CONTENT_SAFETY_MULTIMODAL_EVALUATOR,
+        )

azure/ai/evaluation/_constants.py

@@ -1,18 +1,19 @@
 # ---------------------------------------------------------
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
+from typing import Literal
 
 
 class EvaluationMetrics:
     """Metrics for model evaluation."""
 
-    GPT_GROUNDEDNESS = "gpt_groundedness"
-    GPT_RELEVANCE = "gpt_relevance"
-    GPT_COHERENCE = "gpt_coherence"
-    GPT_FLUENCY = "gpt_fluency"
-    GPT_SIMILARITY = "gpt_similarity"
+    GROUNDEDNESS = "groundedness"
+    RELEVANCE = "relevance"
+    COHERENCE = "coherence"
+    FLUENCY = "fluency"
+    SIMILARITY = "similarity"
     F1_SCORE = "f1_score"
-    GPT_RETRIEVAL_SCORE = "gpt_retrieval_score"
+    RETRIEVAL_SCORE = "retrieval_score"
     HATE_FAIRNESS = "hate_fairness"
     HATE_UNFAIRNESS = "hate_unfairness"
     VIOLENCE = "violence"
@@ -48,6 +49,13 @@ class DefaultOpenEncoding:
     """SDK Default Encoding when writing a file"""
 
 
+class EvaluationRunProperties:
+    """Defines properties used to identify an evaluation run by UI"""
+
+    RUN_TYPE = "runType"
+    EVALUATION_RUN = "_azureml.evaluation_run"
+
+
 DEFAULT_EVALUATION_RESULTS_FILE_NAME = "evaluation_results.json"
 
 CONTENT_SAFETY_DEFECT_RATE_THRESHOLD_DEFAULT = 4
@@ -58,6 +66,6 @@ PF_BATCH_TIMEOUT_SEC = "PF_BATCH_TIMEOUT_SEC"
 OTEL_EXPORTER_OTLP_TRACES_TIMEOUT = "OTEL_EXPORTER_OTLP_TRACES_TIMEOUT"
 OTEL_EXPORTER_OTLP_TRACES_TIMEOUT_DEFAULT = 60
 
-AZURE_OPENAI_TYPE = "azure_openai"
+AZURE_OPENAI_TYPE: Literal["azure_openai"] = "azure_openai"
 
-OPENAI_TYPE = "openai"
+OPENAI_TYPE: Literal["openai"] = "openai"

azure/ai/evaluation/_evaluate/{_batch_run_client → _batch_run}/__init__.py

@@ -1,8 +1,9 @@
 # ---------------------------------------------------------
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
-from .batch_run_context import BatchRunContext
+from .eval_run_context import EvalRunContext
 from .code_client import CodeClient
 from .proxy_client import ProxyClient
+from .target_run_context import TargetRunContext
 
-__all__ = ["CodeClient", "ProxyClient", "BatchRunContext"]
+__all__ = ["CodeClient", "ProxyClient", "EvalRunContext", "TargetRunContext"]

azure/ai/evaluation/_evaluate/{_batch_run_client → _batch_run}/code_client.py

@@ -5,8 +5,9 @@ import inspect
 import json
 import logging
 import os
+from concurrent.futures import Future
 from pathlib import Path
-from typing import Callable, Dict, Optional, Union
+from typing import Any, Callable, Dict, Optional, Union, cast
 
 import pandas as pd
 from promptflow.contracts.types import AttrDict
@@ -22,25 +23,31 @@ LOGGER = logging.getLogger(__name__)
 
 class CodeRun:
     def __init__(
-        self, run, input_data, evaluator_name=None, aggregated_metrics=None, **kwargs  # pylint: disable=unused-argument
-    ):
+        self,
+        *,
+        run: Future,
+        input_data,
+        evaluator_name: Optional[str] = None,
+        aggregator: Callable[["CodeRun"], Future],
+        **kwargs,  # pylint: disable=unused-argument
+    ) -> None:
         self.run = run
         self.evaluator_name = evaluator_name if evaluator_name is not None else ""
         self.input_data = input_data
-        self.aggregated_metrics = aggregated_metrics
+        self.aggregated_metrics = aggregator(self)
 
-    def get_result_df(self, exclude_inputs=False):
+    def get_result_df(self, exclude_inputs: bool = False) -> pd.DataFrame:
         batch_run_timeout = get_int_env_var(PF_BATCH_TIMEOUT_SEC, PF_BATCH_TIMEOUT_SEC_DEFAULT)
-        result_df = self.run.result(timeout=batch_run_timeout)
+        result_df = cast(pd.DataFrame, self.run.result(timeout=batch_run_timeout))
         if exclude_inputs:
             result_df = result_df.drop(columns=[col for col in result_df.columns if col.startswith("inputs.")])
         return result_df
 
-    def get_aggregated_metrics(self):
+    def get_aggregated_metrics(self) -> Dict[str, Any]:
         try:
             batch_run_timeout = get_int_env_var(PF_BATCH_TIMEOUT_SEC, PF_BATCH_TIMEOUT_SEC_DEFAULT)
-            aggregated_metrics = (
-                self.aggregated_metrics.result(timeout=batch_run_timeout)
+            aggregated_metrics: Optional[Any] = (
+                cast(Dict, self.aggregated_metrics.result(timeout=batch_run_timeout))
                 if self.aggregated_metrics is not None
                 else None
             )
@@ -104,10 +111,10 @@ class CodeClient:  # pylint: disable=client-accepts-api-version-keyword
             verify_integrity=True,
         )
 
-    def _calculate_aggregations(self, evaluator, run):
+    @staticmethod
+    def _calculate_aggregations(evaluator: Callable, run: CodeRun) -> Any:
         try:
             if _has_aggregator(evaluator):
-                aggregate_input = None
                 evaluator_output = run.get_result_df(exclude_inputs=True)
                 if len(evaluator_output.columns) == 1 and evaluator_output.columns[0] == "output":
                     aggregate_input = evaluator_output["output"].tolist()
@@ -152,21 +159,30 @@ class CodeClient:  # pylint: disable=client-accepts-api-version-keyword
             column_mapping=column_mapping,
             evaluator_name=evaluator_name,
         )
-        run = CodeRun(run=eval_future, input_data=data, evaluator_name=evaluator_name, aggregated_metrics=None)
-        aggregation_future = self._thread_pool.submit(self._calculate_aggregations, evaluator=flow, run=run)
-        run.aggregated_metrics = aggregation_future
-        return run
+
+        return CodeRun(
+            run=eval_future,
+            input_data=data,
+            evaluator_name=evaluator_name,
+            aggregator=lambda code_run: self._thread_pool.submit(
+                self._calculate_aggregations, evaluator=flow, run=code_run
+            ),
+        )
 
     def get_details(self, run: CodeRun, all_results: bool = False) -> pd.DataFrame:
         result_df = run.get_result_df(exclude_inputs=not all_results)
         return result_df
 
-    def get_metrics(self, run: CodeRun) -> Optional[None]:
+    def get_metrics(self, run: CodeRun) -> Dict[str, Any]:
         try:
             aggregated_metrics = run.get_aggregated_metrics()
             print("Aggregated metrics")
             print(aggregated_metrics)
         except Exception as ex:  # pylint: disable=broad-exception-caught
             LOGGER.debug("Error calculating metrics for evaluator %s, failed with error %s", run.evaluator_name, ex)
-            return None
+            return {}
         return aggregated_metrics
+
+    def get_run_summary(self, run: CodeRun) -> Any:  # pylint: disable=unused-argument
+        # Not implemented
+        return None

azure/ai/evaluation/_evaluate/{_batch_run_client/batch_run_context.py → _batch_run/eval_run_context.py}

@@ -2,6 +2,8 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
 import os
+import types
+from typing import Optional, Type, Union
 
 from promptflow._sdk._constants import PF_FLOW_ENTRY_IN_TMP, PF_FLOW_META_LOAD_IN_SUBPROCESS
 from promptflow._utils.user_agent_utils import ClientUserAgentUtil
@@ -20,22 +22,22 @@ from .code_client import CodeClient
 from .proxy_client import ProxyClient
 
 
-class BatchRunContext:
-    """Context manager for batch run clients.
+class EvalRunContext:
+    """Context manager for eval batch run.
 
     :param client: The client to run in the context.
     :type client: Union[
-        ~azure.ai.evaluation._evaluate._batch_run_client.code_client.CodeClient,
-        ~azure.ai.evaluation._evaluate._batch_run_client.proxy_client.ProxyClient
+        ~azure.ai.evaluation._evaluate._batch_run.code_client.CodeClient,
+        ~azure.ai.evaluation._evaluate._batch_run.proxy_client.ProxyClient
     ]
     """
 
-    def __init__(self, client) -> None:
+    def __init__(self, client: Union[CodeClient, ProxyClient]) -> None:
         self.client = client
         self._is_batch_timeout_set_by_system = False
         self._is_otel_timeout_set_by_system = False
 
-    def __enter__(self):
+    def __enter__(self) -> None:
         if isinstance(self.client, CodeClient):
             ClientUserAgentUtil.append_user_agent(USER_AGENT)
             inject_openai_api()
@@ -56,7 +58,12 @@ class BatchRunContext:
             # For addressing the issue of asyncio event loop closed on Windows
             set_event_loop_policy()
 
-    def __exit__(self, exc_type, exc_val, exc_tb):
+    def __exit__(
+        self,
+        exc_type: Optional[Type[BaseException]],
+        exc_value: Optional[BaseException],
+        exc_tb: Optional[types.TracebackType],
+    ) -> None:
         if isinstance(self.client, CodeClient):
             recover_openai_api()
 

azure/ai/evaluation/_evaluate/{_batch_run_client → _batch_run}/proxy_client.py

@@ -1,13 +1,17 @@
 # ---------------------------------------------------------
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
+
+# pylint: disable=protected-access
+
 import inspect
 import logging
+import math
 import os
+from collections import OrderedDict
 from concurrent.futures import Future
 from typing import Any, Callable, Dict, Optional, Union
 
-import numpy as np
 import pandas as pd
 from promptflow.client import PFClient
 from promptflow.entities import Run
@@ -36,7 +40,7 @@ class ProxyClient:  # pylint: disable=client-accepts-api-version-keyword
         **kwargs
     ) -> ProxyRun:
         flow_to_run = flow
-        if hasattr(flow, "_to_async"):
+        if os.getenv("AI_EVALS_BATCH_USE_ASYNC", "true").lower() == "true" and hasattr(flow, "_to_async"):
             flow_to_run = flow._to_async()  # pylint: disable=protected-access
 
         batch_use_async = self._should_batch_use_async(flow_to_run)
@@ -53,16 +57,30 @@ class ProxyClient:  # pylint: disable=client-accepts-api-version-keyword
     def get_details(self, proxy_run: ProxyRun, all_results: bool = False) -> pd.DataFrame:
         run: Run = proxy_run.run.result()
         result_df = self._pf_client.get_details(run, all_results=all_results)
-        result_df.replace("(Failed)", np.nan, inplace=True)
+        result_df.replace("(Failed)", math.nan, inplace=True)
         return result_df
 
     def get_metrics(self, proxy_run: ProxyRun) -> Dict[str, Any]:
         run: Run = proxy_run.run.result()
         return self._pf_client.get_metrics(run)
 
+    def get_run_summary(self, proxy_run: ProxyRun) -> Dict[str, Any]:
+        run = proxy_run.run.result()
+
+        # pylint: disable=protected-access
+        return OrderedDict(
+            [
+                ("status", run.status),
+                ("duration", str(run._end_time - run._created_on)),
+                ("completed_lines", run._properties.get("system_metrics", {}).get("__pf__.lines.completed", "NA")),
+                ("failed_lines", run._properties.get("system_metrics", {}).get("__pf__.lines.failed", "NA")),
+                ("log_path", str(run._output_path)),
+            ]
+        )
+
     @staticmethod
     def _should_batch_use_async(flow):
-        if os.getenv("PF_EVALS_BATCH_USE_ASYNC", "true").lower() == "true":
+        if os.getenv("AI_EVALS_BATCH_USE_ASYNC", "true").lower() == "true":
             if hasattr(flow, "__call__") and inspect.iscoroutinefunction(flow.__call__):
                 return True
             if inspect.iscoroutinefunction(flow):