azure-ai-evaluation 1.0.0__py3-none-any.whl → 1.0.0b2__py3-none-any.whl

azure/ai/evaluation/_evaluate/_evaluate.py

@@ -2,32 +2,26 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
 import inspect
-import json
-import logging
 import os
 import re
-from typing import Any, Callable, Dict, List, Optional, Set, Tuple, TypedDict, TypeVar, Union
+from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Type
 
+import numpy as np
 import pandas as pd
 from promptflow._sdk._constants import LINE_NUMBER
-from promptflow._sdk._errors import UserAuthenticationError, UploadInternalError
 from promptflow.client import PFClient
-from promptflow.entities import Run
 
-from azure.ai.evaluation._common.math import list_mean_nan_safe, apply_transform_nan_safe
-from azure.ai.evaluation._common.utils import validate_azure_ai_project
 from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
 
 from .._constants import (
     CONTENT_SAFETY_DEFECT_RATE_THRESHOLD_DEFAULT,
     EvaluationMetrics,
-    EvaluationRunProperties,
     Prefixes,
     _InternalEvaluationMetrics,
 )
-from .._model_configurations import AzureAIProject, EvaluationResult, EvaluatorConfig
+from .._model_configurations import AzureAIProject
 from .._user_agent import USER_AGENT
-from ._batch_run import EvalRunContext, CodeClient, ProxyClient, TargetRunContext
+from ._batch_run_client import BatchRunContext, CodeClient, ProxyClient
 from ._utils import (
     _apply_column_mapping,
     _log_metrics_and_instance_results,
@@ -35,54 +29,10 @@ from ._utils import (
     _write_output,
 )
 
-TClient = TypeVar("TClient", ProxyClient, CodeClient)
-LOGGER = logging.getLogger(__name__)
-
-# For metrics (aggregates) whose metric names intentionally differ from their
-# originating column name, usually because the aggregation of the original value
-# means something sufficiently different.
-# Note that content safety metrics are handled seprately.
-METRIC_COLUMN_NAME_REPLACEMENTS = {
-    "groundedness_pro_label": "groundedness_pro_passing_rate",
-}
-
-
-class __EvaluatorInfo(TypedDict):
-    result: pd.DataFrame
-    metrics: Dict[str, Any]
-    run_summary: Dict[str, Any]
-
-
-def _aggregate_other_metrics(df: pd.DataFrame) -> Tuple[List[str], Dict[str, float]]:
-    """Identify and average various metrics that need to have the metric name be replaced,
-    instead of having the metric match the originating column name.
-    :param df: The dataframe of evaluation results.
-    :type df: ~pandas.DataFrame
-    :return: A tuple; the first element is a list of dataframe columns that were aggregated,
-        and the second element is a dictionary of resultant new metric column names and their values.
-    :rtype: Tuple[List[str], Dict[str, float]]
-    """
-    renamed_cols = []
-    metric_columns = {}
-    for col in df.columns:
-        metric_prefix = col.split(".")[0]
-        metric_name = col.split(".")[1]
-        if metric_name in METRIC_COLUMN_NAME_REPLACEMENTS:
-            renamed_cols.append(col)
-            new_col_name = metric_prefix + "." + METRIC_COLUMN_NAME_REPLACEMENTS[metric_name]
-            col_with_numeric_values = pd.to_numeric(df[col], errors="coerce")
-            try:
-                metric_columns[new_col_name] = round(list_mean_nan_safe(col_with_numeric_values), 2)
-            except EvaluationException:  # only exception that can be cause is all NaN values
-                msg = f"All score evaluations are NaN/None for column {col}. No aggregation can be performed."
-                LOGGER.warning(msg)
-
-    return renamed_cols, metric_columns
-
 
 # pylint: disable=line-too-long
 def _aggregate_content_safety_metrics(
-    df: pd.DataFrame, evaluators: Dict[str, Callable]
+    df: pd.DataFrame, evaluators: Dict[str, Type]
 ) -> Tuple[List[str], Dict[str, float]]:
     """Find and aggregate defect rates for content safety metrics. Returns both a list
     of columns that were used to calculate defect rates and the defect rates themselves.
@@ -122,15 +72,11 @@ def _aggregate_content_safety_metrics(
     for col in content_safety_df.columns:
         defect_rate_name = col.replace("_score", "_defect_rate")
         col_with_numeric_values = pd.to_numeric(content_safety_df[col], errors="coerce")
-        try:
-            col_with_boolean_values = apply_transform_nan_safe(
-                col_with_numeric_values, lambda x: 1 if x >= CONTENT_SAFETY_DEFECT_RATE_THRESHOLD_DEFAULT else 0
-            )
-            defect_rates[defect_rate_name] = round(list_mean_nan_safe(col_with_boolean_values), 2)
-        except EvaluationException:  # only exception that can be cause is all NaN values
-            msg = f"All score evaluations are NaN/None for column {col}. No aggregation can be performed."
-            LOGGER.warning(msg)
-
+        defect_rates[defect_rate_name] = round(
+            np.sum(col_with_numeric_values >= CONTENT_SAFETY_DEFECT_RATE_THRESHOLD_DEFAULT)
+            / col_with_numeric_values.count(),
+            2,
+        )
     return content_safety_cols, defect_rates
 
 
@@ -160,15 +106,14 @@ def _aggregate_label_defect_metrics(df: pd.DataFrame) -> Tuple[List[str], Dict[s
     for col in label_df.columns:
         defect_rate_name = col.replace("_label", "_defect_rate")
         col_with_boolean_values = pd.to_numeric(label_df[col], errors="coerce")
-        try:
-            defect_rates[defect_rate_name] = round(list_mean_nan_safe(col_with_boolean_values), 2)
-        except EvaluationException:  # only exception that can be cause is all NaN values
-            msg = f"All score evaluations are NaN/None for column {col}. No aggregation can be performed."
-            LOGGER.warning(msg)
+        defect_rates[defect_rate_name] = round(
+            np.sum(col_with_boolean_values) / col_with_boolean_values.count(),
+            2,
+        )
     return label_cols, defect_rates
 
 
-def _aggregate_metrics(df: pd.DataFrame, evaluators: Dict[str, Callable]) -> Dict[str, float]:
+def _aggregate_metrics(df: pd.DataFrame, evaluators: Dict[str, Type]) -> Dict[str, float]:
     """Aggregate metrics from the evaluation results.
     On top of naively calculating the mean of most metrics, this function also identifies certain columns
     that represent defect rates and renames them accordingly. Other columns in the dataframe are dropped.
@@ -177,7 +122,7 @@ def _aggregate_metrics(df: pd.DataFrame, evaluators: Dict[str, Callable]) -> Dic
     :param df: The dataframe of evaluation results.
     :type df: ~pandas.DataFrame
     :param evaluators:  A dictionary mapping of strings to evaluator classes.
-    :type evaluators: Dict[str, Callable]
+    :type evaluators: Dict[str, Type]
     :return: The aggregated metrics.
     :rtype: Dict[str, float]
     """
@@ -188,11 +133,8 @@ def _aggregate_metrics(df: pd.DataFrame, evaluators: Dict[str, Callable]) -> Dic
     # Rename certain columns as defect rates if we know that's what their aggregates represent
     # Content safety metrics
     content_safety_cols, cs_defect_rates = _aggregate_content_safety_metrics(df, evaluators)
-    other_renamed_cols, renamed_cols = _aggregate_other_metrics(df)
     handled_columns.extend(content_safety_cols)
-    handled_columns.extend(other_renamed_cols)
     defect_rates.update(cs_defect_rates)
-    defect_rates.update(renamed_cols)
     # Label-based (true/false) metrics where 'true' means 'something is wrong'
     label_cols, label_defect_rates = _aggregate_label_defect_metrics(df)
     handled_columns.extend(label_cols)
@@ -201,9 +143,6 @@ def _aggregate_metrics(df: pd.DataFrame, evaluators: Dict[str, Callable]) -> Dic
     # For rest of metrics, we will calculate mean
     df.drop(columns=handled_columns, inplace=True)
 
-    # NOTE: nan/None values don't count as as booleans, so boolean columns with
-    # nan/None values won't have a mean produced from them.
-    # This is different from label-based known evaluators, which have special handling.
     mean_value = df.mean(numeric_only=True)
     metrics = mean_value.to_dict()
     # Add defect rates back into metrics
@@ -211,133 +150,28 @@ def _aggregate_metrics(df: pd.DataFrame, evaluators: Dict[str, Callable]) -> Dic
     return metrics
 
 
-def _validate_columns_for_target(
-    df: pd.DataFrame,
-    target: Callable,
-) -> None:
-    """
-    Check that all columns needed by target function are present.
-
-    :param df: The data frame to be validated.
-    :type df: pd.DataFrame
-    :param target: The callable to be applied to data set.
-    :type target: Optional[Callable]
-    :raises EvaluationException: If the column starts with "__outputs." or if the input data contains missing fields.
-    """
-    if any(c.startswith(Prefixes.TSG_OUTPUTS) for c in df.columns):
-        msg = "The column cannot start from " f'"{Prefixes.TSG_OUTPUTS}" if target was defined.'
-        raise EvaluationException(
-            message=msg,
-            internal_message=msg,
-            target=ErrorTarget.EVALUATE,
-            category=ErrorCategory.INVALID_VALUE,
-            blame=ErrorBlame.USER_ERROR,
-        )
-    # If the target function is given, it may return
-    # several columns and hence we cannot check the availability of columns
-    # without knowing target function semantics.
-    # Instead, here we will validate the columns, taken by target.
+def _validate_input_data_for_evaluator(evaluator, evaluator_name, df_data, is_target_fn=False):
     required_inputs = [
         param.name
-        for param in inspect.signature(target).parameters.values()
+        for param in inspect.signature(evaluator).parameters.values()
         if param.default == inspect.Parameter.empty and param.name not in ["kwargs", "args", "self"]
     ]
 
-    missing_inputs = [col for col in required_inputs if col not in df.columns]
+    missing_inputs = [col for col in required_inputs if col not in df_data.columns]
     if missing_inputs:
-        msg = f"Missing required inputs for target: {missing_inputs}."
+        if not is_target_fn:
+            msg = f"Missing required inputs for evaluator {evaluator_name} : {missing_inputs}."
+            raise EvaluationException(
+                message=msg,
+                internal_message=msg,
+                target=ErrorTarget.EVALUATE,
+                category=ErrorCategory.MISSING_FIELD,
+                blame=ErrorBlame.USER_ERROR,
+            )
+        msg = f"Missing required inputs for target : {missing_inputs}."
         raise EvaluationException(
             message=msg,
-            target=ErrorTarget.EVALUATE,
-            category=ErrorCategory.MISSING_FIELD,
-            blame=ErrorBlame.USER_ERROR,
-        )
-
-
-def _validate_columns_for_evaluators(
-    df: pd.DataFrame,
-    evaluators: Dict[str, Callable],
-    target: Optional[Callable],
-    target_generated_columns: Optional[Set[str]],
-    column_mapping: Dict[str, Dict[str, str]],
-) -> None:
-    """
-    Check that all columns needed by evaluators are present.
-
-    :param df: The data frame to be validated.
-    :type df: pd.DataFrame
-    :param evaluators: The dictionary of evaluators.
-    :type evaluators: Dict[str, Callable]
-    :param target: The callable to be applied to data set.
-    :type target: Optional[Callable]
-    :param target_generated_columns: The set of columns generated by the target callable.
-    :type target_generated_columns: Optional[Set[str]]
-    :param column_mapping: Dictionary mapping evaluator name to evaluator column mapping.
-    :type column_mapping: Dict[str, Dict[str, str]]
-    :raises EvaluationException: If data is missing required inputs or if the target callable did not generate the necessary columns.
-    """
-    missing_inputs_per_evaluator = {}
-
-    for evaluator_name, evaluator in evaluators.items():
-        # Apply column mapping
-        mapping_config = column_mapping.get(evaluator_name, column_mapping.get("default", None))
-        new_df = _apply_column_mapping(df, mapping_config)
-
-        # Validate input data for evaluator
-        is_built_in = evaluator.__module__.startswith("azure.ai.evaluation")
-        if is_built_in:
-            # Note that for built-in evaluators supporting the "conversation" parameter,
-            # input parameters are now optional.
-            evaluator_params = [
-                param.name
-                for param in inspect.signature(evaluator).parameters.values()
-                if param.name not in ["kwargs", "args", "self"]
-            ]
-
-            if "conversation" in evaluator_params and "conversation" in new_df.columns:
-                # Ignore the missing fields if "conversation" presents in the input data
-                missing_inputs = []
-            else:
-                optional_params = (
-                    evaluator._OPTIONAL_PARAMS  # pylint: disable=protected-access
-                    if hasattr(evaluator, "_OPTIONAL_PARAMS")
-                    else []
-                )
-                excluded_params = set(new_df.columns).union(optional_params)
-                missing_inputs = [col for col in evaluator_params if col not in excluded_params]
-
-                # If "conversation" is the only parameter and it is missing, keep it in the missing inputs
-                # Otherwise, remove it from the missing inputs
-                if "conversation" in missing_inputs:
-                    if not (evaluator_params == ["conversation"] and missing_inputs == ["conversation"]):
-                        missing_inputs.remove("conversation")
-        else:
-            evaluator_params = [
-                param.name
-                for param in inspect.signature(evaluator).parameters.values()
-                if param.default == inspect.Parameter.empty and param.name not in ["kwargs", "args", "self"]
-            ]
-
-            missing_inputs = [col for col in evaluator_params if col not in new_df.columns]
-
-        if missing_inputs:
-            missing_inputs_per_evaluator[evaluator_name] = missing_inputs
-
-    if missing_inputs_per_evaluator:
-        msg = "Some evaluators are missing required inputs:\n"
-        for evaluator_name, missing in missing_inputs_per_evaluator.items():
-            msg += f"- {evaluator_name}: {missing}\n"
-
-        # Add the additional notes
-        msg += "\nTo resolve this issue:\n"
-        msg += "- Ensure the data contains required inputs.\n"
-        if target is not None:
-            msg += "- Verify that the target is generating the necessary columns for the evaluators. "
-            msg += f"Currently generated columns: {target_generated_columns} \n"
-        msg += "- Check that the column mapping is correctly configured."
-
-        raise EvaluationException(
-            message=msg.strip(),
+            internal_message=msg,
             target=ErrorTarget.EVALUATE,
             category=ErrorCategory.MISSING_FIELD,
             blame=ErrorBlame.USER_ERROR,
@@ -346,85 +180,76 @@ def _validate_columns_for_evaluators(
 
 def _validate_and_load_data(target, data, evaluators, output_path, azure_ai_project, evaluation_name):
     if data is None:
-        msg = "The 'data' parameter is required for evaluation."
-        raise EvaluationException(
-            message=msg,
-            target=ErrorTarget.EVALUATE,
-            category=ErrorCategory.INVALID_VALUE,
-            blame=ErrorBlame.USER_ERROR,
-        )
-    if not isinstance(data, (os.PathLike, str)):
-        msg = "The 'data' parameter must be a string or a path-like object."
-        raise EvaluationException(
-            message=msg,
-            target=ErrorTarget.EVALUATE,
-            category=ErrorCategory.INVALID_VALUE,
-            blame=ErrorBlame.USER_ERROR,
-        )
-    if not os.path.exists(data):
-        msg = f"The input data file path '{data}' does not exist."
+        msg = "data parameter must be provided for evaluation."
         raise EvaluationException(
             message=msg,
+            internal_message=msg,
             target=ErrorTarget.EVALUATE,
-            category=ErrorCategory.INVALID_VALUE,
+            category=ErrorCategory.MISSING_FIELD,
             blame=ErrorBlame.USER_ERROR,
         )
 
     if target is not None:
         if not callable(target):
-            msg = "The 'target' parameter must be a callable function."
+            msg = "target parameter must be a callable function."
             raise EvaluationException(
                 message=msg,
+                internal_message=msg,
                 target=ErrorTarget.EVALUATE,
                 category=ErrorCategory.INVALID_VALUE,
                 blame=ErrorBlame.USER_ERROR,
             )
 
-    if not evaluators:
-        msg = "The 'evaluators' parameter is required and cannot be None or empty."
-        raise EvaluationException(
-            message=msg,
-            target=ErrorTarget.EVALUATE,
-            category=ErrorCategory.INVALID_VALUE,
-            blame=ErrorBlame.USER_ERROR,
-        )
-    if not isinstance(evaluators, dict):
-        msg = "The 'evaluators' parameter must be a dictionary."
-        raise EvaluationException(
-            message=msg,
-            target=ErrorTarget.EVALUATE,
-            category=ErrorCategory.INVALID_VALUE,
-            blame=ErrorBlame.USER_ERROR,
-        )
+    if data is not None:
+        if not isinstance(data, str):
+            msg = "data parameter must be a string."
+            raise EvaluationException(
+                message=msg,
+                internal_message=msg,
+                target=ErrorTarget.EVALUATE,
+                category=ErrorCategory.INVALID_VALUE,
+                blame=ErrorBlame.USER_ERROR,
+            )
 
-    if output_path is not None:
-        if not isinstance(output_path, (os.PathLike, str)):
-            msg = "The 'output_path' parameter must be a string or a path-like object."
+    if evaluators is not None:
+        if not isinstance(evaluators, dict):
+            msg = "evaluators parameter must be a dictionary."
             raise EvaluationException(
                 message=msg,
+                internal_message=msg,
                 target=ErrorTarget.EVALUATE,
                 category=ErrorCategory.INVALID_VALUE,
                 blame=ErrorBlame.USER_ERROR,
             )
 
-        output_dir = output_path if os.path.isdir(output_path) else os.path.dirname(output_path)
-        if output_dir and not os.path.exists(output_dir):
-            msg = f"The output directory '{output_dir}' does not exist. Please create the directory manually."
+    if output_path is not None:
+        if not isinstance(output_path, str):
+            msg = "output_path parameter must be a string."
             raise EvaluationException(
                 message=msg,
+                internal_message=msg,
                 target=ErrorTarget.EVALUATE,
                 category=ErrorCategory.INVALID_VALUE,
                 blame=ErrorBlame.USER_ERROR,
             )
 
     if azure_ai_project is not None:
-        validate_azure_ai_project(azure_ai_project)
+        if not isinstance(azure_ai_project, Dict):
+            msg = "azure_ai_project parameter must be a dictionary."
+            raise EvaluationException(
+                message=msg,
+                internal_message=msg,
+                target=ErrorTarget.EVALUATE,
+                category=ErrorCategory.INVALID_VALUE,
+                blame=ErrorBlame.USER_ERROR,
+            )
 
     if evaluation_name is not None:
-        if not isinstance(evaluation_name, str) or not evaluation_name.strip():
-            msg = "The 'evaluation_name' parameter must be a non-empty string."
+        if not isinstance(evaluation_name, str):
+            msg = "evaluation_name parameter must be a string."
             raise EvaluationException(
                 message=msg,
+                internal_message=msg,
                 target=ErrorTarget.EVALUATE,
                 category=ErrorCategory.INVALID_VALUE,
                 blame=ErrorBlame.USER_ERROR,
@@ -434,7 +259,8 @@ def _validate_and_load_data(target, data, evaluators, output_path, azure_ai_proj
         initial_data_df = pd.read_json(data, lines=True)
     except Exception as e:
         raise EvaluationException(
-            message=f"Unable to load data from '{data}'. Please ensure the input is valid JSONL format. Detailed error: {e}.",
+            message=f"Failed to load data from {data}. Confirm that it is valid jsonl data. Error: {str(e)}.",
+            internal_message="Failed to load data. Confirm that it is valid jsonl data.",
             target=ErrorTarget.EVALUATE,
             category=ErrorCategory.INVALID_VALUE,
             blame=ErrorBlame.USER_ERROR,
@@ -443,60 +269,88 @@ def _validate_and_load_data(target, data, evaluators, output_path, azure_ai_proj
     return initial_data_df
 
 
+def _validate_columns(
+    df: pd.DataFrame,
+    evaluators: Dict[str, Any],
+    target: Optional[Callable],
+    evaluator_config: Dict[str, Dict[str, str]],
+) -> None:
+    """
+    Check that all columns needed by evaluator or target function are present.
+
+    :param df: The data frame to be validated.
+    :type df: pd.DataFrame
+    :param evaluators: The dictionary of evaluators.
+    :type evaluators: Dict[str, Any]
+    :param target: The callable to be applied to data set.
+    :type target: Optional[Callable]
+    :param evaluator_config: The configuration for evaluators.
+    :type evaluator_config: Dict[str, Dict[str, str]]
+    :raises EvaluationException: If column starts from "__outputs." while target is defined.
+    """
+    if target:
+        if any(c.startswith(Prefixes.TSG_OUTPUTS) for c in df.columns):
+            msg = "The column cannot start from " f'"{Prefixes.TSG_OUTPUTS}" if target was defined.'
+            raise EvaluationException(
+                message=msg,
+                internal_message=msg,
+                target=ErrorTarget.EVALUATE,
+                category=ErrorCategory.INVALID_VALUE,
+                blame=ErrorBlame.USER_ERROR,
+            )
+        # If the target function is given, it may return
+        # several columns and hence we cannot check the availability of columns
+        # without knowing target function semantics.
+        # Instead, here we will validate the columns, taken by target.
+        _validate_input_data_for_evaluator(target, None, df, is_target_fn=True)
+    else:
+        for evaluator_name, evaluator in evaluators.items():
+            # Apply column mapping
+            mapping_config = evaluator_config.get(evaluator_name, evaluator_config.get("default", None))
+            new_df = _apply_column_mapping(df, mapping_config)
+
+            # Validate input data for evaluator
+            _validate_input_data_for_evaluator(evaluator, evaluator_name, new_df)
+
+
 def _apply_target_to_data(
     target: Callable,
-    data: Union[str, os.PathLike],
+    data: str,
     pf_client: PFClient,
     initial_data: pd.DataFrame,
     evaluation_name: Optional[str] = None,
-    **kwargs,
-) -> Tuple[pd.DataFrame, Set[str], Run]:
+    _run_name: Optional[str] = None,
+) -> Tuple[pd.DataFrame, Set[str]]:
     """
     Apply the target function to the data set and return updated data and generated columns.
 
     :param target: The function to be applied to data.
     :type target: Callable
     :param data: The path to input jsonl file.
-    :type data: Union[str, os.PathLike]
+    :type data: str
     :param pf_client: The promptflow client to be used.
     :type pf_client: PFClient
     :param initial_data: The data frame with the loaded data.
     :type initial_data: pd.DataFrame
     :param evaluation_name: The name of the evaluation.
     :type evaluation_name: Optional[str]
+    :param _run_name: The name of target run. Used for testing only.
+    :type _run_name: Optional[str]
     :return: The tuple, containing data frame and the list of added columns.
     :rtype: Tuple[pandas.DataFrame, List[str]]
     """
-    _run_name = kwargs.get("_run_name")
-    upload_target_snaphot = kwargs.get("_upload_target_snapshot", False)
-
-    try:
-        with TargetRunContext(upload_target_snaphot):
-            run: Run = pf_client.run(
-                flow=target,
-                display_name=evaluation_name,
-                data=data,
-                properties={EvaluationRunProperties.RUN_TYPE: "eval_run", "isEvaluatorRun": "true"},
-                stream=True,
-                name=_run_name,
-            )
-    except (UserAuthenticationError, UploadInternalError) as ex:
-        if "Failed to upload run" in ex.message:
-            msg = (
-                "Failed to upload the target run to the cloud. "
-                "This may be caused by insufficient permission to access storage or other errors."
-            )
-            raise EvaluationException(
-                message=msg,
-                target=ErrorTarget.EVALUATE,
-                category=ErrorCategory.FAILED_REMOTE_TRACKING,
-                blame=ErrorBlame.USER_ERROR,
-                tsg_link="https://aka.ms/azsdk/python/evaluation/remotetracking/troubleshoot",
-            ) from ex
-
-        raise ex
-
-    target_output: pd.DataFrame = pf_client.runs.get_details(run, all_results=True)
+    # We are manually creating the temporary directory for the flow
+    # because the way tempdir remove temporary directories will
+    # hang the debugger, because promptflow will keep flow directory.
+    run = pf_client.run(
+        flow=target,
+        display_name=evaluation_name,
+        data=data,
+        properties={"runType": "eval_run", "isEvaluatorRun": "true"},
+        stream=True,
+        name=_run_name,
+    )
+    target_output = pf_client.runs.get_details(run, all_results=True)
     # Remove input and output prefix
     generated_columns = {
         col[len(Prefixes.OUTPUTS) :] for col in target_output.columns if col.startswith(Prefixes.OUTPUTS)
@@ -518,30 +372,28 @@ def _apply_target_to_data(
     return target_output, generated_columns, run
 
 
-def _process_column_mappings(
-    column_mapping: Dict[str, Optional[Dict[str, str]]],
-) -> Dict[str, Dict[str, str]]:
-    """Process column_mapping to replace ${target.} with ${data.}
+def _process_evaluator_config(evaluator_config: Dict[str, Dict[str, str]]) -> Dict[str, Dict[str, str]]:
+    """Process evaluator_config to replace ${target.} with ${data.}
 
-    :param column_mapping: The configuration for evaluators.
-    :type column_mapping: Dict[str, Optional[Dict[str, str]]]
+    :param evaluator_config: The configuration for evaluators.
+    :type evaluator_config: Dict[str, Dict[str, str]]
     :return: The processed configuration.
     :rtype: Dict[str, Dict[str, str]]
     """
 
-    processed_config: Dict[str, Dict[str, str]] = {}
+    processed_config = {}
 
     unexpected_references = re.compile(r"\${(?!target\.|data\.).+?}")
 
-    if column_mapping:
-        for evaluator, mapping_config in column_mapping.items():
+    if evaluator_config:
+        for evaluator, mapping_config in evaluator_config.items():
             if isinstance(mapping_config, dict):
                 processed_config[evaluator] = {}
 
                 for map_to_key, map_value in mapping_config.items():
                     # Check if there's any unexpected reference other than ${target.} or ${data.}
                     if unexpected_references.search(map_value):
-                        msg = "Unexpected references detected in 'column_mapping'. Ensure only ${target.} and ${data.} are used."
+                        msg = "Unexpected references detected in 'evaluator_config'. Ensure only ${target.} and ${data.} are used."
                         raise EvaluationException(
                             message=msg,
                             internal_message=msg,
@@ -583,15 +435,15 @@ def _rename_columns_conditionally(df: pd.DataFrame) -> pd.DataFrame:
 # @log_evaluate_activity
 def evaluate(
     *,
-    data: Union[str, os.PathLike],
+    data: str,
     evaluators: Dict[str, Callable],
     evaluation_name: Optional[str] = None,
     target: Optional[Callable] = None,
-    evaluator_config: Optional[Dict[str, EvaluatorConfig]] = None,
+    evaluator_config: Optional[Dict[str, Dict[str, str]]] = None,
     azure_ai_project: Optional[AzureAIProject] = None,
-    output_path: Optional[Union[str, os.PathLike]] = None,
+    output_path: Optional[str] = None,
     **kwargs,
-) -> EvaluationResult:
+):
     """Evaluates target or data with built-in or custom evaluators. If both target and data are provided,
         data will be run through target function and then results will be evaluated.
 
@@ -606,26 +458,56 @@ def evaluate(
     :keyword target: Target to be evaluated. `target` and `data` both cannot be None
     :paramtype target: Optional[Callable]
     :keyword evaluator_config: Configuration for evaluators. The configuration should be a dictionary with evaluator
-        names as keys and a values that are dictionaries containing the column mappings. The column mappings should
-        be a dictionary with keys as the column names in the evaluator input and values as the column names in the
-        input data or data generated by target.
-    :paramtype evaluator_config: Optional[Dict[str, ~azure.ai.evaluation.EvaluatorConfig]]
+        names as keys and a dictionary of column mappings as values. The column mappings should be a dictionary with
+        keys as the column names in the evaluator input and values as the column names in the input data or data
+        generated by target.
+    :paramtype evaluator_config: Optional[Dict[str, Dict[str, str]]
     :keyword output_path: The local folder or file path to save evaluation results to if set. If folder path is provided
           the results will be saved to a file named `evaluation_results.json` in the folder.
     :paramtype output_path: Optional[str]
     :keyword azure_ai_project: Logs evaluation results to AI Studio if set.
     :paramtype azure_ai_project: Optional[~azure.ai.evaluation.AzureAIProject]
     :return: Evaluation results.
-    :rtype: ~azure.ai.evaluation.EvaluationResult
+    :rtype: dict
+
+    :Example:
+
+    Evaluate API can be used as follows:
+
+    .. code-block:: python
+
+            from azure.ai.evaluation import evaluate, RelevanceEvaluator, CoherenceEvaluator
+
 
-    .. admonition:: Example:
+            model_config = {
+                "azure_endpoint": os.environ.get("AZURE_OPENAI_ENDPOINT"),
+                "api_key": os.environ.get("AZURE_OPENAI_KEY"),
+                "azure_deployment": os.environ.get("AZURE_OPENAI_DEPLOYMENT")
+            }
+
+            coherence_eval = CoherenceEvaluator(model_config=model_config)
+            relevance_eval = RelevanceEvaluator(model_config=model_config)
+
+            path = "evaluate_test_data.jsonl"
+            result = evaluate(
+                data=path,
+                evaluators={
+                    "coherence": coherence_eval,
+                    "relevance": relevance_eval,
+                },
+                evaluator_config={
+                    "coherence": {
+                        "response": "${data.response}",
+                        "query": "${data.query}"
+                    },
+                    "relevance": {
+                        "response": "${data.response}",
+                        "context": "${data.context}",
+                        "query": "${data.query}"
+                    }
+                }
+            )
 
-        .. literalinclude:: ../samples/evaluation_samples_evaluate.py
-            :start-after: [START evaluate_method]
-            :end-before: [END evaluate_method]
-            :language: python
-            :dedent: 8
-            :caption: Run an evaluation on local data with Coherence and Relevance evaluators.
     """
     try:
         return _evaluate(
@@ -656,90 +538,69 @@ def evaluate(
                 internal_message=error_message,
                 target=ErrorTarget.EVALUATE,
                 category=ErrorCategory.FAILED_EXECUTION,
-                blame=ErrorBlame.USER_ERROR,
-            ) from e
-
-        # Ensure a consistent user experience when encountering errors by converting
-        # all other exceptions to EvaluationException.
-        if not isinstance(e, EvaluationException):
-            raise EvaluationException(
-                message=str(e),
-                target=ErrorTarget.EVALUATE,
-                category=ErrorCategory.FAILED_EXECUTION,
-                blame=ErrorBlame.SYSTEM_ERROR,
+                blame=ErrorBlame.UNKNOWN,
             ) from e
 
         raise e
 
 
-def _print_summary(per_evaluator_results: Dict[str, Any]) -> None:
-    # Extract evaluators with a non-empty "run_summary"
-    output_dict = {
-        name: result["run_summary"] for name, result in per_evaluator_results.items() if result.get("run_summary")
-    }
-
-    if output_dict:
-        print("======= Combined Run Summary (Per Evaluator) =======\n")
-        print(json.dumps(output_dict, indent=4))
-        print("\n====================================================\n")
-
-
-def _evaluate(  # pylint: disable=too-many-locals,too-many-statements
+def _evaluate(  # pylint: disable=too-many-locals
     *,
-    evaluators: Dict[str, Callable],
     evaluation_name: Optional[str] = None,
     target: Optional[Callable] = None,
-    data: Union[str, os.PathLike],
-    evaluator_config: Optional[Dict[str, EvaluatorConfig]] = None,
+    data: Optional[str] = None,
+    evaluators: Optional[Dict[str, Callable]] = None,
+    evaluator_config: Optional[Dict[str, Dict[str, str]]] = None,
     azure_ai_project: Optional[AzureAIProject] = None,
-    output_path: Optional[Union[str, os.PathLike]] = None,
+    output_path: Optional[str] = None,
     **kwargs,
-) -> EvaluationResult:
+):
     input_data_df = _validate_and_load_data(target, data, evaluators, output_path, azure_ai_project, evaluation_name)
 
     # Process evaluator config to replace ${target.} with ${data.}
     if evaluator_config is None:
         evaluator_config = {}
-    # extract column mapping dicts into dictionary mapping evaluator name to column mapping
-    column_mapping = _process_column_mappings(
-        {
-            evaluator_name: evaluator_configuration.get("column_mapping", None)
-            for evaluator_name, evaluator_configuration in evaluator_config.items()
-        }
+    evaluator_config = _process_evaluator_config(evaluator_config)
+    _validate_columns(input_data_df, evaluators, target, evaluator_config)
+
+    # Target Run
+    pf_client = PFClient(
+        config=(
+            {"trace.destination": _trace_destination_from_project_scope(azure_ai_project)} if azure_ai_project else None
+        ),
+        user_agent=USER_AGENT,
     )
 
-    if target is not None:
-        _validate_columns_for_target(input_data_df, target)
-
-    pf_client = PFClient(user_agent=USER_AGENT)
-    target_run: Optional[Run] = None
+    trace_destination = pf_client._config.get_trace_destination()  # pylint: disable=protected-access
+    target_run = None
+    target_generated_columns = set()
 
     # Create default configuration for evaluators that directly maps
     # input data names to keyword inputs of the same name in the evaluators.
-    column_mapping = column_mapping or {}
-    column_mapping.setdefault("default", {})
+    evaluator_config = evaluator_config or {}
+    evaluator_config.setdefault("default", {})
 
     # If target is set, apply 1-1 column mapping from target outputs to evaluator inputs
-    target_generated_columns: Set[str] = set()
     if data is not None and target is not None:
         input_data_df, target_generated_columns, target_run = _apply_target_to_data(
-            target, data, pf_client, input_data_df, evaluation_name, **kwargs
+            target, data, pf_client, input_data_df, evaluation_name, _run_name=kwargs.get("_run_name")
         )
 
-        for evaluator_name, mapping in column_mapping.items():
+        for evaluator_name, mapping in evaluator_config.items():
             mapped_to_values = set(mapping.values())
             for col in target_generated_columns:
                 # If user defined mapping differently, do not change it.
                 # If it was mapped to target, we have already changed it
-                # in _process_column_mappings
+                # in _process_evaluator_config
                 run_output = f"${{run.outputs.{col}}}"
                 # We will add our mapping only if
                 # customer did not mapped target output.
                 if col not in mapping and run_output not in mapped_to_values:
-                    column_mapping[evaluator_name][col] = run_output  # pylint: disable=unnecessary-dict-index-lookup
+                    evaluator_config[evaluator_name][col] = run_output  # pylint: disable=unnecessary-dict-index-lookup
 
-    # After we have generated all columns, we can check if we have everything we need for evaluators.
-    _validate_columns_for_evaluators(input_data_df, evaluators, target, target_generated_columns, column_mapping)
+        # After we have generated all columns we can check if we have
+        # everything we need for evaluators.
+        _validate_columns(input_data_df, evaluators, target=None, evaluator_config=evaluator_config)
 
     # Apply 1-1 mapping from input data to evaluator inputs, excluding values already assigned
     # via target mapping.
@@ -749,52 +610,44 @@ def _evaluate(  # pylint: disable=too-many-locals,too-many-statements
         for col in input_data_df.columns:
             # Ignore columns added by target mapping. These are formatted as "__outputs.<column_name>"
             # Also ignore columns that are already in config, since they've been covered by target mapping.
-            if not col.startswith(Prefixes.TSG_OUTPUTS) and col not in column_mapping["default"].keys():
-                column_mapping["default"][col] = f"${{data.{col}}}"
-
-    def eval_batch_run(
-        batch_run_client: TClient, *, data=Union[str, os.PathLike, pd.DataFrame]
-    ) -> Dict[str, __EvaluatorInfo]:
-        with EvalRunContext(batch_run_client):
-            runs = {
-                evaluator_name: batch_run_client.run(
-                    flow=evaluator,
-                    run=target_run,
-                    evaluator_name=evaluator_name,
-                    column_mapping=column_mapping.get(evaluator_name, column_mapping.get("default", None)),
-                    data=data,
-                    stream=True,
-                    name=kwargs.get("_run_name"),
-                )
-                for evaluator_name, evaluator in evaluators.items()
-            }
-
-            # get_details needs to be called within EvalRunContext scope in order to have user agent populated
-            return {
-                evaluator_name: {
-                    "result": batch_run_client.get_details(run, all_results=True),
-                    "metrics": batch_run_client.get_metrics(run),
-                    "run_summary": batch_run_client.get_run_summary(run),
-                }
-                for evaluator_name, run in runs.items()
-            }
-
+            if not col.startswith(Prefixes.TSG_OUTPUTS) and col not in evaluator_config["default"].keys():
+                evaluator_config["default"][col] = f"${{data.{col}}}"
     # Batch Run
+    evaluators_info = {}
     use_pf_client = kwargs.get("_use_pf_client", True)
     if use_pf_client:
+        batch_run_client = ProxyClient(pf_client)
+
         # Ensure the absolute path is passed to pf.run, as relative path doesn't work with
         # multiple evaluators. If the path is already absolute, abspath will return the original path.
         data = os.path.abspath(data)
-        per_evaluator_results = eval_batch_run(ProxyClient(pf_client), data=data)
     else:
+        batch_run_client = CodeClient()
         data = input_data_df
-        per_evaluator_results = eval_batch_run(CodeClient(), data=input_data_df)
+
+    with BatchRunContext(batch_run_client):
+        for evaluator_name, evaluator in evaluators.items():
+            evaluators_info[evaluator_name] = {}
+            evaluators_info[evaluator_name]["run"] = batch_run_client.run(
+                flow=evaluator,
+                run=target_run,
+                evaluator_name=evaluator_name,
+                column_mapping=evaluator_config.get(evaluator_name, evaluator_config.get("default", None)),
+                data=data,
+                stream=True,
+                name=kwargs.get("_run_name"),
+            )
+
+        # get_details needs to be called within BatchRunContext scope in order to have user agent populated
+        for evaluator_name, evaluator_info in evaluators_info.items():
+            evaluator_info["result"] = batch_run_client.get_details(evaluator_info["run"], all_results=True)
+            evaluator_info["metrics"] = batch_run_client.get_metrics(evaluator_info["run"])
 
     # Concatenate all results
     evaluators_result_df = None
     evaluators_metric = {}
-    for evaluator_name, evaluator_result in per_evaluator_results.items():
-        evaluator_result_df = evaluator_result["result"]
+    for evaluator_name, evaluator_info in evaluators_info.items():
+        evaluator_result_df = evaluator_info["result"]
 
         # drop input columns
         evaluator_result_df = evaluator_result_df.drop(
@@ -817,7 +670,7 @@ def _evaluate(  # pylint: disable=too-many-locals,too-many-statements
             else evaluator_result_df
         )
 
-        evaluators_metric.update({f"{evaluator_name}.{k}": v for k, v in evaluator_result["metrics"].items()})
+        evaluators_metric.update({f"{evaluator_name}.{k}": v for k, v in evaluator_info["metrics"].items()})
 
     # Rename columns, generated by target function to outputs instead of inputs.
     # If target generates columns, already present in the input data, these columns
@@ -827,10 +680,6 @@ def _evaluate(  # pylint: disable=too-many-locals,too-many-statements
     result_df = pd.concat([input_data_df, evaluators_result_df], axis=1, verify_integrity=True)
     metrics = _aggregate_metrics(evaluators_result_df, evaluators)
     metrics.update(evaluators_metric)
-
-    # Since tracing is disabled, pass None for target_run so a dummy evaluation run will be created each time.
-    target_run = None
-    trace_destination = _trace_destination_from_project_scope(azure_ai_project) if azure_ai_project else None
     studio_url = _log_metrics_and_instance_results(
         metrics,
         result_df,
@@ -839,10 +688,7 @@ def _evaluate(  # pylint: disable=too-many-locals,too-many-statements
         evaluation_name,
     )
 
-    result_df_dict = result_df.to_dict("records")
-    result: EvaluationResult = {"rows": result_df_dict, "metrics": metrics, "studio_url": studio_url}  # type: ignore
-
-    _print_summary(per_evaluator_results)
+    result = {"rows": result_df.to_dict("records"), "metrics": metrics, "studio_url": studio_url}
 
     if output_path:
         _write_output(output_path, result)