azure-ai-evaluation 1.0.0b1__py3-none-any.whl → 1.0.0b3__py3-none-any.whl

azure/ai/evaluation/_evaluate/_evaluate.py

@@ -8,27 +8,26 @@ from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Type
 
 import numpy as np
 import pandas as pd
-
 from promptflow._sdk._constants import LINE_NUMBER
 from promptflow.client import PFClient
 
-from .._model_configurations import AzureAIProject
+from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
+
 from .._constants import (
     CONTENT_SAFETY_DEFECT_RATE_THRESHOLD_DEFAULT,
     EvaluationMetrics,
     Prefixes,
     _InternalEvaluationMetrics,
 )
+from .._model_configurations import AzureAIProject, EvaluatorConfig
 from .._user_agent import USER_AGENT
 from ._batch_run_client import BatchRunContext, CodeClient, ProxyClient
-from ._telemetry import log_evaluate_activity
 from ._utils import (
     _apply_column_mapping,
     _log_metrics_and_instance_results,
     _trace_destination_from_project_scope,
     _write_output,
 )
-from azure.ai.evaluation._exceptions import EvaluationException, ErrorBlame, ErrorCategory, ErrorTarget
 
 
 # pylint: disable=line-too-long
@@ -159,6 +158,12 @@ def _validate_input_data_for_evaluator(evaluator, evaluator_name, df_data, is_ta
     ]
 
     missing_inputs = [col for col in required_inputs if col not in df_data.columns]
+    if missing_inputs and "conversation" in required_inputs:
+        non_conversation_inputs = [val for val in required_inputs if val != "conversation"]
+        if len(missing_inputs) == len(non_conversation_inputs) and [
+            input in non_conversation_inputs for input in missing_inputs
+        ]:
+            missing_inputs = []
     if missing_inputs:
         if not is_target_fn:
             msg = f"Missing required inputs for evaluator {evaluator_name} : {missing_inputs}."
@@ -260,12 +265,12 @@ def _validate_and_load_data(target, data, evaluators, output_path, azure_ai_proj
         initial_data_df = pd.read_json(data, lines=True)
     except Exception as e:
         raise EvaluationException(
-                message=f"Failed to load data from {data}. Confirm that it is valid jsonl data. Error: {str(e)}.",
-                internal_message="Failed to load data. Confirm that it is valid jsonl data.",
-                target=ErrorTarget.EVALUATE,
-                category=ErrorCategory.INVALID_VALUE,
-                blame=ErrorBlame.USER_ERROR,
-            ) from e
+            message=f"Failed to load data from {data}. Confirm that it is valid jsonl data. Error: {str(e)}.",
+            internal_message="Failed to load data. Confirm that it is valid jsonl data.",
+            target=ErrorTarget.EVALUATE,
+            category=ErrorCategory.INVALID_VALUE,
+            blame=ErrorBlame.USER_ERROR,
+        ) from e
 
     return initial_data_df
 
@@ -274,7 +279,7 @@ def _validate_columns(
     df: pd.DataFrame,
     evaluators: Dict[str, Any],
     target: Optional[Callable],
-    evaluator_config: Dict[str, Dict[str, str]],
+    column_mapping: Dict[str, Dict[str, str]],
 ) -> None:
     """
     Check that all columns needed by evaluator or target function are present.
@@ -285,8 +290,8 @@ def _validate_columns(
     :type evaluators: Dict[str, Any]
     :param target: The callable to be applied to data set.
     :type target: Optional[Callable]
-    :param evaluator_config: The configuration for evaluators.
-    :type evaluator_config: Dict[str, Dict[str, str]]
+    :param column_mapping: Dictionary mapping evaluator name to evaluator column mapping
+    :type column_mapping: Dict[str, Dict[str, str]]
     :raises EvaluationException: If column starts from "__outputs." while target is defined.
     """
     if target:
@@ -307,7 +312,7 @@ def _validate_columns(
     else:
         for evaluator_name, evaluator in evaluators.items():
             # Apply column mapping
-            mapping_config = evaluator_config.get(evaluator_name, evaluator_config.get("default", None))
+            mapping_config = column_mapping.get(evaluator_name, column_mapping.get("default", None))
             new_df = _apply_column_mapping(df, mapping_config)
 
             # Validate input data for evaluator
@@ -373,11 +378,11 @@ def _apply_target_to_data(
     return target_output, generated_columns, run
 
 
-def _process_evaluator_config(evaluator_config: Dict[str, Dict[str, str]]) -> Dict[str, Dict[str, str]]:
-    """Process evaluator_config to replace ${target.} with ${data.}
+def _process_column_mappings(column_mapping: Dict[str, Dict[str, str]]) -> Dict[str, Dict[str, str]]:
+    """Process column_mapping to replace ${target.} with ${data.}
 
-    :param evaluator_config: The configuration for evaluators.
-    :type evaluator_config: Dict[str, Dict[str, str]]
+    :param column_mapping: The configuration for evaluators.
+    :type column_mapping: Dict[str, Dict[str, str]]
     :return: The processed configuration.
     :rtype: Dict[str, Dict[str, str]]
     """
@@ -386,15 +391,15 @@ def _process_evaluator_config(evaluator_config: Dict[str, Dict[str, str]]) -> Di
 
     unexpected_references = re.compile(r"\${(?!target\.|data\.).+?}")
 
-    if evaluator_config:
-        for evaluator, mapping_config in evaluator_config.items():
+    if column_mapping:
+        for evaluator, mapping_config in column_mapping.items():
             if isinstance(mapping_config, dict):
                 processed_config[evaluator] = {}
 
                 for map_to_key, map_value in mapping_config.items():
                     # Check if there's any unexpected reference other than ${target.} or ${data.}
                     if unexpected_references.search(map_value):
-                        msg = "Unexpected references detected in 'evaluator_config'. Ensure only ${target.} and ${data.} are used."
+                        msg = "Unexpected references detected in 'column_mapping'. Ensure only ${target.} and ${data.} are used."
                         raise EvaluationException(
                             message=msg,
                             internal_message=msg,
@@ -436,11 +441,11 @@ def _rename_columns_conditionally(df: pd.DataFrame) -> pd.DataFrame:
 # @log_evaluate_activity
 def evaluate(
     *,
+    data: str,
+    evaluators: Dict[str, Callable],
     evaluation_name: Optional[str] = None,
     target: Optional[Callable] = None,
-    data: Optional[str] = None,
-    evaluators: Optional[Dict[str, Callable]] = None,
-    evaluator_config: Optional[Dict[str, Dict[str, str]]] = None,
+    evaluator_config: Optional[Dict[str, EvaluatorConfig]] = None,
     azure_ai_project: Optional[AzureAIProject] = None,
     output_path: Optional[str] = None,
     **kwargs,
@@ -448,21 +453,21 @@ def evaluate(
     """Evaluates target or data with built-in or custom evaluators. If both target and data are provided,
         data will be run through target function and then results will be evaluated.
 
+    :keyword data: Path to the data to be evaluated or passed to target if target is set.
+        Only .jsonl format files are supported.  `target` and `data` both cannot be None. Required.
+    :paramtype data: str
+    :keyword evaluators: Evaluators to be used for evaluation. It should be a dictionary with key as alias for evaluator
+        and value as the evaluator function. Required.
+    :paramtype evaluators: Dict[str, Callable]
     :keyword evaluation_name: Display name of the evaluation.
     :paramtype evaluation_name: Optional[str]
     :keyword target: Target to be evaluated. `target` and `data` both cannot be None
     :paramtype target: Optional[Callable]
-    :keyword data: Path to the data to be evaluated or passed to target if target is set.
-        Only .jsonl format files are supported.  `target` and `data` both cannot be None
-    :paramtype data: Optional[str]
-    :keyword evaluators: Evaluators to be used for evaluation. It should be a dictionary with key as alias for evaluator
-        and value as the evaluator function.
-    :paramtype evaluators: Optional[Dict[str, Callable]
     :keyword evaluator_config: Configuration for evaluators. The configuration should be a dictionary with evaluator
-        names as keys and a dictionary of column mappings as values. The column mappings should be a dictionary with
-        keys as the column names in the evaluator input and values as the column names in the input data or data
-        generated by target.
-    :paramtype evaluator_config: Optional[Dict[str, Dict[str, str]]
+        names as keys and a values that are dictionaries containing the column mappings. The column mappings should
+        be a dictionary with keys as the column names in the evaluator input and values as the column names in the
+        input data or data generated by target.
+    :paramtype evaluator_config: Optional[Dict[str, ~azure.ai.evaluation.EvaluatorConfig]]
     :keyword output_path: The local folder or file path to save evaluation results to if set. If folder path is provided
           the results will be saved to a file named `evaluation_results.json` in the folder.
     :paramtype output_path: Optional[str]
@@ -483,7 +488,7 @@ def evaluate(
             model_config = {
                 "azure_endpoint": os.environ.get("AZURE_OPENAI_ENDPOINT"),
                 "api_key": os.environ.get("AZURE_OPENAI_KEY"),
-                "azure_deployment": os.environ.get("AZURE_OPENAI_DEPLOYMENT")
+                "azure_deployment": os.environ.get("AZURE_OPENAI_DEPLOYMENT"),
             }
 
             coherence_eval = CoherenceEvaluator(model_config=model_config)
@@ -498,15 +503,19 @@ def evaluate(
                 },
                 evaluator_config={
                     "coherence": {
-                        "response": "${data.response}",
-                        "query": "${data.query}"
+                        "column_mapping": {
+                            "response": "${data.response}",
+                            "query": "${data.query}",
+                        },
                     },
                     "relevance": {
-                        "response": "${data.response}",
-                        "context": "${data.context}",
-                        "query": "${data.query}"
-                    }
-                }
+                        "column_mapping": {
+                            "response": "${data.response}",
+                            "context": "${data.context}",
+                            "query": "${data.query}",
+                        },
+                    },
+                },
             )
 
     """
@@ -545,13 +554,13 @@ def evaluate(
         raise e
 
 
-def _evaluate(  # pylint: disable=too-many-locals
+def _evaluate(  # pylint: disable=too-many-locals,too-many-statements
     *,
     evaluation_name: Optional[str] = None,
     target: Optional[Callable] = None,
     data: Optional[str] = None,
     evaluators: Optional[Dict[str, Callable]] = None,
-    evaluator_config: Optional[Dict[str, Dict[str, str]]] = None,
+    evaluator_config: Optional[Dict[str, EvaluatorConfig]] = None,
     azure_ai_project: Optional[AzureAIProject] = None,
     output_path: Optional[str] = None,
     **kwargs,
@@ -561,8 +570,13 @@ def _evaluate(  # pylint: disable=too-many-locals
     # Process evaluator config to replace ${target.} with ${data.}
     if evaluator_config is None:
         evaluator_config = {}
-    evaluator_config = _process_evaluator_config(evaluator_config)
-    _validate_columns(input_data_df, evaluators, target, evaluator_config)
+    # extract column mapping dicts into dictionary mapping evaluator name to column mapping
+    column_mapping = {
+        evaluator_name: evaluator_configuration.get("column_mapping", None)
+        for evaluator_name, evaluator_configuration in evaluator_config.items()
+    }
+    column_mapping = _process_column_mappings(column_mapping)
+    _validate_columns(input_data_df, evaluators, target, column_mapping)
 
     # Target Run
     pf_client = PFClient(
@@ -572,43 +586,55 @@ def _evaluate(  # pylint: disable=too-many-locals
         user_agent=USER_AGENT,
     )
 
-    trace_destination = pf_client._config.get_trace_destination()
-
+    trace_destination = pf_client._config.get_trace_destination()  # pylint: disable=protected-access
     target_run = None
-
     target_generated_columns = set()
+
+    # Create default configuration for evaluators that directly maps
+    # input data names to keyword inputs of the same name in the evaluators.
+    column_mapping = column_mapping or {}
+    column_mapping.setdefault("default", {})
+
+    # If target is set, apply 1-1 column mapping from target outputs to evaluator inputs
     if data is not None and target is not None:
         input_data_df, target_generated_columns, target_run = _apply_target_to_data(
             target, data, pf_client, input_data_df, evaluation_name, _run_name=kwargs.get("_run_name")
         )
 
-        # Make sure, the default is always in the configuration.
-        if not evaluator_config:
-            evaluator_config = {}
-        if "default" not in evaluator_config:
-            evaluator_config["default"] = {}
-
-        for evaluator_name, mapping in evaluator_config.items():
+        for evaluator_name, mapping in column_mapping.items():
             mapped_to_values = set(mapping.values())
             for col in target_generated_columns:
                 # If user defined mapping differently, do not change it.
                 # If it was mapped to target, we have already changed it
-                # in _process_evaluator_config
+                # in _process_column_mappings
                 run_output = f"${{run.outputs.{col}}}"
                 # We will add our mapping only if
                 # customer did not mapped target output.
                 if col not in mapping and run_output not in mapped_to_values:
-                    evaluator_config[evaluator_name][col] = run_output  # pylint: disable=unnecessary-dict-index-lookup
+                    column_mapping[evaluator_name][col] = run_output  # pylint: disable=unnecessary-dict-index-lookup
 
         # After we have generated all columns we can check if we have
         # everything we need for evaluators.
-        _validate_columns(input_data_df, evaluators, target=None, evaluator_config=evaluator_config)
-
+        _validate_columns(input_data_df, evaluators, target=None, column_mapping=column_mapping)
+
+    # Apply 1-1 mapping from input data to evaluator inputs, excluding values already assigned
+    # via target mapping.
+    # If both the data and the output dictionary of the target function
+    # have the same column, then the target function value is used.
+    if input_data_df is not None:
+        for col in input_data_df.columns:
+            # Ignore columns added by target mapping. These are formatted as "__outputs.<column_name>"
+            # Also ignore columns that are already in config, since they've been covered by target mapping.
+            if not col.startswith(Prefixes.TSG_OUTPUTS) and col not in column_mapping["default"].keys():
+                column_mapping["default"][col] = f"${{data.{col}}}"
     # Batch Run
     evaluators_info = {}
     use_pf_client = kwargs.get("_use_pf_client", True)
     if use_pf_client:
-        batch_run_client = ProxyClient(pf_client)
+        # A user reported intermittent errors when PFClient uploads evaluation runs to the cloud.
+        # The root cause is still unclear, but it seems related to a conflict between the async run uploader
+        # and the async batch run. As a quick mitigation, use a PFClient without a trace destination for batch runs.
+        batch_run_client = ProxyClient(PFClient(user_agent=USER_AGENT))
 
         # Ensure the absolute path is passed to pf.run, as relative path doesn't work with
         # multiple evaluators. If the path is already absolute, abspath will return the original path.
@@ -624,7 +650,7 @@ def _evaluate(  # pylint: disable=too-many-locals
                 flow=evaluator,
                 run=target_run,
                 evaluator_name=evaluator_name,
-                column_mapping=evaluator_config.get(evaluator_name, evaluator_config.get("default", None)),
+                column_mapping=column_mapping.get(evaluator_name, column_mapping.get("default", None)),
                 data=data,
                 stream=True,
                 name=kwargs.get("_run_name"),
@@ -672,7 +698,6 @@ def _evaluate(  # pylint: disable=too-many-locals
     result_df = pd.concat([input_data_df, evaluators_result_df], axis=1, verify_integrity=True)
     metrics = _aggregate_metrics(evaluators_result_df, evaluators)
     metrics.update(evaluators_metric)
-
     studio_url = _log_metrics_and_instance_results(
         metrics,
         result_df,

azure/ai/evaluation/_evaluate/_telemetry/__init__.py

@@ -6,21 +6,24 @@ import functools
 import inspect
 import json
 import logging
-from typing import Callable, Dict
+from typing import Callable, Dict, TypeVar
 
 import pandas as pd
-
 from promptflow._sdk.entities._flows import FlexFlow as flex_flow
 from promptflow._sdk.entities._flows import Prompty as prompty_sdk
 from promptflow._sdk.entities._flows.dag import Flow as dag_flow
 from promptflow.client import PFClient
 from promptflow.core import Prompty as prompty_core
+from typing_extensions import ParamSpec
 
 from ..._user_agent import USER_AGENT
 from .._utils import _trace_destination_from_project_scope
 
 LOGGER = logging.getLogger(__name__)
 
+P = ParamSpec("P")
+R = TypeVar("R")
+
 
 def _get_evaluator_type(evaluator: Dict[str, Callable]):
     """
@@ -82,7 +85,7 @@ def _get_evaluator_properties(evaluator, evaluator_name):
             name = str(evaluator)
             pf_type = "Unknown"
     except Exception as e:  # pylint: disable=broad-exception-caught
-        LOGGER.debug(f"Failed to get evaluator properties: {e}")
+        LOGGER.debug("Failed to get evaluator properties: %s", e)
         name = str(evaluator)
         pf_type = "Unknown"
 
@@ -95,15 +98,17 @@ def _get_evaluator_properties(evaluator, evaluator_name):
 
 
 # cspell:ignore isna
-def log_evaluate_activity(func) -> None:
+def log_evaluate_activity(func: Callable[P, R]) -> Callable[P, R]:
     """Decorator to log evaluate activity
 
     :param func: The function to be decorated
     :type func: Callable
+    :returns: The decorated function
+    :rtype: Callable[P, R]
     """
 
     @functools.wraps(func)
-    def wrapper(*args, **kwargs) -> Callable:
+    def wrapper(*args: P.args, **kwargs: P.kwargs) -> R:
         from promptflow._sdk._telemetry import ActivityType, log_activity
         from promptflow._sdk._telemetry.telemetry import get_telemetry_logger
 
@@ -119,7 +124,7 @@ def log_evaluate_activity(func) -> None:
             user_agent=USER_AGENT,
         )
 
-        track_in_cloud = bool(pf_client._config.get_trace_destination())
+        track_in_cloud = bool(pf_client._config.get_trace_destination())  # pylint: disable=protected-access
         evaluate_target = bool(kwargs.get("target", None))
         evaluator_config = bool(kwargs.get("evaluator_config", None))
         custom_dimensions = {
@@ -154,7 +159,7 @@ def log_evaluate_activity(func) -> None:
                         evaluator_info["failed_rows"] = failed_rows
                         evaluator_info["total_rows"] = total_rows
                     except Exception as e:  # pylint: disable=broad-exception-caught
-                        LOGGER.debug(f"Failed to collect evaluate failed row info for {evaluator_name}: {e}")
+                        LOGGER.debug("Failed to collect evaluate failed row info for %s: %s", evaluator_name, e)
                     evaluators_info.append(evaluator_info)
 
                 custom_dimensions = {"evaluators_info": json.dumps(evaluators_info)}
@@ -167,7 +172,7 @@ def log_evaluate_activity(func) -> None:
                 ):
                     pass
             except Exception as e:  # pylint: disable=broad-exception-caught
-                LOGGER.debug(f"Failed to collect evaluate usage info: {e}")
+                LOGGER.debug("Failed to collect evaluate usage info: %s", e)
 
             return result
 

azure/ai/evaluation/_evaluate/_utils.py

@@ -8,12 +8,13 @@ import re
 import tempfile
 from collections import namedtuple
 from pathlib import Path
+from typing import Dict
 
 import pandas as pd
 
-from azure.ai.evaluation._constants import DEFAULT_EVALUATION_RESULTS_FILE_NAME, Prefixes
-from azure.ai.evaluation._exceptions import EvaluationException, ErrorBlame, ErrorCategory, ErrorTarget
+from azure.ai.evaluation._constants import DEFAULT_EVALUATION_RESULTS_FILE_NAME, DefaultOpenEncoding, Prefixes
 from azure.ai.evaluation._evaluate._eval_run import EvalRun
+from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
 
 LOGGER = logging.getLogger(__name__)
 
@@ -33,16 +34,16 @@ def extract_workspace_triad_from_trace_provider(trace_provider: str):  # pylint:
     match = re.match(AZURE_WORKSPACE_REGEX_FORMAT, trace_provider)
     if not match or len(match.groups()) != 5:
         raise EvaluationException(
-                message="Malformed trace provider string, expected azureml://subscriptions/<subscription_id>/"
-                "resourceGroups/<resource_group>/providers/Microsoft.MachineLearningServices/"
-                f"workspaces/<workspace_name>, got {trace_provider}",
-                internal_message="Malformed trace provider string, expected azureml://subscriptions/<subscription_id>/"
-                "resourceGroups/<resource_group>/providers/Microsoft.MachineLearningServices/"
-                "workspaces/<workspace_name>,",
-                target=ErrorTarget.UNKNOWN,
-                category=ErrorCategory.INVALID_VALUE,
-                blame=ErrorBlame.UNKNOWN,
-            )
+            message="Malformed trace provider string, expected azureml://subscriptions/<subscription_id>/"
+            "resourceGroups/<resource_group>/providers/Microsoft.MachineLearningServices/"
+            f"workspaces/<workspace_name>, got {trace_provider}",
+            internal_message="Malformed trace provider string, expected azureml://subscriptions/<subscription_id>/"
+            "resourceGroups/<resource_group>/providers/Microsoft.MachineLearningServices/"
+            "workspaces/<workspace_name>,",
+            target=ErrorTarget.UNKNOWN,
+            category=ErrorCategory.INVALID_VALUE,
+            blame=ErrorBlame.UNKNOWN,
+        )
     subscription_id = match.group(1)
     resource_group_name = match.group(3)
     workspace_name = match.group(5)
@@ -50,7 +51,7 @@ def extract_workspace_triad_from_trace_provider(trace_provider: str):  # pylint:
 
 
 def load_jsonl(path):
-    with open(path, "r", encoding="utf-8") as f:
+    with open(path, "r", encoding=DefaultOpenEncoding.READ) as f:
         return [json.loads(line) for line in f.readlines()]
 
 
@@ -99,7 +100,7 @@ def _log_metrics_and_instance_results(
         with tempfile.TemporaryDirectory() as tmpdir:
             tmp_path = os.path.join(tmpdir, artifact_name)
 
-            with open(tmp_path, "w", encoding="utf-8") as f:
+            with open(tmp_path, "w", encoding=DefaultOpenEncoding.WRITE) as f:
                 f.write(instance_results.to_json(orient="records", lines=True))
 
             ev_run.log_artifact(tmpdir, artifact_name)
@@ -155,11 +156,13 @@ def _write_output(path, data_dict):
     if os.path.isdir(path):
         p = p / DEFAULT_EVALUATION_RESULTS_FILE_NAME
 
-    with open(p, "w") as f:
+    with open(p, "w", encoding=DefaultOpenEncoding.WRITE) as f:
         json.dump(data_dict, f)
 
 
-def _apply_column_mapping(source_df: pd.DataFrame, mapping_config: dict, inplace: bool = False) -> pd.DataFrame:
+def _apply_column_mapping(
+    source_df: pd.DataFrame, mapping_config: Dict[str, str], inplace: bool = False
+) -> pd.DataFrame:
     """
     Apply column mapping to source_df based on mapping_config.
 
@@ -167,10 +170,11 @@ def _apply_column_mapping(source_df: pd.DataFrame, mapping_config: dict, inplace
     :param source_df: the data frame to be changed.
     :type source_df: pd.DataFrame
     :param mapping_config: The configuration, containing column mapping.
-    :type mapping_config: dict.
+    :type mapping_config: Dict[str, str].
     :param inplace: If true, the source_df will be changed inplace.
     :type inplace: bool
     :return: The modified data frame.
+    :rtype: pd.DataFrame
     """
     result_df = source_df
 
@@ -211,19 +215,22 @@ def _has_aggregator(evaluator):
     return hasattr(evaluator, "__aggregate__")
 
 
-def get_int_env_var(env_var_name, default_value=None):
+def get_int_env_var(env_var_name: str, default_value: int) -> int:
     """
-    The function `get_int_env_var` retrieves an integer environment variable value, with an optional
+    The function `get_int_env_var` retrieves an integer environment variable value, with a
     default value if the variable is not set or cannot be converted to an integer.
 
     :param env_var_name: The name of the environment variable you want to retrieve the value of
+    :type env_var_name: str
     :param default_value: The default value is the value that will be returned if the environment
-    variable is not found or if it cannot be converted to an integer
+        variable is not found or if it cannot be converted to an integer
+    :type default_value: int
     :return: an integer value.
+    :rtype: int
     """
     try:
-        return int(os.environ.get(env_var_name, default_value))
-    except Exception:
+        return int(os.environ[env_var_name])
+    except (ValueError, KeyError):
         return default_value
 
 

azure/ai/evaluation/_evaluators/_bleu/_bleu.py

@@ -2,8 +2,8 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
 from nltk.translate.bleu_score import SmoothingFunction, sentence_bleu
-
 from promptflow._utils.async_utils import async_run_allowing_running_loop
+
 from azure.ai.evaluation._common.utils import nltk_tokenize