azure-ai-evaluation 0.0.0b0__py3-none-any.whl → 1.0.0__py3-none-any.whl

azure/ai/evaluation/_evaluate/_batch_run/code_client.py

@@ -0,0 +1,188 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+import inspect
+import json
+import logging
+import os
+from concurrent.futures import Future
+from pathlib import Path
+from typing import Any, Callable, Dict, Optional, Union, cast
+
+import pandas as pd
+from promptflow.contracts.types import AttrDict
+from promptflow.tracing import ThreadPoolExecutorWithContext as ThreadPoolExecutor
+
+from azure.ai.evaluation._evaluate._utils import _apply_column_mapping, _has_aggregator, get_int_env_var, load_jsonl
+from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
+
+from ..._constants import PF_BATCH_TIMEOUT_SEC, PF_BATCH_TIMEOUT_SEC_DEFAULT
+
+LOGGER = logging.getLogger(__name__)
+
+
+class CodeRun:
+    def __init__(
+        self,
+        *,
+        run: Future,
+        input_data,
+        evaluator_name: Optional[str] = None,
+        aggregator: Callable[["CodeRun"], Future],
+        **kwargs,  # pylint: disable=unused-argument
+    ) -> None:
+        self.run = run
+        self.evaluator_name = evaluator_name if evaluator_name is not None else ""
+        self.input_data = input_data
+        self.aggregated_metrics = aggregator(self)
+
+    def get_result_df(self, exclude_inputs: bool = False) -> pd.DataFrame:
+        batch_run_timeout = get_int_env_var(PF_BATCH_TIMEOUT_SEC, PF_BATCH_TIMEOUT_SEC_DEFAULT)
+        result_df = cast(pd.DataFrame, self.run.result(timeout=batch_run_timeout))
+        if exclude_inputs:
+            result_df = result_df.drop(columns=[col for col in result_df.columns if col.startswith("inputs.")])
+        return result_df
+
+    def get_aggregated_metrics(self) -> Dict[str, Any]:
+        try:
+            batch_run_timeout = get_int_env_var(PF_BATCH_TIMEOUT_SEC, PF_BATCH_TIMEOUT_SEC_DEFAULT)
+            aggregated_metrics: Optional[Any] = (
+                cast(Dict, self.aggregated_metrics.result(timeout=batch_run_timeout))
+                if self.aggregated_metrics is not None
+                else None
+            )
+        except Exception as ex:  # pylint: disable=broad-exception-caught
+            LOGGER.debug("Error calculating metrics for evaluator %s, failed with error %s", self.evaluator_name, ex)
+            aggregated_metrics = None
+
+        if not isinstance(aggregated_metrics, dict):
+            LOGGER.warning(
+                "Aggregated metrics for evaluator %s is not a dictionary will not be logged as metrics",
+                self.evaluator_name,
+            )
+
+        aggregated_metrics = aggregated_metrics if isinstance(aggregated_metrics, dict) else {}
+
+        return aggregated_metrics
+
+
+class CodeClient:  # pylint: disable=client-accepts-api-version-keyword
+    def __init__(  # pylint: disable=missing-client-constructor-parameter-credential,missing-client-constructor-parameter-kwargs
+        self,
+    ) -> None:
+        self._thread_pool = ThreadPoolExecutor(thread_name_prefix="evaluators_thread")
+
+    def _calculate_metric(
+        self, evaluator: Callable, input_df: pd.DataFrame, column_mapping: Optional[Dict[str, str]], evaluator_name: str
+    ) -> pd.DataFrame:
+        row_metric_futures = []
+        row_metric_results = []
+        input_df = _apply_column_mapping(input_df, column_mapping)
+        # Ignoring args and kwargs from the signature since they are usually catching extra arguments
+        parameters = {
+            param.name
+            for param in inspect.signature(evaluator).parameters.values()
+            if param.name not in ["args", "kwargs"]
+        }
+        for value in input_df.to_dict("records"):
+            # Filter out only the parameters that are present in the input data
+            # if no parameters then pass data as is
+            filtered_values = {k: v for k, v in value.items() if k in parameters} if len(parameters) > 0 else value
+            row_metric_futures.append(self._thread_pool.submit(evaluator, **filtered_values))
+
+        for row_number, row_metric_future in enumerate(row_metric_futures):
+            try:
+                result = row_metric_future.result()
+                if not isinstance(result, dict):
+                    result = {"output": result}
+                row_metric_results.append(result)
+            except Exception as ex:  # pylint: disable=broad-except
+                msg_1 = f"Error calculating value for row {row_number} for metric {evaluator_name}, "
+                msg_2 = f"failed with error {str(ex)} : Stack trace : {str(ex.__traceback__)}"
+                LOGGER.info(msg_1 + msg_2)
+                # If a row fails to calculate, add an empty dict to maintain the row index
+                # This is to ensure the output dataframe has the same number of rows as the input dataframe
+                # pd concat will fill NaN for missing values
+                row_metric_results.append({})
+
+        return pd.concat(
+            [input_df.add_prefix("inputs."), pd.DataFrame(row_metric_results)],
+            axis=1,
+            verify_integrity=True,
+        )
+
+    @staticmethod
+    def _calculate_aggregations(evaluator: Callable, run: CodeRun) -> Any:
+        try:
+            if _has_aggregator(evaluator):
+                evaluator_output = run.get_result_df(exclude_inputs=True)
+                if len(evaluator_output.columns) == 1 and evaluator_output.columns[0] == "output":
+                    aggregate_input = evaluator_output["output"].tolist()
+                else:
+                    aggregate_input = [AttrDict(item) for item in evaluator_output.to_dict("records")]
+
+                aggr_func = getattr(evaluator, "__aggregate__")
+                aggregated_output = aggr_func(aggregate_input)
+                return aggregated_output
+        except Exception as ex:  # pylint: disable=broad-exception-caught
+            LOGGER.warning(
+                "Error calculating aggregations for evaluator %s, failed with error %s", run.evaluator_name, ex
+            )
+        return None
+
+    def run(
+        self,  # pylint: disable=unused-argument
+        flow: Callable,
+        data: Union[os.PathLike, Path, pd.DataFrame],
+        evaluator_name: Optional[str] = None,
+        column_mapping: Optional[Dict[str, str]] = None,
+        **kwargs,
+    ) -> CodeRun:
+        input_df = data
+        if not isinstance(input_df, pd.DataFrame):
+            try:
+                json_data = load_jsonl(data)
+            except json.JSONDecodeError as exc:
+                raise EvaluationException(
+                    message=f"Failed to parse data as JSON: {data}. Provide valid json lines data.",
+                    internal_message="Failed to parse data as JSON",
+                    target=ErrorTarget.CODE_CLIENT,
+                    category=ErrorCategory.INVALID_VALUE,
+                    blame=ErrorBlame.USER_ERROR,
+                ) from exc
+
+            input_df = pd.DataFrame(json_data)
+        eval_future = self._thread_pool.submit(
+            self._calculate_metric,
+            evaluator=flow,
+            input_df=input_df,
+            column_mapping=column_mapping,
+            evaluator_name=evaluator_name,
+        )
+
+        return CodeRun(
+            run=eval_future,
+            input_data=data,
+            evaluator_name=evaluator_name,
+            aggregator=lambda code_run: self._thread_pool.submit(
+                self._calculate_aggregations, evaluator=flow, run=code_run
+            ),
+        )
+
+    def get_details(self, run: CodeRun, all_results: bool = False) -> pd.DataFrame:
+        result_df = run.get_result_df(exclude_inputs=not all_results)
+        return result_df
+
+    def get_metrics(self, run: CodeRun) -> Dict[str, Any]:
+        try:
+            aggregated_metrics = run.get_aggregated_metrics()
+            print("Aggregated metrics")
+            print(aggregated_metrics)
+        except Exception as ex:  # pylint: disable=broad-exception-caught
+            LOGGER.debug("Error calculating metrics for evaluator %s, failed with error %s", run.evaluator_name, ex)
+            return {}
+        return aggregated_metrics
+
+    def get_run_summary(self, run: CodeRun) -> Any:  # pylint: disable=unused-argument
+        # Not implemented
+        return None

azure/ai/evaluation/_evaluate/_batch_run/eval_run_context.py

@@ -0,0 +1,89 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+import os
+import types
+from typing import Optional, Type, Union
+
+from promptflow._sdk._constants import PF_FLOW_ENTRY_IN_TMP, PF_FLOW_META_LOAD_IN_SUBPROCESS
+from promptflow._utils.user_agent_utils import ClientUserAgentUtil
+from promptflow.tracing._integrations._openai_injector import inject_openai_api, recover_openai_api
+
+from azure.ai.evaluation._constants import (
+    OTEL_EXPORTER_OTLP_TRACES_TIMEOUT,
+    OTEL_EXPORTER_OTLP_TRACES_TIMEOUT_DEFAULT,
+    PF_BATCH_TIMEOUT_SEC,
+    PF_BATCH_TIMEOUT_SEC_DEFAULT,
+    PF_DISABLE_TRACING,
+)
+
+from ..._user_agent import USER_AGENT
+from .._utils import set_event_loop_policy
+from .code_client import CodeClient
+from .proxy_client import ProxyClient
+
+
+class EvalRunContext:
+    """Context manager for eval batch run.
+
+    :param client: The client to run in the context.
+    :type client: Union[
+        ~azure.ai.evaluation._evaluate._batch_run.code_client.CodeClient,
+        ~azure.ai.evaluation._evaluate._batch_run.proxy_client.ProxyClient
+    ]
+    """
+
+    def __init__(self, client: Union[CodeClient, ProxyClient]) -> None:
+        self.client = client
+        self._is_batch_timeout_set_by_system = False
+        self._is_otel_timeout_set_by_system = False
+        self._original_cwd = os.getcwd()
+
+    def __enter__(self) -> None:
+        # Preserve current working directory, as PF may change it without restoring it afterward
+        self._original_cwd = os.getcwd()
+
+        if isinstance(self.client, CodeClient):
+            ClientUserAgentUtil.append_user_agent(USER_AGENT)
+            inject_openai_api()
+
+        if isinstance(self.client, ProxyClient):
+            os.environ[PF_FLOW_ENTRY_IN_TMP] = "true"
+            os.environ[PF_FLOW_META_LOAD_IN_SUBPROCESS] = "false"
+            os.environ[PF_DISABLE_TRACING] = "true"
+
+            if os.environ.get(PF_BATCH_TIMEOUT_SEC) is None:
+                os.environ[PF_BATCH_TIMEOUT_SEC] = str(PF_BATCH_TIMEOUT_SEC_DEFAULT)
+                self._is_batch_timeout_set_by_system = True
+
+            # For dealing with the timeout issue of OpenTelemetry exporter when multiple evaluators are running
+            if os.environ.get(OTEL_EXPORTER_OTLP_TRACES_TIMEOUT) is None:
+                os.environ[OTEL_EXPORTER_OTLP_TRACES_TIMEOUT] = str(OTEL_EXPORTER_OTLP_TRACES_TIMEOUT_DEFAULT)
+                self._is_otel_timeout_set_by_system = True
+
+            # For addressing the issue of asyncio event loop closed on Windows
+            set_event_loop_policy()
+
+    def __exit__(
+        self,
+        exc_type: Optional[Type[BaseException]],
+        exc_value: Optional[BaseException],
+        exc_tb: Optional[types.TracebackType],
+    ) -> None:
+        os.chdir(self._original_cwd)
+
+        if isinstance(self.client, CodeClient):
+            recover_openai_api()
+
+        if isinstance(self.client, ProxyClient):
+            os.environ.pop(PF_FLOW_ENTRY_IN_TMP, None)
+            os.environ.pop(PF_FLOW_META_LOAD_IN_SUBPROCESS, None)
+            os.environ.pop(PF_DISABLE_TRACING, None)
+
+            if self._is_batch_timeout_set_by_system:
+                os.environ.pop(PF_BATCH_TIMEOUT_SEC, None)
+                self._is_batch_timeout_set_by_system = False
+
+            if self._is_otel_timeout_set_by_system:
+                os.environ.pop(OTEL_EXPORTER_OTLP_TRACES_TIMEOUT, None)
+                self._is_otel_timeout_set_by_system = False

azure/ai/evaluation/_evaluate/_batch_run/proxy_client.py

@@ -0,0 +1,99 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+# pylint: disable=protected-access
+
+import inspect
+import logging
+import math
+import os
+from collections import OrderedDict
+from concurrent.futures import Future
+from typing import Any, Callable, Dict, Optional, Union
+
+import pandas as pd
+from promptflow.client import PFClient
+from promptflow.entities import Run
+from promptflow.tracing import ThreadPoolExecutorWithContext as ThreadPoolExecutor
+
+LOGGER = logging.getLogger(__name__)
+
+
+class ProxyRun:
+    def __init__(self, run: Future, **kwargs) -> None:  # pylint: disable=unused-argument
+        self.run = run
+
+
+class ProxyClient:  # pylint: disable=client-accepts-api-version-keyword
+    def __init__(  # pylint: disable=missing-client-constructor-parameter-credential,missing-client-constructor-parameter-kwargs
+        self, pf_client: PFClient
+    ) -> None:
+        self._pf_client = pf_client
+        self._thread_pool = ThreadPoolExecutor(thread_name_prefix="evaluators_thread")
+
+    def run(
+        self,
+        flow: Union[str, os.PathLike, Callable],
+        data: Union[str, os.PathLike],
+        column_mapping: Optional[Dict[str, str]] = None,
+        **kwargs
+    ) -> ProxyRun:
+        flow_to_run = flow
+        if os.getenv("AI_EVALS_BATCH_USE_ASYNC", "true").lower() == "true" and hasattr(flow, "_to_async"):
+            flow_to_run = flow._to_async()  # pylint: disable=protected-access
+
+        batch_use_async = self._should_batch_use_async(flow_to_run)
+        eval_future = self._thread_pool.submit(
+            self._pf_client.run,
+            flow_to_run,
+            data=data,
+            column_mapping=column_mapping,
+            batch_use_async=batch_use_async,
+            **kwargs
+        )
+        return ProxyRun(run=eval_future)
+
+    def get_details(self, proxy_run: ProxyRun, all_results: bool = False) -> pd.DataFrame:
+        run: Run = proxy_run.run.result()
+        result_df = self._pf_client.get_details(run, all_results=all_results)
+        result_df.replace("(Failed)", math.nan, inplace=True)
+        return result_df
+
+    def get_metrics(self, proxy_run: ProxyRun) -> Dict[str, Any]:
+        run: Run = proxy_run.run.result()
+        return self._pf_client.get_metrics(run)
+
+    def get_run_summary(self, proxy_run: ProxyRun) -> Dict[str, Any]:
+        run = proxy_run.run.result()
+
+        # pylint: disable=protected-access
+        completed_lines = run._properties.get("system_metrics", {}).get("__pf__.lines.completed", "NA")
+        failed_lines = run._properties.get("system_metrics", {}).get("__pf__.lines.failed", "NA")
+
+        # Update status to "Completed with Errors" if the original status is "Completed" and there are failed lines
+        if run.status == "Completed" and failed_lines != "NA" and int(failed_lines) > 0:
+            status = "Completed with Errors"
+        else:
+            status = run.status
+
+        # Return the ordered dictionary with the updated status
+        return OrderedDict(
+            [
+                ("status", status),
+                ("duration", str(run._end_time - run._created_on)),
+                ("completed_lines", completed_lines),
+                ("failed_lines", failed_lines),
+                ("log_path", str(run._output_path)),
+            ]
+        )
+
+    @staticmethod
+    def _should_batch_use_async(flow):
+        if os.getenv("AI_EVALS_BATCH_USE_ASYNC", "true").lower() == "true":
+            if hasattr(flow, "__call__") and inspect.iscoroutinefunction(flow.__call__):
+                return True
+            if inspect.iscoroutinefunction(flow):
+                return True
+            return False
+        return False

azure/ai/evaluation/_evaluate/_batch_run/target_run_context.py

@@ -0,0 +1,46 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+import os
+import types
+from typing import Optional, Type
+
+from promptflow._sdk._constants import PF_FLOW_ENTRY_IN_TMP
+from azure.ai.evaluation._constants import PF_DISABLE_TRACING
+
+
+class TargetRunContext:
+    """Context manager for target batch run.
+
+    :param upload_snapshot: Whether to upload target snapshot.
+    :type upload_snapshot: bool
+    """
+
+    def __init__(self, upload_snapshot: bool) -> None:
+        self._upload_snapshot = upload_snapshot
+        self._original_cwd = os.getcwd()
+
+    def __enter__(self) -> None:
+        # Preserve current working directory, as PF may change it without restoring it afterward
+        self._original_cwd = os.getcwd()
+
+        # Address "[WinError 32] The process cannot access the file" error,
+        # caused by conflicts when the venv and target function are in the same directory.
+        # Setting PF_FLOW_ENTRY_IN_TMP to true uploads only the flex entry file (flow.flex.yaml).
+        if not self._upload_snapshot:
+            os.environ[PF_FLOW_ENTRY_IN_TMP] = "true"
+
+        os.environ[PF_DISABLE_TRACING] = "true"
+
+    def __exit__(
+        self,
+        exc_type: Optional[Type[BaseException]],
+        exc_value: Optional[BaseException],
+        exc_tb: Optional[types.TracebackType],
+    ) -> None:
+        os.chdir(self._original_cwd)
+
+        if not self._upload_snapshot:
+            os.environ.pop(PF_FLOW_ENTRY_IN_TMP, None)
+
+        os.environ.pop(PF_DISABLE_TRACING, None)