azure-ai-evaluation 1.0.1__py3-none-any.whl → 1.13.3__py3-none-any.whl

azure/ai/evaluation/_evaluate/_batch_run/_run_submitter_client.py

@@ -0,0 +1,176 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+import asyncio
+import logging
+import pandas as pd
+import sys
+import itertools
+from collections import defaultdict
+from concurrent.futures import Future
+from os import PathLike
+from typing import Any, Callable, Dict, Final, List, Mapping, Optional, Sequence, Union, cast
+
+from .batch_clients import BatchClientRun, HasAsyncCallable
+from ..._legacy._batch_engine._run_submitter import RunSubmitter
+from ..._legacy._batch_engine._config import BatchEngineConfig
+from ..._legacy._batch_engine._run import Run
+from ..._legacy._adapters._constants import LINE_NUMBER
+from ..._legacy._adapters.types import AttrDict
+from ..._legacy._common._thread_pool_executor_with_context import ThreadPoolExecutorWithContext
+from ..._evaluate._utils import _has_aggregator
+from ..._constants import Prefixes, PF_BATCH_TIMEOUT_SEC
+
+from .._utils import get_int_env_var as get_int
+
+
+LOGGER = logging.getLogger("run")
+MISSING_VALUE: Final[int] = sys.maxsize
+
+
+class RunSubmitterClient:
+    def __init__(self, *, raise_on_errors: bool = False, config: Optional[BatchEngineConfig] = None) -> None:
+        if config:
+            self._config = config
+        else:
+            # Generate default config and apply any overrides to the configuration from environment variables
+            self._config = BatchEngineConfig(LOGGER, use_async=True)
+            if (val := get_int(PF_BATCH_TIMEOUT_SEC, MISSING_VALUE)) != MISSING_VALUE:
+                self._config.batch_timeout_seconds = val
+            if (val := get_int("PF_LINE_TIMEOUT_SEC", MISSING_VALUE)) != MISSING_VALUE:
+                self._config.line_timeout_seconds = val
+            if (val := get_int("PF_WORKER_COUNT", MISSING_VALUE)) != MISSING_VALUE:
+                self._config.max_concurrency = val
+
+        self._config.raise_on_error = raise_on_errors
+
+        self._thread_pool = ThreadPoolExecutorWithContext(
+            thread_name_prefix="evaluators_thread", max_workers=self._config.max_concurrency
+        )
+
+    def run(
+        self,
+        flow: Callable,
+        data: Union[str, PathLike, pd.DataFrame],
+        column_mapping: Optional[Dict[str, str]] = None,
+        evaluator_name: Optional[str] = None,
+        **kwargs: Any,
+    ) -> BatchClientRun:
+        if not isinstance(data, pd.DataFrame):
+            raise ValueError("Data must be a pandas DataFrame")
+
+        # The column mappings are indexed by data to indicate they come from the data
+        # input. Update the inputs so that each entry is a dictionary with a data key
+        # that contains the original input data.
+        inputs = [{"data": input_data} for input_data in data.to_dict(orient="records")]
+        # Pass the correct previous run to the evaluator
+        run: Optional[BatchClientRun] = kwargs.pop("run", None)
+        if run:
+            kwargs["run"] = self._get_run(run)
+
+        # Try to get async function to use
+        if isinstance(flow, HasAsyncCallable):
+            flow = flow._to_async()  # pylint: disable=protected-access
+
+        # Start an event loop for async execution on a thread pool thread to separate it
+        # from the caller's thread.
+        run_submitter = RunSubmitter(self._config, self._thread_pool)
+        run_future = self._thread_pool.submit(
+            asyncio.run,
+            run_submitter.submit(
+                dynamic_callable=flow,
+                inputs=inputs,
+                column_mapping=column_mapping,
+                name_prefix=evaluator_name,
+                created_on=kwargs.pop("created_on", None),
+                storage_creator=kwargs.pop("storage_creator", None),
+                **kwargs,
+            ),
+        )
+
+        return run_future
+
+    def get_details(self, client_run: BatchClientRun, all_results: bool = False) -> pd.DataFrame:
+        run = self._get_run(client_run)
+
+        def concat(*dataframes: pd.DataFrame) -> pd.DataFrame:
+            return pd.concat(dataframes, axis=1, verify_integrity=True)
+
+        def to_dataframe(items: Sequence[Mapping[str, Any]], *, max_length: Optional[int] = None) -> pd.DataFrame:
+            """Convert a sequence of dictionaries to a DataFrame.
+
+            :param items: Sequence of dictionaries to convert.
+            :type items: Sequence[Mapping[str, Any]]
+            :param max_length: Maximum number of items to include in the DataFrame. If None, include all items.
+            :type max_length: Optional[int]
+            :return: DataFrame containing the items.
+            :rtype: pd.DataFrame
+            """
+            max_length = None if all_results else self._config.default_num_results
+            return pd.DataFrame(data=items if all_results else itertools.islice(items, max_length))
+
+        inputs = concat(
+            to_dataframe(run.inputs), to_dataframe([{LINE_NUMBER: i} for i in range(len(run.inputs))])
+        ).add_prefix(Prefixes.INPUTS)
+
+        outputs = to_dataframe(run.outputs).add_prefix(Prefixes.OUTPUTS)
+
+        return concat(inputs, outputs)
+
+    def get_metrics(self, client_run: BatchClientRun) -> Dict[str, Any]:
+        run = self._get_run(client_run)
+        return {**run.metrics, **self._get_aggregated_metrics(client_run)}
+
+    def _get_aggregated_metrics(self, client_run: BatchClientRun) -> Dict[str, Any]:
+        aggregated_metrics = None
+        run = self._get_run(client_run)
+        try:
+            if _has_aggregator(run.dynamic_callable):
+                result_df = pd.DataFrame(run.outputs)
+                if len(result_df.columns) == 1 and result_df.columns[0] == "output":
+                    aggregate_input = result_df["output"].tolist()
+                else:
+                    aggregate_input = [AttrDict(item) for item in result_df.to_dict("records")]
+
+                aggr_func = getattr(run.dynamic_callable, "__aggregate__")
+                aggregated_metrics = aggr_func(aggregate_input)
+
+        except Exception as ex:  # pylint: disable=broad-exception-caught
+            LOGGER.warning("Error calculating aggregations for evaluator, failed with error %s", ex)
+
+        if not isinstance(aggregated_metrics, dict):
+            LOGGER.warning(
+                "Aggregated metrics for evaluator is not a dictionary will not be logged as metrics",
+            )
+            return {}
+
+        return aggregated_metrics
+
+    def get_run_summary(self, client_run: BatchClientRun) -> Dict[str, Any]:
+        run = self._get_run(client_run)
+
+        total_lines = run.result.total_lines if run.result else 0
+        failed_lines = run.result.failed_lines if run.result else 0
+
+        return {
+            "status": run.status.value,
+            "duration": str(run.duration),
+            "completed_lines": total_lines - failed_lines,
+            "failed_lines": failed_lines,
+            "log_path": None,
+            "error_message": (
+                f"({run.result.error.blame.value}) {run.result.error.message}"
+                if run.result and run.result.error and run.result.error.blame
+                else None
+            ),
+            "error_code": (
+                f"{run.result.error.category.value}"
+                if run.result and run.result.error and run.result.error.category
+                else None
+            ),
+        }
+
+    @staticmethod
+    def _get_run(run: BatchClientRun) -> Run:
+        return cast(Future[Run], run).result()

azure/ai/evaluation/_evaluate/_batch_run/batch_clients.py

@@ -0,0 +1,82 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+import pandas
+from os import PathLike
+from typing import Any, Awaitable, Callable, Dict, Optional, Protocol, Union, runtime_checkable
+
+
+class BatchClientRun(Protocol):
+    """The protocol for the batch client run."""
+
+    pass
+
+
+@runtime_checkable
+class HasAsyncCallable(Protocol):
+    """The protocol for an object that has an async callable."""
+
+    def _to_async(self) -> Callable[[Any, Any], Awaitable[Any]]: ...
+
+
+class BatchClient(Protocol):
+    """The protocol for the batch client. This allows for running a flow on a data source
+    and getting the details of the run."""
+
+    def run(
+        self,
+        flow: Callable,
+        data: Union[str, PathLike, pandas.DataFrame],
+        column_mapping: Optional[Dict[str, str]] = None,
+        evaluator_name: Optional[str] = None,
+        **kwargs: Any,
+    ) -> BatchClientRun:
+        """Run the given flow on the data with the given column mapping.
+
+        :param flow: The flow to run.
+        :type flow: Union[Callable, HasAsyncCallable]
+        :param data: The JSONL file containing the data to run the flow on,
+                     or the loaded data
+        :type data: Union[str, PathLike]
+        :param column_mapping: The column mapping to use.
+        :type column_mapping: Mapping[str, str]
+        :param name: The name of the run.
+        :type name: Optional[str]
+        :param kwargs: Additional keyword arguments to pass to the flow.
+        :return: The result of the batch client run.
+        :rtype: BatchClientRun
+        """
+        ...
+
+    def get_details(self, client_run: BatchClientRun, all_results: bool = False) -> pandas.DataFrame:
+        """Get the details of the run.
+
+        :param client_run: The run to get the details of.
+        :type client_run: BatchClientRun
+        :param all_results: Whether to get all results.
+        :type all_results: bool
+        :return: The details of the run.
+        :rtype: pandas.DataFrame
+        """
+        ...
+
+    def get_metrics(self, client_run: BatchClientRun) -> Dict[str, Any]:
+        """Get the metrics of the run.
+
+        :param client_run: The run to get the metrics of.
+        :type client_run: BatchClientRun
+        :return: The metrics of the run.
+        :rtype: Mapping[str, Any]
+        """
+        ...
+
+    def get_run_summary(self, client_run: BatchClientRun) -> Dict[str, Any]:
+        """Get the summary of the run.
+
+        :param client_run: The run to get the summary of.
+        :type client_run: BatchClientRun
+        :return: The summary of the run.
+        :rtype: Mapping[str, Any]
+        """
+        ...

azure/ai/evaluation/_evaluate/_batch_run/code_client.py

@@ -6,17 +6,17 @@ import json
 import logging
 import os
 from concurrent.futures import Future
-from pathlib import Path
-from typing import Any, Callable, Dict, Optional, Union, cast
+from typing import Any, Callable, Dict, Optional, Sequence, Union, cast
 
 import pandas as pd
-from promptflow.contracts.types import AttrDict
-from promptflow.tracing import ThreadPoolExecutorWithContext as ThreadPoolExecutor
+from azure.ai.evaluation._legacy._adapters.types import AttrDict
+from azure.ai.evaluation._legacy._adapters.tracing import ThreadPoolExecutorWithContext as ThreadPoolExecutor
 
 from azure.ai.evaluation._evaluate._utils import _apply_column_mapping, _has_aggregator, get_int_env_var, load_jsonl
 from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
 
 from ..._constants import PF_BATCH_TIMEOUT_SEC, PF_BATCH_TIMEOUT_SEC_DEFAULT
+from .batch_clients import BatchClientRun
 
 LOGGER = logging.getLogger(__name__)
 
@@ -84,7 +84,7 @@ class CodeClient:  # pylint: disable=client-accepts-api-version-keyword
             for param in inspect.signature(evaluator).parameters.values()
             if param.name not in ["args", "kwargs"]
         }
-        for value in input_df.to_dict("records"):
+        for value in cast(Sequence[Dict[str, Any]], input_df.to_dict("records")):
             # Filter out only the parameters that are present in the input data
             # if no parameters then pass data as is
             filtered_values = {k: v for k, v in value.items() if k in parameters} if len(parameters) > 0 else value
@@ -133,10 +133,10 @@ class CodeClient:  # pylint: disable=client-accepts-api-version-keyword
     def run(
         self,  # pylint: disable=unused-argument
         flow: Callable,
-        data: Union[os.PathLike, Path, pd.DataFrame],
-        evaluator_name: Optional[str] = None,
+        data: Union[str, os.PathLike, pd.DataFrame],
         column_mapping: Optional[Dict[str, str]] = None,
-        **kwargs,
+        evaluator_name: Optional[str] = None,
+        **kwargs: Any,
     ) -> CodeRun:
         input_df = data
         if not isinstance(input_df, pd.DataFrame):
@@ -157,7 +157,7 @@ class CodeClient:  # pylint: disable=client-accepts-api-version-keyword
             evaluator=flow,
             input_df=input_df,
             column_mapping=column_mapping,
-            evaluator_name=evaluator_name,
+            evaluator_name=evaluator_name or "",
         )
 
         return CodeRun(
@@ -169,11 +169,13 @@ class CodeClient:  # pylint: disable=client-accepts-api-version-keyword
             ),
         )
 
-    def get_details(self, run: CodeRun, all_results: bool = False) -> pd.DataFrame:
+    def get_details(self, client_run: BatchClientRun, all_results: bool = False) -> pd.DataFrame:
+        run = self._get_result(client_run)
         result_df = run.get_result_df(exclude_inputs=not all_results)
         return result_df
 
-    def get_metrics(self, run: CodeRun) -> Dict[str, Any]:
+    def get_metrics(self, client_run: BatchClientRun) -> Dict[str, Any]:
+        run = self._get_result(client_run)
         try:
             aggregated_metrics = run.get_aggregated_metrics()
             print("Aggregated metrics")
@@ -183,6 +185,10 @@ class CodeClient:  # pylint: disable=client-accepts-api-version-keyword
             return {}
         return aggregated_metrics
 
-    def get_run_summary(self, run: CodeRun) -> Any:  # pylint: disable=unused-argument
+    def get_run_summary(self, client_run: BatchClientRun) -> Any:  # pylint: disable=unused-argument
         # Not implemented
         return None
+
+    @staticmethod
+    def _get_result(run: BatchClientRun) -> CodeRun:
+        return cast(CodeRun, run)

azure/ai/evaluation/_evaluate/_batch_run/eval_run_context.py

@@ -5,9 +5,13 @@ import os
 import types
 from typing import Optional, Type, Union
 
-from promptflow._sdk._constants import PF_FLOW_ENTRY_IN_TMP, PF_FLOW_META_LOAD_IN_SUBPROCESS
-from promptflow._utils.user_agent_utils import ClientUserAgentUtil
-from promptflow.tracing._integrations._openai_injector import inject_openai_api, recover_openai_api
+from azure.ai.evaluation._legacy._adapters._constants import PF_FLOW_ENTRY_IN_TMP, PF_FLOW_META_LOAD_IN_SUBPROCESS
+from azure.ai.evaluation._legacy._adapters.utils import ClientUserAgentUtil
+from azure.ai.evaluation._legacy._adapters.tracing import inject_openai_api, recover_openai_api
+from azure.ai.evaluation._legacy._batch_engine._openai_injector import (
+    inject_openai_api as ported_inject_openai_api,
+    recover_openai_api as ported_recover_openai_api,
+)
 
 from azure.ai.evaluation._constants import (
     OTEL_EXPORTER_OTLP_TRACES_TIMEOUT,
@@ -17,8 +21,10 @@ from azure.ai.evaluation._constants import (
     PF_DISABLE_TRACING,
 )
 
-from ..._user_agent import USER_AGENT
+from ..._user_agent import UserAgentSingleton
 from .._utils import set_event_loop_policy
+from .batch_clients import BatchClient
+from ._run_submitter_client import RunSubmitterClient
 from .code_client import CodeClient
 from .proxy_client import ProxyClient
 
@@ -33,7 +39,7 @@ class EvalRunContext:
     ]
     """
 
-    def __init__(self, client: Union[CodeClient, ProxyClient]) -> None:
+    def __init__(self, client: BatchClient) -> None:
         self.client = client
         self._is_batch_timeout_set_by_system = False
         self._is_otel_timeout_set_by_system = False
@@ -44,7 +50,7 @@ class EvalRunContext:
         self._original_cwd = os.getcwd()
 
         if isinstance(self.client, CodeClient):
-            ClientUserAgentUtil.append_user_agent(USER_AGENT)
+            ClientUserAgentUtil.append_user_agent(UserAgentSingleton().value)
             inject_openai_api()
 
         if isinstance(self.client, ProxyClient):
@@ -64,6 +70,10 @@ class EvalRunContext:
             # For addressing the issue of asyncio event loop closed on Windows
             set_event_loop_policy()
 
+        if isinstance(self.client, RunSubmitterClient):
+            set_event_loop_policy()
+            ported_inject_openai_api()
+
     def __exit__(
         self,
         exc_type: Optional[Type[BaseException]],
@@ -87,3 +97,6 @@ class EvalRunContext:
             if self._is_otel_timeout_set_by_system:
                 os.environ.pop(OTEL_EXPORTER_OTLP_TRACES_TIMEOUT, None)
                 self._is_otel_timeout_set_by_system = False
+
+        if isinstance(self.client, RunSubmitterClient):
+            ported_recover_openai_api()

azure/ai/evaluation/_evaluate/_batch_run/proxy_client.py

@@ -8,15 +8,21 @@ import inspect
 import logging
 import math
 import os
+from datetime import datetime
 from collections import OrderedDict
 from concurrent.futures import Future
-from typing import Any, Callable, Dict, Optional, Union
+from typing import Any, Callable, Dict, Optional, Union, cast
 
+from azure.ai.evaluation._legacy._adapters.entities import Run
+from azure.ai.evaluation._legacy._adapters._configuration import Configuration
+from azure.ai.evaluation._legacy._adapters.client import PFClient
+from azure.ai.evaluation._legacy._adapters.tracing import ThreadPoolExecutorWithContext
 import pandas as pd
-from promptflow.client import PFClient
-from promptflow.entities import Run
-from promptflow.tracing import ThreadPoolExecutorWithContext as ThreadPoolExecutor
 
+from azure.ai.evaluation._evaluate._batch_run.batch_clients import BatchClientRun, HasAsyncCallable
+
+
+Configuration.get_instance().set_config("trace.destination", "none")
 LOGGER = logging.getLogger(__name__)
 
 
@@ -26,46 +32,61 @@ class ProxyRun:
 
 
 class ProxyClient:  # pylint: disable=client-accepts-api-version-keyword
-    def __init__(  # pylint: disable=missing-client-constructor-parameter-credential,missing-client-constructor-parameter-kwargs
-        self, pf_client: PFClient
+    def __init__(  # pylint: disable=missing-client-constructor-parameter-credential
+        self,
+        **kwargs: Any,
     ) -> None:
-        self._pf_client = pf_client
-        self._thread_pool = ThreadPoolExecutor(thread_name_prefix="evaluators_thread")
+        self._pf_client = PFClient(**kwargs)
+        self._thread_pool = ThreadPoolExecutorWithContext(thread_name_prefix="evaluators_thread")
 
     def run(
         self,
-        flow: Union[str, os.PathLike, Callable],
-        data: Union[str, os.PathLike],
+        flow: Callable,
+        data: Union[str, os.PathLike, pd.DataFrame],
         column_mapping: Optional[Dict[str, str]] = None,
-        **kwargs
+        evaluator_name: Optional[str] = None,
+        **kwargs: Any,
     ) -> ProxyRun:
-        flow_to_run = flow
-        if os.getenv("AI_EVALS_BATCH_USE_ASYNC", "true").lower() == "true" and hasattr(flow, "_to_async"):
+        if isinstance(data, pd.DataFrame):
+            raise ValueError("Data cannot be a pandas DataFrame")
+
+        flow_to_run: Callable = flow
+        if os.getenv("AI_EVALS_BATCH_USE_ASYNC", "true").lower() == "true" and isinstance(flow, HasAsyncCallable):
             flow_to_run = flow._to_async()  # pylint: disable=protected-access
 
+        name: str = kwargs.pop("name", "")
+        if not name:
+            name = f"azure_ai_evaluation_evaluators_{evaluator_name}_{datetime.now().strftime('%Y%m%d_%H%M%S_%f')}"
+
+        # Pass the correct previous run to the evaluator
+        run: Optional[BatchClientRun] = kwargs.pop("run", None)
+        if run:
+            kwargs["run"] = self.get_result(run)
+
         batch_use_async = self._should_batch_use_async(flow_to_run)
         eval_future = self._thread_pool.submit(
             self._pf_client.run,
             flow_to_run,
             data=data,
-            column_mapping=column_mapping,
+            column_mapping=column_mapping,  # type: ignore
             batch_use_async=batch_use_async,
-            **kwargs
+            name=name,
+            **kwargs,
         )
         return ProxyRun(run=eval_future)
 
-    def get_details(self, proxy_run: ProxyRun, all_results: bool = False) -> pd.DataFrame:
-        run: Run = proxy_run.run.result()
+    def get_details(self, client_run: BatchClientRun, all_results: bool = False) -> pd.DataFrame:
+        run: Run = self.get_result(client_run)
         result_df = self._pf_client.get_details(run, all_results=all_results)
         result_df.replace("(Failed)", math.nan, inplace=True)
         return result_df
 
-    def get_metrics(self, proxy_run: ProxyRun) -> Dict[str, Any]:
-        run: Run = proxy_run.run.result()
+    def get_metrics(self, client_run: BatchClientRun) -> Dict[str, Any]:
+        run: Run = self.get_result(client_run)
         return self._pf_client.get_metrics(run)
 
-    def get_run_summary(self, proxy_run: ProxyRun) -> Dict[str, Any]:
-        run = proxy_run.run.result()
+    def get_run_summary(self, client_run: BatchClientRun) -> Dict[str, Any]:
+        run: Run = self.get_result(client_run)
 
         # pylint: disable=protected-access
         completed_lines = run._properties.get("system_metrics", {}).get("__pf__.lines.completed", "NA")
@@ -81,13 +102,17 @@ class ProxyClient:  # pylint: disable=client-accepts-api-version-keyword
         return OrderedDict(
             [
                 ("status", status),
-                ("duration", str(run._end_time - run._created_on)),
+                ("duration", str((run._end_time or run._created_on) - run._created_on)),
                 ("completed_lines", completed_lines),
                 ("failed_lines", failed_lines),
                 ("log_path", str(run._output_path)),
             ]
         )
 
+    @staticmethod
+    def get_result(run: BatchClientRun) -> Run:
+        return cast(ProxyRun, run).run.result()
+
     @staticmethod
     def _should_batch_use_async(flow):
         if os.getenv("AI_EVALS_BATCH_USE_ASYNC", "true").lower() == "true":

azure/ai/evaluation/_evaluate/_batch_run/target_run_context.py

@@ -5,8 +5,15 @@ import os
 import types
 from typing import Optional, Type
 
-from promptflow._sdk._constants import PF_FLOW_ENTRY_IN_TMP
+from azure.ai.evaluation._evaluate._batch_run.batch_clients import BatchClient
+from azure.ai.evaluation._evaluate._batch_run import RunSubmitterClient
+from azure.ai.evaluation._legacy._adapters._constants import PF_FLOW_ENTRY_IN_TMP
+from azure.ai.evaluation._legacy._batch_engine._openai_injector import (
+    inject_openai_api as ported_inject_openai_api,
+    recover_openai_api as ported_recover_openai_api,
+)
 from azure.ai.evaluation._constants import PF_DISABLE_TRACING
+from azure.ai.evaluation._evaluate._utils import set_event_loop_policy
 
 
 class TargetRunContext:
@@ -16,7 +23,8 @@ class TargetRunContext:
     :type upload_snapshot: bool
     """
 
-    def __init__(self, upload_snapshot: bool) -> None:
+    def __init__(self, client: BatchClient, upload_snapshot: bool = False) -> None:
+        self._client = client
         self._upload_snapshot = upload_snapshot
         self._original_cwd = os.getcwd()
 
@@ -32,6 +40,11 @@ class TargetRunContext:
 
         os.environ[PF_DISABLE_TRACING] = "true"
 
+        if isinstance(self._client, RunSubmitterClient):
+            ported_inject_openai_api()
+            # For addressing the issue of asyncio event loop closed on Windows
+            set_event_loop_policy()
+
     def __exit__(
         self,
         exc_type: Optional[Type[BaseException]],
@@ -44,3 +57,6 @@ class TargetRunContext:
             os.environ.pop(PF_FLOW_ENTRY_IN_TMP, None)
 
         os.environ.pop(PF_DISABLE_TRACING, None)
+
+        if isinstance(self._client, RunSubmitterClient):
+            ported_recover_openai_api()