azure-ai-evaluation 1.8.0__py3-none-any.whl → 1.10.0__py3-none-any.whl

azure/ai/evaluation/_legacy/_batch_engine/_run_submitter.py

@@ -5,6 +5,7 @@
 import dataclasses
 import inspect
 import sys
+import traceback
 
 from concurrent.futures import Executor
 from datetime import datetime, timezone
@@ -46,11 +47,6 @@ class RunSubmitter:
         **kwargs,
     ) -> Run:
 
-        # if the column mappings are not provided, generate them based on the arguments to the
-        # flow function.
-        if column_mapping is None:
-            column_mapping = self._generate_column_mapping(dynamic_callable)
-
         # The old code always spun up two threads here using a ThreadPoolExecutor:
         # 1. One thread essentially did nothing of value (since tracing was disabled, and we
         #    don't care about checking for the latest PromptFlow version number now)
@@ -84,7 +80,7 @@ class RunSubmitter:
             # unnecessary Flow loading code was removed here. Instead do direct calls to _submit_bulk_run
             await self._submit_bulk_run(run=run, local_storage=local_storage, **kwargs)
 
-        self.stream_run(run=run, storage=local_storage, raise_on_error=True)
+        self.stream_run(run=run, storage=local_storage, raise_on_error=self._config.raise_on_error)
         return run
 
     async def _submit_bulk_run(self, run: Run, local_storage: AbstractRunStorage, **kwargs) -> None:
@@ -108,16 +104,13 @@ class RunSubmitter:
                         f"Referenced run {previous.name} has {len(previous.outputs)} outputs, "
                         f"but {len(run.inputs)} inputs are provided."
                     )
-                
+
                 # load in the previous run's outputs and inputs into the list of dictionaries to allow for
                 # the previous run's outputs to be used as inputs for the current run
                 run.inputs = [
-                    {
-                        "run.outputs": previous.outputs[i],
-                        "run.inputs": previous.inputs[i],
-                        **run.inputs[i]
-                    }
-                    for i in range(len(run.inputs))]
+                    {"run.outputs": previous.outputs[i], "run.inputs": previous.inputs[i], **run.inputs[i]}
+                    for i in range(len(run.inputs))
+                ]
 
         self._validate_column_mapping(run.column_mapping)
 
@@ -128,10 +121,8 @@ class RunSubmitter:
         try:
             batch_engine = BatchEngine(
                 run.dynamic_callable,
+                config=self._config,
                 storage=local_storage,
-                batch_timeout_sec=self._config.batch_timeout_seconds,
-                line_timeout_sec=self._config.run_timeout_seconds,
-                max_worker_count=self._config.max_concurrency,
                 executor=self._executor,
             )
 
@@ -163,10 +154,10 @@ class RunSubmitter:
             # system metrics
             system_metrics = {}
             if batch_result:
-                system_metrics.update(dataclasses.asdict(batch_result.tokens))  # token related
+                # system_metrics.update(dataclasses.asdict(batch_result.tokens))  # token related
                 system_metrics.update(
                     {
-                        "duration": batch_result.duration.total_seconds(),
+                        # "duration": batch_result.duration.total_seconds(),
                         # "__pf__.lines.completed": batch_result.total_lines - batch_result.failed_lines,
                         # "__pf__.lines.failed": batch_result.failed_lines,
                     }
@@ -176,33 +167,16 @@ class RunSubmitter:
             run.metrics = system_metrics
             run.result = batch_result
 
-    @staticmethod
-    def _generate_column_mapping(function: Callable) -> Mapping[str, Any]:
-        args = inspect.signature(function).parameters
-        default_values: Dict[str, Any] = {}
-        mapping: Dict[str, Any] = {}
-        for key, value in args.items():
-            if key in ["self", "cls"] or value.kind in [value.VAR_POSITIONAL, value.VAR_KEYWORD]:
-                continue
-
-            mapping[key] = f"${{data.{key}}}"
-            if value.default != inspect.Parameter.empty:
-                default_values[key] = value.default
-
-        return {
-            **mapping,
-            DEFAULTS_KEY: default_values,
-        }
-
     @staticmethod
     def _validate_inputs(run: Run):
         if not run.inputs and not run.previous_run:
-            raise BatchEngineValidationError(
-                "Either data, or a previous run must be specified for the evaluation run."
-            )
+            raise BatchEngineValidationError("Either data, or a previous run must be specified for the evaluation run.")
 
     @staticmethod
-    def _validate_column_mapping(column_mapping: Mapping[str, str]):
+    def _validate_column_mapping(column_mapping: Optional[Mapping[str, str]]):
+        if not column_mapping:
+            return
+
         if not isinstance(column_mapping, Mapping):
             raise BatchEngineValidationError(f"Column mapping must be a dict, got {type(column_mapping)}.")
 
@@ -226,6 +200,7 @@ class RunSubmitter:
             return
 
         file_handler = sys.stdout
+        error_message: Optional[str] = None
         try:
             printed = 0
             available_logs = storage.logger.get_logs()
@@ -237,7 +212,24 @@ class RunSubmitter:
 
         if run.status == RunStatus.FAILED or run.status == RunStatus.CANCELED:
             if run.status == RunStatus.FAILED:
-                error_message = storage.load_exception().get("message", "Run fails with unknown error.")
+                # Get the first error message from the results, or use a default one
+                if run.result and run.result.error:
+                    error_message = "".join(
+                        traceback.format_exception(
+                            type(run.result.error), run.result.error, run.result.error.__traceback__
+                        )
+                    )
+                elif run.result and run.result.details:
+                    err = next((r.error for r in run.result.details if r.error), None)
+                    if err and err.exception:
+                        error_message = "".join(
+                            traceback.format_exception(type(err.exception), err.exception, err.exception.__traceback__)
+                        )
+                    elif err and err.details:
+                        error_message = err.details
+
+                if not error_message:
+                    error_message = "Run fails with unknown error."
             else:
                 error_message = "Run is canceled."
             if raise_on_error:

azure/ai/evaluation/_legacy/_batch_engine/_utils.py

@@ -94,7 +94,4 @@ def is_async_callable(obj: Any) -> bool:
     :return: True if the object is an async callable.
     :rtype: bool
     """
-    return (
-        inspect.iscoroutinefunction(obj)
-        or inspect.iscoroutinefunction(getattr(obj, "__call__", None))
-    )
+    return inspect.iscoroutinefunction(obj) or inspect.iscoroutinefunction(getattr(obj, "__call__", None))

azure/ai/evaluation/_legacy/_common/_async_token_provider.py

@@ -11,21 +11,15 @@ from azure.identity import AzureCliCredential, DefaultAzureCredential, ManagedId
 from azure.ai.evaluation._exceptions import EvaluationException, ErrorBlame, ErrorCategory, ErrorTarget
 from azure.ai.evaluation._azure._envs import AzureEnvironmentClient
 
+
 class AsyncAzureTokenProvider(AsyncContextManager["AsyncAzureTokenProvider"]):
     """Asynchronous token provider for Azure services that supports non-default Azure clouds
     (e.g. Azure China, Azure US Government, etc.)."""
 
-    def __init__(
-        self,
-        *,
-        base_url: Optional[str] = None,
-        **kwargs: Any
-    ) -> None:
+    def __init__(self, *, base_url: Optional[str] = None, **kwargs: Any) -> None:
         """Initialize the AsyncAzureTokenProvider."""
         self._credential: Optional[TokenCredential] = None
-        self._env_client: Optional[AzureEnvironmentClient] = AzureEnvironmentClient(
-            base_url=base_url,
-            **kwargs)
+        self._env_client: Optional[AzureEnvironmentClient] = AzureEnvironmentClient(base_url=base_url, **kwargs)
 
     async def close(self) -> None:
         if self._env_client:
@@ -50,14 +44,10 @@ class AsyncAzureTokenProvider(AsyncContextManager["AsyncAzureTokenProvider"]):
                 f"{self.__class__.__name__} could not determine the credential to use.",
                 target=ErrorTarget.UNKNOWN,
                 category=ErrorCategory.INVALID_VALUE,
-                blame=ErrorBlame.SYSTEM_ERROR)
+                blame=ErrorBlame.SYSTEM_ERROR,
+            )
 
-        return self._credential.get_token(
-            *scopes,
-            claims=claims,
-            tenant_id=tenant_id,
-            enable_cae=enable_cae,
-            **kwargs)
+        return self._credential.get_token(*scopes, claims=claims, tenant_id=tenant_id, enable_cae=enable_cae, **kwargs)
 
     async def __aenter__(self) -> "AsyncAzureTokenProvider":
         self._credential = await self._initialize_async(self._env_client)
@@ -67,7 +57,7 @@ class AsyncAzureTokenProvider(AsyncContextManager["AsyncAzureTokenProvider"]):
         self,
         exc_type: Optional[type] = None,
         exc_value: Optional[BaseException] = None,
-        traceback: Optional[Any] = None
+        traceback: Optional[Any] = None,
     ) -> None:
         await self.close()
 
@@ -80,7 +70,8 @@ class AsyncAzureTokenProvider(AsyncContextManager["AsyncAzureTokenProvider"]):
                 f"{AsyncAzureTokenProvider.__name__} instance has already been closed.",
                 target=ErrorTarget.UNKNOWN,
                 category=ErrorCategory.INVALID_VALUE,
-                blame=ErrorBlame.USER_ERROR)
+                blame=ErrorBlame.USER_ERROR,
+            )
 
         cloud_name: str = await client.get_default_cloud_name_async()
         if cloud_name != client.DEFAULT_AZURE_CLOUD_NAME:
@@ -92,7 +83,8 @@ class AsyncAzureTokenProvider(AsyncContextManager["AsyncAzureTokenProvider"]):
                     f"Failed to get metadata for cloud '{cloud_name}'.",
                     target=ErrorTarget.UNKNOWN,
                     category=ErrorCategory.INVALID_VALUE,
-                    blame=ErrorBlame.USER_ERROR)
+                    blame=ErrorBlame.USER_ERROR,
+                )
 
             authority = metadata.get("active_directory_endpoint")
             return DefaultAzureCredential(authority=authority, exclude_shared_token_cache_credential=True)
@@ -100,6 +92,7 @@ class AsyncAzureTokenProvider(AsyncContextManager["AsyncAzureTokenProvider"]):
             # using Azure on behalf of credentials requires the use of the azure-ai-ml package
             try:
                 from azure.ai.ml.identity import AzureMLOnBehalfOfCredential
+
                 return AzureMLOnBehalfOfCredential()  # type: ignore
             except (ModuleNotFoundError, ImportError):
                 raise EvaluationException(  # pylint: disable=raise-missing-from

azure/ai/evaluation/_legacy/_common/_thread_pool_executor_with_context.py

@@ -7,8 +7,10 @@ from concurrent.futures import ThreadPoolExecutor
 from functools import partial
 from typing_extensions import override
 
+
 class ThreadPoolExecutorWithContext(ThreadPoolExecutor):
     """ThreadPoolExecutor that preserves context variables across threads."""
+
     @override
     def submit(self, fn, *args, **kwargs):
         context = contextvars.copy_context()

azure/ai/evaluation/_legacy/prompty/_prompty.py

@@ -40,7 +40,7 @@ from azure.ai.evaluation._legacy.prompty._utils import (
 from azure.ai.evaluation._constants import DEFAULT_MAX_COMPLETION_TOKENS_REASONING_MODELS
 from azure.ai.evaluation._legacy._common._logging import get_logger
 from azure.ai.evaluation._legacy._common._async_token_provider import AsyncAzureTokenProvider
-
+from azure.ai.evaluation._user_agent import UserAgentSingleton
 
 PROMPTY_EXTENSION: Final[str] = ".prompty"
 
@@ -168,8 +168,9 @@ class AsyncPrompty:
         self._outputs: Dict[str, Any] = configs.get("outputs", {})
         self._name: str = configs.get("name", path.stem)
         self._logger = logger or get_logger(__name__)
-        self._token_credential: Union[TokenCredential, AsyncTokenCredential] = \
+        self._token_credential: Union[TokenCredential, AsyncTokenCredential] = (
             token_credential or AsyncAzureTokenProvider()
+        )
 
     @property
     def path(self) -> Path:
@@ -290,6 +291,8 @@ class AsyncPrompty:
         # for better debugging and real-time status updates.
         max_retries = 0
 
+        default_headers = {"User-Agent": UserAgentSingleton().value}
+
         api_client: Union[AsyncAzureOpenAI, AsyncOpenAI]
         if isinstance(connection, AzureOpenAIConnection):
             api_client = AsyncAzureOpenAI(
@@ -298,9 +301,10 @@ class AsyncPrompty:
                 azure_deployment=connection.azure_deployment,
                 api_version=connection.api_version,
                 max_retries=max_retries,
-                azure_ad_token_provider=(self.get_token_provider(self._token_credential)
-                    if not connection.api_key
-                    else None),
+                azure_ad_token_provider=(
+                    self.get_token_provider(self._token_credential) if not connection.api_key else None
+                ),
+                default_headers=default_headers,
             )
         elif isinstance(connection, OpenAIConnection):
             api_client = AsyncOpenAI(
@@ -308,6 +312,7 @@ class AsyncPrompty:
                 api_key=connection.api_key,
                 organization=connection.organization,
                 max_retries=max_retries,
+                default_headers=default_headers,
             )
         else:
             raise NotSupportedError(
@@ -414,6 +419,7 @@ class AsyncPrompty:
         :return: The token provider if a credential is provided, otherwise None.
         :rtype: Optional[AsyncAzureADTokenProvider]
         """
+
         async def _wrapper() -> str:
             token = cred.get_token(TokenScope.COGNITIVE_SERVICES_MANAGEMENT)
             if isinstance(token, Awaitable):

azure/ai/evaluation/_safety_evaluation/__init__.py

@@ -1,3 +1,3 @@
 # ---------------------------------------------------------
 # Copyright (c) Microsoft Corporation. All rights reserved.
-# ---------------------------------------------------------
+# ---------------------------------------------------------