azure-ai-evaluation 1.0.0b2__py3-none-any.whl → 1.13.3__py3-none-any.whl

azure/ai/evaluation/_legacy/_common/_async_token_provider.py

@@ -0,0 +1,117 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+import os
+from typing import Any, AsyncContextManager, Optional
+
+from azure.core.credentials import AccessToken, TokenCredential
+from azure.identity import AzureCliCredential, DefaultAzureCredential, ManagedIdentityCredential
+
+from azure.ai.evaluation._exceptions import EvaluationException, ErrorBlame, ErrorCategory, ErrorTarget
+from azure.ai.evaluation._azure._envs import AzureEnvironmentClient
+
+
+class AsyncAzureTokenProvider(AsyncContextManager["AsyncAzureTokenProvider"]):
+    """Asynchronous token provider for Azure services that supports non-default Azure clouds
+    (e.g. Azure China, Azure US Government, etc.)."""
+
+    def __init__(self, *, base_url: Optional[str] = None, **kwargs: Any) -> None:
+        """Initialize the AsyncAzureTokenProvider."""
+        self._credential: Optional[TokenCredential] = None
+        self._env_client: Optional[AzureEnvironmentClient] = AzureEnvironmentClient(base_url=base_url, **kwargs)
+
+    async def close(self) -> None:
+        if self._env_client:
+            await self._env_client.close()
+            self._env_client = None
+
+        self._credential = None
+
+    async def get_token(
+        self,
+        *scopes: str,
+        claims: Optional[str] = None,
+        tenant_id: Optional[str] = None,
+        enable_cae: bool = False,
+        **kwargs: Any,
+    ) -> AccessToken:
+        if self._credential is None:
+            self._credential = await self._initialize_async(self._env_client)
+
+        if self._credential is None:
+            raise EvaluationException(
+                f"{self.__class__.__name__} could not determine the credential to use.",
+                target=ErrorTarget.UNKNOWN,
+                category=ErrorCategory.INVALID_VALUE,
+                blame=ErrorBlame.SYSTEM_ERROR,
+            )
+
+        return self._credential.get_token(*scopes, claims=claims, tenant_id=tenant_id, enable_cae=enable_cae, **kwargs)
+
+    async def __aenter__(self) -> "AsyncAzureTokenProvider":
+        self._credential = await self._initialize_async(self._env_client)
+        return self
+
+    async def __aexit__(
+        self,
+        exc_type: Optional[type] = None,
+        exc_value: Optional[BaseException] = None,
+        traceback: Optional[Any] = None,
+    ) -> None:
+        await self.close()
+
+    @staticmethod
+    async def _initialize_async(client: Optional[AzureEnvironmentClient]) -> TokenCredential:
+        # Determine which credential to use based on the configured Azure cloud environment variables
+        # and possibly making network calls to Azure to get the correct Azure cloud metadata.
+        if client is None:
+            raise EvaluationException(
+                f"{AsyncAzureTokenProvider.__name__} instance has already been closed.",
+                target=ErrorTarget.UNKNOWN,
+                category=ErrorCategory.INVALID_VALUE,
+                blame=ErrorBlame.USER_ERROR,
+            )
+
+        cloud_name: str = await client.get_default_cloud_name_async()
+        if cloud_name != client.DEFAULT_AZURE_CLOUD_NAME:
+            # If the cloud name is not the default, we need to get the metadata for the specified cloud
+            # and set it in the environment client.
+            metadata = await client.get_cloud_async(cloud_name)
+            if metadata is None:
+                raise EvaluationException(
+                    f"Failed to get metadata for cloud '{cloud_name}'.",
+                    target=ErrorTarget.UNKNOWN,
+                    category=ErrorCategory.INVALID_VALUE,
+                    blame=ErrorBlame.USER_ERROR,
+                )
+
+            authority = metadata.get("active_directory_endpoint")
+            return DefaultAzureCredential(authority=authority, exclude_shared_token_cache_credential=True)
+        elif os.getenv("AZUREML_OBO_ENABLED"):
+            # using Azure on behalf of credentials requires the use of the azure-ai-ml package
+            try:
+                from azure.ai.ml.identity import AzureMLOnBehalfOfCredential
+
+                return AzureMLOnBehalfOfCredential()  # type: ignore
+            except (ModuleNotFoundError, ImportError):
+                raise EvaluationException(  # pylint: disable=raise-missing-from
+                    message=(
+                        "The required packages for OBO credentials are missing.\n"
+                        'To resolve this, please install them by running "pip install azure-ai-ml".'
+                    ),
+                    target=ErrorTarget.EVALUATE,
+                    category=ErrorCategory.MISSING_PACKAGE,
+                    blame=ErrorBlame.USER_ERROR,
+                )
+        elif os.environ.get("PF_USE_AZURE_CLI_CREDENTIAL", "false").lower() == "true":
+            # TODO ralphe: Is this still needed? DefaultAzureCredential already includes CLI credentials
+            #              albeit with a lower priority
+            return AzureCliCredential()
+        elif os.environ.get("IS_IN_CI_PIPELINE", "false").lower() == "true":
+            # use managed identity when executing in CI pipeline.
+            return AzureCliCredential()
+        elif identity_client_id := os.environ.get("DEFAULT_IDENTITY_CLIENT_ID"):
+            return ManagedIdentityCredential(client_id=identity_client_id)
+        else:
+            return DefaultAzureCredential()

azure/ai/evaluation/_legacy/_common/_logging.py

@@ -0,0 +1,292 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+# Original source:
+# - promptflow-core/promptflow/_core/log_manager.py
+# - promptflow-core/promptflow/_utils/logger_utils.py
+
+import os
+import logging
+import re
+import sys
+from re import Pattern
+from contextvars import ContextVar
+from datetime import datetime, timezone
+from dataclasses import dataclass
+from io import StringIO, TextIOBase
+from typing import Any, Dict, Final, Mapping, Optional, Set, TextIO, Tuple, Union
+
+
+valid_logging_level: Final[Set[str]] = {"CRITICAL", "FATAL", "ERROR", "WARN", "WARNING", "INFO", "DEBUG", "NOTSET"}
+
+
+def get_pf_logging_level(default=logging.INFO):
+    logging_level = os.environ.get("PF_LOGGING_LEVEL", None)
+    if logging_level not in valid_logging_level:
+        # Fall back to info if user input is invalid.
+        logging_level = default
+    return logging_level
+
+
+def _get_format_for_logger(
+    default_log_format: Optional[str] = None, default_date_format: Optional[str] = None
+) -> Tuple[str, str]:
+    """
+    Get the logging format and date format for logger.
+
+    This function attempts to find the handler of the root logger with a configured formatter.
+    If such a handler is found, it returns the format and date format used by this handler.
+    This can be configured through logging.basicConfig. If no configured formatter is found,
+    it defaults to LOG_FORMAT and DATETIME_FORMAT.
+    """
+    log_format = (
+        os.environ.get("PF_LOG_FORMAT")
+        or default_log_format
+        or "%(asctime)s %(thread)7d %(name)-18s %(levelname)-8s %(message)s"
+    )
+    datetime_format = os.environ.get("PF_LOG_DATETIME_FORMAT") or default_date_format or "%Y-%m-%d %H:%M:%S %z"
+    return log_format, datetime_format
+
+
+def get_logger(name: str) -> logging.Logger:
+    """Get logger used during execution."""
+    logger = logging.Logger(name)
+    logger.setLevel(get_pf_logging_level())
+    # logger.addHandler(FileHandlerConcurrentWrapper())
+    stdout_handler = logging.StreamHandler(sys.stdout)
+    fmt, datefmt = _get_format_for_logger()
+    # TODO ralphe: Do we need a credentials scrubber here like the old code had? We are not logging
+    #              logging anything that sensitive here.
+    stdout_handler.setFormatter(logging.Formatter(fmt=fmt, datefmt=datefmt))
+    logger.addHandler(stdout_handler)
+    return logger
+
+
+def scrub_credentials(s: str):
+    """Scrub credentials in string s.
+
+    For example, for input string: "print accountkey=accountKey", the output will be:
+    "print accountkey=**data_scrubbed**"
+    """
+    # for h in logger.handlers:
+    #     if isinstance(h, FileHandlerConcurrentWrapper):
+    #         if h.handler and h.handler._formatter:
+    #             credential_scrubber = h.handler._formatter.credential_scrubber
+    #             if credential_scrubber:
+    #                 return credential_scrubber.scrub(s)
+    return CredentialScrubber.scrub(s)
+
+
+class CredentialScrubber:
+    """Scrub sensitive information in string."""
+
+    PLACE_HOLDER = "**data_scrubbed**"
+    LENGTH_THRESHOLD = 2
+    DEFAULT_REGEX_SET: Final[Set[Pattern[str]]] = {
+        re.compile(r"(?<=sig=)[^\s;&]+", flags=re.IGNORECASE),  # Replace signature.
+        re.compile(r"(?<=key=)[^\s;&]+", flags=re.IGNORECASE),  # Replace key.
+    }
+
+    @staticmethod
+    def scrub(input: str) -> str:
+        """Replace sensitive information in input string with PLACE_HOLDER.
+
+        For example, for input string: "print accountkey=accountKey", the output will be:
+        "print accountkey=**data_scrubbed**"
+        """
+        output = input
+        for regex in CredentialScrubber.DEFAULT_REGEX_SET:
+            output = regex.sub(CredentialScrubber.PLACE_HOLDER, output)
+        return output
+
+
+# Logs by flow_logger will only be shown in flow mode.
+# These logs should contain all detailed logs from executor and runtime.
+flow_logger = get_logger("execution.flow")
+
+# Logs by bulk_logger will only be shown in bulktest and eval modes.
+# These logs should contain overall progress logs and error logs.
+bulk_logger = get_logger("execution.bulk")
+
+# Logs by logger will be shown in all the modes above,
+# such as error logs.
+logger = get_logger("execution")
+
+
+def log_progress(
+    run_start_time: datetime,
+    total_count: int,
+    current_count: int,
+    logger: logging.Logger = bulk_logger,
+    formatter="Finished {count} / {total_count} lines.",
+) -> None:
+    if current_count > 0:
+        delta = datetime.now(timezone.utc).timestamp() - run_start_time.timestamp()
+        average_execution_time = round(delta / current_count, 2)
+        estimated_execution_time = round(average_execution_time * (total_count - current_count), 2)
+        logger.info(formatter.format(count=current_count, total_count=total_count))
+        logger.info(
+            f"Average execution time for completed lines: {average_execution_time} seconds. "
+            f"Estimated time for incomplete lines: {estimated_execution_time} seconds."
+        )
+
+
+def incremental_print(log: str, printed: int, fileout: Union[TextIO, Any]) -> int:
+    count = 0
+    for line in log.splitlines():
+        if count >= printed:
+            fileout.write(line + "\n")
+            printed += 1
+        count += 1
+    return printed
+
+
+def print_red_error(message):
+    try:
+        from colorama import Fore, init
+
+        init(autoreset=True)
+        print(Fore.RED + message)
+    except ImportError:
+        print(message)
+
+
+@dataclass
+class NodeInfo:
+    run_id: str
+    node_name: str
+    line_number: int
+
+
+class NodeLogManager:
+    """Replace sys.stdout and sys.stderr with NodeLogWriter.
+
+    This class intercepts and saves logs to stdout/stderr when executing a node. For example:
+    with NodeLogManager() as log_manager:
+        print('test stdout')
+        print('test stderr', file=sys.stderr)
+
+    log_manager.get_logs() will return: {'stdout': 'test stdout\n', 'stderr': 'test stderr\n'}
+    """
+
+    def __init__(self, record_datetime: bool = True):
+        self.stdout_logger = NodeLogWriter(sys.stdout, record_datetime)
+        self.stderr_logger = NodeLogWriter(sys.stderr, record_datetime, is_stderr=True)
+
+    def __enter__(self) -> "NodeLogManager":
+        """Replace sys.stdout and sys.stderr with NodeLogWriter."""
+        self._prev_stdout = sys.stdout
+        self._prev_stderr = sys.stderr
+        sys.stdout = self.stdout_logger
+        sys.stderr = self.stderr_logger
+        return self
+
+    def __exit__(self, *args) -> None:
+        """Restore sys.stdout and sys.stderr."""
+        sys.stdout = self._prev_stdout
+        sys.stderr = self._prev_stderr
+
+    def set_node_context(self, run_id: str, node_name: str, line_number: int) -> None:
+        """Set node context."""
+        self.stdout_logger.set_node_info(run_id, node_name, line_number)
+        self.stderr_logger.set_node_info(run_id, node_name, line_number)
+
+    def clear_node_context(self, run_id: str) -> None:
+        """Clear node context."""
+        self.stdout_logger.clear_node_info(run_id)
+        self.stderr_logger.clear_node_info(run_id)
+
+    def get_logs(self, run_id: str) -> Mapping[str, str]:
+        return {
+            "stdout": self.stdout_logger.get_log(run_id),
+            "stderr": self.stderr_logger.get_log(run_id),
+        }
+
+
+class NodeLogWriter(TextIOBase):
+    """Record node run logs."""
+
+    DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%S%z"
+
+    def __init__(self, prev_stdout: Union[TextIOBase, Any], record_datetime: bool = True, is_stderr: bool = False):
+        self.run_id_to_stdout: Dict[str, StringIO] = {}
+        self._context: ContextVar[Optional[NodeInfo]] = ContextVar("run_log_info", default=None)
+        self._prev_out: Union[TextIOBase, Any] = prev_stdout
+        self._record_datetime: bool = record_datetime
+        self._is_stderr: bool = is_stderr
+
+    def set_node_info(self, run_id: str, node_name: str, line_number: int) -> None:
+        """Set node info to a context variable.
+
+        After set node info, write method will write to string IO associated with this node.
+        """
+        run_log_info = NodeInfo(run_id, node_name, line_number)
+        self._context.set(run_log_info)
+        self.run_id_to_stdout.update({run_id: StringIO()})
+
+    def clear_node_info(self, run_id: str):
+        """Clear context variable associated with run id."""
+        log_info: Optional[NodeInfo] = self._context.get()
+        if log_info and log_info.run_id == run_id:
+            self._context.set(None)
+
+        if run_id in self.run_id_to_stdout:
+            self.run_id_to_stdout.pop(run_id)
+
+    def get_log(self, run_id: str) -> str:
+        """Get log associated with run id."""
+        string_io: Optional[StringIO] = self.run_id_to_stdout.get(run_id)
+        if string_io is None:
+            return ""
+
+        return string_io.getvalue()
+
+    def write(self, s: str) -> int:
+        """Override TextIO's write method and writes input string into a string IO
+
+        The written string is compliant without any credentials.
+        The string is also recorded to flow/bulk logger.
+        If node info is not set, write to previous stdout.
+        """
+        log_info: Optional[NodeInfo] = self._context.get()
+        s = scrub_credentials(s)  # Remove credential from string.
+        if log_info is None:
+            return self._prev_out.write(s)
+        else:
+            self._write_to_flow_log(log_info, s)
+            stdout: Optional[StringIO] = self.run_id_to_stdout.get(log_info.run_id)
+            # When the line execution timeout is reached, all running nodes will be cancelled and node info will
+            # be cleared. This will remove StringIO from self.run_id_to_stdout. For sync tools running in a worker
+            # thread, they can't be stopped and self._context won't change in the worker
+            # thread because it's a thread-local variable. Therefore, we need to check if StringIO is None here.
+            if stdout is None:
+                return 0
+            if self._record_datetime and s != "\n":  # For line breaker, do not add datetime prefix.
+                s = f"[{datetime.now(timezone.utc).strftime(self.DATETIME_FORMAT)}] {s}"
+            return stdout.write(s)
+
+    def flush(self):
+        """Override TextIO's flush method."""
+        node_info: Optional[NodeInfo] = self._context.get()
+        if node_info is None:
+            self._prev_out.flush()
+        else:
+            string_io = self.run_id_to_stdout.get(node_info.run_id)
+            if string_io is not None:
+                string_io.flush()
+
+    def _write_to_flow_log(self, log_info: NodeInfo, s: str):
+        """Save stdout log to flow_logger and stderr log to logger."""
+        # If user uses "print('log message.')" to log, then
+        # "write" method will be called twice and the second time input is only '\n'.
+        # For this case, should not log '\n' in flow_logger.
+        if s != "\n":
+            if self._is_stderr:
+                flow_log = f"[{str(log_info)}] stderr> " + s.rstrip("\n")
+                # Log stderr in all scenarios so we can diagnose problems.
+                logger.warning(flow_log)
+            else:
+                flow_log = f"[{str(log_info)}] stdout> " + s.rstrip("\n")
+                # Log stdout only in flow mode.
+                flow_logger.info(flow_log)

azure/ai/evaluation/_legacy/_common/_thread_pool_executor_with_context.py

@@ -0,0 +1,17 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+import contextvars
+from concurrent.futures import ThreadPoolExecutor
+from functools import partial
+from typing_extensions import override
+
+
+class ThreadPoolExecutorWithContext(ThreadPoolExecutor):
+    """ThreadPoolExecutor that preserves context variables across threads."""
+
+    @override
+    def submit(self, fn, *args, **kwargs):
+        context = contextvars.copy_context()
+        return super().submit(context.run, partial(fn, *args, **kwargs))

azure/ai/evaluation/_legacy/prompty/__init__.py

@@ -0,0 +1,36 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from azure.ai.evaluation._legacy.prompty._prompty import AsyncPrompty
+from azure.ai.evaluation._legacy.prompty._connection import Connection, OpenAIConnection, AzureOpenAIConnection
+from azure.ai.evaluation._legacy.prompty._exceptions import (
+    PromptyException,
+    MissingRequiredInputError,
+    InvalidInputError,
+    JinjaTemplateError,
+    NotSupportedError,
+)
+
+# =========================================================================================================
+# NOTE: All of the code here is largely copy of code from Promptflow. Generally speaking, the following
+#       changes were made:
+#       - Added type annotations
+#       - Legacy or deprecated functionality has been removed (e.g. no more support for completions API)
+#       - Reworked the way images are handled to 1) Reduce the amount of code brought over, 2) remove
+#         the need to do two passes over the template to insert images, 3) remove the completely unnecessary
+#         loading of image data from the internet when it is not actually needed
+#       - Minor obvious tweaks to improve code readability, and removal of unused code paths
+# =========================================================================================================
+
+__all__ = [
+    "AsyncPrompty",
+    "Connection",
+    "AzureOpenAIConnection",
+    "OpenAIConnection",
+    "PromptyException",
+    "MissingRequiredInputError",
+    "InvalidInputError",
+    "JinjaTemplateError",
+    "NotSupportedError",
+]

azure/ai/evaluation/_legacy/prompty/_connection.py

@@ -0,0 +1,119 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+import os
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import Any, ClassVar, Mapping, Optional
+
+from azure.ai.evaluation._legacy.prompty._exceptions import MissingRequiredInputError
+from azure.ai.evaluation._legacy.prompty._utils import dataclass_from_dict
+
+
+def _is_empty_connection_config(connection_dict: Mapping[str, Any]) -> bool:
+    return any(key not in {"azure_deployment", "model", "type"} for key in connection_dict.keys())
+
+
+@dataclass
+class Connection(ABC):
+    """Base class for all connection classes."""
+
+    @property
+    @abstractmethod
+    def type(self) -> str:
+        """Gets the type of the connection.
+
+        :return: The type of the connection.
+        :rtype: str"""
+        ...
+
+    @staticmethod
+    def parse_from_config(model_configuration: Mapping[str, Any]) -> "Connection":
+        """Parse a connection from a model configuration.
+
+        :param model_configuration: The model configuration.
+        :type model_configuration: Mapping[str, Any]
+        :return: The connection.
+        :rtype: Connection
+        """
+        connection_dict = {**model_configuration}
+        connection_type = connection_dict.pop("type", "")
+
+        connection: Connection
+        if connection_type in [AzureOpenAIConnection.TYPE, "azure_openai"]:
+            if not _is_empty_connection_config(connection_dict):
+                connection = AzureOpenAIConnection.from_env()
+            else:
+                connection = dataclass_from_dict(AzureOpenAIConnection, connection_dict)
+
+        elif connection_type in [OpenAIConnection.TYPE, "openai"]:
+            if not _is_empty_connection_config(connection_dict):
+                connection = OpenAIConnection.from_env()
+            else:
+                connection = dataclass_from_dict(OpenAIConnection, connection_dict)
+
+        else:
+            error_message = (
+                f"'{connection_type}' is not a supported connection type. Valid values are "
+                f"[{AzureOpenAIConnection.TYPE}, {OpenAIConnection.TYPE}]"
+            )
+            raise MissingRequiredInputError(error_message)
+
+        return connection
+
+
+@dataclass
+class OpenAIConnection(Connection):
+    """Connection class for OpenAI endpoints."""
+
+    base_url: str
+    model: str
+    api_key: Optional[str] = None
+    organization: Optional[str] = None
+
+    TYPE: ClassVar[str] = "openai"
+
+    @property
+    def type(self) -> str:
+        return OpenAIConnection.TYPE
+
+    @classmethod
+    def from_env(cls) -> "OpenAIConnection":
+        return cls(
+            base_url=os.environ.get("OPENAI_BASE_URL", ""),
+            model=os.environ.get("OPENAI_MODEL", ""),
+            api_key=os.environ.get("OPENAI_API_KEY"),
+            organization=os.environ.get("OPENAI_ORG_ID"),
+        )
+
+
+@dataclass
+class AzureOpenAIConnection(Connection):
+    """Connection class for Azure OpenAI endpoints."""
+
+    azure_endpoint: str
+    azure_deployment: str
+    api_key: Optional[str] = None
+    api_version: Optional[str] = None
+    resource_id: Optional[str] = None
+
+    TYPE: ClassVar[str] = "azure_openai"
+
+    @property
+    def type(self) -> str:
+        return AzureOpenAIConnection.TYPE
+
+    @classmethod
+    def from_env(cls) -> "AzureOpenAIConnection":
+        return cls(
+            azure_endpoint=os.environ.get("AZURE_OPENAI_ENDPOINT", ""),
+            azure_deployment=os.environ.get("AZURE_OPENAI_DEPLOYMENT", ""),
+            api_key=os.environ.get("AZURE_OPENAI_API_KEY"),
+            api_version=os.environ.get("AZURE_OPENAI_API_VERSION", "2024-02-01"),
+        )
+
+    def __post_init__(self):
+        # set default API version
+        if not self.api_version:
+            self.api_version = "2024-02-01"