azure-ai-evaluation 1.0.1__py3-none-any.whl → 1.13.5__py3-none-any.whl

azure/ai/evaluation/_legacy/_common/_logging.py

@@ -0,0 +1,292 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+# Original source:
+# - promptflow-core/promptflow/_core/log_manager.py
+# - promptflow-core/promptflow/_utils/logger_utils.py
+
+import os
+import logging
+import re
+import sys
+from re import Pattern
+from contextvars import ContextVar
+from datetime import datetime, timezone
+from dataclasses import dataclass
+from io import StringIO, TextIOBase
+from typing import Any, Dict, Final, Mapping, Optional, Set, TextIO, Tuple, Union
+
+
+valid_logging_level: Final[Set[str]] = {"CRITICAL", "FATAL", "ERROR", "WARN", "WARNING", "INFO", "DEBUG", "NOTSET"}
+
+
+def get_pf_logging_level(default=logging.INFO):
+    logging_level = os.environ.get("PF_LOGGING_LEVEL", None)
+    if logging_level not in valid_logging_level:
+        # Fall back to info if user input is invalid.
+        logging_level = default
+    return logging_level
+
+
+def _get_format_for_logger(
+    default_log_format: Optional[str] = None, default_date_format: Optional[str] = None
+) -> Tuple[str, str]:
+    """
+    Get the logging format and date format for logger.
+
+    This function attempts to find the handler of the root logger with a configured formatter.
+    If such a handler is found, it returns the format and date format used by this handler.
+    This can be configured through logging.basicConfig. If no configured formatter is found,
+    it defaults to LOG_FORMAT and DATETIME_FORMAT.
+    """
+    log_format = (
+        os.environ.get("PF_LOG_FORMAT")
+        or default_log_format
+        or "%(asctime)s %(thread)7d %(name)-18s %(levelname)-8s %(message)s"
+    )
+    datetime_format = os.environ.get("PF_LOG_DATETIME_FORMAT") or default_date_format or "%Y-%m-%d %H:%M:%S %z"
+    return log_format, datetime_format
+
+
+def get_logger(name: str) -> logging.Logger:
+    """Get logger used during execution."""
+    logger = logging.Logger(name)
+    logger.setLevel(get_pf_logging_level())
+    # logger.addHandler(FileHandlerConcurrentWrapper())
+    stdout_handler = logging.StreamHandler(sys.stdout)
+    fmt, datefmt = _get_format_for_logger()
+    # TODO ralphe: Do we need a credentials scrubber here like the old code had? We are not logging
+    #              logging anything that sensitive here.
+    stdout_handler.setFormatter(logging.Formatter(fmt=fmt, datefmt=datefmt))
+    logger.addHandler(stdout_handler)
+    return logger
+
+
+def scrub_credentials(s: str):
+    """Scrub credentials in string s.
+
+    For example, for input string: "print accountkey=accountKey", the output will be:
+    "print accountkey=**data_scrubbed**"
+    """
+    # for h in logger.handlers:
+    #     if isinstance(h, FileHandlerConcurrentWrapper):
+    #         if h.handler and h.handler._formatter:
+    #             credential_scrubber = h.handler._formatter.credential_scrubber
+    #             if credential_scrubber:
+    #                 return credential_scrubber.scrub(s)
+    return CredentialScrubber.scrub(s)
+
+
+class CredentialScrubber:
+    """Scrub sensitive information in string."""
+
+    PLACE_HOLDER = "**data_scrubbed**"
+    LENGTH_THRESHOLD = 2
+    DEFAULT_REGEX_SET: Final[Set[Pattern[str]]] = {
+        re.compile(r"(?<=sig=)[^\s;&]+", flags=re.IGNORECASE),  # Replace signature.
+        re.compile(r"(?<=key=)[^\s;&]+", flags=re.IGNORECASE),  # Replace key.
+    }
+
+    @staticmethod
+    def scrub(input: str) -> str:
+        """Replace sensitive information in input string with PLACE_HOLDER.
+
+        For example, for input string: "print accountkey=accountKey", the output will be:
+        "print accountkey=**data_scrubbed**"
+        """
+        output = input
+        for regex in CredentialScrubber.DEFAULT_REGEX_SET:
+            output = regex.sub(CredentialScrubber.PLACE_HOLDER, output)
+        return output
+
+
+# Logs by flow_logger will only be shown in flow mode.
+# These logs should contain all detailed logs from executor and runtime.
+flow_logger = get_logger("execution.flow")
+
+# Logs by bulk_logger will only be shown in bulktest and eval modes.
+# These logs should contain overall progress logs and error logs.
+bulk_logger = get_logger("execution.bulk")
+
+# Logs by logger will be shown in all the modes above,
+# such as error logs.
+logger = get_logger("execution")
+
+
+def log_progress(
+    run_start_time: datetime,
+    total_count: int,
+    current_count: int,
+    logger: logging.Logger = bulk_logger,
+    formatter="Finished {count} / {total_count} lines.",
+) -> None:
+    if current_count > 0:
+        delta = datetime.now(timezone.utc).timestamp() - run_start_time.timestamp()
+        average_execution_time = round(delta / current_count, 2)
+        estimated_execution_time = round(average_execution_time * (total_count - current_count), 2)
+        logger.info(formatter.format(count=current_count, total_count=total_count))
+        logger.info(
+            f"Average execution time for completed lines: {average_execution_time} seconds. "
+            f"Estimated time for incomplete lines: {estimated_execution_time} seconds."
+        )
+
+
+def incremental_print(log: str, printed: int, fileout: Union[TextIO, Any]) -> int:
+    count = 0
+    for line in log.splitlines():
+        if count >= printed:
+            fileout.write(line + "\n")
+            printed += 1
+        count += 1
+    return printed
+
+
+def print_red_error(message):
+    try:
+        from colorama import Fore, init
+
+        init(autoreset=True)
+        print(Fore.RED + message)
+    except ImportError:
+        print(message)
+
+
+@dataclass
+class NodeInfo:
+    run_id: str
+    node_name: str
+    line_number: int
+
+
+class NodeLogManager:
+    """Replace sys.stdout and sys.stderr with NodeLogWriter.
+
+    This class intercepts and saves logs to stdout/stderr when executing a node. For example:
+    with NodeLogManager() as log_manager:
+        print('test stdout')
+        print('test stderr', file=sys.stderr)
+
+    log_manager.get_logs() will return: {'stdout': 'test stdout\n', 'stderr': 'test stderr\n'}
+    """
+
+    def __init__(self, record_datetime: bool = True):
+        self.stdout_logger = NodeLogWriter(sys.stdout, record_datetime)
+        self.stderr_logger = NodeLogWriter(sys.stderr, record_datetime, is_stderr=True)
+
+    def __enter__(self) -> "NodeLogManager":
+        """Replace sys.stdout and sys.stderr with NodeLogWriter."""
+        self._prev_stdout = sys.stdout
+        self._prev_stderr = sys.stderr
+        sys.stdout = self.stdout_logger
+        sys.stderr = self.stderr_logger
+        return self
+
+    def __exit__(self, *args) -> None:
+        """Restore sys.stdout and sys.stderr."""
+        sys.stdout = self._prev_stdout
+        sys.stderr = self._prev_stderr
+
+    def set_node_context(self, run_id: str, node_name: str, line_number: int) -> None:
+        """Set node context."""
+        self.stdout_logger.set_node_info(run_id, node_name, line_number)
+        self.stderr_logger.set_node_info(run_id, node_name, line_number)
+
+    def clear_node_context(self, run_id: str) -> None:
+        """Clear node context."""
+        self.stdout_logger.clear_node_info(run_id)
+        self.stderr_logger.clear_node_info(run_id)
+
+    def get_logs(self, run_id: str) -> Mapping[str, str]:
+        return {
+            "stdout": self.stdout_logger.get_log(run_id),
+            "stderr": self.stderr_logger.get_log(run_id),
+        }
+
+
+class NodeLogWriter(TextIOBase):
+    """Record node run logs."""
+
+    DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%S%z"
+
+    def __init__(self, prev_stdout: Union[TextIOBase, Any], record_datetime: bool = True, is_stderr: bool = False):
+        self.run_id_to_stdout: Dict[str, StringIO] = {}
+        self._context: ContextVar[Optional[NodeInfo]] = ContextVar("run_log_info", default=None)
+        self._prev_out: Union[TextIOBase, Any] = prev_stdout
+        self._record_datetime: bool = record_datetime
+        self._is_stderr: bool = is_stderr
+
+    def set_node_info(self, run_id: str, node_name: str, line_number: int) -> None:
+        """Set node info to a context variable.
+
+        After set node info, write method will write to string IO associated with this node.
+        """
+        run_log_info = NodeInfo(run_id, node_name, line_number)
+        self._context.set(run_log_info)
+        self.run_id_to_stdout.update({run_id: StringIO()})
+
+    def clear_node_info(self, run_id: str):
+        """Clear context variable associated with run id."""
+        log_info: Optional[NodeInfo] = self._context.get()
+        if log_info and log_info.run_id == run_id:
+            self._context.set(None)
+
+        if run_id in self.run_id_to_stdout:
+            self.run_id_to_stdout.pop(run_id)
+
+    def get_log(self, run_id: str) -> str:
+        """Get log associated with run id."""
+        string_io: Optional[StringIO] = self.run_id_to_stdout.get(run_id)
+        if string_io is None:
+            return ""
+
+        return string_io.getvalue()
+
+    def write(self, s: str) -> int:
+        """Override TextIO's write method and writes input string into a string IO
+
+        The written string is compliant without any credentials.
+        The string is also recorded to flow/bulk logger.
+        If node info is not set, write to previous stdout.
+        """
+        log_info: Optional[NodeInfo] = self._context.get()
+        s = scrub_credentials(s)  # Remove credential from string.
+        if log_info is None:
+            return self._prev_out.write(s)
+        else:
+            self._write_to_flow_log(log_info, s)
+            stdout: Optional[StringIO] = self.run_id_to_stdout.get(log_info.run_id)
+            # When the line execution timeout is reached, all running nodes will be cancelled and node info will
+            # be cleared. This will remove StringIO from self.run_id_to_stdout. For sync tools running in a worker
+            # thread, they can't be stopped and self._context won't change in the worker
+            # thread because it's a thread-local variable. Therefore, we need to check if StringIO is None here.
+            if stdout is None:
+                return 0
+            if self._record_datetime and s != "\n":  # For line breaker, do not add datetime prefix.
+                s = f"[{datetime.now(timezone.utc).strftime(self.DATETIME_FORMAT)}] {s}"
+            return stdout.write(s)
+
+    def flush(self):
+        """Override TextIO's flush method."""
+        node_info: Optional[NodeInfo] = self._context.get()
+        if node_info is None:
+            self._prev_out.flush()
+        else:
+            string_io = self.run_id_to_stdout.get(node_info.run_id)
+            if string_io is not None:
+                string_io.flush()
+
+    def _write_to_flow_log(self, log_info: NodeInfo, s: str):
+        """Save stdout log to flow_logger and stderr log to logger."""
+        # If user uses "print('log message.')" to log, then
+        # "write" method will be called twice and the second time input is only '\n'.
+        # For this case, should not log '\n' in flow_logger.
+        if s != "\n":
+            if self._is_stderr:
+                flow_log = f"[{str(log_info)}] stderr> " + s.rstrip("\n")
+                # Log stderr in all scenarios so we can diagnose problems.
+                logger.warning(flow_log)
+            else:
+                flow_log = f"[{str(log_info)}] stdout> " + s.rstrip("\n")
+                # Log stdout only in flow mode.
+                flow_logger.info(flow_log)

azure/ai/evaluation/_legacy/_common/_thread_pool_executor_with_context.py

@@ -0,0 +1,17 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+import contextvars
+from concurrent.futures import ThreadPoolExecutor
+from functools import partial
+from typing_extensions import override
+
+
+class ThreadPoolExecutorWithContext(ThreadPoolExecutor):
+    """ThreadPoolExecutor that preserves context variables across threads."""
+
+    @override
+    def submit(self, fn, *args, **kwargs):
+        context = contextvars.copy_context()
+        return super().submit(context.run, partial(fn, *args, **kwargs))

azure/ai/evaluation/_legacy/prompty/__init__.py

@@ -0,0 +1,36 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from azure.ai.evaluation._legacy.prompty._prompty import AsyncPrompty
+from azure.ai.evaluation._legacy.prompty._connection import Connection, OpenAIConnection, AzureOpenAIConnection
+from azure.ai.evaluation._legacy.prompty._exceptions import (
+    PromptyException,
+    MissingRequiredInputError,
+    InvalidInputError,
+    JinjaTemplateError,
+    NotSupportedError,
+)
+
+# =========================================================================================================
+# NOTE: All of the code here is largely copy of code from Promptflow. Generally speaking, the following
+#       changes were made:
+#       - Added type annotations
+#       - Legacy or deprecated functionality has been removed (e.g. no more support for completions API)
+#       - Reworked the way images are handled to 1) Reduce the amount of code brought over, 2) remove
+#         the need to do two passes over the template to insert images, 3) remove the completely unnecessary
+#         loading of image data from the internet when it is not actually needed
+#       - Minor obvious tweaks to improve code readability, and removal of unused code paths
+# =========================================================================================================
+
+__all__ = [
+    "AsyncPrompty",
+    "Connection",
+    "AzureOpenAIConnection",
+    "OpenAIConnection",
+    "PromptyException",
+    "MissingRequiredInputError",
+    "InvalidInputError",
+    "JinjaTemplateError",
+    "NotSupportedError",
+]

azure/ai/evaluation/_legacy/prompty/_connection.py

@@ -0,0 +1,119 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+import os
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import Any, ClassVar, Mapping, Optional
+
+from azure.ai.evaluation._legacy.prompty._exceptions import MissingRequiredInputError
+from azure.ai.evaluation._legacy.prompty._utils import dataclass_from_dict
+
+
+def _is_empty_connection_config(connection_dict: Mapping[str, Any]) -> bool:
+    return any(key not in {"azure_deployment", "model", "type"} for key in connection_dict.keys())
+
+
+@dataclass
+class Connection(ABC):
+    """Base class for all connection classes."""
+
+    @property
+    @abstractmethod
+    def type(self) -> str:
+        """Gets the type of the connection.
+
+        :return: The type of the connection.
+        :rtype: str"""
+        ...
+
+    @staticmethod
+    def parse_from_config(model_configuration: Mapping[str, Any]) -> "Connection":
+        """Parse a connection from a model configuration.
+
+        :param model_configuration: The model configuration.
+        :type model_configuration: Mapping[str, Any]
+        :return: The connection.
+        :rtype: Connection
+        """
+        connection_dict = {**model_configuration}
+        connection_type = connection_dict.pop("type", "")
+
+        connection: Connection
+        if connection_type in [AzureOpenAIConnection.TYPE, "azure_openai"]:
+            if not _is_empty_connection_config(connection_dict):
+                connection = AzureOpenAIConnection.from_env()
+            else:
+                connection = dataclass_from_dict(AzureOpenAIConnection, connection_dict)
+
+        elif connection_type in [OpenAIConnection.TYPE, "openai"]:
+            if not _is_empty_connection_config(connection_dict):
+                connection = OpenAIConnection.from_env()
+            else:
+                connection = dataclass_from_dict(OpenAIConnection, connection_dict)
+
+        else:
+            error_message = (
+                f"'{connection_type}' is not a supported connection type. Valid values are "
+                f"[{AzureOpenAIConnection.TYPE}, {OpenAIConnection.TYPE}]"
+            )
+            raise MissingRequiredInputError(error_message)
+
+        return connection
+
+
+@dataclass
+class OpenAIConnection(Connection):
+    """Connection class for OpenAI endpoints."""
+
+    base_url: str
+    model: str
+    api_key: Optional[str] = None
+    organization: Optional[str] = None
+
+    TYPE: ClassVar[str] = "openai"
+
+    @property
+    def type(self) -> str:
+        return OpenAIConnection.TYPE
+
+    @classmethod
+    def from_env(cls) -> "OpenAIConnection":
+        return cls(
+            base_url=os.environ.get("OPENAI_BASE_URL", ""),
+            model=os.environ.get("OPENAI_MODEL", ""),
+            api_key=os.environ.get("OPENAI_API_KEY"),
+            organization=os.environ.get("OPENAI_ORG_ID"),
+        )
+
+
+@dataclass
+class AzureOpenAIConnection(Connection):
+    """Connection class for Azure OpenAI endpoints."""
+
+    azure_endpoint: str
+    azure_deployment: str
+    api_key: Optional[str] = None
+    api_version: Optional[str] = None
+    resource_id: Optional[str] = None
+
+    TYPE: ClassVar[str] = "azure_openai"
+
+    @property
+    def type(self) -> str:
+        return AzureOpenAIConnection.TYPE
+
+    @classmethod
+    def from_env(cls) -> "AzureOpenAIConnection":
+        return cls(
+            azure_endpoint=os.environ.get("AZURE_OPENAI_ENDPOINT", ""),
+            azure_deployment=os.environ.get("AZURE_OPENAI_DEPLOYMENT", ""),
+            api_key=os.environ.get("AZURE_OPENAI_API_KEY"),
+            api_version=os.environ.get("AZURE_OPENAI_API_VERSION", "2024-02-01"),
+        )
+
+    def __post_init__(self):
+        # set default API version
+        if not self.api_version:
+            self.api_version = "2024-02-01"

azure/ai/evaluation/_legacy/prompty/_exceptions.py

@@ -0,0 +1,139 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from typing import Optional
+from openai import OpenAIError
+from azure.ai.evaluation._exceptions import ErrorCategory, ErrorBlame, ErrorTarget, EvaluationException
+
+
+class PromptyException(EvaluationException):
+    """Exception class for Prompty related errors.
+
+    This exception is used to indicate that the error was caused by Prompty execution.
+
+    :param message: The error message.
+    :type message: str
+    """
+
+    def __init__(self, message: str, **kwargs):
+        kwargs.setdefault("category", ErrorCategory.INVALID_VALUE)
+        kwargs.setdefault("target", ErrorTarget.UNKNOWN)
+        kwargs.setdefault("blame", ErrorBlame.USER_ERROR)
+
+        super().__init__(message, **kwargs)
+
+
+class MissingRequiredInputError(PromptyException):
+    """Exception raised when missing required input"""
+
+    def __init__(self, message: str, **kwargs):
+        kwargs.setdefault("category", ErrorCategory.MISSING_FIELD)
+        kwargs.setdefault("target", ErrorTarget.EVAL_RUN)
+        super().__init__(message, **kwargs)
+
+
+class InvalidInputError(PromptyException):
+    """Exception raised when an input is invalid, could not be loaded, or is not the expected format."""
+
+    def __init__(self, message: str, **kwargs):
+        kwargs.setdefault("category", ErrorCategory.INVALID_VALUE)
+        kwargs.setdefault("target", ErrorTarget.EVAL_RUN)
+        super().__init__(message, **kwargs)
+
+
+class JinjaTemplateError(PromptyException):
+    """Exception raised when the Jinja template is invalid or could not be rendered."""
+
+    def __init__(self, message: str, **kwargs):
+        kwargs.setdefault("category", ErrorCategory.INVALID_VALUE)
+        kwargs.setdefault("target", ErrorTarget.EVAL_RUN)
+        super().__init__(message, **kwargs)
+
+
+class NotSupportedError(PromptyException):
+    """Exception raised when the operation is not supported."""
+
+    def __init__(self, message: str, **kwargs):
+        kwargs.setdefault("category", ErrorCategory.INVALID_VALUE)
+        kwargs.setdefault("target", ErrorTarget.UNKNOWN)
+        kwargs.setdefault("blame", ErrorBlame.SYSTEM_ERROR)
+        super().__init__(message, **kwargs)
+
+
+class WrappedOpenAIError(PromptyException):
+    """Exception raised when an OpenAI error is encountered."""
+
+    def __init__(self, *, message: Optional[str] = None, error: Optional[OpenAIError] = None, **kwargs):
+        kwargs.setdefault("category", ErrorCategory.FAILED_EXECUTION)
+        kwargs.setdefault("target", ErrorTarget.EVAL_RUN)
+        kwargs.setdefault("blame", ErrorBlame.USER_ERROR)
+
+        message = (
+            message or self.to_openai_error_message(error)
+            if error
+            else "An error occurred while executing the OpenAI API."
+        )
+        super().__init__(message, **kwargs)
+
+    @staticmethod
+    def to_openai_error_message(e: OpenAIError) -> str:
+        # TODO ralphe: Error handling that relies on string matching is fragile and should be replaced
+        #              with a more robust solution that examines the actual error type since that provies
+        #              more than enough information to handle errors.
+        ex_type = type(e).__name__
+        error_message = str(e)
+        # https://learn.microsoft.com/en-gb/azure/ai-services/openai/reference
+        if error_message == "<empty message>":
+            msg = "The api key is invalid or revoked. " "You can correct or regenerate the api key of your connection."
+            return f"OpenAI API hits {ex_type}: {msg}"
+        # for models that do not support the `functions` parameter.
+        elif "Unrecognized request argument supplied: functions" in error_message:
+            msg = (
+                "Current model does not support the `functions` parameter. If you are using openai connection, then "
+                "please use gpt-3.5-turbo, gpt-4, gpt-4-32k, gpt-3.5-turbo-0613 or gpt-4-0613. You can refer to "
+                "https://platform.openai.com/docs/guides/gpt/function-calling. If you are using azure openai "
+                "connection, then please first go to your Azure OpenAI resource, deploy model 'gpt-35-turbo' or "
+                "'gpt-4' with version 0613. You can refer to "
+                "https://learn.microsoft.com/en-us/azure/ai-services/openai/how-to/function-calling."
+            )
+            return f"OpenAI API hits {ex_type}: {msg}"
+        elif "Invalid content type. image_url is only supported by certain models" in error_message:
+            msg = (
+                "Current model does not support the image input. If you are using openai connection, then please use "
+                "gpt-4-vision-preview. You can refer to https://platform.openai.com/docs/guides/vision."
+                "If you are using azure openai connection, then please first go to your Azure OpenAI resource, "
+                'create a GPT-4 Turbo with Vision deployment by selecting model name: "gpt-4" and '
+                'model version "vision-preview". You can refer to '
+                "https://learn.microsoft.com/en-us/azure/ai-services/openai/how-to/gpt-with-vision"
+            )
+            return f"OpenAI API hits {ex_type}: {msg}"
+        elif (
+            "'response_format' of type" in error_message and "is not supported with this model." in error_message
+        ) or (
+            "Additional properties are not allowed" in error_message
+            and "unexpected) - 'response_format'" in error_message
+        ):
+            msg = (
+                'The response_format parameter needs to be a dictionary such as {"type": "text"}. '
+                "The value associated with the type key should be either 'text' or 'json_object' "
+                'If you are using openai connection, you can only set response_format to { "type": "json_object" } '
+                "when calling gpt-3.5-turbo-1106 or gpt-4-1106-preview to enable JSON mode. You can refer to "
+                "https://platform.openai.com/docs/guides/text-generation/json-mode. If you are using azure openai "
+                "connection, then please first go to your Azure OpenAI resource, compatible with GPT-4 Turbo and "
+                "all GPT-3.5 Turbo models newer than gpt-35-turbo-1106. You can refer to "
+                "https://learn.microsoft.com/en-us/azure/ai-services/openai/how-to/json-mode?tabs=python."
+            )
+            return f"OpenAI API hits {ex_type}: {msg}"
+        elif "Principal does not have access to API/Operation" in error_message:
+            msg = (
+                "Principal does not have access to API/Operation. If you are using azure openai connection, "
+                "please make sure you have proper role assignment on your azure openai resource. You can refer to "
+                "https://learn.microsoft.com/en-us/azure/ai-services/openai/how-to/role-based-access-control"
+            )
+            return f"OpenAI API hits {ex_type}: {msg}"
+        else:
+            return (
+                f"OpenAI API hits {ex_type}: {error_message} [Error reference: "
+                "https://platform.openai.com/docs/guides/error-codes/api-errors]"
+            )