azure-ai-evaluation 1.2.0__py3-none-any.whl → 1.4.0__py3-none-any.whl

azure/ai/evaluation/_legacy/prompty/__init__.py

@@ -0,0 +1,36 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from azure.ai.evaluation._legacy.prompty._prompty import AsyncPrompty
+from azure.ai.evaluation._legacy.prompty._connection import Connection, OpenAIConnection, AzureOpenAIConnection
+from azure.ai.evaluation._legacy.prompty._exceptions import (
+    PromptyException,
+    MissingRequiredInputError,
+    InvalidInputError,
+    JinjaTemplateError,
+    NotSupportedError,
+)
+
+# =========================================================================================================
+# NOTE: All of the code here is largely copy of code from Promptflow. Generally speaking, the following
+#       changes were made:
+#       - Added type annotations
+#       - Legacy or deprecated functionality has been removed (e.g. no more support for completions API)
+#       - Reworked the way images are handled to 1) Reduce the amount of code brought over, 2) remove
+#         the need to do two passes over the template to insert images, 3) remove the completely unnecessary
+#         loading of image data from the internet when it is not actually needed
+#       - Minor obvious tweaks to improve code readability, and removal of unused code paths
+# =========================================================================================================
+
+__all__ = [
+    "AsyncPrompty",
+    "Connection",
+    "AzureOpenAIConnection",
+    "OpenAIConnection",
+    "PromptyException",
+    "MissingRequiredInputError",
+    "InvalidInputError",
+    "JinjaTemplateError",
+    "NotSupportedError",
+]

azure/ai/evaluation/_legacy/prompty/_connection.py

@@ -0,0 +1,182 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+import os
+import re
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import Any, ClassVar, Mapping, Optional, Set, Union
+
+from azure.ai.evaluation._legacy.prompty._exceptions import MissingRequiredInputError
+from azure.ai.evaluation._legacy.prompty._utils import dataclass_from_dict
+
+
+ENV_VAR_PATTERN = re.compile(r"^\$\{env:(.*)\}$")
+
+
+def _parse_environment_variable(value: Union[str, Any]) -> Union[str, Any]:
+    """Get environment variable from ${env:ENV_NAME}. If not found, return original value.
+
+    :param value: The value to parse.
+    :type value: str | Any
+    :return: The parsed value
+    :rtype: str | Any"""
+    if not isinstance(value, str):
+        return value
+
+    result = re.match(ENV_VAR_PATTERN, value)
+    if result:
+        env_name = result.groups()[0]
+        return os.environ.get(env_name, value)
+
+    return value
+
+
+def _is_empty_connection_config(connection_dict: Mapping[str, Any]) -> bool:
+    ignored_fields = set(["azure_deployment", "model"])
+    keys = {k for k, v in connection_dict.items() if v}
+    return len(keys - ignored_fields) == 0
+
+
+@dataclass
+class Connection(ABC):
+    """Base class for all connection classes."""
+
+    @property
+    @abstractmethod
+    def type(self) -> str:
+        """Gets the type of the connection.
+
+        :return: The type of the connection.
+        :rtype: str"""
+        ...
+
+    @abstractmethod
+    def is_valid(self, missing_fields: Optional[Set[str]] = None) -> bool:
+        """Check if the connection is valid.
+
+        :param missing_fields: If set, this will be populated with the missing required fields.
+        :type missing_fields: Set[str] | None
+        :return: True if the connection is valid, False otherwise.
+        :rtype: bool"""
+        ...
+
+    @staticmethod
+    def parse_from_config(model_configuration: Mapping[str, Any]) -> "Connection":
+        """Parse a connection from a model configuration.
+
+        :param model_configuration: The model configuration.
+        :type model_configuration: Mapping[str, Any]
+        :return: The connection.
+        :rtype: Connection
+        """
+        connection: Connection
+        connection_dict = {k: _parse_environment_variable(v) for k, v in model_configuration.items()}
+        connection_type = connection_dict.pop("type", "")
+
+        if connection_type in [AzureOpenAIConnection.TYPE, "azure_openai"]:
+            if _is_empty_connection_config(connection_dict):
+                connection = AzureOpenAIConnection.from_env()
+            else:
+                connection = dataclass_from_dict(AzureOpenAIConnection, connection_dict)
+
+        elif connection_type in [OpenAIConnection.TYPE, "openai"]:
+            if _is_empty_connection_config(connection_dict):
+                connection = OpenAIConnection.from_env()
+            else:
+                connection = dataclass_from_dict(OpenAIConnection, connection_dict)
+
+        else:
+            error_message = (
+                f"'{connection_type}' is not a supported connection type. Valid values are "
+                f"[{AzureOpenAIConnection.TYPE}, {OpenAIConnection.TYPE}]"
+            )
+            raise MissingRequiredInputError(error_message)
+
+        missing_fields: Set[str] = set()
+        if not connection.is_valid(missing_fields):
+            raise MissingRequiredInputError(
+                f"The following required fields are missing for connection {connection.type}: "
+                f"{', '.join(missing_fields)}"
+            )
+
+        return connection
+
+
+@dataclass
+class OpenAIConnection(Connection):
+    """Connection class for OpenAI endpoints."""
+
+    base_url: str
+    api_key: Optional[str] = None
+    organization: Optional[str] = None
+
+    TYPE: ClassVar[str] = "openai"
+
+    @property
+    def type(self) -> str:
+        return OpenAIConnection.TYPE
+
+    @classmethod
+    def from_env(cls) -> "OpenAIConnection":
+        return cls(
+            base_url=os.environ.get("OPENAI_BASE_URL", ""),
+            api_key=os.environ.get("OPENAI_API_KEY"),
+            organization=os.environ.get("OPENAI_ORG_ID"),
+        )
+
+    def is_valid(self, missing_fields: Optional[Set[str]] = None) -> bool:
+        if missing_fields is None:
+            missing_fields = set()
+        if not self.base_url:
+            missing_fields.add("base_url")
+        if not self.api_key:
+            missing_fields.add("api_key")
+        if not self.organization:
+            missing_fields.add("organization")
+        return not bool(missing_fields)
+
+
+@dataclass
+class AzureOpenAIConnection(Connection):
+    """Connection class for Azure OpenAI endpoints."""
+
+    azure_endpoint: str
+    api_key: Optional[str] = None  # TODO ralphe: Replace this TokenCredential to allow for more flexible authentication
+    azure_deployment: Optional[str] = None
+    api_version: Optional[str] = None
+    resource_id: Optional[str] = None
+
+    TYPE: ClassVar[str] = "azure_openai"
+
+    @property
+    def type(self) -> str:
+        return AzureOpenAIConnection.TYPE
+
+    @classmethod
+    def from_env(cls) -> "AzureOpenAIConnection":
+        return cls(
+            azure_endpoint=os.environ.get("AZURE_OPENAI_ENDPOINT", ""),
+            api_key=os.environ.get("AZURE_OPENAI_API_KEY"),
+            azure_deployment=os.environ.get("AZURE_OPENAI_DEPLOYMENT"),
+            api_version=os.environ.get("AZURE_OPENAI_API_VERSION", "2024-02-01"),
+        )
+
+    def __post_init__(self):
+        # set default API version
+        if not self.api_version:
+            self.api_version = "2024-02-01"
+
+    def is_valid(self, missing_fields: Optional[Set[str]] = None) -> bool:
+        if missing_fields is None:
+            missing_fields = set()
+        if not self.azure_endpoint:
+            missing_fields.add("azure_endpoint")
+        if not self.api_key:
+            missing_fields.add("api_key")
+        if not self.azure_deployment:
+            missing_fields.add("azure_deployment")
+        if not self.api_version:
+            missing_fields.add("api_version")
+        return not bool(missing_fields)

azure/ai/evaluation/_legacy/prompty/_exceptions.py

@@ -0,0 +1,59 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from azure.ai.evaluation._exceptions import ErrorCategory, ErrorBlame, ErrorTarget, EvaluationException
+
+
+class PromptyException(EvaluationException):
+    """Exception class for Prompty related errors.
+
+    This exception is used to indicate that the error was caused by Prompty execution.
+
+    :param message: The error message.
+    :type message: str
+    """
+
+    def __init__(self, message: str, **kwargs):
+        kwargs.setdefault("category", ErrorCategory.INVALID_VALUE)
+        kwargs.setdefault("target", ErrorTarget.UNKNOWN)
+        kwargs.setdefault("blame", ErrorBlame.USER_ERROR)
+
+        super().__init__(message, **kwargs)
+
+
+class MissingRequiredInputError(PromptyException):
+    """Exception raised when missing required input"""
+
+    def __init__(self, message: str, **kwargs):
+        kwargs.setdefault("category", ErrorCategory.MISSING_FIELD)
+        kwargs.setdefault("target", ErrorTarget.EVAL_RUN)
+        super().__init__(message, **kwargs)
+
+
+class InvalidInputError(PromptyException):
+    """Exception raised when an input is invalid, could not be loaded, or is not the expected format."""
+
+    def __init__(self, message: str, **kwargs):
+        kwargs.setdefault("category", ErrorCategory.INVALID_VALUE)
+        kwargs.setdefault("target", ErrorTarget.EVAL_RUN)
+        super().__init__(message, **kwargs)
+
+
+class JinjaTemplateError(PromptyException):
+    """Exception raised when the Jinja template is invalid or could not be rendered."""
+
+    def __init__(self, message: str, **kwargs):
+        kwargs.setdefault("category", ErrorCategory.INVALID_VALUE)
+        kwargs.setdefault("target", ErrorTarget.EVAL_RUN)
+        super().__init__(message, **kwargs)
+
+
+class NotSupportedError(PromptyException):
+    """Exception raised when the operation is not supported."""
+
+    def __init__(self, message: str, **kwargs):
+        kwargs.setdefault("category", ErrorCategory.INVALID_VALUE)
+        kwargs.setdefault("target", ErrorTarget.UNKNOWN)
+        kwargs.setdefault("blame", ErrorBlame.SYSTEM_ERROR)
+        super().__init__(message, **kwargs)

azure/ai/evaluation/_legacy/prompty/_prompty.py

@@ -0,0 +1,313 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+import re
+from os import PathLike
+from pathlib import Path
+from typing import Any, AsyncGenerator, Dict, Final, List, Mapping, Optional, Sequence, Tuple, Union, cast
+
+from openai import AsyncAzureOpenAI, AsyncOpenAI, NotGiven
+
+from azure.ai.evaluation._exceptions import ErrorTarget
+from azure.ai.evaluation._constants import DefaultOpenEncoding
+from azure.ai.evaluation._legacy.prompty._exceptions import (
+    InvalidInputError,
+    PromptyException,
+    MissingRequiredInputError,
+    NotSupportedError,
+)
+from azure.ai.evaluation._legacy.prompty._connection import AzureOpenAIConnection, Connection, OpenAIConnection
+from azure.ai.evaluation._legacy.prompty._yaml_utils import load_yaml_string
+from azure.ai.evaluation._legacy.prompty._utils import (
+    dataclass_from_dict,
+    PromptyModelConfiguration,
+    OpenAIChatResponseType,
+    build_messages,
+    format_llm_response,
+    prepare_open_ai_request_params,
+    resolve_references,
+    update_dict_recursively,
+)
+
+
+PROMPTY_EXTENSION: Final[str] = ".prompty"
+
+
+class AsyncPrompty:
+    """A prompty is a prompt with predefined metadata like inputs, and can be executed directly like a flow.
+    A prompty is represented as a templated markdown file with a modified front matter.
+    The front matter is a yaml file that contains meta fields like model configuration, inputs, etc..
+
+    Prompty example:
+    .. code-block::
+
+        ---
+        name: Hello Prompty
+        description: A basic prompt
+        model:
+            api: chat
+            configuration:
+              type: azure_openai
+              azure_deployment: gpt-35-turbo
+              api_key="${env:AZURE_OPENAI_API_KEY}",
+              api_version=${env:AZURE_OPENAI_API_VERSION}",
+              azure_endpoint="${env:AZURE_OPENAI_ENDPOINT}",
+            parameters:
+              max_tokens: 128
+              temperature: 0.2
+        inputs:
+          text:
+            type: string
+        ---
+        system:
+        Write a simple {{text}} program that displays the greeting message.
+
+    Prompty as function example:
+
+    .. code-block:: python
+
+        from azure.ai.evaluation._legacy.prompty import AsyncPrompty
+        prompty = Prompty(path="path/to/prompty.prompty")
+        result = prompty(input_a=1, input_b=2)
+
+        # Override model config with dict
+        model_config = {
+            "api": "chat",
+            "configuration": {
+                "type": "azure_openai",
+                "azure_deployment": "gpt-35-turbo",
+                "api_key": "${env:AZURE_OPENAI_API_KEY}",
+                "api_version": "${env:AZURE_OPENAI_API_VERSION}",
+                "azure_endpoint": "${env:AZURE_OPENAI_ENDPOINT}",
+            },
+            "parameters": {
+                "max_token": 512
+            }
+        }
+        prompty = Prompty.load(source="path/to/prompty.prompty", model=model_config)
+        result = prompty(input_a=1, input_b=2)
+
+        # Override model config with configuration
+        from azure.ai.evaluation._legacy.prompty._connection import AzureOpenAIConnection
+        model_config = {
+            "api": "chat",
+            "configuration": AzureOpenAIModelConfiguration(
+                azure_deployment="gpt-35-turbo",
+                api_key="${env:AZURE_OPENAI_API_KEY}",
+                api_version="${env:AZURE_OPENAI_API_VERSION}",
+                azure_endpoint="${env:AZURE_OPENAI_ENDPOINT}",
+            ),
+            "parameters": {
+                "max_token": 512
+            }
+        }
+        prompty = Prompty(path="path/to/prompty.prompty", model=model_config)
+        result = prompty(input_a=1, input_b=2)
+
+        # Override model config with created connection
+        from azure.ai.evaluation._legacy.prompty._connection import AzureOpenAIConnection
+        model_config = {
+            "api": "chat",
+            "configuration": AzureOpenAIModelConfiguration(
+                connection="azure_open_ai_connection",
+                azure_deployment="gpt-35-turbo",
+            ),
+            "parameters": {
+                "max_token": 512
+            }
+        }
+        prompty = Prompty(path="path/to/prompty.prompty", model=model_config)
+        result = prompty(input_a=1, input_b=2)
+    """
+
+    def __init__(
+        self,
+        path: Union[str, PathLike],
+        **kwargs: Any,
+    ):
+        path = Path(path)
+        configs, self._template = self._parse_prompty(path)
+        configs = resolve_references(configs, base_path=path.parent)
+        configs = update_dict_recursively(configs, resolve_references(kwargs, base_path=path.parent))
+
+        if configs["model"].get("api") == "completion":
+            raise InvalidInputError(
+                "Prompty does not support the completion API. Please use the 'chat' completions API instead."
+            )
+
+        self._data = configs
+        self._path = path
+        self._model = dataclass_from_dict(PromptyModelConfiguration, configs["model"])
+        self._inputs: Dict[str, Any] = configs.get("inputs", {})
+        self._outputs: Dict[str, Any] = configs.get("outputs", {})
+        self._name: str = configs.get("name", path.stem)
+
+    @property
+    def path(self) -> Path:
+        """Path of the prompty file.
+
+        :return: The path of the prompty file.
+        :rtype: Path
+        """
+        return self._path
+
+    @property
+    def name(self) -> str:
+        """Name of the prompty.
+
+        :return: The name of the prompty.
+        :rtype: str
+        """
+        return self._name
+
+    @property
+    def description(self) -> Optional[str]:
+        """Description of the prompty.
+
+        :return: The description of the prompty.
+        :rtype: str
+        """
+        return self._data.get("description")
+
+    @classmethod
+    def load(
+        cls,
+        source: Union[str, PathLike],
+        **kwargs,
+    ) -> "AsyncPrompty":
+        """
+        Loads the prompty file.
+
+        :param source: The local prompty file. Must be a path to a local file.
+            An exception is raised if the file does not exist.
+        :type source: Union[PathLike, str]
+        :return: A Prompty object
+        :rtype: Prompty
+        """
+        source_path = Path(source)
+        if not source_path.exists():
+            raise PromptyException(f"Source {source_path.absolute().as_posix()} does not exist")
+
+        if source_path.suffix != PROMPTY_EXTENSION:
+            raise PromptyException("Source must be a file with .prompty extension.")
+
+        return cls(path=source_path, **kwargs)
+
+    @staticmethod
+    def _parse_prompty(path) -> Tuple[Dict[str, Any], str]:
+        with open(path, "r", encoding=DefaultOpenEncoding.READ) as f:
+            prompty_content = f.read()
+        pattern = r"-{3,}\n(.*)-{3,}\n(.*)"
+        result = re.search(pattern, prompty_content, re.DOTALL)
+        if not result:
+            raise PromptyException(
+                "Illegal formatting of prompty. The prompt file is in markdown format and can be divided into two "
+                "parts, the first part is in YAML format and contains connection and model information. The second "
+                "part is the prompt template."
+            )
+        config_content, prompt_template = result.groups()
+        configs = load_yaml_string(config_content)
+        return configs, prompt_template
+
+    def _resolve_inputs(self, input_values: Dict[str, Any]) -> Mapping[str, Any]:
+        """
+        Resolve prompty inputs. If not provide input_values, sample data will be regarded as input value.
+        For inputs are not provided, the default value in the input signature will be used.
+
+        :param Dict[str, Any] input_values: The input values provided by the user.
+        :return: The resolved inputs.
+        :rtype: Mapping[str, Any]
+        """
+
+        resolved_inputs: Dict[str, Any] = {}
+        missing_inputs: List[str] = []
+        for input_name, value in self._inputs.items():
+            if input_name not in input_values and "default" not in value:
+                missing_inputs.append(input_name)
+                continue
+
+            resolved_inputs[input_name] = input_values.get(input_name, value.get("default", None))
+
+        if missing_inputs:
+            raise MissingRequiredInputError(f"Missing required inputs: {missing_inputs}")
+
+        return resolved_inputs
+
+    # TODO ralphe: error handling
+    # @trace
+    # @handle_openai_error()
+    async def __call__(  # pylint: disable=docstring-keyword-should-match-keyword-only
+        self,
+        **kwargs: Any,
+    ) -> Union[OpenAIChatResponseType, AsyncGenerator[str, None], str, Mapping[str, Any]]:
+        """Calling prompty as a function in async, the inputs should be provided with key word arguments.
+        Returns the output of the prompty.
+
+        The function call throws PromptyException if the Prompty file is not valid or the inputs are not valid.
+
+        :keyword kwargs: Additional keyword arguments passed to the parent class.
+        :paramtype kwargs: Any
+        :return: The output of the prompty.
+        :rtype: ChatCompletion | AsyncStream[ChatCompletionChunk] | AsyncGenerator[str] | str | Mapping[str, Any]
+        """
+
+        inputs = self._resolve_inputs(kwargs)
+        connection = Connection.parse_from_config(self._model.configuration)
+        messages = build_messages(prompt=self._template, working_dir=self.path.parent, **inputs)
+        params = prepare_open_ai_request_params(self._model, messages)
+
+        timeout: Union[NotGiven, float] = NotGiven()
+        if timeout_val := cast(Any, kwargs.get("timeout", None)):
+            timeout = float(timeout_val)
+
+        # disable OpenAI's built-in retry mechanism by using our own retry
+        # for better debugging and real-time status updates.
+        max_retries = 0
+
+        api_client: Union[AsyncAzureOpenAI, AsyncOpenAI]
+        if isinstance(connection, AzureOpenAIConnection):
+            api_client = AsyncAzureOpenAI(
+                azure_endpoint=connection.azure_endpoint,
+                api_key=connection.api_key,
+                azure_deployment=connection.azure_deployment,
+                api_version=connection.api_version,
+                max_retries=max_retries,
+            )
+        elif isinstance(connection, OpenAIConnection):
+            api_client = AsyncOpenAI(
+                base_url=connection.base_url,
+                api_key=connection.api_key,
+                organization=connection.organization,
+                max_retries=max_retries,
+            )
+        else:
+            raise NotSupportedError(
+                f"'{type(connection).__name__}' is not a supported connection type.", target=ErrorTarget.EVAL_RUN
+            )
+
+        response: OpenAIChatResponseType = await api_client.with_options(timeout=timeout).chat.completions.create(
+            **params
+        )
+
+        return await format_llm_response(
+            response=response,
+            is_first_choice=self._data.get("model", {}).get("response", "first").lower() == "first",
+            response_format=params.get("response_format", {}),
+            outputs=self._outputs,
+        )
+
+    def render(  # pylint: disable=docstring-keyword-should-match-keyword-only
+        self, **kwargs: Any
+    ) -> Sequence[Mapping[str, Any]]:
+        """Render the prompt content.
+
+        :keyword kwargs: Additional keyword arguments passed to the parent class.
+        :paramtype kwargs: Any
+        :return: Prompt content
+        :rtype: Sequence[Mapping[str, Any]]
+        """
+
+        inputs = self._resolve_inputs(kwargs)
+        messages = build_messages(prompt=self._template, working_dir=self.path.parent, **inputs)
+        return messages