azure-ai-evaluation 1.0.0b2__py3-none-any.whl → 1.13.3__py3-none-any.whl

azure/ai/evaluation/_legacy/prompty/_yaml_utils.py

@@ -0,0 +1,99 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from os import PathLike
+from typing import IO, Any, Dict, Optional, Union, cast
+
+from ruamel.yaml import YAML, YAMLError  # cspell:ignore ruamel
+
+from azure.ai.evaluation._constants import DefaultOpenEncoding
+from azure.ai.evaluation._legacy.prompty._exceptions import MissingRequiredInputError
+
+
+def load_yaml(source: Optional[Union[str, PathLike, IO]]) -> Dict:
+    # null check - just return an empty dict.
+    # Certain CLI commands rely on this behavior to produce a resource
+    # via CLI, which is then populated through CLArgs.
+    """Load a local YAML file or a readable stream object.
+
+    .. note::
+
+        1. For a local file yaml
+
+        .. code-block:: python
+
+            yaml_path = "path/to/yaml"
+            content = load_yaml(yaml_path)
+
+        2. For a readable stream object
+
+        .. code-block:: python
+
+            with open("path/to/yaml", "r", encoding="utf-8") as f:
+                content = load_yaml(f)
+
+
+    :param source: The relative or absolute path to the local file, or a readable stream object.
+    :type source: str
+    :return: A dictionary representation of the local file's contents.
+    :rtype: Dict
+    """
+
+    if source is None:
+        return {}
+
+    # pylint: disable=redefined-builtin
+    input: Optional[IO] = None
+    must_open_file = False
+    try:  # check source type by duck-typing it as an IOBase
+        readable = cast(IO, source).readable()
+        if not readable:  # source is misformatted stream or file
+            msg = "File Permissions Error: The already-open \n\n inputted file is not readable."
+            raise PermissionError(msg)
+        # source is an already-open stream or file, we can read() from it directly.
+        input = cast(IO, source)
+    except AttributeError:
+        # source has no writable() function, assume it's a string or file path.
+        must_open_file = True
+
+    if must_open_file:  # If supplied a file path, open it.
+        try:
+            input = open(  # pylint: disable=consider-using-with
+                cast(Union[PathLike, str], source), "r", encoding=DefaultOpenEncoding.READ
+            )
+        except OSError:  # FileNotFoundError introduced in Python 3
+            e = FileNotFoundError(f"No such file or directory: {source}")
+            raise MissingRequiredInputError(str(e), privacy_info=[str(source)]) from e
+    # input should now be a readable file or stream. Parse it.
+    try:
+        yaml = YAML()
+        yaml.preserve_quotes = True
+        cfg = yaml.load(input)
+    except YAMLError as e:
+        msg = f"Error while parsing yaml file: {source} \n\n {str(e)}"
+        raise YAMLError(msg) from e
+    finally:
+        if input and must_open_file:
+            input.close()
+
+    return cfg or {}
+
+
+def load_yaml_string(yaml_string: str) -> Dict[str, Any]:
+    """Load a yaml string.
+
+    .. code-block:: python
+
+        yaml_string = "some yaml string"
+        object = load_yaml_string(yaml_string)
+
+
+    :param yaml_string: A yaml string.
+    :type yaml_string: str
+    :return: A dictionary representation of the yaml string.
+    :rtype: Dict
+    """
+    yaml = YAML()
+    yaml.preserve_quotes = True
+    return yaml.load(yaml_string)

azure/ai/evaluation/_model_configurations.py

@@ -2,26 +2,148 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
 
-from typing import TypedDict
+from typing import Any, Dict, List, Literal, TypedDict, Union
 
+from typing_extensions import NotRequired
+from ._evaluator_definition import EvaluatorDefinition
+from typing import Dict, List, Optional, Any
 
-class AzureOpenAIModelConfigurationBase(TypedDict):
+
+class AzureOpenAIModelConfiguration(TypedDict):
+    """Model configuration for Azure OpenAI models
+
+    :param type: The type of the model configuration. Should be 'azure_openai' for AzureOpenAIModelConfiguration
+    :type type: NotRequired[Literal["azure_openai"]]
+    :param azure_deployment: Name of Azure OpenAI deployment to make requests to
+    :type azure_deployment: str
+    :param azure_endpoint: Endpoint of Azure OpenAI resource to make requests to
+    :type azure_endpoint: str
+    :param api_key: API key of Azure OpenAI resource
+    :type api_key: str
+    :param api_version: API version to use in request to Azure OpenAI deployment. Optional.
+    :type api_version: NotRequired[str]
+
+    .. admonition:: Example:
+
+        .. literalinclude:: ../samples/evaluation_samples_common.py
+            :start-after: [START create_AOAI_model_config]
+            :end-before: [END create_AOAI_model_config]
+            :language: python
+            :dedent: 8
+            :caption: Creating an AzureOpenAIModelConfiguration object.
+
+    """
+
+    type: NotRequired[Literal["azure_openai"]]
     azure_deployment: str
     azure_endpoint: str
-    api_key: str
+    """Endpoint of Azure OpenAI resource to make request to"""
+    api_key: NotRequired[str]
+    """API key of Azure OpenAI resource"""
+    api_version: NotRequired[str]
+
 
+class OpenAIModelConfiguration(TypedDict):
+    """Model configuration for OpenAI models
 
-class AzureOpenAIModelConfiguration(AzureOpenAIModelConfigurationBase, total=False):
-    api_version: str
+    :param type: The type of the model configuration. Should be 'openai' for OpenAIModelConfiguration
+    :type type: NotRequired[Literal["openai"]]
+    :param api_key: API key needed to make requests to model
+    :type api_key: str
+    :param model: Name of model to be used in OpenAI request
+    :type model: str
+    :param base_url: Base URL to be used in OpenAI request. Optional.
+    :type base_url: NotRequired[str]
+    :param organization: OpenAI organization. Optional.
+    :type organization: NotRequired[str]
 
+    .. admonition:: Example:
 
-class OpenAIModelConfiguration(TypedDict):
+        .. literalinclude:: ../samples/evaluation_samples_common.py
+            :start-after: [START create_OAI_model_config]
+            :end-before: [END create_OAI_model_config]
+            :language: python
+            :dedent: 8
+            :caption: Creating an OpenAIModelConfiguration object.
+
+    """
+
+    type: NotRequired[Literal["openai"]]
     api_key: str
-    base_url: str
-    organization: str
+    model: str
+    base_url: NotRequired[str]
+    organization: NotRequired[str]
 
 
 class AzureAIProject(TypedDict):
+    """Information about the Azure AI project
+
+    :param subscription_id: ID of the Azure subscription the project is in
+    :type subscription_id: str
+    :param resource_group_name: Name of the Azure resource group the project is in
+    :type resource_group_name: str
+    :param project_name: Name of the Azure project
+    :type project_name: str
+
+    .. admonition:: Example:
+
+        .. literalinclude:: ../samples/evaluation_samples_common.py
+            :start-after: [START create_azure_ai_project_object]
+            :end-before: [END create_azure_ai_project_object]
+            :language: python
+            :dedent: 8
+            :caption: Creating an AzureAIProject object.
+
+    """
+
     subscription_id: str
     resource_group_name: str
     project_name: str
+
+
+class EvaluatorConfig(TypedDict, total=False):
+    """Configuration for an evaluator"""
+
+    column_mapping: Dict[str, str]
+    """Dictionary mapping evaluator input name to column in data"""
+
+    _evaluator_name: NotRequired[Optional[str]]
+    """Name of the evaluator from the evaluator asset, currently only used for Otel emission"""
+
+    _evaluator_version: NotRequired[Optional[str]]
+    """Version of the evaluator from the evaluator asset, currently only used for Otel emission"""
+
+    _evaluator_id: NotRequired[Optional[str]]
+    """ID of the evaluator from the evaluator asset, currently only used for Otel emission"""
+
+    _evaluator_definition: NotRequired[Optional[EvaluatorDefinition]]
+    """Definition of the evaluator to be used from the evaluator asset"""
+    """Currently only used for Otel emission, will be changed to used in AOAI eval results converter as well in the future."""
+
+
+class Message(TypedDict):
+    role: str
+    content: Union[str, List[Dict]]
+    context: NotRequired[Dict[str, Any]]
+
+
+class Conversation(TypedDict):
+    messages: Union[List[Message], List[Dict]]
+    context: NotRequired[Dict[str, Any]]
+
+
+class EvaluationResult(TypedDict):
+    metrics: Dict
+    studio_url: NotRequired[str]
+    rows: List[Dict]
+    _evaluation_results_list: List[Dict]
+    _evaluation_summary: Dict
+
+
+class AppInsightsConfig(TypedDict):
+    connection_string: str
+    project_id: NotRequired[str]
+    run_type: NotRequired[str]
+    schedule_type: NotRequired[str]
+    run_id: NotRequired[str]
+    extra_attributes: NotRequired[Dict[str, Any]]

azure/ai/evaluation/_safety_evaluation/__init__.py

@@ -0,0 +1,3 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------