azure-ai-evaluation 0.0.0b0__py3-none-any.whl → 1.0.0__py3-none-any.whl

azure/ai/evaluation/__init__.py

@@ -0,0 +1,82 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from ._evaluate._evaluate import evaluate
+from ._evaluators._bleu import BleuScoreEvaluator
+from ._evaluators._coherence import CoherenceEvaluator
+from ._evaluators._content_safety import (
+    ContentSafetyEvaluator,
+    HateUnfairnessEvaluator,
+    SelfHarmEvaluator,
+    SexualEvaluator,
+    ViolenceEvaluator,
+)
+from ._evaluators._multimodal._content_safety_multimodal import (
+    ContentSafetyMultimodalEvaluator,
+    HateUnfairnessMultimodalEvaluator,
+    SelfHarmMultimodalEvaluator,
+    SexualMultimodalEvaluator,
+    ViolenceMultimodalEvaluator,
+)
+from ._evaluators._multimodal._protected_material import ProtectedMaterialMultimodalEvaluator
+from ._evaluators._f1_score import F1ScoreEvaluator
+from ._evaluators._fluency import FluencyEvaluator
+from ._evaluators._gleu import GleuScoreEvaluator
+from ._evaluators._groundedness import GroundednessEvaluator
+from ._evaluators._service_groundedness import GroundednessProEvaluator
+from ._evaluators._meteor import MeteorScoreEvaluator
+from ._evaluators._protected_material import ProtectedMaterialEvaluator
+from ._evaluators._qa import QAEvaluator
+from ._evaluators._relevance import RelevanceEvaluator
+from ._evaluators._retrieval import RetrievalEvaluator
+from ._evaluators._rouge import RougeScoreEvaluator, RougeType
+from ._evaluators._similarity import SimilarityEvaluator
+from ._evaluators._xpia import IndirectAttackEvaluator
+from ._model_configurations import (
+    AzureAIProject,
+    AzureOpenAIModelConfiguration,
+    Conversation,
+    EvaluationResult,
+    EvaluatorConfig,
+    Message,
+    OpenAIModelConfiguration,
+)
+
+__all__ = [
+    "evaluate",
+    "CoherenceEvaluator",
+    "F1ScoreEvaluator",
+    "FluencyEvaluator",
+    "GroundednessEvaluator",
+    "GroundednessProEvaluator",
+    "RelevanceEvaluator",
+    "SimilarityEvaluator",
+    "QAEvaluator",
+    "ViolenceEvaluator",
+    "SexualEvaluator",
+    "SelfHarmEvaluator",
+    "HateUnfairnessEvaluator",
+    "ContentSafetyEvaluator",
+    "IndirectAttackEvaluator",
+    "BleuScoreEvaluator",
+    "GleuScoreEvaluator",
+    "MeteorScoreEvaluator",
+    "RetrievalEvaluator",
+    "RougeScoreEvaluator",
+    "RougeType",
+    "ProtectedMaterialEvaluator",
+    "AzureAIProject",
+    "AzureOpenAIModelConfiguration",
+    "OpenAIModelConfiguration",
+    "EvaluatorConfig",
+    "Conversation",
+    "Message",
+    "EvaluationResult",
+    "ContentSafetyMultimodalEvaluator",
+    "HateUnfairnessMultimodalEvaluator",
+    "SelfHarmMultimodalEvaluator",
+    "SexualMultimodalEvaluator",
+    "ViolenceMultimodalEvaluator",
+    "ProtectedMaterialMultimodalEvaluator",
+]

azure/ai/evaluation/_common/__init__.py

@@ -0,0 +1,16 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+# To minimize relative imports in our evaluators, the scope of this package also includes anything
+# that would have otherwise been a relative import scoped to single evaluator directories.
+
+from . import constants
+from .rai_service import evaluate_with_rai_service
+from .utils import get_harm_severity_level
+
+__all__ = [
+    "get_harm_severity_level",
+    "evaluate_with_rai_service",
+    "constants",
+]

azure/ai/evaluation/_common/_experimental.py

@@ -0,0 +1,172 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+import os
+import functools
+import inspect
+import logging
+import sys
+from typing import Callable, Type, TypeVar, Union, overload
+
+from typing_extensions import ParamSpec, TypeGuard
+
+DOCSTRING_TEMPLATE = ".. note::    {0} {1}\n\n"
+DOCSTRING_DEFAULT_INDENTATION = 8
+EXPERIMENTAL_CLASS_MESSAGE = "This is an experimental class,"
+EXPERIMENTAL_METHOD_MESSAGE = "This is an experimental method,"
+EXPERIMENTAL_FIELD_MESSAGE = "This is an experimental field,"
+EXPERIMENTAL_LINK_MESSAGE = (
+    "and may change at any time. Please see https://aka.ms/azuremlexperimental for more information."
+)
+
+_warning_cache = set()
+module_logger = logging.getLogger(__name__)
+
+P = ParamSpec("P")
+T = TypeVar("T")
+
+
+@overload
+def experimental(wrapped: Type[T]) -> Type[T]: ...
+
+
+@overload
+def experimental(wrapped: Callable[P, T]) -> Callable[P, T]: ...
+
+
+def experimental(wrapped: Union[Type[T], Callable[P, T]]) -> Union[Type[T], Callable[P, T]]:
+    """Add experimental tag to a class or a method.
+
+    :param wrapped: Either a Class or Function to mark as experimental
+    :type wrapped: Union[Type[T], Callable[P, T]]
+    :return: The wrapped class or method
+    :rtype: Union[Type[T], Callable[P, T]]
+    """
+
+    def is_class(t: Union[Type[T], Callable[P, T]]) -> TypeGuard[Type[T]]:
+        return isinstance(t, type)
+
+    if is_class(wrapped):
+        return _add_class_docstring(wrapped)
+    if inspect.isfunction(wrapped):
+        return _add_method_docstring(wrapped)
+    return wrapped
+
+
+def _add_class_docstring(cls: Type[T]) -> Type[T]:
+    """Add experimental tag to the class doc string.
+
+    :return: The updated class
+    :rtype: Type[T]
+    """
+
+    P2 = ParamSpec("P2")
+
+    def _add_class_warning(func: Callable[P2, None]) -> Callable[P2, None]:
+        """Add warning message for class __init__.
+
+        :param func: The original __init__ function
+        :type func: Callable[P2, None]
+        :return: Updated __init__
+        :rtype: Callable[P2, None]
+        """
+
+        @functools.wraps(func)
+        def wrapped(*args, **kwargs):
+            message = "Class {0}: {1} {2}".format(cls.__name__, EXPERIMENTAL_CLASS_MESSAGE, EXPERIMENTAL_LINK_MESSAGE)
+            if not _should_skip_warning() and not _is_warning_cached(message):
+                module_logger.warning(message)
+            return func(*args, **kwargs)
+
+        return wrapped
+
+    doc_string = DOCSTRING_TEMPLATE.format(EXPERIMENTAL_CLASS_MESSAGE, EXPERIMENTAL_LINK_MESSAGE)
+    if cls.__doc__:
+        cls.__doc__ = _add_note_to_docstring(cls.__doc__, doc_string)
+    else:
+        cls.__doc__ = doc_string + ">"
+    cls.__init__ = _add_class_warning(cls.__init__)  # type: ignore[method-assign]
+    return cls
+
+
+def _add_method_docstring(func: Callable[P, T]) -> Callable[P, T]:
+    """Add experimental tag to the method doc string.
+
+    :param func: The function to update
+    :type func: Callable[P, T]
+    :return: A wrapped method marked as experimental
+    :rtype: Callable[P,T]
+    """
+    doc_string = DOCSTRING_TEMPLATE.format(EXPERIMENTAL_METHOD_MESSAGE, EXPERIMENTAL_LINK_MESSAGE)
+    if func.__doc__:
+        func.__doc__ = _add_note_to_docstring(func.__doc__, doc_string)
+    else:
+        # '>' is required. Otherwise the note section can't be generated
+        func.__doc__ = doc_string + ">"
+
+    @functools.wraps(func)
+    def wrapped(*args: P.args, **kwargs: P.kwargs) -> T:
+        message = "Method {0}: {1} {2}".format(func.__name__, EXPERIMENTAL_METHOD_MESSAGE, EXPERIMENTAL_LINK_MESSAGE)
+        if not _should_skip_warning() and not _is_warning_cached(message):
+            module_logger.warning(message)
+        return func(*args, **kwargs)
+
+    return wrapped
+
+
+def _add_note_to_docstring(doc_string: str, note: str) -> str:
+    """Adds experimental note to docstring at the top and correctly indents original docstring.
+
+    :param doc_string: The docstring
+    :type doc_string: str
+    :param note: The note to add to the docstring
+    :type note: str
+    :return: Updated docstring
+    :rtype: str
+    """
+    indent = _get_indentation_size(doc_string)
+    doc_string = doc_string.rjust(len(doc_string) + indent)
+    return note + doc_string
+
+
+def _get_indentation_size(doc_string: str) -> int:
+    """Finds the minimum indentation of all non-blank lines after the first line.
+
+    :param doc_string: The docstring
+    :type doc_string: str
+    :return: Minimum number of indentation of the docstring
+    :rtype: int
+    """
+    lines = doc_string.expandtabs().splitlines()
+    indent = sys.maxsize
+    for line in lines[1:]:
+        stripped = line.lstrip()
+        if stripped:
+            indent = min(indent, len(line) - len(stripped))
+    return indent if indent < sys.maxsize else DOCSTRING_DEFAULT_INDENTATION
+
+
+def _should_skip_warning():
+    skip_warning_msg = False
+
+    if os.getenv("AI_EVALS_DISABLE_EXPERIMENTAL_WARNING", "false").lower() == "true":
+        skip_warning_msg = True
+
+    # Cases where we want to suppress the warning:
+    # 1. When converting from REST object to SDK object
+    for frame in inspect.stack():
+        if frame.function == "_from_rest_object":
+            skip_warning_msg = True
+            break
+
+    return skip_warning_msg
+
+
+def _is_warning_cached(warning_msg):
+    # use cache to make sure we only print same warning message once under same session
+    # this prevents duplicated warnings got printed when user does a loop call on a method or a class
+    if warning_msg in _warning_cache:
+        return True
+    _warning_cache.add(warning_msg)
+    return False

azure/ai/evaluation/_common/constants.py

@@ -0,0 +1,72 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+from enum import Enum
+
+from azure.core import CaseInsensitiveEnumMeta
+
+
+PROMPT_BASED_REASON_EVALUATORS = ["coherence", "relevance", "retrieval", "groundedness", "fluency"]
+
+
+class CommonConstants:
+    """Define common constants."""
+
+    DEFAULT_HTTP_TIMEOUT = 60
+
+
+class RAIService:
+    """Define constants related to RAI service"""
+
+    TIMEOUT = 1800
+    SLEEP_TIME = 2
+    HARM_SEVERITY_THRESHOLD = 4
+
+
+class HarmSeverityLevel(Enum):
+    """Harm severity levels."""
+
+    VeryLow = "Very low"
+    Low = "Low"
+    Medium = "Medium"
+    High = "High"
+
+
+class Tasks:
+    """Defines types of annotation tasks supported by RAI Service."""
+
+    CONTENT_HARM = "content harm"
+    PROTECTED_MATERIAL = "protected material"
+    XPIA = "xpia"
+    GROUNDEDNESS = "groundedness"
+
+
+class _InternalAnnotationTasks:
+    """Defines types of annotation tasks that are supported for internal use. Such tasks are
+    experimental and subject to potential change or migration to the main Evaluation Metrics
+    enum over time."""
+
+    ECI = "eci"
+
+
+class EvaluationMetrics(str, Enum, metaclass=CaseInsensitiveEnumMeta):
+    """Evaluation metrics to aid the RAI service in determining what
+    metrics to request, and how to present them back to the user."""
+
+    HATE_FAIRNESS = "hate_fairness"
+    HATE_UNFAIRNESS = "hate_unfairness"
+    VIOLENCE = "violence"
+    SELF_HARM = "self_harm"
+    SEXUAL = "sexual"
+    PROTECTED_MATERIAL = "protected_material"
+    XPIA = "xpia"
+    GROUNDEDNESS = "generic_groundedness"
+
+
+class _InternalEvaluationMetrics(str, Enum, metaclass=CaseInsensitiveEnumMeta):
+    """Evaluation metrics that are not publicly supported.
+    These metrics are experimental and subject to potential change or migration to the main
+    enum over time.
+    """
+
+    ECI = "eci"

azure/ai/evaluation/_common/math.py

@@ -0,0 +1,89 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+import math
+from typing import List, Callable, Any
+
+from azure.ai.evaluation._exceptions import EvaluationException, ErrorBlame, ErrorCategory, ErrorTarget
+
+
+def list_sum(lst: List[float]) -> float:
+    """Given a list of floats, return the sum of the values.
+
+    :param lst: A list of floats.
+    :type lst: List[float]
+    :return: The sum of the values in the list.
+    :rtype: float
+    """
+
+    return sum(lst)
+
+
+def list_mean(lst: List[float]) -> float:
+    """Given a list of floats, calculate the mean of the values.
+
+    :param lst: A list of floats.
+    :type lst: List[float]
+    :return: The mean of the values in the list.
+    :rtype: float
+    """
+
+    return list_sum(lst) / len(lst)
+
+
+def list_mean_nan_safe(lst: List[float]) -> float:
+    """Given a list of floats, remove all nan or None values, then calculate the mean of the remaining values.
+
+    :param lst: A list of floats.
+    :type lst: List[float]
+    :return: The mean of the values in the list.
+    :rtype: float
+    """
+
+    msg = "All score values are NaN. The mean cannot be calculated."
+    if all(math.isnan(l) for l in lst):
+        raise EvaluationException(
+            message=msg,
+            internal_message=msg,
+            blame=ErrorBlame.USER_ERROR,
+            category=ErrorCategory.INVALID_VALUE,
+            target=ErrorTarget.CONVERSATION,
+        )
+    return list_mean([l for l in lst if not is_none_or_nan(l)])
+
+
+def apply_transform_nan_safe(lst: List[float], transform_fn: Callable[[float], Any]) -> List[Any]:
+    """Given a list of floats, remove all nan values, then apply the inputted transform function
+    to the remaining values, and return the resulting list of outputted values.
+
+    :param lst: A list of floats.
+    :type lst: List[float]
+    :param transform_fn: A function that produces something when applied to a float.
+    :type transform_fn: Callable[[float], Any]
+    :return: A list of the transformed values.
+    :rtype: List[Any]
+    """
+
+    msg = "All score values are NaN. The mean cannot be calculated."
+    if all(math.isnan(l) for l in lst):
+        raise EvaluationException(
+            message=msg,
+            internal_message=msg,
+            blame=ErrorBlame.USER_ERROR,
+            category=ErrorCategory.INVALID_VALUE,
+            target=ErrorTarget.CONVERSATION,
+        )
+    return [transform_fn(l) for l in lst if not is_none_or_nan(l)]
+
+
+def is_none_or_nan(val: float) -> bool:
+    """math.isnan raises an error if None is inputted. This is a more robust wrapper.
+
+    :param val: The value to check.
+    :type val: float
+    :return: Whether the value is None or NaN.
+    :rtype: bool
+    """
+
+    return val is None or math.isnan(val)