azure-ai-evaluation 1.0.0__py3-none-any.whl → 1.0.0b2__py3-none-any.whl

azure/ai/evaluation/_common/math.py

@@ -1,89 +0,0 @@
-# ---------------------------------------------------------
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# ---------------------------------------------------------
-
-import math
-from typing import List, Callable, Any
-
-from azure.ai.evaluation._exceptions import EvaluationException, ErrorBlame, ErrorCategory, ErrorTarget
-
-
-def list_sum(lst: List[float]) -> float:
-    """Given a list of floats, return the sum of the values.
-
-    :param lst: A list of floats.
-    :type lst: List[float]
-    :return: The sum of the values in the list.
-    :rtype: float
-    """
-
-    return sum(lst)
-
-
-def list_mean(lst: List[float]) -> float:
-    """Given a list of floats, calculate the mean of the values.
-
-    :param lst: A list of floats.
-    :type lst: List[float]
-    :return: The mean of the values in the list.
-    :rtype: float
-    """
-
-    return list_sum(lst) / len(lst)
-
-
-def list_mean_nan_safe(lst: List[float]) -> float:
-    """Given a list of floats, remove all nan or None values, then calculate the mean of the remaining values.
-
-    :param lst: A list of floats.
-    :type lst: List[float]
-    :return: The mean of the values in the list.
-    :rtype: float
-    """
-
-    msg = "All score values are NaN. The mean cannot be calculated."
-    if all(math.isnan(l) for l in lst):
-        raise EvaluationException(
-            message=msg,
-            internal_message=msg,
-            blame=ErrorBlame.USER_ERROR,
-            category=ErrorCategory.INVALID_VALUE,
-            target=ErrorTarget.CONVERSATION,
-        )
-    return list_mean([l for l in lst if not is_none_or_nan(l)])
-
-
-def apply_transform_nan_safe(lst: List[float], transform_fn: Callable[[float], Any]) -> List[Any]:
-    """Given a list of floats, remove all nan values, then apply the inputted transform function
-    to the remaining values, and return the resulting list of outputted values.
-
-    :param lst: A list of floats.
-    :type lst: List[float]
-    :param transform_fn: A function that produces something when applied to a float.
-    :type transform_fn: Callable[[float], Any]
-    :return: A list of the transformed values.
-    :rtype: List[Any]
-    """
-
-    msg = "All score values are NaN. The mean cannot be calculated."
-    if all(math.isnan(l) for l in lst):
-        raise EvaluationException(
-            message=msg,
-            internal_message=msg,
-            blame=ErrorBlame.USER_ERROR,
-            category=ErrorCategory.INVALID_VALUE,
-            target=ErrorTarget.CONVERSATION,
-        )
-    return [transform_fn(l) for l in lst if not is_none_or_nan(l)]
-
-
-def is_none_or_nan(val: float) -> bool:
-    """math.isnan raises an error if None is inputted. This is a more robust wrapper.
-
-    :param val: The value to check.
-    :type val: float
-    :return: Whether the value is None or NaN.
-    :rtype: bool
-    """
-
-    return val is None or math.isnan(val)

azure/ai/evaluation/_evaluate/_batch_run/target_run_context.py

@@ -1,46 +0,0 @@
-# ---------------------------------------------------------
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# ---------------------------------------------------------
-import os
-import types
-from typing import Optional, Type
-
-from promptflow._sdk._constants import PF_FLOW_ENTRY_IN_TMP
-from azure.ai.evaluation._constants import PF_DISABLE_TRACING
-
-
-class TargetRunContext:
-    """Context manager for target batch run.
-
-    :param upload_snapshot: Whether to upload target snapshot.
-    :type upload_snapshot: bool
-    """
-
-    def __init__(self, upload_snapshot: bool) -> None:
-        self._upload_snapshot = upload_snapshot
-        self._original_cwd = os.getcwd()
-
-    def __enter__(self) -> None:
-        # Preserve current working directory, as PF may change it without restoring it afterward
-        self._original_cwd = os.getcwd()
-
-        # Address "[WinError 32] The process cannot access the file" error,
-        # caused by conflicts when the venv and target function are in the same directory.
-        # Setting PF_FLOW_ENTRY_IN_TMP to true uploads only the flex entry file (flow.flex.yaml).
-        if not self._upload_snapshot:
-            os.environ[PF_FLOW_ENTRY_IN_TMP] = "true"
-
-        os.environ[PF_DISABLE_TRACING] = "true"
-
-    def __exit__(
-        self,
-        exc_type: Optional[Type[BaseException]],
-        exc_value: Optional[BaseException],
-        exc_tb: Optional[types.TracebackType],
-    ) -> None:
-        os.chdir(self._original_cwd)
-
-        if not self._upload_snapshot:
-            os.environ.pop(PF_FLOW_ENTRY_IN_TMP, None)
-
-        os.environ.pop(PF_DISABLE_TRACING, None)

azure/ai/evaluation/_evaluators/_common/__init__.py

@@ -1,13 +0,0 @@
-# ---------------------------------------------------------
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# ---------------------------------------------------------
-
-from ._base_eval import EvaluatorBase
-from ._base_prompty_eval import PromptyEvaluatorBase
-from ._base_rai_svc_eval import RaiServiceEvaluatorBase
-
-__all__ = [
-    "EvaluatorBase",
-    "PromptyEvaluatorBase",
-    "RaiServiceEvaluatorBase",
-]

azure/ai/evaluation/_evaluators/_common/_base_eval.py

@@ -1,344 +0,0 @@
-# ---------------------------------------------------------
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# ---------------------------------------------------------
-
-import inspect
-from abc import ABC, abstractmethod
-from typing import Any, Callable, Dict, Generic, List, TypedDict, TypeVar, Union, cast, final
-
-from promptflow._utils.async_utils import async_run_allowing_running_loop
-from typing_extensions import ParamSpec, TypeAlias, get_overloads
-
-from azure.ai.evaluation._common.math import list_mean
-from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
-from azure.ai.evaluation._common.utils import remove_optional_singletons
-
-P = ParamSpec("P")
-T = TypeVar("T")
-T_EvalValue = TypeVar("T_EvalValue")
-
-
-class DerivedEvalInput(TypedDict, total=False):
-    """The eval input generated by EvaluatorBase._derive_conversation_starter."""
-
-    query: Dict[str, Any]
-    response: Dict[str, Any]
-    context: str
-
-
-AggregateResult: TypeAlias = Dict[str, Union[float, Dict[str, List[T]]]]
-"""TypeAlias that models the return value of EvaluatorBase._aggregate_results
-
-    .. code-block:: python
-
-    foo: AggregateResult[float] = {
-        "evaluation_per_turn": {
-            "coherence": [1.0, 2.0, 3.0]
-        },
-        "coherence": 2.0
-    }
-"""
-
-DoEvalResult: TypeAlias = Dict[str, T]
-"""TypeAlias that models the return value of EvaluatorBase._do_eval
-
-    .. code-block:: python
-
-    foo: DoEvalResult[float] = {
-        "coherence": 2.0
-    }
-"""
-
-
-# TODO exception target pass down?
-class EvaluatorBase(ABC, Generic[T_EvalValue]):
-    """Base class for all evaluators that are capable of accepting either a group of single values,
-    or conversation as input. All such evaluators need to implement two functions of their own:
-        - _convert_conversation_to_eval_input
-        - _do_eval
-
-    Additionally, __call__ should be overridden to reshape the function header as needed to produce more informative
-    documentation, although ideally the actual child implementation of __call__ should just amount to
-    'super().__init__()'.
-
-
-    :param not_singleton_inputs: A list of strings that represent the names of
-        inputs to the child evaluator's __call__ function that are NOT singleton inputs. By default, this
-        is ["conversation", "kwargs"].
-    :type not_singleton_inputs: List[str]
-    :param eval_last_turn: If True, only the last turn of the conversation will be evaluated. Default is False.
-    :type eval_last_turn: bool
-    """
-
-    # ~~~ METHODS THAT ALMOST ALWAYS NEED TO BE OVERRIDDEN BY CHILDREN~~~
-
-    # Make sure to call super().__init__() in the child class's __init__ method.
-    # pylint: disable=dangerous-default-value
-    def __init__(
-        self,
-        *,
-        not_singleton_inputs: List[str] = ["conversation", "kwargs"],
-        eval_last_turn: bool = False,
-    ):
-        self._not_singleton_inputs = not_singleton_inputs
-        self._eval_last_turn = eval_last_turn
-        self._singleton_inputs = self._derive_singleton_inputs()
-        self._async_evaluator = AsyncEvaluatorBase(self._real_call)
-
-    # This needs to be overridden just to change the function header into something more informative,
-    # and to be able to add a more specific docstring. The actual function contents should just be
-    # super().__call__(<inputs>)
-    def __call__(  # pylint: disable=docstring-missing-param
-        self,
-        *args,
-        **kwargs,
-    ) -> Union[DoEvalResult[T_EvalValue], AggregateResult[T_EvalValue]]:
-        """Evaluate a given input. This method serves as a wrapper and is meant to be overridden by child classes for
-        one main reason - to overwrite the method headers and docstring to include additional inputs as needed.
-        The actual behavior of this function shouldn't change beyond adding more inputs to the
-        async_run_allowing_running_loop call.
-
-        :keyword kwargs: A dictionary that contains inputs needed to evaluate a conversation.
-        :type kwargs: Dict
-        :return: The evaluation result
-        :rtype: Union[DoEvalResult[T_EvalValue], AggregateResult[T_EvalValue]]
-        """
-        return async_run_allowing_running_loop(self._async_evaluator, **kwargs)
-
-    @abstractmethod
-    async def _do_eval(self, eval_input: Any) -> DoEvalResult[T_EvalValue]:
-        """Evaluate the input and produce a response. Must be overridden to produce a functional evaluator.
-        In the default case, all required inputs are assumed to be within eval_input, as user-friendly
-        typing is handled above this function in favor of polymorphic simplicity. This function must be
-        asynchronous.
-
-        :param eval_input: Whatever inputs are needed for this evaluator to perform a single evaluation.
-        :type eval_input: Any
-        :return: A single evaluation result
-        :rtype: DoEvalResult[T_EvalValue]
-        """
-
-    # ~~~ METHODS THAT MIGHT NEED TO BE OVERRIDDEN BY CHILDREN~~~
-
-    def _derive_singleton_inputs(self) -> List[str]:
-        """Inspect the evaluator's __call__ function to determine what singleton inputs are expected
-        when the evaluator is being used in a non-conversation context.
-        By default, it's assumed that any input that is NOT kwargs or a conversation are singleton inputs.
-        Thankfully this works the way you'd hope, with the call_signature being based on the child
-        function's signature, not the parent's.
-
-        :return: A list of strings representing the names of singleton inputs.
-        :rtype: List[str]
-        """
-
-        overloads = get_overloads(self.__call__)
-        if not overloads:
-            call_signatures = [inspect.signature(self.__call__)]
-        else:
-            call_signatures = [inspect.signature(overload) for overload in overloads]
-        call_signature = inspect.signature(self.__call__)
-        singletons = []
-        for call_signature in call_signatures:
-            params = call_signature.parameters
-            if any(not_singleton_input in params for not_singleton_input in self._not_singleton_inputs):
-                continue
-            # exclude self since it is not a singleton input
-            singletons.extend([p for p in params if p != "self"])
-        return singletons
-
-    def _derive_conversation_converter(self) -> Callable[[Dict], List[DerivedEvalInput]]:
-        """Produce the function that will be used to convert conversations to a list of evaluable inputs.
-        This uses the inputs derived from the _derive_singleton_inputs function to determine which
-        aspects of a conversation ought to be extracted.
-
-        :return: The function that will be used to convert conversations to evaluable inputs.
-        :rtype: Callable
-        """
-        include_context = "context" in self._singleton_inputs
-        include_query = "query" in self._singleton_inputs
-        include_response = "response" in self._singleton_inputs
-
-        def converter(conversation: Dict) -> List[DerivedEvalInput]:
-            messages = cast(List[Dict[str, Any]], conversation["messages"])
-            global_context = conversation.get("context", None)
-            # Extract queries, responses from conversation
-            queries: List[Dict[str, Any]] = []
-            responses: List[Dict[str, Any]] = []
-
-            # Convert conversation slice into queries and responses.
-            # Assume that 'user' role is asking queries and 'assistant' role is responding.
-            if self._eval_last_turn and len(messages) > 1:
-                messages = messages[-2:]
-
-            for each_turn in messages:
-                role = each_turn["role"]
-                if role == "user":
-                    queries.append(each_turn)
-                elif role == "assistant":
-                    responses.append(each_turn)
-            # TODO complain if len(queries) != len(responses)?
-            eval_inputs = []
-            for query, response in zip(queries, responses):
-                context = {}
-                if include_context:
-                    query_context = query.get("context", None)
-                    response_context = response.get("context", None)
-                    if global_context:
-                        context["global_context"] = global_context
-                    if query_context and include_query:
-                        context["query_context"] = query_context
-                    if response_context and include_response:
-                        context["response_context"] = response_context
-
-                eval_input: DerivedEvalInput = {}
-                if include_query:
-                    eval_input["query"] = query.get("content", "")
-                if include_response:
-                    eval_input["response"] = response.get("content", "")
-                if include_context:
-                    eval_input["context"] = str(context)
-                eval_inputs.append(eval_input)
-            return eval_inputs
-
-        return converter
-
-    def _convert_kwargs_to_eval_input(self, **kwargs) -> Union[List[Dict], List[DerivedEvalInput]]:
-        """Convert an arbitrary input into a list of inputs for evaluators.
-        It is assumed that evaluators generally make use of their inputs in one of two ways.
-        Either they receive a collection of keyname inputs that are all single values
-        (like a query and response), or they receive conversation that iss a list of dictionary
-        values.
-
-        The self._singleton_inputs list assigned during initialization is used to find and extract
-        singleton keywords, and self._allow_converssation_input is used to determine if a conversation
-        is a valid input.
-
-        If both conversations and singletons are allowed, the function will raise an exception if both
-        are inputted.
-
-        This function must be overridden by child classes IF they need to both a conversation and
-        other inputs to be passed in.
-
-        :keyword kwargs: The inputs to convert.
-        :type kwargs: Dict
-        :return: A list of arbitrary values that are valid inputs for this evaluator's do_eval function.
-        :rtype: List
-        """
-
-        # Collect inputs
-        conversation = kwargs.get("conversation", None)
-        singletons = {}
-        if len(self._singleton_inputs) > 0:
-            singletons = {key: kwargs.get(key, None) for key in self._singleton_inputs}
-        # Check that both conversation and other inputs aren't set
-        if conversation is not None and any(singletons.values()):
-            msg = f"{type(self).__name__}: Cannot provide both 'conversation' and individual inputs at the same time."
-            raise EvaluationException(
-                message=msg,
-                blame=ErrorBlame.USER_ERROR,
-                category=ErrorCategory.INVALID_VALUE,
-                target=ErrorTarget.CONVERSATION,
-            )
-        # Handle Conversation
-        if conversation is not None:
-            return self._derive_conversation_converter()(conversation)
-        # Handle Singletons
-        required_singletons = remove_optional_singletons(self, singletons)
-        if all(value is not None for value in required_singletons.values()):
-            return [singletons]
-        # Missing input
-        msg = f"{type(self).__name__}: Either 'conversation' or individual inputs must be provided."
-        raise EvaluationException(
-            message=msg,
-            blame=ErrorBlame.USER_ERROR,
-            category=ErrorCategory.INVALID_VALUE,
-            target=ErrorTarget.CONVERSATION,
-        )
-
-    def _aggregate_results(self, per_turn_results: List[DoEvalResult[T_EvalValue]]) -> AggregateResult[T_EvalValue]:
-        """Aggregate the evaluation results of each conversation turn into a single result.
-
-        Exact implementation might need to vary slightly depending on the results produced.
-        Default behavior is to average the all number-based outputs.
-
-        :param per_turn_results: List of evaluation results for each turn in the conversation.
-        :type per_turn_results: List[Dict]
-        :return: A dictionary containing aggregated results, with numeric metrics having their
-        means as top-level values in the dictionary, and all original
-        values (including non-numerics) located in under the "evaluation_per_turn" key,
-        which each sub-key being a metric and each sub-value being a the list of that metric's
-        per-turn values.
-        :rtype: AggregateResult[T_EvalValue]
-        """
-
-        aggregated: Dict[str, Union[float, Dict[str, List[T_EvalValue]]]] = {}
-        evaluation_per_turn: Dict[str, List[T_EvalValue]] = {}
-
-        # Go over each turn, and rotate the results into a
-        # metric: List[values] format for the evals_per_turn dictionary.
-        for turn in per_turn_results:
-            for metric, value in turn.items():
-                if metric not in evaluation_per_turn:
-                    evaluation_per_turn[metric] = []
-                evaluation_per_turn[metric].append(value)
-
-        # Find and average all numeric values
-        for metric, values in evaluation_per_turn.items():
-            if all(isinstance(value, (int, float)) for value in values):
-                aggregated[metric] = list_mean(cast(List[Union[int, float]], values))
-        # Slap the per-turn results back in.
-        aggregated["evaluation_per_turn"] = evaluation_per_turn
-        return aggregated
-
-    async def _real_call(self, **kwargs) -> Union[DoEvalResult[T_EvalValue], AggregateResult[T_EvalValue]]:
-        """The asynchronous call where real end-to-end evaluation logic is performed.
-
-        :keyword kwargs: The inputs to evaluate.
-        :type kwargs: Dict
-        :return: The evaluation result.
-        :rtype: Union[DoEvalResult[T_EvalValue], AggregateResult[T_EvalValue]]
-        """
-        # Convert inputs into list of evaluable inputs.
-        eval_input_list = self._convert_kwargs_to_eval_input(**kwargs)
-        per_turn_results = []
-        # Evaluate all inputs.
-        for eval_input in eval_input_list:
-            per_turn_results.append(await self._do_eval(eval_input))
-        # Return results as-is if only one result was produced.
-
-        if len(per_turn_results) == 1:
-            return per_turn_results[0]
-        if len(per_turn_results) == 0:
-            return {}  # TODO raise something?
-        # Otherwise, aggregate results.
-        return self._aggregate_results(per_turn_results=per_turn_results)
-
-    @final
-    def _to_async(self) -> "AsyncEvaluatorBase":
-        return self._async_evaluator
-
-
-class AsyncEvaluatorBase:
-    """The asynchronous evaluator hidden underneath all evaluators. This makes generous use passing functions
-    to ensure that no one ever needs to extend or otherwise modify this class directly.
-    """
-
-    def __init__(self, real_call):  # DO NOT ADD TYPEHINT PROMPT FLOW WILL SCREAM AT YOU ABOUT META GENERATION
-        self._real_call = real_call
-
-    # Don't look at my shame. Nothing to see here....
-    # Oh, you're still here? Ok, the reason this has such a gross call signature and behavior is due
-    # to our broken async code not properly handling inputs; keyword arguments that aren't in the signature
-    # are just not passed into this function instead of ending up in kwargs.
-    # Since we want this to be relatively call-agnostic, we just account for every input that any children
-    # are known to throw at this, mash them into kwargs, and then pass them into the real call.
-    async def __call__(self, *, query=None, response=None, context=None, conversation=None, **kwargs):
-        if conversation is not None:
-            kwargs["conversation"] = conversation
-        if query is not None:
-            kwargs["query"] = query
-        if response is not None:
-            kwargs["response"] = response
-        if context is not None:
-            kwargs["context"] = context
-        return await self._real_call(**kwargs)

azure/ai/evaluation/_evaluators/_common/_base_prompty_eval.py

@@ -1,88 +0,0 @@
-# ---------------------------------------------------------
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# ---------------------------------------------------------
-
-import math
-import re
-from typing import Dict, TypeVar, Union
-
-from promptflow.core import AsyncPrompty
-from typing_extensions import override
-
-from azure.ai.evaluation._common.constants import PROMPT_BASED_REASON_EVALUATORS
-from ..._common.utils import construct_prompty_model_config, validate_model_config, parse_quality_evaluator_reason_score
-from . import EvaluatorBase
-
-try:
-    from ..._user_agent import USER_AGENT
-except ImportError:
-    USER_AGENT = "None"
-
-T = TypeVar("T")
-
-
-class PromptyEvaluatorBase(EvaluatorBase[T]):
-    """Base class for all evaluators that make use of context as an input. It's also assumed that such evaluators
-    make use of a prompty file, and return their results as a dictionary, with a single key-value pair
-    linking the result name to a float value (unless multi-turn evaluation occurs, in which case the
-    per-turn results are stored in a list under the key "evaluation_per_turn").
-
-    :param result_key: The key to use for the result of the evaluation. Single turn evaluations will return
-        a dictionary in the format {result_key: float}.
-    :type result_key: str
-    :param prompty_file: The path to the prompty file to use for evaluation.
-    :type prompty_file: str
-    :param model_config: The model configuration to use for evaluation.
-    :type model_config: Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration]
-    :param ignore_queries: If True, queries will be ignored in conversation evaluations. Default is False.
-        Useful since some evaluators of this format are response-only.
-    :type ignore_queries: bool
-    """
-
-    _LLM_CALL_TIMEOUT = 600
-    _DEFAULT_OPEN_API_VERSION = "2024-02-15-preview"
-
-    def __init__(self, *, result_key: str, prompty_file: str, model_config: dict, eval_last_turn: bool = False):
-        self._result_key = result_key
-        self._prompty_file = prompty_file
-        super().__init__(eval_last_turn=eval_last_turn)
-
-        prompty_model_config = construct_prompty_model_config(
-            validate_model_config(model_config),
-            self._DEFAULT_OPEN_API_VERSION,
-            USER_AGENT,
-        )
-
-        self._flow = AsyncPrompty.load(source=prompty_file, model=prompty_model_config)
-
-    # __call__ not overridden here because child classes have such varied signatures that there's no point
-    # defining a default here.
-
-    @override
-    async def _do_eval(self, eval_input: Dict) -> Dict[str, Union[float, str]]:  # type: ignore[override]
-        """Do a relevance evaluation.
-
-        :param eval_input: The input to the evaluator. Expected to contain
-        whatever inputs are needed for the _flow method, including context
-        and other fields depending on the child class.
-        :type eval_input: Dict
-        :return: The evaluation result.
-        :rtype: Dict
-        """
-        llm_output = await self._flow(timeout=self._LLM_CALL_TIMEOUT, **eval_input)
-
-        score = math.nan
-        if llm_output:
-            # Parse out score and reason from evaluators known to possess them.
-            if self._result_key in PROMPT_BASED_REASON_EVALUATORS:
-                score, reason = parse_quality_evaluator_reason_score(llm_output)
-                return {
-                    self._result_key: float(score),
-                    f"gpt_{self._result_key}": float(score),
-                    f"{self._result_key}_reason": reason,
-                }
-            match = re.search(r"\d", llm_output)
-            if match:
-                score = float(match.group())
-            return {self._result_key: float(score), f"gpt_{self._result_key}": float(score)}
-        return {self._result_key: float(score), f"gpt_{self._result_key}": float(score)}