azure-ai-evaluation 1.0.0b2__py3-none-any.whl → 1.0.0b4__py3-none-any.whl

azure/ai/evaluation/_evaluate/_utils.py

@@ -6,15 +6,22 @@ import logging
 import os
 import re
 import tempfile
-from collections import namedtuple
 from pathlib import Path
-from typing import Dict
+from typing import Any, Dict, List, NamedTuple, Optional, Tuple, TypedDict, Union
 
 import pandas as pd
-
-from azure.ai.evaluation._constants import DEFAULT_EVALUATION_RESULTS_FILE_NAME, DefaultOpenEncoding, Prefixes
+from promptflow.client import PFClient
+from promptflow.entities import Run
+
+from azure.ai.evaluation._constants import (
+    DEFAULT_EVALUATION_RESULTS_FILE_NAME,
+    DefaultOpenEncoding,
+    EvaluationRunProperties,
+    Prefixes,
+)
 from azure.ai.evaluation._evaluate._eval_run import EvalRun
 from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
+from azure.ai.evaluation._model_configurations import AzureAIProject
 
 LOGGER = logging.getLogger(__name__)
 
@@ -23,14 +30,26 @@ AZURE_WORKSPACE_REGEX_FORMAT = (
     "(/providers/Microsoft.MachineLearningServices)?/workspaces/([^/]+)$"
 )
 
-AzureMLWorkspaceTriad = namedtuple("AzureMLWorkspace", ["subscription_id", "resource_group_name", "workspace_name"])
+
+class AzureMLWorkspace(NamedTuple):
+    subscription_id: str
+    resource_group_name: str
+    workspace_name: str
 
 
-def is_none(value):
+class EvaluateResult(TypedDict):
+    metrics: Dict[str, float]
+    studio_url: Optional[str]
+    rows: List[Dict]
+
+
+def is_none(value) -> bool:
     return value is None or str(value).lower() == "none"
 
 
-def extract_workspace_triad_from_trace_provider(trace_provider: str):  # pylint: disable=name-too-long
+def extract_workspace_triad_from_trace_provider(  # pylint: disable=name-too-long
+    trace_provider: str,
+) -> AzureMLWorkspace:
     match = re.match(AZURE_WORKSPACE_REGEX_FORMAT, trace_provider)
     if not match or len(match.groups()) != 5:
         raise EvaluationException(
@@ -47,7 +66,7 @@ def extract_workspace_triad_from_trace_provider(trace_provider: str):  # pylint:
     subscription_id = match.group(1)
     resource_group_name = match.group(3)
     workspace_name = match.group(5)
-    return AzureMLWorkspaceTriad(subscription_id, resource_group_name, workspace_name)
+    return AzureMLWorkspace(subscription_id, resource_group_name, workspace_name)
 
 
 def load_jsonl(path):
@@ -55,7 +74,7 @@ def load_jsonl(path):
         return [json.loads(line) for line in f.readlines()]
 
 
-def _azure_pf_client_and_triad(trace_destination):
+def _azure_pf_client_and_triad(trace_destination) -> Tuple[PFClient, AzureMLWorkspace]:
     from promptflow.azure._cli._utils import _get_azure_pf_client
 
     ws_triad = extract_workspace_triad_from_trace_provider(trace_destination)
@@ -69,14 +88,14 @@ def _azure_pf_client_and_triad(trace_destination):
 
 
 def _log_metrics_and_instance_results(
-    metrics,
-    instance_results,
-    trace_destination,
-    run,
-    evaluation_name,
-) -> str:
+    metrics: Dict[str, Any],
+    instance_results: pd.DataFrame,
+    trace_destination: Optional[str],
+    run: Run,
+    evaluation_name: Optional[str],
+) -> Optional[str]:
     if trace_destination is None:
-        LOGGER.error("Unable to log traces as trace destination was not defined.")
+        LOGGER.debug("Skip uploading evaluation results to AI Studio since no trace destination was provided.")
         return None
 
     azure_pf_client, ws_triad = _azure_pf_client_and_triad(trace_destination)
@@ -94,7 +113,6 @@ def _log_metrics_and_instance_results(
         ml_client=azure_pf_client.ml_client,
         promptflow_run=run,
     ) as ev_run:
-
         artifact_name = EvalRun.EVALUATION_ARTIFACT if run else EvalRun.EVALUATION_ARTIFACT_DUMMY_RUN
 
         with tempfile.TemporaryDirectory() as tmpdir:
@@ -112,7 +130,8 @@ def _log_metrics_and_instance_results(
             if run is None:
                 ev_run.write_properties_to_run_history(
                     properties={
-                        "_azureml.evaluation_run": "azure-ai-generative-parent",
+                        EvaluationRunProperties.RUN_TYPE: "eval_run",
+                        EvaluationRunProperties.EVALUATION_RUN: "azure-ai-generative-parent",
                         "_azureml.evaluate_artifacts": json.dumps([{"path": artifact_name, "type": "table"}]),
                         "isEvaluatorRun": "true",
                     }
@@ -138,7 +157,7 @@ def _get_ai_studio_url(trace_destination: str, evaluation_id: str) -> str:
     return studio_url
 
 
-def _trace_destination_from_project_scope(project_scope: dict) -> str:
+def _trace_destination_from_project_scope(project_scope: AzureAIProject) -> str:
     subscription_id = project_scope["subscription_id"]
     resource_group_name = project_scope["resource_group_name"]
     workspace_name = project_scope["project_name"]
@@ -151,9 +170,9 @@ def _trace_destination_from_project_scope(project_scope: dict) -> str:
     return trace_destination
 
 
-def _write_output(path, data_dict):
+def _write_output(path: Union[str, os.PathLike], data_dict: Any) -> None:
     p = Path(path)
-    if os.path.isdir(path):
+    if p.is_dir():
         p = p / DEFAULT_EVALUATION_RESULTS_FILE_NAME
 
     with open(p, "w", encoding=DefaultOpenEncoding.WRITE) as f:
@@ -161,7 +180,7 @@ def _write_output(path, data_dict):
 
 
 def _apply_column_mapping(
-    source_df: pd.DataFrame, mapping_config: Dict[str, str], inplace: bool = False
+    source_df: pd.DataFrame, mapping_config: Optional[Dict[str, str]], inplace: bool = False
 ) -> pd.DataFrame:
     """
     Apply column mapping to source_df based on mapping_config.
@@ -211,7 +230,7 @@ def _apply_column_mapping(
     return result_df
 
 
-def _has_aggregator(evaluator):
+def _has_aggregator(evaluator: object) -> bool:
     return hasattr(evaluator, "__aggregate__")
 
 
@@ -234,11 +253,11 @@ def get_int_env_var(env_var_name: str, default_value: int) -> int:
         return default_value
 
 
-def set_event_loop_policy():
+def set_event_loop_policy() -> None:
     import asyncio
     import platform
 
     if platform.system().lower() == "windows":
         # Reference: https://stackoverflow.com/questions/45600579/asyncio-event-loop-is-closed-when-getting-loop
         # On Windows seems to be a problem with EventLoopPolicy, use this snippet to work around it
-        asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
+        asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())  # type: ignore[attr-defined]

azure/ai/evaluation/_evaluators/_coherence/_coherence.py

@@ -1,77 +1,15 @@
 # ---------------------------------------------------------
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
-
 import os
-import re
-
-import numpy as np
-from promptflow._utils.async_utils import async_run_allowing_running_loop
-from promptflow.core import AsyncPrompty
-
-from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
-
-from ..._common.utils import ensure_api_version_in_aoai_model_config, ensure_user_agent_in_aoai_model_config
-
-try:
-    from ..._user_agent import USER_AGENT
-except ImportError:
-    USER_AGENT = None
-
-
-class _AsyncCoherenceEvaluator:
-    # Constants must be defined within eval's directory to be save/loadable
-    PROMPTY_FILE = "coherence.prompty"
-    LLM_CALL_TIMEOUT = 600
-    DEFAULT_OPEN_API_VERSION = "2024-02-15-preview"
-
-    def __init__(self, model_config: dict):
-        ensure_api_version_in_aoai_model_config(model_config, self.DEFAULT_OPEN_API_VERSION)
-
-        prompty_model_config = {"configuration": model_config, "parameters": {"extra_headers": {}}}
-
-        # Handle "RuntimeError: Event loop is closed" from httpx AsyncClient
-        # https://github.com/encode/httpx/discussions/2959
-        prompty_model_config["parameters"]["extra_headers"].update({"Connection": "close"})
-
-        ensure_user_agent_in_aoai_model_config(
-            model_config,
-            prompty_model_config,
-            USER_AGENT,
-        )
+from typing import Optional
 
-        current_dir = os.path.dirname(__file__)
-        prompty_path = os.path.join(current_dir, self.PROMPTY_FILE)
-        self._flow = AsyncPrompty.load(source=prompty_path, model=prompty_model_config)
-
-    async def __call__(self, *, query: str, response: str, **kwargs):
-        # Validate input parameters
-        query = str(query or "")
-        response = str(response or "")
-
-        if not (query.strip() and response.strip()):
-            msg = "Both 'query' and 'response' must be non-empty strings."
-            raise EvaluationException(
-                message=msg,
-                internal_message=msg,
-                error_category=ErrorCategory.INVALID_VALUE,
-                error_blame=ErrorBlame.USER_ERROR,
-                error_target=ErrorTarget.COHERENCE_EVALUATOR,
-            )
+from typing_extensions import override
 
-        # Run the evaluation flow
-        llm_output = await self._flow(query=query, response=response, timeout=self.LLM_CALL_TIMEOUT, **kwargs)
+from azure.ai.evaluation._evaluators._common import PromptyEvaluatorBase
 
-        score = np.nan
-        if llm_output:
-            match = re.search(r"\d", llm_output)
-            if match:
-                score = float(match.group())
 
-        return {"gpt_coherence": float(score)}
-
-
-class CoherenceEvaluator:
+class CoherenceEvaluator(PromptyEvaluatorBase):
     """
     Initialize a coherence evaluator configured for a specific Azure OpenAI model.
 
@@ -97,21 +35,37 @@ class CoherenceEvaluator:
         }
     """
 
-    def __init__(self, model_config: dict):
-        self._async_evaluator = _AsyncCoherenceEvaluator(model_config)
+    PROMPTY_FILE = "coherence.prompty"
+    RESULT_KEY = "gpt_coherence"
 
-    def __call__(self, *, query: str, response: str, **kwargs):
-        """
-        Evaluate coherence.
+    @override
+    def __init__(self, model_config: dict):
+        current_dir = os.path.dirname(__file__)
+        prompty_path = os.path.join(current_dir, self.PROMPTY_FILE)
+        super().__init__(model_config=model_config, prompty_file=prompty_path, result_key=self.RESULT_KEY)
+
+    @override
+    def __call__(
+        self,
+        *,
+        query: Optional[str] = None,
+        response: Optional[str] = None,
+        conversation: Optional[dict] = None,
+        **kwargs,
+    ):
+        """Evaluate coherence. Accepts either a query and response for a single evaluation,
+        or a conversation for a potentially multi-turn evaluation. If the conversation has more than one pair of
+        turns, the evaluator will aggregate the results of each turn.
 
-        :keyword query: The query to be evaluated.
-        :paramtype query: str
         :keyword response: The response to be evaluated.
-        :paramtype response: str
-        :return: The coherence score.
+        :paramtype response: Optional[str]
+        :keyword context: The context to be evaluated.
+        :paramtype context: Optional[str]
+        :keyword conversation: The conversation to evaluate. Expected to contain a list of conversation turns under the
+            key "messages". Conversation turns are expected
+            to be dictionaries with keys "content" and "role".
+        :paramtype conversation: Optional[Dict]
+        :return: The relevance score.
         :rtype: Dict[str, float]
         """
-        return async_run_allowing_running_loop(self._async_evaluator, query=query, response=response, **kwargs)
-
-    def _to_async(self):
-        return self._async_evaluator
+        return super().__call__(query=query, response=response, conversation=conversation, **kwargs)

azure/ai/evaluation/_evaluators/_coherence/coherence.prompty

@@ -3,11 +3,6 @@ name: Coherence
 description: Evaluates coherence score for QA scenario
 model:
   api: chat
-  configuration:
-    type: azure_openai
-    azure_deployment: ${env:AZURE_DEPLOYMENT}
-    api_key: ${env:AZURE_OPENAI_API_KEY}
-    azure_endpoint: ${env:AZURE_OPENAI_ENDPOINT}
   parameters:
     temperature: 0.0
     max_tokens: 1

azure/ai/evaluation/_evaluators/_common/__init__.py

@@ -0,0 +1,13 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from ._base_eval import EvaluatorBase
+from ._base_prompty_eval import PromptyEvaluatorBase
+from ._base_rai_svc_eval import RaiServiceEvaluatorBase
+
+__all__ = [
+    "EvaluatorBase",
+    "PromptyEvaluatorBase",
+    "RaiServiceEvaluatorBase",
+]

azure/ai/evaluation/_evaluators/_common/_base_eval.py

@@ -0,0 +1,331 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+import inspect
+from abc import ABC, abstractmethod
+from typing import Any, Callable, Dict, Generic, List, TypedDict, TypeVar, Union, cast, final
+
+from promptflow._utils.async_utils import async_run_allowing_running_loop
+from typing_extensions import ParamSpec, TypeAlias
+
+from azure.ai.evaluation._common.math import list_mean
+from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
+
+P = ParamSpec("P")
+T = TypeVar("T")
+T_EvalValue = TypeVar("T_EvalValue")
+
+
+class DerivedEvalInput(TypedDict, total=False):
+    """The eval input generated by EvaluatorBase._derive_conversation_starter."""
+
+    query: Dict[str, Any]
+    response: Dict[str, Any]
+    context: str
+
+
+AggregateResult: TypeAlias = Dict[str, Union[float, Dict[str, List[T]]]]
+"""TypeAlias that models the return value of EvaluatorBase._aggregate_results
+
+    .. code-block:: python
+
+    foo: AggregateResult[float] = {
+        "evaluation_per_turn": {
+            "gpt_coherence": [1.0, 2.0, 3.0]
+        },
+        "gpt_coherence": 2.0
+    }
+"""
+
+DoEvalResult: TypeAlias = Dict[str, T]
+"""TypeAlias that models the return value of EvaluatorBase._do_eval
+
+    .. code-block:: python
+
+    foo: DoEvalResult[float] = {
+        "gpt_coherence": 2.0
+    }
+"""
+
+
+# TODO exception target pass down?
+class EvaluatorBase(ABC, Generic[T_EvalValue]):
+    """Base class for all evaluators that are capable of accepting either a group of single values,
+    or conversation as input. All such evaluators need to implement two functions of their own:
+        - _convert_conversation_to_eval_input
+        - _do_eval
+
+    Additionally, __call__ should be overridden to reshape the function header as needed to produce more informative
+    documentation, although ideally the actual child implementation of __call__ should just amount to
+    'super().__init__()'.
+
+
+    :param not_singleton_inputs: A list of strings that represent the names of
+        inputs to the child evaluator's __call__ function that are NOT singleton inputs. By default, this
+        is ["conversation", "kwargs"].
+    :type not_singleton_inputs: List[str]
+    :param eval_last_turn: If True, only the last turn of the conversation will be evaluated. Default is False.
+    :type eval_last_turn: bool
+    """
+
+    # ~~~ METHODS THAT ALMOST ALWAYS NEED TO BE OVERRIDDEN BY CHILDREN~~~
+
+    # Make sure to call super().__init__() in the child class's __init__ method.
+    # pylint: disable=dangerous-default-value
+    def __init__(
+        self,
+        *,
+        not_singleton_inputs: List[str] = ["conversation", "kwargs"],
+        eval_last_turn: bool = False,
+    ):
+        self._not_singleton_inputs = not_singleton_inputs
+        self._eval_last_turn = eval_last_turn
+        self._singleton_inputs = self._derive_singleton_inputs()
+        self._async_evaluator = AsyncEvaluatorBase(self._real_call)
+
+    # This needs to be overridden just to change the function header into something more informative,
+    # and to be able to add a more specific docstring. The actual function contents should just be
+    # super().__call__(<inputs>)
+    def __call__(self, **kwargs) -> Union[DoEvalResult[T_EvalValue], AggregateResult[T_EvalValue]]:
+        """Evaluate a given input. This method serves as a wrapper and is meant to be overridden by child classes for
+        one main reason - to overwrite the method headers and docstring to include additional inputs as needed.
+        The actual behavior of this function shouldn't change beyond adding more inputs to the
+        async_run_allowing_running_loop call.
+
+        :keyword kwargs: A dictionary that contains inputs needed to evaluate a conversation.
+        :type kwargs: Dict
+        :return: The evaluation result
+        :rtype: Dict
+        """
+        return async_run_allowing_running_loop(self._async_evaluator, **kwargs)
+
+    @abstractmethod
+    async def _do_eval(self, eval_input: Any) -> DoEvalResult[T_EvalValue]:
+        """Evaluate the input and produce a response. Must be overridden to produce a functional evaluator.
+        In the default case, all required inputs are assumed to be within eval_input, as user-friendly
+        typing is handled above this function in favor of polymorphic simplicity. This function must be
+        asynchronous.
+
+        :param eval_input: Whatever inputs are needed for this evaluator to perform a single evaluation.
+        :type eval_input: Any
+        :return: A single evaluation result
+        :rtype: Dict
+        """
+
+    # ~~~ METHODS THAT MIGHT NEED TO BE OVERRIDDEN BY CHILDREN~~~
+
+    def _derive_singleton_inputs(self) -> List[str]:
+        """Inspect the evaluator's __call__ function to determine what singleton inputs are expected
+        when the evaluator is being used in a non-conversation context.
+        By default, it's assumed that any input that is NOT kwargs or a conversation are singleton inputs.
+        Thankfully this works the way you'd hope, with the call_signature being based on the child
+        function's signature, not the parent's.
+
+        :return: A list of strings representing the names of singleton inputs.
+        :rtype: List[str]
+        """
+
+        call_signature = inspect.signature(self.__call__)
+        singletons = []
+        for param in call_signature.parameters:
+            if param not in self._not_singleton_inputs:
+                singletons.append(param)
+        return singletons
+
+    def _derive_conversation_converter(self) -> Callable[[Dict], List[DerivedEvalInput]]:
+        """Produce the function that will be used to convert conversations to a list of evaluable inputs.
+        This uses the inputs derived from the _derive_singleton_inputs function to determine which
+        aspects of a conversation ought to be extracted.
+
+        :return: The function that will be used to convert conversations to evaluable inputs.
+        :rtype: Callable
+        """
+        include_context = "context" in self._singleton_inputs
+        include_query = "query" in self._singleton_inputs
+        include_response = "response" in self._singleton_inputs
+
+        def converter(conversation: Dict) -> List[DerivedEvalInput]:
+            messages = cast(List[Dict[str, Any]], conversation["messages"])
+            global_context = conversation.get("context", None)
+            # Extract queries, responses from conversation
+            queries: List[Dict[str, Any]] = []
+            responses: List[Dict[str, Any]] = []
+
+            # Convert conversation slice into queries and responses.
+            # Assume that 'user' role is asking queries and 'assistant' role is responding.
+            if self._eval_last_turn and len(messages) > 1:
+                messages = messages[-2:]
+
+            for each_turn in messages:
+                role = each_turn["role"]
+                if role == "user":
+                    queries.append(each_turn)
+                elif role == "assistant":
+                    responses.append(each_turn)
+            # TODO complain if len(queries) != len(responses)?
+            eval_inputs = []
+            for query, response in zip(queries, responses):
+                context = {}
+                if include_context:
+                    query_context = query.get("context", None)
+                    response_context = response.get("context", None)
+                    if global_context:
+                        context["global_context"] = global_context
+                    if query_context and not include_query:
+                        context["query_context"] = query_context
+                    if response_context and not include_response:
+                        context["response_context"] = response_context
+
+                eval_input: DerivedEvalInput = {}
+                if include_query:
+                    eval_input["query"] = query
+                if include_response:
+                    eval_input["response"] = response
+                if include_context:
+                    eval_input["context"] = str(context)
+                eval_inputs.append(eval_input)
+            return eval_inputs
+
+        return converter
+
+    def _convert_kwargs_to_eval_input(self, **kwargs) -> Union[List[Dict], List[DerivedEvalInput]]:
+        """Convert an arbitrary input into a list of inputs for evaluators.
+        It is assumed that evaluators generally make use of their inputs in one of two ways.
+        Either they receive a collection of keyname inputs that are all single values
+        (like a query and response), or they receive conversation that iss a list of dictionary
+        values.
+
+        The self._singleton_inputs list assigned during initialization is used to find and extract
+        singleton keywords, and self._allow_converssation_input is used to determine if a conversation
+        is a valid input.
+
+        If both conversations and singletons are allowed, the function will raise an exception if both
+        are inputted.
+
+        This function must be overridden by child classes IF they need to both a conversation and
+        other inputs to be passed in.
+
+        :keyword kwargs: The inputs to convert.
+        :type kwargs: Dict
+        :return: A list of arbitrary values that are valid inputs for this evaluator's do_eval function.
+        :rtype: List
+        """
+
+        # Collect inputs
+        conversation = kwargs.get("conversation", None)
+        singletons = {}
+        if len(self._singleton_inputs) > 0:
+            singletons = {key: kwargs.get(key, None) for key in self._singleton_inputs}
+        # Check that both conversation and other inputs aren't set
+        if conversation is not None and any(singletons.values()):
+            raise EvaluationException(
+                message="Invalid input",
+                internal_message=f"Both conversation and individual inputs were provided to {type(self).__name__}",
+                blame=ErrorBlame.USER_ERROR,
+                category=ErrorCategory.INVALID_VALUE,
+                target=ErrorTarget.CONVERSATION,
+            )
+        # Handle Conversation
+        if conversation is not None:
+            return self._derive_conversation_converter()(conversation)
+        # Handle Singletons
+        if all(value is not None for value in singletons.values()):
+            return [singletons]  # TODO loosen requirements to allow for optional singletons?
+        # Missing input
+        raise EvaluationException(
+            message="Missing input",
+            internal_message=f"Neither conversation nor individual inputs provided to {type(self).__name__}.",
+            blame=ErrorBlame.USER_ERROR,
+            category=ErrorCategory.INVALID_VALUE,
+            target=ErrorTarget.CONVERSATION,
+        )
+
+    def _aggregate_results(self, per_turn_results: List[DoEvalResult[T_EvalValue]]) -> AggregateResult[T_EvalValue]:
+        """Aggregate the evaluation results of each conversation turn into a single result.
+
+        Exact implementation might need to vary slightly depending on the results produced.
+        Default behavior is to average the all number-based outputs.
+
+        :param per_turn_results: List of evaluation results for each turn in the conversation.
+        :type per_turn_results: List[Dict]
+        :return: A dictionary containing aggregated results, with numeric metrics having their
+        means as top-level values in the dictionary, and all original
+        values (including non-numerics) located in under the "evaluation_per_turn" key,
+        which each sub-key being a metric and each sub-value being a the list of that metric's
+        per-turn values.
+        :rtype: Dict
+        """
+
+        aggregated: Dict[str, Union[float, Dict[str, List[T_EvalValue]]]] = {}
+        evaluation_per_turn: Dict[str, List[T_EvalValue]] = {}
+
+        # Go over each turn, and rotate the results into a
+        # metric: List[values] format for the evals_per_turn dictionary.
+        for turn in per_turn_results:
+            for metric, value in turn.items():
+                if metric not in evaluation_per_turn:
+                    evaluation_per_turn[metric] = []
+                evaluation_per_turn[metric].append(value)
+
+        # Find and average all numeric values
+        for metric, values in evaluation_per_turn.items():
+            if all(isinstance(value, (int, float)) for value in values):
+                aggregated[metric] = list_mean(cast(List[Union[int, float]], values))
+        # Slap the per-turn results back in.
+        aggregated["evaluation_per_turn"] = evaluation_per_turn
+
+        return aggregated
+
+    async def _real_call(self, **kwargs) -> Union[DoEvalResult[T_EvalValue], AggregateResult[T_EvalValue]]:
+        """The asynchronous call where real end-to-end evaluation logic is performed.
+
+        :keyword kwargs: The inputs to evaluate.
+        :type kwargs: Dict
+        :return: The evaluation result.
+        :rtype: Dict
+        """
+        # Convert inputs into list of evaluable inputs.
+        eval_input_list = self._convert_kwargs_to_eval_input(**kwargs)
+        per_turn_results = []
+        # Evaluate all inputs.
+        for eval_input in eval_input_list:
+            per_turn_results.append(await self._do_eval(eval_input))
+        # Return results as-is if only one result was produced.
+
+        if len(per_turn_results) == 1:
+            return per_turn_results[0]
+        if len(per_turn_results) == 0:
+            return {}  # TODO raise something?
+        # Otherwise, aggregate results.
+        return self._aggregate_results(per_turn_results=per_turn_results)
+
+    @final
+    def _to_async(self) -> "AsyncEvaluatorBase":
+        return self._async_evaluator
+
+
+class AsyncEvaluatorBase:
+    """The asynchronous evaluator hidden underneath all evaluators. This makes generous use passing functions
+    to ensure that no one ever needs to extend or otherwise modify this class directly.
+    """
+
+    def __init__(self, real_call):  # DO NOT ADD TYPEHINT PROMPT FLOW WILL SCREAM AT YOU ABOUT META GENERATION
+        self._real_call = real_call
+
+    # Don't look at my shame. Nothing to see here....
+    # Oh, you're still here? Ok, the reason this has such a gross call signature and behavior is due
+    # to our broken async code not properly handling inputs; keyword arguments that aren't in the signature#
+    # are just not passed into this function instead of ending up in kwargs.
+    # Since we want this to be relatively call-agnostic, we just account for every input that any children
+    # are known to throw at this, mash them into kwargs, and then pass them into the real call.
+    async def __call__(self, *, query=None, response=None, context=None, conversation=None, **kwargs):
+        if conversation is not None:
+            kwargs["conversation"] = conversation
+        if query is not None:
+            kwargs["query"] = query
+        if response is not None:
+            kwargs["response"] = response
+        if context is not None:
+            kwargs["context"] = context
+        return await self._real_call(**kwargs)