azure-ai-evaluation 0.0.0b0__py3-none-any.whl → 1.0.0__py3-none-any.whl

azure/ai/evaluation/_evaluate/_telemetry/__init__.py

@@ -0,0 +1,179 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+import functools
+import inspect
+import json
+import logging
+from typing import Callable, Dict, Literal, Optional, Union, cast
+
+import pandas as pd
+from promptflow._sdk.entities._flows import FlexFlow as flex_flow
+from promptflow._sdk.entities._flows import Prompty as prompty_sdk
+from promptflow._sdk.entities._flows.dag import Flow as dag_flow
+from promptflow.client import PFClient
+from promptflow.core import Prompty as prompty_core
+from typing_extensions import ParamSpec
+
+from azure.ai.evaluation._model_configurations import AzureAIProject, EvaluationResult
+
+from ..._user_agent import USER_AGENT
+from .._utils import _trace_destination_from_project_scope
+
+LOGGER = logging.getLogger(__name__)
+
+P = ParamSpec("P")
+
+
+def _get_evaluator_type(evaluator: Dict[str, Callable]) -> Literal["content-safety", "built-in", "custom"]:
+    """
+    Get evaluator type for telemetry.
+
+    :param evaluator: The evaluator object
+    :type evaluator: Dict[str, Callable]
+    :return: The evaluator type. Possible values are "built-in", "custom", and "content-safety".
+    :rtype: Literal["content-safety", "built-in", "custom"]
+    """
+    module = inspect.getmodule(evaluator)
+    module_name = module.__name__ if module else ""
+
+    built_in = module_name.startswith("azure.ai.evaluation._evaluators.")
+    content_safety = built_in and module_name.startswith("azure.ai.evaluation._evaluators._content_safety")
+
+    if content_safety:
+        return "content-safety"
+    if built_in:
+        return "built-in"
+    return "custom"
+
+
+def _get_evaluator_properties(evaluator, evaluator_name):
+    """
+    Get evaluator properties for telemetry.
+
+    :param: evaluator: The evaluator object
+    :param: evaluator_name: The alias for the evaluator
+    :type: str
+    :raises Exception: If the evaluator properties cannot be retrieved
+    :return: A dictionary containing the evaluator properties, including
+        "name": A name for the evaluator
+        "pf_type": The promptflow type being used
+        "type": The evaluator type. Accepted values are "built-in", "custom", and "content-safety"
+        "alias": The alias for the evaluator. Defaults to an empty string.
+    :rtype: Dict[str, str]
+    """
+
+    try:
+        # Cover flex flow and prompty based evaluator
+        if isinstance(evaluator, (prompty_sdk, prompty_core, flex_flow)):
+            name = evaluator.name
+            pf_type = evaluator.__class__.__name__
+        # Cover dag flow based evaluator
+        elif isinstance(evaluator, dag_flow):
+            name = evaluator.name
+            pf_type = "DagFlow"
+        elif inspect.isfunction(evaluator):
+            name = evaluator.__name__
+            pf_type = flex_flow.__name__
+        elif hasattr(evaluator, "__class__") and callable(evaluator):
+            name = evaluator.__class__.__name__
+            pf_type = flex_flow.__name__
+        else:
+            # fallback option
+            name = str(evaluator)
+            pf_type = "Unknown"
+    except Exception as e:  # pylint: disable=broad-exception-caught
+        LOGGER.debug("Failed to get evaluator properties: %s", e)
+        name = str(evaluator)
+        pf_type = "Unknown"
+
+    return {
+        "name": name,
+        "pf_type": pf_type,
+        "type": _get_evaluator_type(evaluator),
+        "alias": evaluator_name if evaluator_name else "",
+    }
+
+
+# cspell:ignore isna
+def log_evaluate_activity(func: Callable[P, EvaluationResult]) -> Callable[P, EvaluationResult]:
+    """Decorator to log evaluate activity
+
+    :param func: The function to be decorated
+    :type func: Callable
+    :returns: The decorated function
+    :rtype: Callable[P, EvaluationResult]
+    """
+
+    @functools.wraps(func)
+    def wrapper(*args: P.args, **kwargs: P.kwargs) -> EvaluationResult:
+        from promptflow._sdk._telemetry import ActivityType, log_activity
+        from promptflow._sdk._telemetry.telemetry import get_telemetry_logger
+
+        evaluators = cast(Optional[Dict[str, Callable]], kwargs.get("evaluators", {})) or {}
+        azure_ai_project = cast(Optional[AzureAIProject], kwargs.get("azure_ai_project", None))
+
+        pf_client = PFClient(
+            config=(
+                {"trace.destination": _trace_destination_from_project_scope(azure_ai_project)}
+                if azure_ai_project
+                else None
+            ),
+            user_agent=USER_AGENT,
+        )
+
+        trace_destination = pf_client._config.get_trace_destination()  # pylint: disable=protected-access
+        track_in_cloud = bool(trace_destination) if trace_destination != "none" else False
+        evaluate_target = bool(kwargs.get("target", None))
+        evaluator_config = bool(kwargs.get("evaluator_config", None))
+        custom_dimensions: Dict[str, Union[str, bool]] = {
+            "track_in_cloud": track_in_cloud,
+            "evaluate_target": evaluate_target,
+            "evaluator_config": evaluator_config,
+        }
+
+        with log_activity(
+            get_telemetry_logger(),
+            "pf.evals.evaluate",
+            activity_type=ActivityType.PUBLICAPI,
+            user_agent=USER_AGENT,
+            custom_dimensions=custom_dimensions,
+        ):
+            result = func(*args, **kwargs)
+
+            try:
+                evaluators_info = []
+                for evaluator_name, evaluator in evaluators.items():
+                    evaluator_info = _get_evaluator_properties(evaluator, evaluator_name)
+                    try:
+                        evaluator_df = pd.DataFrame(result.get("rows", [])).filter(
+                            like=f"outputs.{evaluator_name}", axis=1
+                        )
+
+                        failed_rows = (
+                            evaluator_df.shape[0] if evaluator_df.empty else int(evaluator_df.isna().any(axis=1).sum())
+                        )
+                        total_rows = evaluator_df.shape[0]
+
+                        evaluator_info["failed_rows"] = failed_rows
+                        evaluator_info["total_rows"] = total_rows
+                    except Exception as e:  # pylint: disable=broad-exception-caught
+                        LOGGER.debug("Failed to collect evaluate failed row info for %s: %s", evaluator_name, e)
+                    evaluators_info.append(evaluator_info)
+
+                custom_dimensions = {"evaluators_info": json.dumps(evaluators_info)}
+                with log_activity(
+                    get_telemetry_logger(),
+                    "pf.evals.evaluate_usage_info",
+                    activity_type=ActivityType.PUBLICAPI,
+                    user_agent=USER_AGENT,
+                    custom_dimensions=custom_dimensions,
+                ):
+                    pass
+            except Exception as e:  # pylint: disable=broad-exception-caught
+                LOGGER.debug("Failed to collect evaluate usage info: %s", e)
+
+            return result
+
+    return wrapper

azure/ai/evaluation/_evaluate/_utils.py

@@ -0,0 +1,298 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+import json
+import logging
+import os
+import re
+import tempfile
+from pathlib import Path
+from typing import Any, Dict, NamedTuple, Optional, Tuple, Union
+import uuid
+import base64
+
+import pandas as pd
+from promptflow.client import PFClient
+from promptflow.entities import Run
+
+from azure.ai.evaluation._constants import (
+    DEFAULT_EVALUATION_RESULTS_FILE_NAME,
+    DefaultOpenEncoding,
+    EvaluationRunProperties,
+    Prefixes,
+)
+from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
+from azure.ai.evaluation._model_configurations import AzureAIProject
+
+LOGGER = logging.getLogger(__name__)
+
+AZURE_WORKSPACE_REGEX_FORMAT = (
+    "^azureml:[/]{1,2}subscriptions/([^/]+)/resource(groups|Groups)/([^/]+)"
+    "(/providers/Microsoft.MachineLearningServices)?/workspaces/([^/]+)$"
+)
+
+
+class AzureMLWorkspace(NamedTuple):
+    subscription_id: str
+    resource_group_name: str
+    workspace_name: str
+
+
+def is_none(value) -> bool:
+    return value is None or str(value).lower() == "none"
+
+
+def extract_workspace_triad_from_trace_provider(  # pylint: disable=name-too-long
+    trace_provider: str,
+) -> AzureMLWorkspace:
+    match = re.match(AZURE_WORKSPACE_REGEX_FORMAT, trace_provider)
+    if not match or len(match.groups()) != 5:
+        raise EvaluationException(
+            message="Malformed trace provider string, expected azureml://subscriptions/<subscription_id>/"
+            "resourceGroups/<resource_group>/providers/Microsoft.MachineLearningServices/"
+            f"workspaces/<workspace_name>, got {trace_provider}",
+            internal_message="Malformed trace provider string, expected azureml://subscriptions/<subscription_id>/"
+            "resourceGroups/<resource_group>/providers/Microsoft.MachineLearningServices/"
+            "workspaces/<workspace_name>,",
+            target=ErrorTarget.UNKNOWN,
+            category=ErrorCategory.INVALID_VALUE,
+            blame=ErrorBlame.UNKNOWN,
+        )
+    subscription_id = match.group(1)
+    resource_group_name = match.group(3)
+    workspace_name = match.group(5)
+    return AzureMLWorkspace(subscription_id, resource_group_name, workspace_name)
+
+
+def load_jsonl(path):
+    with open(path, "r", encoding=DefaultOpenEncoding.READ) as f:
+        return [json.loads(line) for line in f.readlines()]
+
+
+def _azure_pf_client_and_triad(trace_destination) -> Tuple[PFClient, AzureMLWorkspace]:
+    from promptflow.azure._cli._utils import _get_azure_pf_client
+
+    ws_triad = extract_workspace_triad_from_trace_provider(trace_destination)
+    azure_pf_client = _get_azure_pf_client(
+        subscription_id=ws_triad.subscription_id,
+        resource_group=ws_triad.resource_group_name,
+        workspace_name=ws_triad.workspace_name,
+    )
+
+    return azure_pf_client, ws_triad
+
+
+def _store_multimodal_content(messages, tmpdir: str):
+    # verify if images folder exists
+    images_folder_path = os.path.join(tmpdir, "images")
+    os.makedirs(images_folder_path, exist_ok=True)
+
+    # traverse all messages and replace base64 image data with new file name.
+    for message in messages:
+        if isinstance(message.get("content", []), list):
+            for content in message.get("content", []):
+                if content.get("type") == "image_url":
+                    image_url = content.get("image_url")
+                    if image_url and "url" in image_url and image_url["url"].startswith("data:image/jpg;base64,"):
+                        # Extract the base64 string
+                        base64image = image_url["url"].replace("data:image/jpg;base64,", "")
+
+                        # Generate a unique filename
+                        image_file_name = f"{str(uuid.uuid4())}.jpg"
+                        image_url["url"] = f"images/{image_file_name}"  # Replace the base64 URL with the file path
+
+                        # Decode the base64 string to binary image data
+                        image_data_binary = base64.b64decode(base64image)
+
+                        # Write the binary image data to the file
+                        image_file_path = os.path.join(images_folder_path, image_file_name)
+                        with open(image_file_path, "wb") as f:
+                            f.write(image_data_binary)
+
+
+def _log_metrics_and_instance_results(
+    metrics: Dict[str, Any],
+    instance_results: pd.DataFrame,
+    trace_destination: Optional[str],
+    run: Run,
+    evaluation_name: Optional[str],
+) -> Optional[str]:
+    from azure.ai.evaluation._evaluate._eval_run import EvalRun
+
+    if trace_destination is None:
+        LOGGER.debug("Skip uploading evaluation results to AI Studio since no trace destination was provided.")
+        return None
+
+    azure_pf_client, ws_triad = _azure_pf_client_and_triad(trace_destination)
+    tracking_uri = azure_pf_client.ml_client.workspaces.get(ws_triad.workspace_name).mlflow_tracking_uri
+
+    # Adding line_number as index column this is needed by UI to form link to individual instance run
+    instance_results["line_number"] = instance_results.index.values
+
+    with EvalRun(
+        run_name=run.name if run is not None else evaluation_name,
+        tracking_uri=tracking_uri,
+        subscription_id=ws_triad.subscription_id,
+        group_name=ws_triad.resource_group_name,
+        workspace_name=ws_triad.workspace_name,
+        ml_client=azure_pf_client.ml_client,
+        promptflow_run=run,
+    ) as ev_run:
+        artifact_name = EvalRun.EVALUATION_ARTIFACT
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            # storing multi_modal images if exists
+            col_name = "inputs.conversation"
+            if col_name in instance_results.columns:
+                for item in instance_results[col_name].items():
+                    value = item[1]
+                    if "messages" in value:
+                        _store_multimodal_content(value["messages"], tmpdir)
+
+            # storing artifact result
+            tmp_path = os.path.join(tmpdir, artifact_name)
+
+            with open(tmp_path, "w", encoding=DefaultOpenEncoding.WRITE) as f:
+                f.write(instance_results.to_json(orient="records", lines=True))
+
+            ev_run.log_artifact(tmpdir, artifact_name)
+
+            # Using mlflow to create a dummy run since once created via PF show traces of dummy run in UI.
+            # Those traces can be confusing.
+            # adding these properties to avoid showing traces if a dummy run is created.
+            # We are doing that only for the pure evaluation runs.
+            if run is None:
+                ev_run.write_properties_to_run_history(
+                    properties={
+                        EvaluationRunProperties.RUN_TYPE: "eval_run",
+                        EvaluationRunProperties.EVALUATION_RUN: "promptflow.BatchRun",
+                        "_azureml.evaluate_artifacts": json.dumps([{"path": artifact_name, "type": "table"}]),
+                    }
+                )
+
+        for metric_name, metric_value in metrics.items():
+            ev_run.log_metric(metric_name, metric_value)
+
+    evaluation_id = ev_run.info.run_name if run is not None else ev_run.info.run_id
+    return _get_ai_studio_url(trace_destination=trace_destination, evaluation_id=evaluation_id)
+
+
+def _get_ai_studio_url(trace_destination: str, evaluation_id: str) -> str:
+    ws_triad = extract_workspace_triad_from_trace_provider(trace_destination)
+    studio_base_url = os.getenv("AI_STUDIO_BASE_URL", "https://ai.azure.com")
+
+    studio_url = (
+        f"{studio_base_url}/build/evaluation/{evaluation_id}?wsid=/subscriptions/{ws_triad.subscription_id}"
+        f"/resourceGroups/{ws_triad.resource_group_name}/providers/Microsoft.MachineLearningServices/"
+        f"workspaces/{ws_triad.workspace_name}"
+    )
+
+    return studio_url
+
+
+def _trace_destination_from_project_scope(project_scope: AzureAIProject) -> str:
+    subscription_id = project_scope["subscription_id"]
+    resource_group_name = project_scope["resource_group_name"]
+    workspace_name = project_scope["project_name"]
+
+    trace_destination = (
+        f"azureml://subscriptions/{subscription_id}/resourceGroups/{resource_group_name}/"
+        f"providers/Microsoft.MachineLearningServices/workspaces/{workspace_name}"
+    )
+
+    return trace_destination
+
+
+def _write_output(path: Union[str, os.PathLike], data_dict: Any) -> None:
+    p = Path(path)
+    if p.is_dir():
+        p = p / DEFAULT_EVALUATION_RESULTS_FILE_NAME
+
+    with open(p, "w", encoding=DefaultOpenEncoding.WRITE) as f:
+        json.dump(data_dict, f)
+
+    print(f'Evaluation results saved to "{p.resolve()}".\n')
+
+
+def _apply_column_mapping(
+    source_df: pd.DataFrame, mapping_config: Optional[Dict[str, str]], inplace: bool = False
+) -> pd.DataFrame:
+    """
+    Apply column mapping to source_df based on mapping_config.
+
+    This function is used for pre-validation of input data for evaluators
+    :param source_df: the data frame to be changed.
+    :type source_df: pd.DataFrame
+    :param mapping_config: The configuration, containing column mapping.
+    :type mapping_config: Dict[str, str].
+    :param inplace: If true, the source_df will be changed inplace.
+    :type inplace: bool
+    :return: The modified data frame.
+    :rtype: pd.DataFrame
+    """
+    result_df = source_df
+
+    if mapping_config:
+        column_mapping = {}
+        columns_to_drop = set()
+        pattern_prefix = "data."
+        run_outputs_prefix = "run.outputs."
+
+        for map_to_key, map_value in mapping_config.items():
+            match = re.search(r"^\${([^{}]+)}$", map_value)
+            if match is not None:
+                pattern = match.group(1)
+                if pattern.startswith(pattern_prefix):
+                    map_from_key = pattern[len(pattern_prefix) :]
+                elif pattern.startswith(run_outputs_prefix):
+                    # Target-generated columns always starts from .outputs.
+                    map_from_key = f"{Prefixes.TSG_OUTPUTS}{pattern[len(run_outputs_prefix) :]}"
+                # if we are not renaming anything, skip.
+                if map_from_key == map_to_key:
+                    continue
+                # If column needs to be mapped to already existing column, we will add it
+                # to the drop list.
+                if map_to_key in source_df.columns:
+                    columns_to_drop.add(map_to_key)
+                column_mapping[map_from_key] = map_to_key
+        # If we map column to another one, which is already present in the data
+        # set and the letter also needs to be mapped, we will not drop it, but map
+        # instead.
+        columns_to_drop = columns_to_drop - set(column_mapping.keys())
+        result_df = source_df.drop(columns=columns_to_drop, inplace=inplace)
+        result_df.rename(columns=column_mapping, inplace=True)
+
+    return result_df
+
+
+def _has_aggregator(evaluator: object) -> bool:
+    return hasattr(evaluator, "__aggregate__")
+
+
+def get_int_env_var(env_var_name: str, default_value: int) -> int:
+    """
+    The function `get_int_env_var` retrieves an integer environment variable value, with a
+    default value if the variable is not set or cannot be converted to an integer.
+
+    :param env_var_name: The name of the environment variable you want to retrieve the value of
+    :type env_var_name: str
+    :param default_value: The default value is the value that will be returned if the environment
+        variable is not found or if it cannot be converted to an integer
+    :type default_value: int
+    :return: an integer value.
+    :rtype: int
+    """
+    try:
+        return int(os.environ[env_var_name])
+    except (ValueError, KeyError):
+        return default_value
+
+
+def set_event_loop_policy() -> None:
+    import asyncio
+    import platform
+
+    if platform.system().lower() == "windows":
+        # Reference: https://stackoverflow.com/questions/45600579/asyncio-event-loop-is-closed-when-getting-loop
+        # On Windows seems to be a problem with EventLoopPolicy, use this snippet to work around it
+        asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())  # type: ignore[attr-defined]

azure/ai/evaluation/_evaluators/__init__.py

@@ -0,0 +1,3 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------

azure/ai/evaluation/_evaluators/_bleu/__init__.py

@@ -0,0 +1,9 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from ._bleu import BleuScoreEvaluator
+
+__all__ = [
+    "BleuScoreEvaluator",
+]

azure/ai/evaluation/_evaluators/_bleu/_bleu.py

@@ -0,0 +1,72 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+from nltk.translate.bleu_score import SmoothingFunction, sentence_bleu
+from promptflow._utils.async_utils import async_run_allowing_running_loop
+
+from azure.ai.evaluation._common.utils import nltk_tokenize
+
+
+class _AsyncBleuScoreEvaluator:
+    def __init__(self):
+        pass
+
+    async def __call__(self, *, response: str, ground_truth: str, **kwargs):
+        reference_tokens = nltk_tokenize(ground_truth)
+        hypothesis_tokens = nltk_tokenize(response)
+
+        # NIST Smoothing
+        smoothing_function = SmoothingFunction().method4
+        score = sentence_bleu([reference_tokens], hypothesis_tokens, smoothing_function=smoothing_function)
+
+        return {
+            "bleu_score": score,
+        }
+
+
+class BleuScoreEvaluator:
+    """
+    Calculate the BLEU score for a given response and ground truth.
+
+    BLEU (Bilingual Evaluation Understudy) score is commonly used in natural language processing (NLP) and machine
+    translation. It is widely used in text summarization and text generation use cases.
+
+    Use the BLEU score when you want to evaluate the similarity between the generated text and reference text,
+    especially in tasks such as machine translation or text summarization, where n-gram overlap is a significant
+    indicator of quality.
+
+    The BLEU score ranges from 0 to 1, with higher scores indicating better quality.
+
+    .. admonition:: Example:
+
+        .. literalinclude:: ../samples/evaluation_samples_evaluate.py
+            :start-after: [START bleu_score_evaluator]
+            :end-before: [END bleu_score_evaluator]
+            :language: python
+            :dedent: 8
+            :caption: Initialize and call an BleuScoreEvaluator.
+    """
+
+    id = "azureml://registries/azureml/models/Bleu-Score-Evaluator/versions/3"
+    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+
+    def __init__(self):
+        self._async_evaluator = _AsyncBleuScoreEvaluator()
+
+    def __call__(self, *, response: str, ground_truth: str, **kwargs):
+        """
+        Evaluate the BLEU score between the response and the ground truth.
+
+        :keyword response: The response to be evaluated.
+        :paramtype response: str
+        :keyword ground_truth: The ground truth to be compared against.
+        :paramtype ground_truth: str
+        :return: The BLEU score.
+        :rtype: Dict[str, float]
+        """
+        return async_run_allowing_running_loop(
+            self._async_evaluator, response=response, ground_truth=ground_truth, **kwargs
+        )
+
+    def _to_async(self):
+        return self._async_evaluator

azure/ai/evaluation/_evaluators/_coherence/__init__.py

@@ -0,0 +1,7 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from ._coherence import CoherenceEvaluator
+
+__all__ = ["CoherenceEvaluator"]

azure/ai/evaluation/_evaluators/_coherence/_coherence.py

@@ -0,0 +1,107 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+import os
+from typing import Dict, Union, List
+
+from typing_extensions import overload, override
+
+from azure.ai.evaluation._evaluators._common import PromptyEvaluatorBase
+from azure.ai.evaluation._model_configurations import Conversation
+
+
+class CoherenceEvaluator(PromptyEvaluatorBase[Union[str, float]]):
+    """
+    Evaluates coherence score for a given query and response or a multi-turn conversation, including reasoning.
+
+    The coherence measure assesses the ability of the language model to generate text that reads naturally,
+    flows smoothly, and resembles human-like language in its responses. Use it when assessing the readability
+    and user-friendliness of a model's generated responses in real-world applications.
+
+    :param model_config: Configuration for the Azure OpenAI model.
+    :type model_config: Union[~azure.ai.evaluation.AzureOpenAIModelConfiguration,
+        ~azure.ai.evaluation.OpenAIModelConfiguration]
+
+    .. admonition:: Example:
+
+        .. literalinclude:: ../samples/evaluation_samples_evaluate.py
+            :start-after: [START coherence_evaluator]
+            :end-before: [END coherence_evaluator]
+            :language: python
+            :dedent: 8
+            :caption: Initialize and call a CoherenceEvaluator with a query and response.
+
+    .. note::
+
+        To align with our support of a diverse set of models, an output key without the `gpt_` prefix has been added.
+        To maintain backwards compatibility, the old key with the `gpt_` prefix is still be present in the output;
+        however, it is recommended to use the new key moving forward as the old key will be deprecated in the future.
+    """
+
+    _PROMPTY_FILE = "coherence.prompty"
+    _RESULT_KEY = "coherence"
+
+    id = "azureml://registries/azureml/models/Coherence-Evaluator/versions/4"
+    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+
+    @override
+    def __init__(self, model_config):
+        current_dir = os.path.dirname(__file__)
+        prompty_path = os.path.join(current_dir, self._PROMPTY_FILE)
+        super().__init__(model_config=model_config, prompty_file=prompty_path, result_key=self._RESULT_KEY)
+
+    @overload
+    def __call__(
+        self,
+        *,
+        query: str,
+        response: str,
+    ) -> Dict[str, Union[str, float]]:
+        """Evaluate coherence for given input of query, response
+
+        :keyword query: The query to be evaluated.
+        :paramtype query: str
+        :keyword response: The response to be evaluated.
+        :paramtype response: str
+        :return: The coherence score.
+        :rtype: Dict[str, float]
+        """
+
+    @overload
+    def __call__(
+        self,
+        *,
+        conversation: Conversation,
+    ) -> Dict[str, Union[float, Dict[str, List[Union[str, float]]]]]:
+        """Evaluate coherence for a conversation
+
+        :keyword conversation: The conversation to evaluate. Expected to contain a list of conversation turns under the
+            key "messages", and potentially a global context under the key "context". Conversation turns are expected
+            to be dictionaries with keys "content", "role", and possibly "context".
+        :paramtype conversation: Optional[~azure.ai.evaluation.Conversation]
+        :return: The coherence score.
+        :rtype: Dict[str, Union[float, Dict[str, List[float]]]]
+        """
+
+    @override
+    def __call__(  # pylint: disable=docstring-missing-param
+        self,
+        *args,
+        **kwargs,
+    ):
+        """Evaluate coherence. Accepts either a query and response for a single evaluation,
+        or a conversation for a potentially multi-turn evaluation. If the conversation has more than one pair of
+        turns, the evaluator will aggregate the results of each turn.
+
+        :keyword query: The query to be evaluated.
+        :paramtype query: str
+        :keyword response: The response to be evaluated.
+        :paramtype response: Optional[str]
+        :keyword conversation: The conversation to evaluate. Expected to contain a list of conversation turns under the
+            key "messages". Conversation turns are expected
+            to be dictionaries with keys "content" and "role".
+        :paramtype conversation: Optional[~azure.ai.evaluation.Conversation]
+        :return: The relevance score.
+        :rtype: Union[Dict[str, float], Dict[str, Union[float, Dict[str, List[float]]]]]
+        """
+        return super().__call__(*args, **kwargs)