azure-ai-evaluation 1.0.0b4__py3-none-any.whl → 1.0.0b5__py3-none-any.whl

azure/ai/evaluation/_evaluate/_utils.py

@@ -7,7 +7,9 @@ import os
 import re
 import tempfile
 from pathlib import Path
-from typing import Any, Dict, List, NamedTuple, Optional, Tuple, TypedDict, Union
+from typing import Any, Dict, NamedTuple, Optional, Tuple, Union
+import uuid
+import base64
 
 import pandas as pd
 from promptflow.client import PFClient
@@ -37,12 +39,6 @@ class AzureMLWorkspace(NamedTuple):
     workspace_name: str
 
 
-class EvaluateResult(TypedDict):
-    metrics: Dict[str, float]
-    studio_url: Optional[str]
-    rows: List[Dict]
-
-
 def is_none(value) -> bool:
     return value is None or str(value).lower() == "none"
 
@@ -87,6 +83,34 @@ def _azure_pf_client_and_triad(trace_destination) -> Tuple[PFClient, AzureMLWork
     return azure_pf_client, ws_triad
 
 
+def _store_multimodal_content(messages, tmpdir: str):
+    # verify if images folder exists
+    images_folder_path = os.path.join(tmpdir, "images")
+    os.makedirs(images_folder_path, exist_ok=True)
+
+    # traverse all messages and replace base64 image data with new file name.
+    for message in messages:
+        if isinstance(message.get("content", []), list):
+            for content in message.get("content", []):
+                if content.get("type") == "image_url":
+                    image_url = content.get("image_url")
+                    if image_url and "url" in image_url and image_url["url"].startswith("data:image/jpg;base64,"):
+                        # Extract the base64 string
+                        base64image = image_url["url"].replace("data:image/jpg;base64,", "")
+
+                        # Generate a unique filename
+                        image_file_name = f"{str(uuid.uuid4())}.jpg"
+                        image_url["url"] = f"images/{image_file_name}"  # Replace the base64 URL with the file path
+
+                        # Decode the base64 string to binary image data
+                        image_data_binary = base64.b64decode(base64image)
+
+                        # Write the binary image data to the file
+                        image_file_path = os.path.join(images_folder_path, image_file_name)
+                        with open(image_file_path, "wb") as f:
+                            f.write(image_data_binary)
+
+
 def _log_metrics_and_instance_results(
     metrics: Dict[str, Any],
     instance_results: pd.DataFrame,
@@ -116,6 +140,15 @@ def _log_metrics_and_instance_results(
         artifact_name = EvalRun.EVALUATION_ARTIFACT if run else EvalRun.EVALUATION_ARTIFACT_DUMMY_RUN
 
         with tempfile.TemporaryDirectory() as tmpdir:
+            # storing multi_modal images if exists
+            col_name = "inputs.conversation"
+            if col_name in instance_results.columns:
+                for item in instance_results[col_name].items():
+                    value = item[1]
+                    if "messages" in value:
+                        _store_multimodal_content(value["messages"], tmpdir)
+
+            # storing artifact result
             tmp_path = os.path.join(tmpdir, artifact_name)
 
             with open(tmp_path, "w", encoding=DefaultOpenEncoding.WRITE) as f:

azure/ai/evaluation/_evaluators/_bleu/_bleu.py

@@ -63,7 +63,7 @@ class BleuScoreEvaluator:
         :keyword ground_truth: The ground truth to be compared against.
         :paramtype ground_truth: str
         :return: The BLEU score.
-        :rtype: dict
+        :rtype: Dict[str, float]
         """
         return async_run_allowing_running_loop(
             self._async_evaluator, response=response, ground_truth=ground_truth, **kwargs

azure/ai/evaluation/_evaluators/_coherence/_coherence.py

@@ -31,18 +31,23 @@ class CoherenceEvaluator(PromptyEvaluatorBase):
     .. code-block:: python
 
         {
-            "gpt_coherence": 1.0
+            "coherence": 1.0,
+            "gpt_coherence": 1.0,
         }
+
+    Note: To align with our support of a diverse set of models, a key without the `gpt_` prefix has been added.
+    To maintain backwards compatibility, the old key with the `gpt_` prefix is still be present in the output;
+    however, it is recommended to use the new key moving forward as the old key will be deprecated in the future.
     """
 
-    PROMPTY_FILE = "coherence.prompty"
-    RESULT_KEY = "gpt_coherence"
+    _PROMPTY_FILE = "coherence.prompty"
+    _RESULT_KEY = "coherence"
 
     @override
-    def __init__(self, model_config: dict):
+    def __init__(self, model_config):
         current_dir = os.path.dirname(__file__)
-        prompty_path = os.path.join(current_dir, self.PROMPTY_FILE)
-        super().__init__(model_config=model_config, prompty_file=prompty_path, result_key=self.RESULT_KEY)
+        prompty_path = os.path.join(current_dir, self._PROMPTY_FILE)
+        super().__init__(model_config=model_config, prompty_file=prompty_path, result_key=self._RESULT_KEY)
 
     @override
     def __call__(
@@ -50,7 +55,7 @@ class CoherenceEvaluator(PromptyEvaluatorBase):
         *,
         query: Optional[str] = None,
         response: Optional[str] = None,
-        conversation: Optional[dict] = None,
+        conversation=None,
         **kwargs,
     ):
         """Evaluate coherence. Accepts either a query and response for a single evaluation,
@@ -64,8 +69,8 @@ class CoherenceEvaluator(PromptyEvaluatorBase):
         :keyword conversation: The conversation to evaluate. Expected to contain a list of conversation turns under the
             key "messages". Conversation turns are expected
             to be dictionaries with keys "content" and "role".
-        :paramtype conversation: Optional[Dict]
+        :paramtype conversation: Optional[~azure.ai.evaluation.Conversation]
         :return: The relevance score.
-        :rtype: Dict[str, float]
+        :rtype: Union[Dict[str, float], Dict[str, Union[float, Dict[str, List[float]]]]]
         """
         return super().__call__(query=query, response=response, conversation=conversation, **kwargs)

azure/ai/evaluation/_evaluators/_coherence/coherence.prompty

@@ -5,7 +5,7 @@ model:
   api: chat
   parameters:
     temperature: 0.0
-    max_tokens: 1
+    max_tokens: 800
     top_p: 1.0
     presence_penalty: 0
     frequency_penalty: 0
@@ -20,38 +20,80 @@ inputs:
 
 ---
 system:
-You are an AI assistant. You will be given the definition of an evaluation metric for assessing the quality of an answer in a question-answering task. Your job is to compute an accurate evaluation score using the provided evaluation metric. You should return a single integer value between 1 to 5 representing the evaluation metric. You will include no other text or information.
+# Instruction
+## Goal
+### You are an expert in evaluating the quality of a RESPONSE from an intelligent system based on provided definition and data. Your goal will involve answering the questions below using the information provided.
+- **Definition**: You are given a definition of the communication trait that is being evaluated to help guide your Score.
+- **Data**: Your input data include a QUERY and a RESPONSE.
+- **Tasks**: To complete your evaluation you will be asked to evaluate the Data in different ways.
 
 user:
-Coherence of an answer is measured by how well all the sentences fit together and sound naturally as a whole. Consider the overall quality of the answer when evaluating coherence. Given the question and answer, score the coherence of answer between one to five stars using the following rating scale:
-One star: the answer completely lacks coherence
-Two stars: the answer mostly lacks coherence
-Three stars: the answer is partially coherent
-Four stars: the answer is mostly coherent
-Five stars: the answer has perfect coherency
-
-This rating value should always be an integer between 1 and 5. So the rating produced should be 1 or 2 or 3 or 4 or 5.
-
-question: What is your favorite indoor activity and why do you enjoy it?
-answer: I like pizza. The sun is shining.
-stars: 1
-
-question: Can you describe your favorite movie without giving away any spoilers?
-answer: It is a science fiction movie. There are dinosaurs. The actors eat cake. People must stop the villain.
-stars: 2
-
-question: What are some benefits of regular exercise?
-answer: Regular exercise improves your mood. A good workout also helps you sleep better. Trees are green.
-stars: 3
-
-question: How do you cope with stress in your daily life?
-answer: I usually go for a walk to clear my head. Listening to music helps me relax as well. Stress is a part of life, but we can manage it through some activities.
-stars: 4
-
-question: What can you tell me about climate change and its effects on the environment?
-answer: Climate change has far-reaching effects on the environment. Rising temperatures result in the melting of polar ice caps, contributing to sea-level rise. Additionally, more frequent and severe weather events, such as hurricanes and heatwaves, can cause disruption to ecosystems and human societies alike.
-stars: 5
-
-question: {{query}}
-answer: {{response}}
-stars:
+# Definition
+**Coherence** refers to the logical and orderly presentation of ideas in a response, allowing the reader to easily follow and understand the writer's train of thought. A coherent answer directly addresses the question with clear connections between sentences and paragraphs, using appropriate transitions and a logical sequence of ideas.
+
+# Ratings
+## [Coherence: 1] (Incoherent Response)
+**Definition:** The response lacks coherence entirely. It consists of disjointed words or phrases that do not form complete or meaningful sentences. There is no logical connection to the question, making the response incomprehensible.
+
+**Examples:**
+  **Query:** What are the benefits of renewable energy?
+  **Response:** Wind sun green jump apple silence over.
+
+  **Query:** Explain the process of photosynthesis.
+  **Response:** Plants light water flying blue music.
+
+## [Coherence: 2] (Poorly Coherent Response)
+**Definition:** The response shows minimal coherence with fragmented sentences and limited connection to the question. It contains some relevant keywords but lacks logical structure and clear relationships between ideas, making the overall message difficult to understand.
+
+**Examples:**
+  **Query:** How does vaccination work?
+  **Response:** Vaccines protect disease. Immune system fight. Health better.
+
+  **Query:** Describe how a bill becomes a law.
+  **Response:** Idea proposed. Congress discuss vote. President signs.
+
+## [Coherence: 3] (Partially Coherent Response)
+**Definition:** The response partially addresses the question with some relevant information but exhibits issues in the logical flow and organization of ideas. Connections between sentences may be unclear or abrupt, requiring the reader to infer the links. The response may lack smooth transitions and may present ideas out of order.
+
+**Examples:**
+  **Query:** What causes earthquakes?
+  **Response:** Earthquakes happen when tectonic plates move suddenly. Energy builds up then releases. Ground shakes and can cause damage.
+
+  **Query:** Explain the importance of the water cycle.
+  **Response:** The water cycle moves water around Earth. Evaporation, then precipitation occurs. It supports life by distributing water.
+
+## [Coherence: 4] (Coherent Response)
+**Definition:** The response is coherent and effectively addresses the question. Ideas are logically organized with clear connections between sentences and paragraphs. Appropriate transitions are used to guide the reader through the response, which flows smoothly and is easy to follow.
+
+**Examples:**
+  **Query:** What is the water cycle and how does it work?
+  **Response:** The water cycle is the continuous movement of water on Earth through processes like evaporation, condensation, and precipitation. Water evaporates from bodies of water, forms clouds through condensation, and returns to the surface as precipitation. This cycle is essential for distributing water resources globally.
+
+  **Query:** Describe the role of mitochondria in cellular function.
+  **Response:** Mitochondria are organelles that produce energy for the cell. They convert nutrients into ATP through cellular respiration. This energy powers various cellular activities, making mitochondria vital for cell survival.
+
+## [Coherence: 5] (Highly Coherent Response)
+**Definition:** The response is exceptionally coherent, demonstrating sophisticated organization and flow. Ideas are presented in a logical and seamless manner, with excellent use of transitional phrases and cohesive devices. The connections between concepts are clear and enhance the reader's understanding. The response thoroughly addresses the question with clarity and precision.
+
+**Examples:**
+  **Query:** Analyze the economic impacts of climate change on coastal cities.
+  **Response:** Climate change significantly affects the economies of coastal cities through rising sea levels, increased flooding, and more intense storms. These environmental changes can damage infrastructure, disrupt businesses, and lead to costly repairs. For instance, frequent flooding can hinder transportation and commerce, while the threat of severe weather may deter investment and tourism. Consequently, cities may face increased expenses for disaster preparedness and mitigation efforts, straining municipal budgets and impacting economic growth.
+
+  **Query:** Discuss the significance of the Monroe Doctrine in shaping U.S. foreign policy.
+  **Response:** The Monroe Doctrine was a pivotal policy declared in 1823 that asserted U.S. opposition to European colonization in the Americas. By stating that any intervention by external powers in the Western Hemisphere would be viewed as a hostile act, it established the U.S. as a protector of the region. This doctrine shaped U.S. foreign policy by promoting isolation from European conflicts while justifying American influence and expansion in the hemisphere. Its long-term significance lies in its enduring influence on international relations and its role in defining the U.S. position in global affairs.
+
+
+# Data
+QUERY: {{query}}
+RESPONSE: {{response}}
+
+
+# Tasks
+## Please provide your assessment Score for the previous RESPONSE in relation to the QUERY based on the Definitions above. Your output should include the following information:
+- **ThoughtChain**: To improve the reasoning process, think step by step and include a step-by-step explanation of your thought process as you analyze the data based on the definitions. Keep it brief and start your ThoughtChain with "Let's think step by step:".
+- **Explanation**: a very short explanation of why you think the input Data should get that Score.
+- **Score**: based on your previous analysis, provide your Score. The Score you give MUST be a integer score (i.e., "1", "2"...) based on the levels of the definitions.
+
+
+## Please provide your answers between the tags: <S0>your chain of thoughts</S0>, <S1>your explanation</S1>, <S2>your Score</S2>.
+# Output

azure/ai/evaluation/_evaluators/_common/_base_eval.py

@@ -11,6 +11,7 @@ from typing_extensions import ParamSpec, TypeAlias
 
 from azure.ai.evaluation._common.math import list_mean
 from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
+from azure.ai.evaluation._common.utils import remove_optional_singletons
 
 P = ParamSpec("P")
 T = TypeVar("T")
@@ -32,9 +33,9 @@ AggregateResult: TypeAlias = Dict[str, Union[float, Dict[str, List[T]]]]
 
     foo: AggregateResult[float] = {
         "evaluation_per_turn": {
-            "gpt_coherence": [1.0, 2.0, 3.0]
+            "coherence": [1.0, 2.0, 3.0]
         },
-        "gpt_coherence": 2.0
+        "coherence": 2.0
     }
 """
 
@@ -44,7 +45,7 @@ DoEvalResult: TypeAlias = Dict[str, T]
     .. code-block:: python
 
     foo: DoEvalResult[float] = {
-        "gpt_coherence": 2.0
+        "coherence": 2.0
     }
 """
 
@@ -96,7 +97,7 @@ class EvaluatorBase(ABC, Generic[T_EvalValue]):
         :keyword kwargs: A dictionary that contains inputs needed to evaluate a conversation.
         :type kwargs: Dict
         :return: The evaluation result
-        :rtype: Dict
+        :rtype: Union[DoEvalResult[T_EvalValue], AggregateResult[T_EvalValue]]
         """
         return async_run_allowing_running_loop(self._async_evaluator, **kwargs)
 
@@ -110,7 +111,7 @@ class EvaluatorBase(ABC, Generic[T_EvalValue]):
         :param eval_input: Whatever inputs are needed for this evaluator to perform a single evaluation.
         :type eval_input: Any
         :return: A single evaluation result
-        :rtype: Dict
+        :rtype: DoEvalResult[T_EvalValue]
         """
 
     # ~~~ METHODS THAT MIGHT NEED TO BE OVERRIDDEN BY CHILDREN~~~
@@ -172,16 +173,16 @@ class EvaluatorBase(ABC, Generic[T_EvalValue]):
                     response_context = response.get("context", None)
                     if global_context:
                         context["global_context"] = global_context
-                    if query_context and not include_query:
+                    if query_context and include_query:
                         context["query_context"] = query_context
-                    if response_context and not include_response:
+                    if response_context and include_response:
                         context["response_context"] = response_context
 
                 eval_input: DerivedEvalInput = {}
                 if include_query:
-                    eval_input["query"] = query
+                    eval_input["query"] = query.get("content", "")
                 if include_response:
-                    eval_input["response"] = response
+                    eval_input["response"] = response.get("content", "")
                 if include_context:
                     eval_input["context"] = str(context)
                 eval_inputs.append(eval_input)
@@ -219,9 +220,9 @@ class EvaluatorBase(ABC, Generic[T_EvalValue]):
             singletons = {key: kwargs.get(key, None) for key in self._singleton_inputs}
         # Check that both conversation and other inputs aren't set
         if conversation is not None and any(singletons.values()):
+            msg = f"{type(self).__name__}: Cannot provide both 'conversation' and individual inputs at the same time."
             raise EvaluationException(
-                message="Invalid input",
-                internal_message=f"Both conversation and individual inputs were provided to {type(self).__name__}",
+                message=msg,
                 blame=ErrorBlame.USER_ERROR,
                 category=ErrorCategory.INVALID_VALUE,
                 target=ErrorTarget.CONVERSATION,
@@ -230,12 +231,13 @@ class EvaluatorBase(ABC, Generic[T_EvalValue]):
         if conversation is not None:
             return self._derive_conversation_converter()(conversation)
         # Handle Singletons
-        if all(value is not None for value in singletons.values()):
-            return [singletons]  # TODO loosen requirements to allow for optional singletons?
+        required_singletons = remove_optional_singletons(self, singletons)
+        if all(value is not None for value in required_singletons.values()):
+            return [singletons]
         # Missing input
+        msg = f"{type(self).__name__}: Either 'conversation' or individual inputs must be provided."
         raise EvaluationException(
-            message="Missing input",
-            internal_message=f"Neither conversation nor individual inputs provided to {type(self).__name__}.",
+            message=msg,
             blame=ErrorBlame.USER_ERROR,
             category=ErrorCategory.INVALID_VALUE,
             target=ErrorTarget.CONVERSATION,
@@ -254,7 +256,7 @@ class EvaluatorBase(ABC, Generic[T_EvalValue]):
         values (including non-numerics) located in under the "evaluation_per_turn" key,
         which each sub-key being a metric and each sub-value being a the list of that metric's
         per-turn values.
-        :rtype: Dict
+        :rtype: AggregateResult[T_EvalValue]
         """
 
         aggregated: Dict[str, Union[float, Dict[str, List[T_EvalValue]]]] = {}
@@ -274,7 +276,6 @@ class EvaluatorBase(ABC, Generic[T_EvalValue]):
                 aggregated[metric] = list_mean(cast(List[Union[int, float]], values))
         # Slap the per-turn results back in.
         aggregated["evaluation_per_turn"] = evaluation_per_turn
-
         return aggregated
 
     async def _real_call(self, **kwargs) -> Union[DoEvalResult[T_EvalValue], AggregateResult[T_EvalValue]]:
@@ -283,7 +284,7 @@ class EvaluatorBase(ABC, Generic[T_EvalValue]):
         :keyword kwargs: The inputs to evaluate.
         :type kwargs: Dict
         :return: The evaluation result.
-        :rtype: Dict
+        :rtype: Union[DoEvalResult[T_EvalValue], AggregateResult[T_EvalValue]]
         """
         # Convert inputs into list of evaluable inputs.
         eval_input_list = self._convert_kwargs_to_eval_input(**kwargs)
@@ -315,7 +316,7 @@ class AsyncEvaluatorBase:
 
     # Don't look at my shame. Nothing to see here....
     # Oh, you're still here? Ok, the reason this has such a gross call signature and behavior is due
-    # to our broken async code not properly handling inputs; keyword arguments that aren't in the signature#
+    # to our broken async code not properly handling inputs; keyword arguments that aren't in the signature
     # are just not passed into this function instead of ending up in kwargs.
     # Since we want this to be relatively call-agnostic, we just account for every input that any children
     # are known to throw at this, mash them into kwargs, and then pass them into the real call.

azure/ai/evaluation/_evaluators/_common/_base_prompty_eval.py

@@ -4,12 +4,13 @@
 
 import math
 import re
-from typing import Dict
+from typing import Dict, Union
 
 from promptflow.core import AsyncPrompty
 from typing_extensions import override
 
-from ..._common.utils import construct_prompty_model_config, validate_model_config
+from azure.ai.evaluation._common.constants import PROMPT_BASED_REASON_EVALUATORS
+from ..._common.utils import construct_prompty_model_config, validate_model_config, parse_quality_evaluator_reason_score
 from . import EvaluatorBase
 
 try:
@@ -36,8 +37,8 @@ class PromptyEvaluatorBase(EvaluatorBase[float]):
     :type ignore_queries: bool
     """
 
-    LLM_CALL_TIMEOUT = 600
-    DEFAULT_OPEN_API_VERSION = "2024-02-15-preview"
+    _LLM_CALL_TIMEOUT = 600
+    _DEFAULT_OPEN_API_VERSION = "2024-02-15-preview"
 
     def __init__(self, *, result_key: str, prompty_file: str, model_config: dict, eval_last_turn: bool = False):
         self._result_key = result_key
@@ -46,7 +47,7 @@ class PromptyEvaluatorBase(EvaluatorBase[float]):
 
         prompty_model_config = construct_prompty_model_config(
             validate_model_config(model_config),
-            self.DEFAULT_OPEN_API_VERSION,
+            self._DEFAULT_OPEN_API_VERSION,
             USER_AGENT,
         )
 
@@ -56,7 +57,7 @@ class PromptyEvaluatorBase(EvaluatorBase[float]):
     # defining a default here.
 
     @override
-    async def _do_eval(self, eval_input: Dict) -> Dict[str, float]:
+    async def _do_eval(self, eval_input: Dict) -> Dict[str, Union[float, str]]:  # type: ignore[override]
         """Do a relevance evaluation.
 
         :param eval_input: The input to the evaluator. Expected to contain
@@ -66,11 +67,20 @@ class PromptyEvaluatorBase(EvaluatorBase[float]):
         :return: The evaluation result.
         :rtype: Dict
         """
-        llm_output = await self._flow(timeout=self.LLM_CALL_TIMEOUT, **eval_input)
+        llm_output = await self._flow(timeout=self._LLM_CALL_TIMEOUT, **eval_input)
 
         score = math.nan
         if llm_output:
+            # Parse out score and reason from evaluators known to possess them.
+            if self._result_key in PROMPT_BASED_REASON_EVALUATORS:
+                score, reason = parse_quality_evaluator_reason_score(llm_output)
+                return {
+                    self._result_key: float(score),
+                    f"gpt_{self._result_key}": float(score),
+                    f"{self._result_key}_reason": reason,
+                }
             match = re.search(r"\d", llm_output)
             if match:
                 score = float(match.group())
-        return {self._result_key: float(score)}
+            return {self._result_key: float(score), f"gpt_{self._result_key}": float(score)}
+        return {self._result_key: float(score), f"gpt_{self._result_key}": float(score)}

azure/ai/evaluation/_evaluators/_common/_base_rai_svc_eval.py

@@ -5,7 +5,12 @@ from typing import Dict, Optional, Union
 
 from typing_extensions import override
 
-from azure.ai.evaluation._common.constants import EvaluationMetrics, _InternalEvaluationMetrics
+from azure.ai.evaluation._common.constants import (
+    EvaluationMetrics,
+    _InternalEvaluationMetrics,
+    Tasks,
+    _InternalAnnotationTasks,
+)
 from azure.ai.evaluation._common.rai_service import evaluate_with_rai_service
 from azure.ai.evaluation._common.utils import validate_azure_ai_project
 from azure.ai.evaluation._exceptions import EvaluationException
@@ -13,8 +18,10 @@ from azure.core.credentials import TokenCredential
 
 from . import EvaluatorBase
 
+T = Union[str, float]
 
-class RaiServiceEvaluatorBase(EvaluatorBase[Union[str, float]]):
+
+class RaiServiceEvaluatorBase(EvaluatorBase[T]):
     """Base class for all evaluators that require the use of the Azure AI RAI service for evaluation.
     This includes content safety evaluators, protected material evaluators, and others. These evaluators
     are all assumed to be of the "query and response or conversation" input variety.
@@ -48,7 +55,7 @@ class RaiServiceEvaluatorBase(EvaluatorBase[Union[str, float]]):
         *,
         query: Optional[str] = None,
         response: Optional[str] = None,
-        conversation: Optional[dict] = None,
+        conversation=None,
         **kwargs,
     ):
         """Evaluate either a query and response or a conversation. Must supply either a query AND response,
@@ -61,14 +68,13 @@ class RaiServiceEvaluatorBase(EvaluatorBase[Union[str, float]]):
         :keyword conversation: The conversation to evaluate. Expected to contain a list of conversation turns under the
             key "messages", and potentially a global context under the key "context". Conversation turns are expected
             to be dictionaries with keys "content", "role", and possibly "context".
-        :paramtype conversation: Optional[Dict]
-        :return: The evaluation result.
-        :rtype: Dict[str, Union[str, float]]
+        :paramtype conversation: Optional[~azure.ai.evaluation.Conversation]
+        :rtype: Union[Dict[str, T], Dict[str, Union[float, Dict[str, List[T]]]]]
         """
         return super().__call__(query=query, response=response, conversation=conversation, **kwargs)
 
     @override
-    async def _do_eval(self, eval_input: Dict) -> Dict[str, Union[str, float]]:
+    async def _do_eval(self, eval_input: Dict) -> Dict[str, T]:
         """Perform the evaluation using the Azure AI RAI service.
         The exact evaluation performed is determined by the evaluation metric supplied
         by the child class initializer.
@@ -88,10 +94,43 @@ class RaiServiceEvaluatorBase(EvaluatorBase[Union[str, float]]):
                     + " This should have failed earlier."
                 ),
             )
+        input_data = {"query": query, "response": response}
+
+        if "context" in self._singleton_inputs:
+            context = eval_input.get("context", None)
+            if context is None:
+                raise EvaluationException(
+                    message="Not implemented",
+                    internal_message=(
+                        "Attempted context-based evaluation without supplying context."
+                        + " This should have failed earlier."
+                    ),
+                )
+            input_data["context"] = context
+
         return await evaluate_with_rai_service(
             metric_name=self._eval_metric,
-            query=query,
-            response=response,
+            data=input_data,
             project_scope=self._azure_ai_project,
             credential=self._credential,
+            annotation_task=self._get_task(),
         )
+
+    def _get_task(self):
+        """Get the annotation task for the current evaluation metric.
+        The annotation task is used by the RAI service script to determine a the message format
+        of the API call, and how the output is processed, among other things.
+
+        :return: The annotation task for the evaluator's self._eval_metric value.
+        :rtype: ~azure.ai.evaluation._common.constants.Tasks
+
+        """
+        if self._eval_metric == EvaluationMetrics.GROUNDEDNESS:
+            return Tasks.GROUNDEDNESS
+        if self._eval_metric == EvaluationMetrics.XPIA:
+            return Tasks.XPIA
+        if self._eval_metric == _InternalEvaluationMetrics.ECI:
+            return _InternalAnnotationTasks.ECI
+        if self._eval_metric == EvaluationMetrics.PROTECTED_MATERIAL:
+            return Tasks.PROTECTED_MATERIAL
+        return Tasks.CONTENT_HARM