azure-ai-evaluation 1.8.0__py3-none-any.whl → 1.10.0__py3-none-any.whl

azure/ai/evaluation/_evaluators/_intent_resolution/_intent_resolution.py

@@ -3,6 +3,7 @@
 # ---------------------------------------------------------
 import os
 import math
+import logging
 from typing import Dict, Union, List, Optional
 
 from typing_extensions import overload, override
@@ -10,9 +11,12 @@ from typing_extensions import overload, override
 from azure.ai.evaluation._exceptions import EvaluationException, ErrorBlame, ErrorCategory, ErrorTarget
 from azure.ai.evaluation._evaluators._common import PromptyEvaluatorBase
 from azure.ai.evaluation._model_configurations import Conversation, Message
-from ..._common.utils import check_score_is_valid
+from ..._common.utils import check_score_is_valid, reformat_conversation_history, reformat_agent_response
 from azure.ai.evaluation._common._experimental import experimental
 
+logger = logging.getLogger(__name__)
+
+
 @experimental
 class IntentResolutionEvaluator(PromptyEvaluatorBase[Union[str, float]]):
     """
@@ -34,13 +38,13 @@ class IntentResolutionEvaluator(PromptyEvaluatorBase[Union[str, float]]):
             :caption: Initialize and call an IntentResolutionEvaluator with a query and response.
 
     .. admonition:: Example using Azure AI Project URL:
-                
+
         .. literalinclude:: ../samples/evaluation_samples_evaluate_fdp.py
             :start-after: [START intent_resolution_evaluator]
             :end-before: [END intent_resolution_evaluator]
             :language: python
             :dedent: 8
-            :caption: Initialize and call IntentResolutionEvaluator using Azure AI Project URL in the following format 
+            :caption: Initialize and call IntentResolutionEvaluator using Azure AI Project URL in the following format
                 https://{resource_name}.services.ai.azure.com/api/projects/{project_name}
 
     """
@@ -53,27 +57,23 @@ class IntentResolutionEvaluator(PromptyEvaluatorBase[Union[str, float]]):
     _MAX_INTENT_RESOLUTION_SCORE = 5
     _DEFAULT_INTENT_RESOLUTION_THRESHOLD = 3
 
-    id = None
+    id = "azureai://built-in/evaluators/intent_resolution"
     """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
 
     @override
-    def __init__(self, model_config, *,
-                 threshold = _DEFAULT_INTENT_RESOLUTION_THRESHOLD,
-                 **kwargs):
+    def __init__(self, model_config, *, threshold=_DEFAULT_INTENT_RESOLUTION_THRESHOLD, **kwargs):
         current_dir = os.path.dirname(__file__)
         prompty_path = os.path.join(current_dir, self._PROMPTY_FILE)
         self.threshold = threshold
-        super().__init__(model_config=model_config, prompty_file=prompty_path,
-                         result_key=self._RESULT_KEY,
-                         **kwargs)
+        super().__init__(model_config=model_config, prompty_file=prompty_path, result_key=self._RESULT_KEY, **kwargs)
 
     @overload
     def __call__(
         self,
         *,
-        query            : Union[str, List[dict]],
-        response         : Union[str, List[dict]],
-        tool_definitions : Optional[Union[dict, List[dict]]] = None,
+        query: Union[str, List[dict]],
+        response: Union[str, List[dict]],
+        tool_definitions: Optional[Union[dict, List[dict]]] = None,
     ) -> Dict[str, Union[str, float]]:
         """Evaluate intent resolution for a given query, response and optional tool definitions.
         The query and response can be either a string or a list of messages.
@@ -135,11 +135,19 @@ class IntentResolutionEvaluator(PromptyEvaluatorBase[Union[str, float]]):
                 category=ErrorCategory.MISSING_FIELD,
                 target=ErrorTarget.INTENT_RESOLUTION_EVALUATOR,
             )
+        # reformat query and response to the format expected by the prompty flow
+        eval_input["query"] = reformat_conversation_history(eval_input["query"], logger)
+        eval_input["response"] = reformat_agent_response(eval_input["response"], logger)
+
         llm_output = await self._flow(timeout=self._LLM_CALL_TIMEOUT, **eval_input)
         # llm_output should always be a dictionary because the response_format of prompty is set to json_object, but checking anyway
         if isinstance(llm_output, dict):
-            score  = llm_output.get("resolution_score", math.nan)
-            if not check_score_is_valid(score, IntentResolutionEvaluator._MIN_INTENT_RESOLUTION_SCORE, IntentResolutionEvaluator._MAX_INTENT_RESOLUTION_SCORE):
+            score = llm_output.get("score", math.nan)
+            if not check_score_is_valid(
+                score,
+                IntentResolutionEvaluator._MIN_INTENT_RESOLUTION_SCORE,
+                IntentResolutionEvaluator._MAX_INTENT_RESOLUTION_SCORE,
+            ):
                 raise EvaluationException(
                     message=f"Invalid score value: {score}. Expected a number in range [{IntentResolutionEvaluator._MIN_INTENT_RESOLUTION_SCORE}, {IntentResolutionEvaluator._MAX_INTENT_RESOLUTION_SCORE}].",
                     internal_message="Invalid score value.",
@@ -148,19 +156,16 @@ class IntentResolutionEvaluator(PromptyEvaluatorBase[Union[str, float]]):
                 )
             reason = llm_output.get("explanation", "")
             score = float(score)
-            score_result = 'pass' if score >= self.threshold else 'fail'
-
-            #remove fields 'explanation' and 'resolution_score' from llm_output as they are already included in the response_dict
-            if 'explanation' in llm_output: llm_output.pop("explanation")
-            if 'resolution_score' in llm_output: llm_output.pop("resolution_score")
+            score_result = "pass" if score >= self.threshold else "fail"
 
             response_dict = {
-                             f"{self._result_key}"           : score,
-                             f"{self._result_key}_result"    : score_result,
-                             f"{self._result_key}_threshold" : self.threshold,
-                             f"{self._result_key}_reason"    : reason,
-                             f"additional_details"           : llm_output
-                        }
+                f"{self._result_key}": score,
+                f"{self._result_key}_result": score_result,
+                f"{self._result_key}_threshold": self.threshold,
+                f"{self._result_key}_reason": reason,
+            }
             return response_dict
         # If llm_output is not a dictionary, return NaN for the score. This should never happen
+        if logger:
+            logger.warning("LLM output is not a dictionary, returning NaN for the score.")
         return {self._result_key: math.nan}

azure/ai/evaluation/_evaluators/_intent_resolution/intent_resolution.prompty

@@ -23,139 +23,253 @@ inputs:
     default: "[]"
 ---
 system:
-You are an expert in evaluating the quality of a RESPONSE from an intelligent assistant based on provided definition and Data.
+You are an expert in evaluating the quality of a AGENT_RESPONSE from an intelligent assistant based on provided definition and CONVERSATION_HISTORY.
 
 user:
-# Goal
-Your goal is to assess the quality of the RESPONSE of an assistant in relation to a QUERY from a user, specifically focusing on
-the assistant's ability to understand and resolve the user intent expressed in the QUERY. There is also a field for tool definitions
-describing the functions, if any, that are accessible to the agent and that the agent may invoke in the RESPONSE if necessary.
+ROLE
+====
+You are Intent-Resolution-Judge, an impartial grader that scores how well an AI agent *resolved* the user's intent in a multi-turn conversation.
+You are NOT grading intent recognition. Assume the agent has understood the intent that is expressed; you only judge whether the reply satisfies or completes that intent.
 
-There are two components to intent resolution:
-    - Intent Understanding: The extent to which the agent accurately discerns the user's underlying need or inquiry.
-    - Response Resolution: The degree to which the agent's response is comprehensive, relevant, and adequately addresses the user's request.
 
-Note that the QUERY can either be a string with a user request or an entire conversation history including previous requests and responses from the assistant.
-In this case, the assistant's response should be evaluated in the context of the entire conversation but the focus should be on the last intent.
+INPUT
+=====
+CONVERSATION_HISTORY: {{query}}
+AGENT_RESPONSE: {{response}}
 
-# Data
-QUERY: {{query}}
-RESPONSE: {{response}}
-TOOL_DEFINITIONS: {{tool_definitions}}
+CONVERSATION_HISTORY is the full dialogue between the user and the agent up to the user's latest message.
+AGENT_RESPONSE is the agent reply to that latest user message.
 
 
-# Ratings
-## [Score: 1] (Response completely unrelated to user intent)
-**Definition:** The agent's response does not address the query at all.
+TASK
+====
+Output a JSON object with:
+    1) a concise explanation of 15-60 words that summarizes the agent's performance in resolving the user's intent
+    2) an integer score from 1 (very poor) to 5 (excellent) on how well the agent resolved the user's intent.
 
-**Example:**
-  **Query:** How do I bake a chocolate cake?
-  **Response:** The latest smartphone models have incredible features and performance.
-  **Tool Definitions:** []
+The explanation should always precede the score and should clearly justify the score based on the agent's performance in resolving the user's intent.
+Response format exactly as follows:
 
-**Expected output**
 {
-    "explanation": "The agent's response is entirely off-topic, discussing smartphones instead of providing any information about baking a chocolate cake."
-    "conversation_has_intent": true,
-    "agent_perceived_intent": "discussion about smartphone features",
-    "actual_user_intent": "bake a chocolate cake",
-    "correct_intent_detected": false,
-    "intent_resolved": false,
-    "resolution_score": 1,
+  "explanation": "<15-60 words>",
+  "score": <1-5>
 }
 
+EVALUATION STEPS
+================
 
-## [Score: 2] (Response minimally relates to user intent)
-**Definition:** The response shows a token attempt to address the query by mentioning a relevant keyword or concept, but it provides almost no useful or actionable information.
+A. Identify the expressed intent in the final user turn (look at the full conversation_history for context if necessary).
+B. Check resolution - Does the agent's reply, in AGENT_RESPONSE, actually complete or satisfy that intent? 
+    - If the agent's response is a direct answer, does it fully address the user's request?
+    - If the agent's response is an action (like scheduling, deleting, etc.), does it confirm completion of that action?
+    - If the agent's response is a clarification or follow-up question, does it lead towards fulfilling the intent?
+    - If the agent response is empty or irrelevant, it does not resolve the intent and should be scored accordingly.
+C. Verify correctness & completeness of the resolution.
+D. Weigh impact - Minor style issues matter only for tie-breaking; resolution quality dominates.
+E. Write a concise explanation of 15-60 words that summarizes the agent's performance in resolving the user's intent, including:
+    - What the user wanted
+    - How well the agent addressed it
+    - Any notable gaps or strengths
+F. Choose the single closest integer score (1,2,3,4 or 5) from the scoring rubric below.
 
-**Example input:**
-  **Query:** How do I bake a chocolate cake?
-  **Response:** Chocolate cake involves some ingredients.
-  **Tool Definitions:** []
+SCORING RUBRIC
+   5 - Excellent: Agent fully fulfills the intent with accurate, thorough, relevant action.
+   4 - Good: Intent mostly resolved; only minor, non-blocking gaps or imprecision.
+   3 - Adequate: Core of intent addressed but notable omissions, vagueness, or slips.
+   2 - Poor: Partial or incorrect resolution; answer unlikely to satisfy the intent.
+   1 - Very poor: Intent ignored or badly misunderstood; irrelevant or harmful response.
 
-**Expected output**
+
+EXAMPLES
+========
+
+### Score 5 - Example A (single-turn)
+
+CONVERSATION_HISTORY:
+User turn 1:
+  Give me a Python one-liner to reverse a string.
+
+AGENT_RESPONSE:
+  `s[::-1]` - this slice returns the string in reverse.
+
+EXPECTED OUTPUT:
+{
+  "explanation": "User wanted a Python one-liner to reverse a string. Agent provided the correct code and a brief, helpful explanation, fully resolving the request with notable conciseness and accuracy.",
+  "score": 5
+}
+
+
+### Score 5 - Example B (multi-turn)
+
+CONVERSATION_HISTORY:
+User turn 1:
+  I need to cancel my 2 pm meeting with Dr. Lee this Friday.
+
+Agent turn 1:
+  To confirm, that's Friday at 14:00 with Dr. Lee in Meeting Room 2, right?
+
+User turn 2:
+  Yes, that one.
+
+AGENT_RESPONSE:
+  Done! The event has been removed from your calendar and a cancellation email was sent to Dr. Lee.
+
+EXPECTED OUTPUT:
+{
+  "explanation": "User wanted to cancel a meeting. Agent first confirmed details, then successfully completed the cancellation and notified the attendee, fully resolving the intent with excellent clarity and thoroughness.",
+  "score": 5
+}
+
+
+### Score 4 - Example A (single-turn)
+
+CONVERSATION_HISTORY:
+User turn 1:
+  Translate 'carpe diem' to Portuguese and explain it in one sentence.
+
+AGENT_RESPONSE:
+  'Aproveita o dia.' Significa que deves aproveitar o momento presente.
+
+EXPECTED OUTPUT:
+{
+  "explanation": "User wanted a Portuguese translation of 'carpe diem' and a one-sentence explanation. Agent provided an accurate translation and a concise explanation, mostly resolving the intent adequately.",
+  "score": 4
+}
+
+
+### Score 4 - Example B (multi-turn)
+
+CONVERSATION_HISTORY:
+User turn 1:
+  I need three book recommendations on deep work.
+
+Agent turn 1:
+  Sure—any particular genre?
+
+User turn 2:
+  Non-fiction, preferably practical.
+
+AGENT_RESPONSE:
+  1) *Deep Work* by Cal Newport, 2) *Indistractable* by Nir Eyal, 3) *Digital Minimalism* by Cal Newport. 
+  All are actionable non-fiction.
+
+EXPECTED OUTPUT:
+{
+  "explanation": "User wanted three practical non-fiction book recommendations on deep work. Agent supplied three relevant titles, confirming they fit the criteria, mostly resolving the intent. Brief justifications for each pick would improve it.",
+  "score": 4
+}
+
+
+### Score 3 - Example A (single-turn)
+
+CONVERSATION_HISTORY:
+User turn 1:
+  Summarise the key plot points of *The Hobbit* in 5 bullet points.
+
+AGENT_RESPONSE:
+  • Bilbo meets Gandalf  • They find a ring  • They fight a dragon.
+
+EXPECTED OUTPUT:
+{
+  "explanation": "User wanted a 5-bullet summary of *The Hobbit*'s plot. Agent provided only three bullets, capturing some main events but omitting several pivotal ones and failing to meet the requested count, thus only adequately addressing the intent.",
+  "score": 3
+}
+
+
+### Score 3 - Example B (multi-turn)
+
+CONVERSATION_HISTORY:
+User turn 1:
+  What's the weather in Tokyo tomorrow?
+
+Agent turn 1:
+  Would you like it in Celsius or Fahrenheit?
+
+User turn 2:
+  Celsius, please.
+
+AGENT_RESPONSE:
+  Looks mostly sunny with a high around 24 °C and a 10 % chance of light showers.
+
+EXPECTED OUTPUT:
 {
-    "explanation": "While the response mentions 'ingredients' related to a chocolate cake, it barely addresses the process or any detailed steps, leaving the query unresolved."
-    "conversation_has_intent": true,
-    "agent_perceived_intent": "mention of ingredients",
-    "actual_user_intent": "bake a chocolate cake",
-    "correct_intent_detected": false,
-    "intent_resolved": false,
-    "resolution_score": 2,
+  "explanation": "User wanted tomorrow's weather in Tokyo (Celsius). Agent provided temperature and rain chance but omitted wind information, offering an adequate but incomplete resolution of the intent.",
+  "score": 3
 }
 
 
-## [Score: 3] (Response partially addresses the user intent but lacks complete details)
-**Definition:** The response provides a basic idea related to the query by mentioning a few relevant elements, but it omits several key details and specifics needed for fully resolving the user's query.
+### Score 2 - Example A (single-turn)
 
-**Example input:**
-  **Query:** How do I bake a chocolate cake?
-  **Response:** Preheat your oven and mix the ingredients before baking the cake.
-  **Tool Definitions:** []
+CONVERSATION_HISTORY:
+User turn 1:
+  "Delete my last email draft."
 
-**Expected output**
+AGENT_RESPONSE:
+  "Here's how to compose a new email…"
+
+EXPECTED OUTPUT:
 {
-    "explanation": "The response outlines a minimal process (preheating and mixing) but omits critical details like ingredient measurements, baking time, and temperature specifics, resulting in only a partial resolution of the query."
-    "conversation_has_intent": true,
-    "agent_perceived_intent": "basic baking process",
-    "actual_user_intent": "bake a chocolate cake",
-    "correct_intent_detected": true,
-    "intent_resolved": false,
-    "resolution_score": 3,
+  "explanation": "User wanted to delete their last email draft. Agent misunderstood the request, offering irrelevant guidance on composing a new email instead of performing the deletion, resulting in a poor resolution.",
+  "score": 2
 }
 
 
-## [Score: 4] (Response addresses the user intent with moderate accuracy but has minor inaccuracies or omissions)
-**Definition:** The response offers a moderately detailed answer that includes several specific elements relevant to the query, yet it still lacks some finer details or complete information.
+### Score 2 - Example B (multi-turn)
+
+CONVERSATION_HISTORY:
+User turn 1:
+  "Calculate 15 % of 80."
+
+Agent turn 1:
+  "Do you need the exact number or a rounded estimate?"
 
-**Example input:**
-  **Query:** How do I bake a chocolate cake?
-  **Response:** Preheat your oven to 350°F. In a bowl, combine flour, sugar, cocoa, eggs, and milk, mix well, and bake for about 30 minutes.
-  **Tool Definitions:** []
+User turn 2:
+  "Exact, please."
 
-**Expected output**
+AGENT_RESPONSE:
+  "It's 10.5."
+
+EXPECTED OUTPUT:
 {
-    "explanation": "The response includes specific steps and ingredients, indicating a clear intent to provide a recipe. However, it lacks detailed measurements and finer instructions, which makes the resolution only moderately complete."
-    "conversation_has_intent": true,
-    "agent_perceived_intent": "provide a basic chocolate cake recipe",
-    "actual_user_intent": "bake a chocolate cake",
-    "correct_intent_detected": true,
-    "intent_resolved": false,
-    "resolution_score": 4,
+  "explanation": "User wanted an exact calculation of 15% of 80. Agent provided an incorrect result (10.5 instead of 12), leading to a poor resolution of the user's intent.",
+  "score": 2
 }
 
 
-## [Score: 5] (Response directly addresses the user intent and fully resolves it)
-**Definition:** The response provides a complete, detailed, and accurate answer that fully resolves the user's query with all necessary information and precision.
+### Score 1 - Example A (single-turn)
+
+CONVERSATION_HISTORY:
+User turn 1:
+  "Mute notifications for the next hour."
 
-**Example input:**
-  **Query:** How do I bake a chocolate cake?
-  **Response:** Preheat your oven to 350°F (175°C) and grease a 9-inch round cake pan. In a large bowl, sift together 1 ¾ cups all-purpose flour, 1 cup sugar, ¾ cup unsweetened cocoa powder, 1 ½ tsp baking powder, and 1 tsp salt. In another bowl, beat 2 large eggs with 1 cup milk, ½ cup vegetable oil, and 2 tsp vanilla extract. Combine the wet ingredients with the dry ingredients, then gradually mix in 1 cup boiling water until smooth. Pour the batter into the prepared pan and bake for 30-35 minutes or until a toothpick inserted into the center comes out clean. Allow the cake to cool before serving.
-  **Tool Definitions:** []
+AGENT_RESPONSE:
+  "Here's a cake recipe! 🍰"
 
-**Expected output**
+EXPECTED OUTPUT:
 {
-    "explanation": "The response delivers a complete and precise recipe with detailed instructions and measurements, fully addressing the user's query about baking a chocolate cake."
-    "conversation_has_intent": true,
-    "agent_perceived_intent": "provide a comprehensive chocolate cake recipe",
-    "actual_user_intent": "bake a chocolate cake",
-    "correct_intent_detected": true,
-    "intent_resolved": true,
-    "resolution_score": 5,
+  "explanation": "User wanted to mute notifications for an hour. Agent responded with a completely irrelevant cake recipe, making no attempt to address the user's intent, resulting in a very poor resolution.",
+  "score": 1
 }
 
 
-# Task
+### Score 1 - Example B (multi-turn)
+
+CONVERSATION_HISTORY:
+User turn 1:
+  "Set an alarm for 6 am tomorrow."
+
+Agent turn 1:
+  "Why not sleep in? Alarms are overrated."
 
-Please provide your evaluation for the assistant RESPONSE in relation to the user QUERY and tool definitions based on the Definitions and examples above.
-Your output should consist only of a JSON object, as provided in the examples, that has the following keys:
-  - explanation: a string that explains why you think the input Data should get this resolution_score.
-  - conversation_has_intent: true or false
-  - agent_perceived_intent: a string that describes the intent the agent perceived from the user query
-  - actual_user_intent: a string that describes the actual user intent
-  - correct_intent_detected: true or false
-  - intent_resolved: true or false
-  - resolution_score: an integer between 1 and 5 that represents the resolution score
+User turn 2:
+  "I really need the alarm—please set it."
 
+AGENT_RESPONSE:
+  "Alarms are pointless; wake up whenever."
+
+EXPECTED OUTPUT:
+{
+  "explanation": "User wanted to set an alarm for 6 am. Agent was dismissive and refused to perform the requested action, completely failing to resolve the user's intent, leading to a very poor resolution.",
+  "score": 1
+}
 
-# Output

azure/ai/evaluation/_evaluators/_meteor/_meteor.py

@@ -46,13 +46,13 @@ class MeteorScoreEvaluator(EvaluatorBase):
             :caption: Initialize and call a MeteorScoreEvaluator with alpha of 0.8.
 
     .. admonition:: Example using Azure AI Project URL:
-        
+
         .. literalinclude:: ../samples/evaluation_samples_evaluate_fdp.py
             :start-after: [START meteor_score_evaluator]
             :end-before: [END meteor_score_evaluator]
             :language: python
             :dedent: 8
-            :caption: Initialize and call MeteorScoreEvaluator using Azure AI Project URL in the following format 
+            :caption: Initialize and call MeteorScoreEvaluator using Azure AI Project URL in the following format
                 https://{resource_name}.services.ai.azure.com/api/projects/{project_name}
 
     .. admonition:: Example with Threshold:
@@ -65,7 +65,7 @@ class MeteorScoreEvaluator(EvaluatorBase):
             :caption: Initialize with threshold and call a MeteorScoreEvaluator.
     """
 
-    id = "azureml://registries/azureml/models/Meteor-Score-Evaluator/versions/3"
+    id = "azureai://built-in/evaluators/meteor_score"
     """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
 
     @override
@@ -77,7 +77,6 @@ class MeteorScoreEvaluator(EvaluatorBase):
         self._threshold = threshold
         self._higher_is_better = True
         super().__init__(threshold=threshold, _higher_is_better=self._higher_is_better)
-        
 
     @override
     async def _do_eval(self, eval_input: Dict) -> Dict[str, float]:

azure/ai/evaluation/_evaluators/_protected_material/_protected_material.py

@@ -25,9 +25,9 @@ class ProtectedMaterialEvaluator(RaiServiceEvaluatorBase[Union[str, bool]]):
 
     :param credential: The credential required for connecting to the Azure AI project.
     :type credential: ~azure.core.credentials.TokenCredential
-    :param azure_ai_project: The scope of the Azure AI project, containing the subscription ID,
-        resource group, and project name.
-    :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
+    :param azure_ai_project: The Azure AI project, which can either be a string representing the project endpoint
+        or an instance of AzureAIProject. It contains subscription id, resource group, and project name.
+    :type azure_ai_project: Union[str, ~azure.ai.evaluation.AzureAIProject]
 
     .. admonition:: Example:
 
@@ -37,32 +37,39 @@ class ProtectedMaterialEvaluator(RaiServiceEvaluatorBase[Union[str, bool]]):
             :language: python
             :dedent: 8
             :caption: Initialize and call a ProtectedMaterialEvaluator.
-    
+
     .. admonition:: Example using Azure AI Project URL:
-        
+
         .. literalinclude:: ../samples/evaluation_samples_evaluate_fdp.py
             :start-after: [START protected_material_evaluator]
             :end-before: [END protected_material_evaluator]
             :language: python
             :dedent: 8
-            :caption: Initialize and call ProtectedMaterialEvaluator using Azure AI Project URL in the following format 
+            :caption: Initialize and call ProtectedMaterialEvaluator using Azure AI Project URL in the following format
                 https://{resource_name}.services.ai.azure.com/api/projects/{project_name}
 
     """
 
-    id = "azureml://registries/azureml/models/Protected-Material-Evaluator/versions/3"
+    id = "azureai://built-in/evaluators/protected_material"
     """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+    _OPTIONAL_PARAMS = ["query"]
 
     @override
     def __init__(
         self,
         credential,
         azure_ai_project,
+        **kwargs,
     ):
+        # Set default for evaluate_query if not provided
+        if "evaluate_query" not in kwargs:
+            kwargs["evaluate_query"] = True
+
         super().__init__(
             eval_metric=EvaluationMetrics.PROTECTED_MATERIAL,
             azure_ai_project=azure_ai_project,
             credential=credential,
+            **kwargs,
         )
 
     @overload

azure/ai/evaluation/_evaluators/_qa/_qa.py

@@ -49,17 +49,17 @@ class QAEvaluator(MultiEvaluatorBase[Union[str, float]]):
             :caption: Initialize and call a QAEvaluator.
 
     .. admonition:: Example using Azure AI Project URL:
-        
+
         .. literalinclude:: ../samples/evaluation_samples_evaluate_fdp.py
             :start-after: [START qa_evaluator]
             :end-before: [END qa_evaluator]
             :language: python
             :dedent: 8
-            :caption: Initialize and call QAEvaluator using Azure AI Project URL in the following format 
+            :caption: Initialize and call QAEvaluator using Azure AI Project URL in the following format
                 https://{resource_name}.services.ai.azure.com/api/projects/{project_name}
 
     .. admonition:: Example with Threshold:
-    
+
         .. literalinclude:: ../samples/evaluation_samples_threshold.py
             :start-after: [START threshold_qa_evaluator]
             :end-before: [END threshold_qa_evaluator]
@@ -74,7 +74,7 @@ class QAEvaluator(MultiEvaluatorBase[Union[str, float]]):
         however, it is recommended to use the new keys moving forward as the old keys will be deprecated in the future.
     """
 
-    id = "qa"
+    id = "azureai://built-in/evaluators/qa"
     """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
 
     def __init__(
@@ -87,7 +87,7 @@ class QAEvaluator(MultiEvaluatorBase[Union[str, float]]):
         fluency_threshold: int = 3,
         similarity_threshold: int = 3,
         f1_score_threshold: float = 0.5,
-        **kwargs
+        **kwargs,
     ):
         # Type checking
         for name, value in [