azure-ai-evaluation 1.9.0__py3-none-any.whl → 1.10.0__py3-none-any.whl

azure/ai/evaluation/_evaluators/_document_retrieval/_document_retrieval.py

@@ -49,6 +49,9 @@ class DocumentRetrievalEvaluator(EvaluatorBase):
             :caption: Initialize with threshold and call a DocumentRetrievalEvaluator.
     """
 
+    id = "azureai://built-in/evaluators/document_retrieval"
+    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+
     def __init__(
         self,
         *,

azure/ai/evaluation/_evaluators/_eci/_eci.py

@@ -52,17 +52,20 @@ class ECIEvaluator(RaiServiceEvaluatorBase):
 
     id = "eci"
     """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+    _OPTIONAL_PARAMS = ["query"]
 
     @override
     def __init__(
         self,
         credential,
         azure_ai_project,
+        **kwargs,
     ):
         super().__init__(
             eval_metric=_InternalEvaluationMetrics.ECI,
             azure_ai_project=azure_ai_project,
             credential=credential,
+            **kwargs,
         )
 
     @overload

azure/ai/evaluation/_evaluators/_f1_score/_f1_score.py

@@ -58,7 +58,7 @@ class F1ScoreEvaluator(EvaluatorBase):
             :caption: Initialize with threshold and call an F1ScoreEvaluator.
     """
 
-    id = "azureml://registries/azureml/models/F1Score-Evaluator/versions/3"
+    id = "azureai://built-in/evaluators/f1_score"
     """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
 
     def __init__(self, *, threshold=0.5):

azure/ai/evaluation/_evaluators/_fluency/_fluency.py

@@ -64,7 +64,7 @@ class FluencyEvaluator(PromptyEvaluatorBase[Union[str, float]]):
     _PROMPTY_FILE = "fluency.prompty"
     _RESULT_KEY = "fluency"
 
-    id = "azureml://registries/azureml/models/Fluency-Evaluator/versions/4"
+    id = "azureai://built-in/evaluators/fluency"
     """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
 
     @override

azure/ai/evaluation/_evaluators/_gleu/_gleu.py

@@ -55,7 +55,7 @@ class GleuScoreEvaluator(EvaluatorBase):
                 https://{resource_name}.services.ai.azure.com/api/projects/{project_name}
     """
 
-    id = "azureml://registries/azureml/models/Gleu-Score-Evaluator/versions/3"
+    id = "azureai://built-in/evaluators/gleu_score"
     """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
 
     @override

azure/ai/evaluation/_evaluators/_groundedness/_groundedness.py

@@ -79,7 +79,7 @@ class GroundednessEvaluator(PromptyEvaluatorBase[Union[str, float]]):
     _RESULT_KEY = "groundedness"
     _OPTIONAL_PARAMS = ["query"]
 
-    id = "azureml://registries/azureml/models/Groundedness-Evaluator/versions/4"
+    id = "azureai://built-in/evaluators/groundedness"
     """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
 
     @override

azure/ai/evaluation/_evaluators/_intent_resolution/_intent_resolution.py

@@ -57,7 +57,7 @@ class IntentResolutionEvaluator(PromptyEvaluatorBase[Union[str, float]]):
     _MAX_INTENT_RESOLUTION_SCORE = 5
     _DEFAULT_INTENT_RESOLUTION_THRESHOLD = 3
 
-    id = None
+    id = "azureai://built-in/evaluators/intent_resolution"
     """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
 
     @override

azure/ai/evaluation/_evaluators/_meteor/_meteor.py

@@ -65,7 +65,7 @@ class MeteorScoreEvaluator(EvaluatorBase):
             :caption: Initialize with threshold and call a MeteorScoreEvaluator.
     """
 
-    id = "azureml://registries/azureml/models/Meteor-Score-Evaluator/versions/3"
+    id = "azureai://built-in/evaluators/meteor_score"
     """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
 
     @override

azure/ai/evaluation/_evaluators/_protected_material/_protected_material.py

@@ -50,19 +50,26 @@ class ProtectedMaterialEvaluator(RaiServiceEvaluatorBase[Union[str, bool]]):
 
     """
 
-    id = "azureml://registries/azureml/models/Protected-Material-Evaluator/versions/3"
+    id = "azureai://built-in/evaluators/protected_material"
     """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+    _OPTIONAL_PARAMS = ["query"]
 
     @override
     def __init__(
         self,
         credential,
         azure_ai_project,
+        **kwargs,
     ):
+        # Set default for evaluate_query if not provided
+        if "evaluate_query" not in kwargs:
+            kwargs["evaluate_query"] = True
+
         super().__init__(
             eval_metric=EvaluationMetrics.PROTECTED_MATERIAL,
             azure_ai_project=azure_ai_project,
             credential=credential,
+            **kwargs,
         )
 
     @overload

azure/ai/evaluation/_evaluators/_qa/_qa.py

@@ -74,7 +74,7 @@ class QAEvaluator(MultiEvaluatorBase[Union[str, float]]):
         however, it is recommended to use the new keys moving forward as the old keys will be deprecated in the future.
     """
 
-    id = "qa"
+    id = "azureai://built-in/evaluators/qa"
     """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
 
     def __init__(

azure/ai/evaluation/_evaluators/_relevance/_relevance.py

@@ -1,15 +1,21 @@
 # ---------------------------------------------------------
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
-
+import logging
+import math
 import os
 from typing import Dict, Union, List
 
 from typing_extensions import overload, override
 
+from azure.ai.evaluation._exceptions import EvaluationException, ErrorBlame, ErrorCategory, ErrorTarget
+from ..._common.utils import reformat_conversation_history, reformat_agent_response
+
 from azure.ai.evaluation._model_configurations import Conversation
 from azure.ai.evaluation._evaluators._common import PromptyEvaluatorBase
 
+logger = logging.getLogger(__name__)
+
 
 class RelevanceEvaluator(PromptyEvaluatorBase):
     """
@@ -69,7 +75,7 @@ class RelevanceEvaluator(PromptyEvaluatorBase):
     _PROMPTY_FILE = "relevance.prompty"
     _RESULT_KEY = "relevance"
 
-    id = "azureml://registries/azureml/models/Relevance-Evaluator/versions/4"
+    id = "azureai://built-in/evaluators/relevance"
     """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
 
     @override
@@ -141,3 +147,49 @@ class RelevanceEvaluator(PromptyEvaluatorBase):
         :rtype: Union[Dict[str, Union[str, float]], Dict[str, Union[float, Dict[str, List[Union[str, float]]]]]]
         """
         return super().__call__(*args, **kwargs)
+
+    async def _do_eval(self, eval_input: Dict) -> Dict[str, Union[float, str]]:  # type: ignore[override]
+        """Do a relevance evaluation.
+
+        :param eval_input: The input to the evaluator. Expected to contain
+        whatever inputs are needed for the _flow method, including context
+        and other fields depending on the child class.
+        :type eval_input: Dict
+        :return: The evaluation result.
+        :rtype: Dict
+        """
+        if "query" not in eval_input and "response" not in eval_input:
+            raise EvaluationException(
+                message="Only text conversation inputs are supported.",
+                internal_message="Only text conversation inputs are supported.",
+                blame=ErrorBlame.USER_ERROR,
+                category=ErrorCategory.INVALID_VALUE,
+                target=ErrorTarget.CONVERSATION,
+            )
+        if not isinstance(eval_input["query"], str):
+            eval_input["query"] = reformat_conversation_history(eval_input["query"], logger)
+        if not isinstance(eval_input["response"], str):
+            eval_input["response"] = reformat_agent_response(eval_input["response"], logger)
+        llm_output = await self._flow(timeout=self._LLM_CALL_TIMEOUT, **eval_input)
+        score = math.nan
+
+        if isinstance(llm_output, dict):
+            score = float(llm_output.get("score", math.nan))
+            reason = llm_output.get("explanation", "")
+            # Parse out score and reason from evaluators known to possess them.
+            binary_result = self._get_binary_result(score)
+            return {
+                self._result_key: float(score),
+                f"gpt_{self._result_key}": float(score),
+                f"{self._result_key}_reason": reason,
+                f"{self._result_key}_result": binary_result,
+                f"{self._result_key}_threshold": self._threshold,
+            }
+
+        binary_result = self._get_binary_result(score)
+        return {
+            self._result_key: float(score),
+            f"gpt_{self._result_key}": float(score),
+            f"{self._result_key}_result": binary_result,
+            f"{self._result_key}_threshold": self._threshold,
+        }

azure/ai/evaluation/_evaluators/_relevance/relevance.prompty

@@ -10,91 +10,172 @@ model:
     presence_penalty: 0
     frequency_penalty: 0
     response_format:
-      type: text
+      type: json_object
 
 inputs:
   query:
     type: string
   response:
     type: string
-
 ---
+
 system:
-# Instruction
-## Goal
-### You are an expert in evaluating the quality of a RESPONSE from an intelligent system based on provided definition and data. Your goal will involve answering the questions below using the information provided.
-- **Definition**: You are given a definition of the communication trait that is being evaluated to help guide your Score.
-- **Data**: Your input data include QUERY and RESPONSE.
-- **Tasks**: To complete your evaluation you will be asked to evaluate the Data in different ways.
+You are a Relevance-Judge, an impartial evaluator that scores how well the RESPONSE addresses the QUERY using the definitions provided.
 
 user:
-# Definition
-**Relevance** refers to how effectively a response addresses a question. It assesses the accuracy, completeness, and direct relevance of the response based solely on the given information.
-
-# Ratings
-## [Relevance: 1] (Irrelevant Response)
-**Definition:** The response is unrelated to the question. It provides information that is off-topic and does not attempt to address the question posed.
-
-**Examples:**
-  **Query:** What is the team preparing for?
-  **Response:** I went grocery shopping yesterday evening.
-
-  **Query:** When will the company's new product line launch?
-  **Response:** International travel can be very rewarding and educational.
+ROLE
+====
+You are a Relevance Evaluator. Your task is to judge how relevant a RESPONSE is to a QUERY using the Relevance definitions provided.
 
-## [Relevance: 2] (Incorrect Response)
-**Definition:** The response attempts to address the question but includes incorrect information. It provides a response that is factually wrong based on the provided information.
+INPUT
+=====
+QUERY: {{query}}
+RESPONSE: {{response}}
 
-**Examples:**
-  **Query:** When was the merger between the two firms finalized?
-  **Response:** The merger was finalized on April 10th.     
+TASK
+====
+Output a JSON object with:
+    1) a concise explanation of 15-60 words justifying your score based on how well the response is relevant to the query.
+    2) an integer score from 1 (very poor) to 5 (excellent) using the rubric below.
 
-  **Query:** Where and when will the solar eclipse be visible?
-  **Response:** The solar eclipse will be visible in Asia on December 14th.
+The explanation should always precede the score and should clearly justify the score based on the rubric definitions.
+Response format exactly as follows:
 
-## [Relevance: 3] (Incomplete Response)
-**Definition:** The response addresses the question but omits key details necessary for a full understanding. It provides a partial response that lacks essential information.      
+{
+  "explanation": "<15-60 words>",
+  "score": <1-5>
+}
 
-**Examples:**
-  **Query:** What type of food does the new restaurant offer?
-  **Response:** The restaurant offers Italian food like pasta.
 
-  **Query:** What topics will the conference cover?
-  **Response:** The conference will cover renewable energy and climate change.
+EVALUATION STEPS
+================
+A. Read the QUERY and RESPONSE carefully.
+B. Compare the RESPONSE against the rubric below:
+    - Does the response directly address the query?
+    - Is the information complete, partial, or off-topic?
+    - Is it vague, generic, or insightful?
+C. Match the response to the best score from the rubric.
+D. Provide a short explanation and the score using the required format.
 
-## [Relevance: 4] (Complete Response)
-**Definition:** The response fully addresses the question with accurate and complete information. It includes all essential details required for a comprehensive understanding, without adding any extraneous information.
+SCORING RUBRIC
+==============
 
-**Examples:**
-  **Query:** What type of food does the new restaurant offer?
-  **Response:** The new restaurant offers Italian cuisine, featuring dishes like pasta, pizza, and risotto.
+### Score 1 - Irrelevant Response
+Definition: The response is unrelated to the question. It provides off-topic information and does not attempt to address the question posed.
 
-  **Query:** What topics will the conference cover?
-  **Response:** The conference will cover renewable energy, climate change, and sustainability practices.
+**Example A**
+QUERY: What is the team preparing for?
+RESPONSE: I went grocery shopping yesterday evening.
 
-## [Relevance: 5] (Comprehensive Response with Insights)    
-**Definition:** The response not only fully and accurately addresses the question but also includes additional relevant insights or elaboration. It may explain the significance, implications, or provide minor inferences that enhance understanding.
+Expected Output:
+{
+    "explanation": "The response is entirely off-topic and doesn't address the question.",
+    "score": 1
+}
 
-**Examples:**
-  **Query:** What type of food does the new restaurant offer?
-  **Response:** The new restaurant offers Italian cuisine, featuring dishes like pasta, pizza, and risotto, aiming to provide customers with an authentic Italian dining experience.
 
-  **Query:** What topics will the conference cover?
-  **Response:** The conference will cover renewable energy, climate change, and sustainability practices, bringing together global experts to discuss these critical issues. 
+**Example B**
+QUERY: When will the company's new product line launch?
+RESPONSE: International travel can be very rewarding and educational.
 
+Expected Output:
+{
+    "explanation": "The response is completely irrelevant to the product launch question.",
+    "score": 1
+}
+
+
+### Score 2 – Related but Unhelpful / Superficial
+Definition: The response is loosely or formally related to the query but fails to deliver any meaningful, specific, or useful information. This includes vague phrases, non-answers, or failure/error messages.
+
+**Example A**
+QUERY: What is the event about?
+RESPONSE: It’s something important.
+
+Expected Output:
+{
+  "explanation": "The response vaguely refers to the query topic but lacks specific or informative content.",
+  "score": 2
+}
+
+**Example B**
+QUERY: What’s the weather in Paris?
+RESPONSE: I tried to find the forecast but the query failed.
+
+Expected Output:
+{
+  "explanation": "The response acknowledges the query but provides no usable weather information. It is related but unhelpful.",
+  "score": 2
+}
+
+### Score 3 - Partially Relevant / Incomplete
+Definition: The response addresses the query and includes relevant information, but omits essential components or detail. The answer is on-topic but insufficient to fully satisfy the request.
+
+**Example A**
+QUERY: What amenities does the new apartment complex provide?
+RESPONSE: The apartment complex has a gym.
+
+Expected Output:
+{
+    "explanation": "The response mentions one amenity but does not provide a fuller list or clarify whether other standard features (like parking or security) are included. It partially addresses the query but lacks completeness.",
+    "score": 3
+}
+
+**Example B**
+QUERY: What services does the premium membership include?
+RESPONSE: It includes priority customer support.
+
+Expected Output:
+{
+    "explanation": "The response identifies one service but omits other likely components of a premium membership (e.g., exclusive content or discounts). The information is relevant but incomplete.",
+    "score": 3
+}
+
+
+
+### Score 4 - Fully Relevant / Sufficient Response
+Definition: The response fully addresses the question with accurate and sufficient information, covering all essential aspects. Very minor omissions are acceptable as long as the core information is intact and the intent is clearly conveyed.
+
+**Example A**
+QUERY: What amenities does the new apartment complex provide?
+RESPONSE: The apartment complex provides a gym, swimming pool, and 24/7 security.
+
+Expected Output:
+{
+    "explanation": "The response mentions multiple key amenities likely to be relevant to most users. While it may not list every feature, it clearly conveys the core offerings of the complex.",
+    "score": 4
+}
 
+**Example B**
+QUERY: What services does the premium membership include?
+RESPONSE: The premium membership includes priority customer support, exclusive content access, and early product releases.
+
+Expected Output:
+{
+    "explanation": "The response outlines all major services expected from a premium membership. Even if a minor service is not mentioned, the core value is clearly and fully represented.",
+    "score": 4
+}
 
-# Data
-QUERY: {{query}}
-RESPONSE: {{response}}
 
+### Score 5 - Comprehensive Response with Insights
+Definition: The response not only fully and accurately answers the question, but also adds meaningful elaboration, interpretation, or context that enhances the user's understanding. This goes beyond just listing relevant details — it offers insight into why the information matters, how it's useful, or what impact it has.
 
-# Tasks
-## Please provide your assessment Score for the previous RESPONSE in relation to the QUERY based on the Definitions above. Your output should include the following information:
-- **ThoughtChain**: To improve the reasoning process, think step by step and include a step-by-step explanation of your thought process as you analyze the data based on the definitions. Keep it brief and start your ThoughtChain with "Let's think step by step:".
-- **Explanation**: a very short explanation of why you think the input Data should get that Score.
-- **Score**: based on your previous analysis, provide your Score. The Score you give MUST be a integer score (i.e., "1", "2"...) based on the levels of the definitions.
+**Example A**
+QUERY: What amenities does the new apartment complex provide?
+RESPONSE: The apartment complex provides a gym, swimming pool, and 24/7 security, designed to offer residents a comfortable and active lifestyle while ensuring their safety.
+
+Expected Output:
+{
+    "explanation": "The response fully lists key amenities and additionally explains how these features contribute to resident experience, enhancing the usefulness of the information.",
+    "score": 5
+}
 
+**Example B**
+QUERY: What services does the premium membership include?
+RESPONSE: The premium membership includes priority customer support, exclusive content access, and early product releases — tailored for users who want quicker resolutions and first access to new features.
 
-## Please provide your answers between the tags: <S0>your chain of thoughts</S0>, <S1>your explanation</S1>, <S2>your Score</S2>.
-# Output
+Expected Output:
+{
+    "explanation": "The response covers all essential services and adds valuable insight about the target user and benefits, enriching the response beyond basic listing.",
+    "score": 5
+}

azure/ai/evaluation/_evaluators/_response_completeness/_response_completeness.py

@@ -64,7 +64,7 @@ class ResponseCompletenessEvaluator(PromptyEvaluatorBase[Union[str, float]]):
     _PROMPTY_FILE = "response_completeness.prompty"
     _RESULT_KEY = "response_completeness"
 
-    id = "completeness"
+    id = "azureai://built-in/evaluators/response_completeness"
 
     _MIN_COMPLETENESS_SCORE = 1
     _MAX_COMPLETENESS_SCORE = 5

azure/ai/evaluation/_evaluators/_retrieval/_retrieval.py

@@ -74,7 +74,7 @@ class RetrievalEvaluator(PromptyEvaluatorBase[Union[str, float]]):
     _PROMPTY_FILE = "retrieval.prompty"
     _RESULT_KEY = "retrieval"
 
-    id = "azureml://registries/azureml/models/Retrieval-Evaluator/versions/1"
+    id = "azureai://built-in/evaluators/retrieval"
     """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
 
     @override

azure/ai/evaluation/_evaluators/_rouge/_rouge.py

@@ -90,7 +90,7 @@ class RougeScoreEvaluator(EvaluatorBase):
             :caption: Initialize with a specified threshold and call a RougeScoreEvaluator with a four-gram rouge type.
     """
 
-    id = "azureml://registries/azureml/models/Rouge-Score-Evaluator/versions/3"
+    id = "azureai://built-in/evaluators/rouge_score"
     """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
 
     @override

azure/ai/evaluation/_evaluators/_service_groundedness/_service_groundedness.py

@@ -66,8 +66,9 @@ class GroundednessProEvaluator(RaiServiceEvaluatorBase[Union[str, bool]]):
         for the groundedness pro label will be "groundedness_pro_passing_rate".
     """
 
-    id = "azureml://registries/azureml/models/Groundedness-Pro-Evaluator/versions/1"
+    id = "azureai://built-in/evaluators/groundedness_pro"
     """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+    _OPTIONAL_PARAMS = ["query"]
 
     @override
     def __init__(

azure/ai/evaluation/_evaluators/_similarity/_similarity.py

@@ -71,7 +71,7 @@ class SimilarityEvaluator(PromptyEvaluatorBase):
     _PROMPTY_FILE = "similarity.prompty"
     _RESULT_KEY = "similarity"
 
-    id = "similarity"
+    id = "azureai://built-in/evaluators/similarity"
     """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
 
     @override

azure/ai/evaluation/_evaluators/_task_adherence/_task_adherence.py

@@ -3,16 +3,19 @@
 # ---------------------------------------------------------
 import os
 import math
+import logging
 from typing import Dict, Union, List, Optional
 
 from typing_extensions import overload, override
 
 from azure.ai.evaluation._exceptions import EvaluationException, ErrorBlame, ErrorCategory, ErrorTarget
 from azure.ai.evaluation._evaluators._common import PromptyEvaluatorBase
-from azure.ai.evaluation._common.utils import parse_quality_evaluator_reason_score
+from ..._common.utils import reformat_conversation_history, reformat_agent_response, reformat_tool_definitions
 from azure.ai.evaluation._model_configurations import Message
 from azure.ai.evaluation._common._experimental import experimental
 
+logger = logging.getLogger(__name__)
+
 
 @experimental
 class TaskAdherenceEvaluator(PromptyEvaluatorBase[Union[str, float]]):
@@ -62,7 +65,7 @@ class TaskAdherenceEvaluator(PromptyEvaluatorBase[Union[str, float]]):
 
     _DEFAULT_TASK_ADHERENCE_SCORE = 3
 
-    id = None
+    id = "azureai://built-in/evaluators/task_adherence"
     """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
 
     @override
@@ -140,20 +143,23 @@ class TaskAdherenceEvaluator(PromptyEvaluatorBase[Union[str, float]]):
                 category=ErrorCategory.MISSING_FIELD,
                 target=ErrorTarget.TASK_ADHERENCE_EVALUATOR,
             )
-
+        eval_input["query"] = reformat_conversation_history(eval_input["query"], logger, include_system_messages=True)
+        eval_input["response"] = reformat_agent_response(eval_input["response"], logger, include_tool_messages=True)
+        if "tool_definitions" in eval_input and eval_input["tool_definitions"] is not None:
+            eval_input["tool_definitions"] = reformat_tool_definitions(eval_input["tool_definitions"], logger)
         llm_output = await self._flow(timeout=self._LLM_CALL_TIMEOUT, **eval_input)
-
-        score = math.nan
-        if llm_output:
-            score, reason = parse_quality_evaluator_reason_score(llm_output, valid_score_range="[1-5]")
-
+        if isinstance(llm_output, dict):
+            score = float(llm_output.get("score", math.nan))
             score_result = "pass" if score >= self.threshold else "fail"
-
+            reason = llm_output.get("explanation", "")
             return {
                 f"{self._result_key}": score,
                 f"{self._result_key}_result": score_result,
                 f"{self._result_key}_threshold": self.threshold,
                 f"{self._result_key}_reason": reason,
+                # Uncomment the following line in the next iteration after UI contracts are validated.
+                # f"{self._result_key}_additional_details": llm_output
             }
-
+        if logger:
+            logger.warning("LLM output is not a dictionary, returning NaN for the score.")
         return {self._result_key: math.nan}