azure-ai-evaluation 1.3.0__py3-none-any.whl → 1.5.0__py3-none-any.whl

azure/ai/evaluation/_evaluators/_qa/_qa.py

@@ -23,6 +23,18 @@ class QAEvaluator(MultiEvaluatorBase[Union[str, float]]):
     :param model_config: Configuration for the Azure OpenAI model.
     :type model_config: Union[~azure.ai.evaluation.AzureOpenAIModelConfiguration,
         ~azure.ai.evaluation.OpenAIModelConfiguration]
+    :param groundedness_threshold: The threshold for groundedness evaluation. Default is 3.
+    :type groundedness_threshold: int
+    :param relevance_threshold: The threshold for relevance evaluation. Default is 3.
+    :type relevance_threshold: int
+    :param coherence_threshold: The threshold for coherence evaluation. Default is 3.
+    :type coherence_threshold: int
+    :param fluency_threshold: The threshold for fluency evaluation. Default is 3.
+    :type fluency_threshold: int
+    :param similarity_threshold: The threshold for similarity evaluation. Default is 3.
+    :type similarity_threshold: int
+    :param f1_score_threshold: The threshold for F1 score evaluation. Default is 0.5.
+    :type f1_score_threshold: float
     :return: A callable class that evaluates and generates metrics for "question-answering" scenario.
     :param kwargs: Additional arguments to pass to the evaluator.
     :type kwargs: Any
@@ -36,6 +48,15 @@ class QAEvaluator(MultiEvaluatorBase[Union[str, float]]):
             :dedent: 8
             :caption: Initialize and call a QAEvaluator.
 
+    .. admonition:: Example with Threshold:
+    
+        .. literalinclude:: ../samples/evaluation_samples_threshold.py
+            :start-after: [START threshold_qa_evaluator]
+            :end-before: [END threshold_qa_evaluator]
+            :language: python
+            :dedent: 8
+            :caption: Initialize with threshold and call a QAEvaluator.
+
     .. note::
 
         To align with our support of a diverse set of models, keys without the `gpt_` prefix has been added.
@@ -46,14 +67,37 @@ class QAEvaluator(MultiEvaluatorBase[Union[str, float]]):
     id = "qa"
     """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
 
-    def __init__(self, model_config, **kwargs):
+    def __init__(
+        self,
+        model_config,
+        *,
+        groundedness_threshold: int = 3,
+        relevance_threshold: int = 3,
+        coherence_threshold: int = 3,
+        fluency_threshold: int = 3,
+        similarity_threshold: int = 3,
+        f1_score_threshold: float = 0.5,
+        **kwargs
+    ):
+        # Type checking
+        for name, value in [
+            ("groundedness_threshold", groundedness_threshold),
+            ("relevance_threshold", relevance_threshold),
+            ("coherence_threshold", coherence_threshold),
+            ("fluency_threshold", fluency_threshold),
+            ("similarity_threshold", similarity_threshold),
+            ("f1_score_threshold", f1_score_threshold),
+        ]:
+            if not isinstance(value, (int, float)):
+                raise TypeError(f"{name} must be an int or float, got {type(value)}")
+
         evaluators = [
-            GroundednessEvaluator(model_config),
-            RelevanceEvaluator(model_config),
-            CoherenceEvaluator(model_config),
-            FluencyEvaluator(model_config),
-            SimilarityEvaluator(model_config),
-            F1ScoreEvaluator(),
+            GroundednessEvaluator(model_config, threshold=groundedness_threshold),
+            RelevanceEvaluator(model_config, threshold=relevance_threshold),
+            CoherenceEvaluator(model_config, threshold=coherence_threshold),
+            FluencyEvaluator(model_config, threshold=fluency_threshold),
+            SimilarityEvaluator(model_config, threshold=similarity_threshold),
+            F1ScoreEvaluator(threshold=f1_score_threshold),
         ]
         super().__init__(evaluators=evaluators, **kwargs)
 

azure/ai/evaluation/_evaluators/_relevance/_relevance.py

@@ -27,6 +27,8 @@ class RelevanceEvaluator(PromptyEvaluatorBase):
     :param model_config: Configuration for the Azure OpenAI model.
     :type model_config: Union[~azure.ai.evaluation.AzureOpenAIModelConfiguration,
         ~azure.ai.evaluation.OpenAIModelConfiguration]
+    :param threshold: The threshold for the relevance evaluator. Default is 5.
+    :type threshold: int
 
     .. admonition:: Example:
 
@@ -37,6 +39,15 @@ class RelevanceEvaluator(PromptyEvaluatorBase):
             :dedent: 8
             :caption: Initialize and call a RelevanceEvaluator with a query, response, and context.
 
+    .. admonition:: Example with Threshold:
+
+        .. literalinclude:: ../samples/evaluation_samples_threshold.py
+            :start-after: [START threshold_relevance_evaluator]
+            :end-before: [END threshold_relevance_evaluator]
+            :language: python
+            :dedent: 8
+            :caption: Initialize with threshold and call a RelevanceEvaluator with a query, response, and context.
+
     .. note::
 
         To align with our support of a diverse set of models, an output key without the `gpt_` prefix has been added.
@@ -52,10 +63,23 @@ class RelevanceEvaluator(PromptyEvaluatorBase):
     """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
 
     @override
-    def __init__(self, model_config):
+    def __init__(
+        self, 
+        model_config, 
+        *,
+        threshold=3
+    ):
         current_dir = os.path.dirname(__file__)
         prompty_path = os.path.join(current_dir, self._PROMPTY_FILE)
-        super().__init__(model_config=model_config, prompty_file=prompty_path, result_key=self._RESULT_KEY)
+        self._threshold = threshold
+        self._higher_is_better = True
+        super().__init__(
+            model_config=model_config, 
+            prompty_file=prompty_path, 
+            result_key=self._RESULT_KEY, 
+            threshold=threshold, 
+            _higher_is_better=self._higher_is_better
+        )
 
     @overload
     def __call__(

azure/ai/evaluation/_evaluators/_response_completeness/__init__.py

@@ -0,0 +1,7 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from ._response_completeness import ResponseCompletenessEvaluator
+
+__all__ = ["ResponseCompletenessEvaluator"]

azure/ai/evaluation/_evaluators/_response_completeness/_response_completeness.py

@@ -0,0 +1,158 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+import os
+import math
+from typing import Dict, List, Union, Optional
+
+from typing_extensions import overload, override
+
+from azure.ai.evaluation._exceptions import EvaluationException, ErrorBlame, ErrorCategory, ErrorTarget
+from azure.ai.evaluation._evaluators._common import PromptyEvaluatorBase
+from azure.ai.evaluation._common.utils import parse_quality_evaluator_reason_score
+from azure.ai.evaluation._model_configurations import Conversation, Message
+from azure.ai.evaluation._common._experimental import experimental
+
+
+@experimental
+class ResponseCompletenessEvaluator(PromptyEvaluatorBase[Union[str, float]]):
+    """
+    Evaluates the extent to which a given response contains all necessary and relevant information with respect to the
+     provided ground truth.
+    The completeness measure assesses how thoroughly an AI model's generated response aligns with the key information,
+    claims, and statements established in the ground truth. This evaluation considers the presence, accuracy,
+    and relevance of the content provided.
+    The assessment spans multiple levels, ranging from fully incomplete to fully complete, ensuring a comprehensive
+    evaluation of the response's content quality.
+    Use this metric when you need to evaluate an AI model's ability to deliver comprehensive and accurate information,
+    particularly in text generation tasks where conveying all essential details is crucial for clarity,
+    context, and correctness.
+    Completeness scores range from 1 to 5:
+    1: Fully incomplete — Contains none of the necessary information.
+    2: Barely complete — Contains only a small portion of the required information.
+    3: Moderately complete — Covers about half of the required content.
+    4: Mostly complete — Includes most of the necessary details with minimal omissions.
+    5: Fully complete — Contains all key information without any omissions.
+    :param model_config: Configuration for the Azure OpenAI model.
+    :type model_config: Union[~azure.ai.evaluation.AzureOpenAIModelConfiguration,
+        ~azure.ai.evaluation.OpenAIModelConfiguration]
+    .. admonition:: Example:
+        .. literalinclude:: ../samples/evaluation_samples_evaluate.py
+            :start-after: [START completeness_evaluator]
+            :end-before: [END completeness_evaluator]
+            :language: python
+            :dedent: 8
+            :caption: Initialize and call a CompletenessEvaluator with a response and groundtruth.
+    """
+
+    # Constants must be defined within eval's directory to be save/loadable
+
+    _PROMPTY_FILE = "response_completeness.prompty"
+    _RESULT_KEY = "response_completeness"
+
+    id = "completeness"
+
+    _MIN_COMPLETENESS_SCORE = 1
+    _MAX_COMPLETENESS_SCORE = 5
+    _DEFAULT_COMPLETENESS_THRESHOLD = 3
+
+    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+
+    @override
+    def __init__(self, model_config, *, threshold: Optional[float] = _DEFAULT_COMPLETENESS_THRESHOLD):
+        current_dir = os.path.dirname(__file__)
+        prompty_path = os.path.join(current_dir, self._PROMPTY_FILE)
+        self.threshold = threshold
+        super().__init__(model_config=model_config, prompty_file=prompty_path, result_key=self._RESULT_KEY)
+
+    @overload
+    def __call__(
+            self,
+            *,
+            ground_truth: str,
+            response: str,
+    ) -> Dict[str, Union[str, float]]:
+        """Evaluate completeness in given response. Accepts ground truth and response for evaluation.
+        Example usage:
+        Evaluating completeness for a response string
+        ```python
+        from azure.ai.evaluation import CompletenessEvaluator
+        completeness_evaluator = CompletenessEvaluator(model_config)
+        ground_truth = "The ground truth to be evaluated."
+        response = "The response to be evaluated."
+        completeness_results = completeness_evaluator(ground_truth=ground_truth, response=response)
+        ```
+        :keword ground_truth: The ground truth to be evaluated.
+        :paramtype ground_truth: str
+        :keyword response: The response to be evaluated.
+        :paramtype response: Union[str, List[Message]]
+        :return: The response completeness score results.
+        :rtype: Dict[str, Union[str, float]]
+        """
+
+    @overload
+    def __call__(
+            self,
+            *,
+            conversation: Conversation,
+    ) -> Dict[str, Union[float, Dict[str, List[Union[str, float]]]]]:
+        """Evaluate completeness for a conversation
+        :keyword conversation: The conversation to evaluate. Expected to contain a list of conversation turns under the
+            key "messages", and potentially a global context under the key "context". Conversation turns are expected
+            to be dictionaries with keys "content", "role", and possibly "context".
+        :paramtype conversation: Optional[~azure.ai.evaluation.Conversation]
+        :return: The fluency score
+        :rtype: Dict[str, Union[float, Dict[str, List[float]]]]
+        """
+
+    @override
+    def __call__(  # pylint: disable=docstring-missing-param
+            self,
+            *args,
+            **kwargs,
+    ):
+        """
+        Invokes the instance using the overloaded __call__ signature.
+
+        For detailed parameter types and return value documentation, see the overloaded __call__ definition.
+        """
+        return super().__call__(*args, **kwargs)
+
+    @override
+    async def _do_eval(self, eval_input: Dict) -> Dict[str, Union[float, str]]:  # type: ignore[override]
+        """Do completeness evaluation.
+        :param eval_input: The input to the evaluator. Expected to contain whatever inputs are needed for the _flow method
+        :type eval_input: Dict
+        :return: The evaluation result.
+        :rtype: Dict
+        """
+        # we override the _do_eval method as we want the output to be a dictionary,
+        # which is a different schema than _base_prompty_eval.py
+        if "ground_truth" not in eval_input or "response" not in eval_input:
+            raise EvaluationException(
+                message=f"Both ground_truth and response must be provided as input to the completeness evaluator.",
+                internal_message=f"Both ground_truth and response must be provided as input to the completeness"
+                                 f" evaluator.",
+                blame=ErrorBlame.USER_ERROR,
+                category=ErrorCategory.MISSING_FIELD,
+                target=ErrorTarget.COMPLETENESS_EVALUATOR,
+            )
+
+        llm_output = await self._flow(timeout=self._LLM_CALL_TIMEOUT, **eval_input)
+
+        score = math.nan
+        if llm_output:
+            score, reason = parse_quality_evaluator_reason_score(llm_output, valid_score_range="[1-5]")
+
+            score_result = 'pass' if score >= self.threshold else 'fail'
+
+            # updating the result key and threshold to int based on the schema
+            return {
+                f"{self._result_key}": int(score),
+                f"{self._result_key}_result": score_result,
+                f"{self._result_key}_threshold": int(self.threshold),
+                f"{self._result_key}_reason": reason,
+            }
+        
+        return {self._result_key: math.nan}

azure/ai/evaluation/_evaluators/_response_completeness/response_completeness.prompty

@@ -0,0 +1,99 @@
+---
+name: Completeness
+description: Evaluates Completeness score for QA scenario
+model:
+  api: chat
+  parameters:
+    temperature: 0.0
+    max_tokens: 800
+    top_p: 1.0
+    seed: 123
+    presence_penalty: 0
+    frequency_penalty: 0
+    response_format:
+      type: text
+
+inputs:
+  response:
+    type: string
+  ground_truth:
+    type: string
+
+---
+system:
+# Instruction
+## Context
+### You are an expert in evaluating the quality of an answer from an intelligent system based on provided definitions and data. Your goal will involve answering the questions below using the information provided.
+- **Definition**: You are given a definition of the response quality that is being evaluated to help guide your Score.
+- **Data**: Your input data include a response and its ground truth.
+- **Questions**: To complete your evaluation you will be asked to evaluate the Data in different ways.
+
+
+# Definition
+
+**Level 1: Fully incomplete**
+
+**Definition:**
+A response is considered fully incomplete if it does not contain any the necessary and relevant information with respect to the ground truth. In other words, it completely misses all the information - especially claims and statements - established in the ground truth. 
+
+**Examples:**
+1. **Response:** "Flu shot cannot cure cancer. Stay healthy requires sleeping exactly 8 hours a day. A few hours of exercise per week will have little benefits for physical and mental health. Physical and mental health benefits are separate topics. Scientists have not studied any of them."
+   **Ground Truth:** "Flu shot can prevent flu-related illnesses. Staying healthy requires proper hydration and moderate exercise. Even a few hours of exercise per week can have long-term benefits for physical and mental health. This is because physical and mental health benefits have intricate relationships through behavioral changes. Scientists are starting to discover them through rigorous studies."
+
+
+**Level 2: Barely complete**
+
+**Definition:**
+A response is considered barely complete if it only contains a small percentage of all the necessary and relevant information with respect to the ground truth. In other words, it misses almost all the information - especially claims and statements - established in the ground truth. 
+
+**Examples:**
+1. **Response:** "Flu shot can prevent flu-related illnesses. Staying healthy requires 2 meals a day. Exercise per week makes not difference to physical and mental health. This is because physical and mental health benefits have low correlation through scientific studies. Scientists are making this observation in studies."
+   **Ground Truth:** "Flu shot can prevent flu-related illnesses. Stay healthy by proper hydration and moderate exercise. Even a few hours of exercise per week can have long-term benefits for physical and mental health. This is because physical and mental health benefits have intricate relationships through behavioral changes. Scientists are starting to discover them through rigorous studies."
+
+
+**Level 3: Moderately complete**
+
+**Definition:**
+A response is considered moderately complete if it contains half of the necessary and relevant information with respect to the ground truth. In other words, it miss half of the information - especially claims and statements - established in the ground truth. 
+
+**Examples:**
+1. **Response:** "Flu shot can prevent flu-related illnesses. Staying healthy requires a few dollar of investments a day. Even a few dollars of investments per week will not make an impact on physical and mental health. This is because physical and mental health benefits have intricate relationships through behavioral changes. Fiction writers are starting to discover them through their works."
+   **Ground Truth:** "Flu shot can prevent flu-related illnesses. Stay healthy by proper hydration and moderate exercise. Even a few hours of exercise per week can have long-term benefits for physical and mental health. This is because physical and mental health benefits have intricate relationships through behavioral changes. Scientists are starting to discover them through rigorous studies."
+
+
+**Level 4: Mostly complete**
+
+**Definition:**
+A response is considered mostly complete if it contains most of the necessary and relevant information with respect to the ground truth. In other words, it misses some minor information - especially claims and statements - established in the ground truth. 
+
+
+**Examples:**
+1. **Response:** "Flu shot can prevent flu-related illnesses. Staying healthy requires keto diet and rigorous athletic training. Even a few hours of exercise per week can have long-term benefits for physical and mental health. This is because physical and mental health benefits have intricate relationships through behavioral changes. Scientists are starting to discover them through rigorous studies."
+   **Ground Truth:** "Flu shot can prevent flu-related illnesses. Stay healthy by proper hydration and moderate exercise. Even a few hours of exercise per week can have long-term benefits for physical and mental health. This is because physical and mental health benefits have intricate relationships through behavioral changes. Scientists are starting to discover them through rigorous studies."
+
+
+**Level 5: Fully complete**
+
+**Definition:**
+A response is considered complete if it perfectly contains all the necessary and relevant information with respect to the ground truth. In other words, it does not miss any information from statements and claims in the ground truth. 
+
+**Examples:**
+1. **Response:** "Flu shot can prevent flu-related illnesses. Stay healthy by proper hydration and moderate exercise. Even a few hours of exercise per week can have long-term benefits for physical and mental health. This is because physical and mental health benefits have intricate relationships through behavioral changes. Scientists are starting to discover them through rigorous studies."
+   **Ground Truth:** "Flu shot can prevent flu-related illnesses. Stay healthy by proper hydration and moderate exercise. Even a few hours of exercise per week can have long-term benefits for physical and mental health. This is because physical and mental health benefits have intricate relationships through behavioral changes. Scientists are starting to discover them through rigorous studies."
+
+
+
+# Data
+Response: {{response}}
+Ground Truth: {{ground_truth}}
+
+
+# Tasks
+## Please provide your assessment Score for the previous answer. Your output should include the following information:
+- **ThoughtChain**: To improve the reasoning process, Think Step by Step and include a step-by-step explanation of your thought process as you analyze the data based on the definitions. Keep it brief and Start your ThoughtChain with "Let's think step by step:".
+- **Explanation**: a very short explanation of why you think the input data should get that Score.
+- **Score**: based on your previous analysis, provide your Score. The answer you give MUST be a integer score ("1", "2", ...) based on the categories of the definitions.
+
+
+## Please provide your answers between the tags: <S0>your chain of thoughts</S0>, <S1>your explanation</S1>, <S2>your score</S2>.
+# Output

azure/ai/evaluation/_evaluators/_retrieval/_retrieval.py

@@ -31,6 +31,8 @@ class RetrievalEvaluator(PromptyEvaluatorBase[Union[str, float]]):
     :param model_config: Configuration for the Azure OpenAI model.
     :type model_config: Union[~azure.ai.evaluation.AzureOpenAIModelConfiguration,
         ~azure.ai.evaluation.OpenAIModelConfiguration]
+    :param threshold: The threshold for the evaluation. Default is 3.
+    :type threshold: float
     :return: A function that evaluates and generates metrics for "chat" scenario.
     :rtype: Callable
 
@@ -43,6 +45,15 @@ class RetrievalEvaluator(PromptyEvaluatorBase[Union[str, float]]):
             :dedent: 8
             :caption: Initialize and call a RetrievalEvaluator.
 
+    .. admonition:: Example with Threshold:
+    
+        .. literalinclude:: ../samples/evaluation_samples_threshold.py
+            :start-after: [START threshold_retrieval_evaluator]
+            :end-before: [END threshold_retrieval_evaluator]
+            :language: python
+            :dedent: 8
+            :caption: Initialize with threshold and call a RetrievalEvaluator.
+
     .. note::
 
         To align with our support of a diverse set of models, an output key without the `gpt_` prefix has been added.
@@ -57,10 +68,18 @@ class RetrievalEvaluator(PromptyEvaluatorBase[Union[str, float]]):
     """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
 
     @override
-    def __init__(self, model_config):  # pylint: disable=super-init-not-called
+    def __init__(self, model_config, *, threshold: float=3):  # pylint: disable=super-init-not-called
         current_dir = os.path.dirname(__file__)
         prompty_path = os.path.join(current_dir, self._PROMPTY_FILE)
-        super().__init__(model_config=model_config, prompty_file=prompty_path, result_key=self._RESULT_KEY)
+        self._threshold = threshold
+        self._higher_is_better = True
+        super().__init__(
+            model_config=model_config,
+            prompty_file=prompty_path,
+            result_key=self._RESULT_KEY,
+            threshold=threshold,
+            _higher_is_better=self._higher_is_better,
+        )
 
     @overload
     def __call__(

azure/ai/evaluation/_evaluators/_rouge/_rouge.py

@@ -8,6 +8,8 @@ from typing_extensions import overload, override
 
 from azure.ai.evaluation._vendor.rouge_score import rouge_scorer
 from azure.ai.evaluation._evaluators._common import EvaluatorBase
+from azure.ai.evaluation._constants import EVALUATION_PASS_FAIL_MAPPING
+import math
 
 
 class RougeType(Enum):
@@ -50,6 +52,14 @@ class RougeScoreEvaluator(EvaluatorBase):
     information from the reference text.
 
     ROUGE scores range from 0 to 1, with higher scores indicating better quality.
+    :param rouge_type: The type of ROUGE score to calculate. Default is "rouge1".
+    :type rouge_type: str
+    :param precision_threshold: The threshold value to determine if the precision evaluation passes or fails. Default is 0.5.
+    :type precision_threshold: float
+    :param recall_threshold: The threshold value to determine if the recall evaluation passes or fails. Default is 0.5.
+    :type recall_threshold: float
+    :param f1_score_threshold: The threshold value to determine if the F1 score evaluation passes or fails. Default is 0.5.
+    :type f1_score_threshold: float
 
     .. admonition:: Example:
 
@@ -59,15 +69,94 @@ class RougeScoreEvaluator(EvaluatorBase):
             :language: python
             :dedent: 8
             :caption: Initialize and call a RougeScoreEvaluator with a four-gram rouge type.
+
+    .. admonition:: Example with threshold:
+
+        .. literalinclude:: ../samples/evaluation_samples_threshold.py
+            :start-after: [START threshold_rouge_score_evaluator]
+            :end-before: [END threshold_rouge_score_evaluator]
+            :language: python
+            :dedent: 8
+            :caption: Initialize with a specified threshold and call a RougeScoreEvaluator with a four-gram rouge type.
     """
 
     id = "azureml://registries/azureml/models/Rouge-Score-Evaluator/versions/3"
     """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
 
     @override
-    def __init__(self, rouge_type: RougeType):
+    def __init__(
+        self, 
+        rouge_type: RougeType,
+        *,
+        precision_threshold: float = 0.5,
+        recall_threshold: float = 0.5,
+        f1_score_threshold: float = 0.5
+    ):
         self._rouge_type = rouge_type
+        self._higher_is_better = True
         super().__init__()
+        
+        # Type checking for threshold parameters
+        for name, value in [
+            ("precision_threshold", precision_threshold),
+            ("recall_threshold", recall_threshold),
+            ("f1_score_threshold", f1_score_threshold),
+        ]:
+            if not isinstance(value, float):
+                raise TypeError(f"{name} must be a float, got {type(value)}")
+                
+        self._threshold = {
+            "precision": precision_threshold,
+            "recall": recall_threshold,
+            "f1_score": f1_score_threshold,
+        }
+
+    def _get_binary_result(
+            self,
+            rouge_precision: float,
+            rouge_recall: float,
+            rouge_f1_score: float,
+    ) -> Dict[str, bool]:
+        """
+        Get binary result based on the threshold.
+
+        :param rouge_precision: The precision score.
+        :type rouge_precision: float
+        :param rouge_recall: The recall score.
+        :type rouge_recall: float
+        :param rouge_f1_score: The F1 score.
+        :type rouge_f1_score: float
+        :return: A dictionary with binary results for precision, recall, and F1 score.
+
+        """
+        # Initialize results with False for NaN values
+        results = {
+            "rouge_precision_result": False,
+            "rouge_recall_result": False,
+            "rouge_f1_score_result": False,
+        }
+
+        # Check if values are valid (not NaN) before comparison
+        precision_valid = not math.isnan(rouge_precision)
+        recall_valid = not math.isnan(rouge_recall)
+        f1_valid = not math.isnan(rouge_f1_score)
+        
+        if self._higher_is_better:
+            if precision_valid:
+                results["rouge_precision_result"] = (rouge_precision >= self._threshold["precision"])
+            if recall_valid:
+                results["rouge_recall_result"] = (rouge_recall >= self._threshold["recall"])
+            if f1_valid:
+                results["rouge_f1_score_result"] = (rouge_f1_score >= self._threshold["f1_score"])
+        else:
+            if precision_valid:
+                results["rouge_precision_result"] = (rouge_precision <= self._threshold["precision"])
+            if recall_valid:
+                results["rouge_recall_result"] = (rouge_recall <= self._threshold["recall"])
+            if f1_valid:
+                results["rouge_f1_score_result"] = (rouge_f1_score <= self._threshold["f1_score"])
+        
+        return results
 
     @override
     async def _do_eval(self, eval_input: Dict) -> Dict[str, float]:
@@ -82,10 +171,30 @@ class RougeScoreEvaluator(EvaluatorBase):
         response = eval_input["response"]
         scorer = rouge_scorer.RougeScorer(rouge_types=[self._rouge_type.value])
         metrics = scorer.score(ground_truth, response)[self._rouge_type.value]
+        binary_results = {
+            "rouge_precision_result": False,
+            "rouge_recall_result": False,
+            "rouge_f1_score_result": False,
+        }
+        # Convert metrics to floats, using nan for None or non-convertible values
+        rouge_precision = float(metrics.precision) if metrics.precision is not None else float('nan')
+        rouge_recall = float(metrics.recall) if metrics.recall is not None else float('nan')
+        rouge_f1_score = float(metrics.fmeasure) if metrics.fmeasure is not None else float('nan')
+        binary_results = self._get_binary_result(
+            rouge_precision=rouge_precision,
+            rouge_recall=rouge_recall,
+            rouge_f1_score=rouge_f1_score,
+        )
         return {
-            "rouge_precision": metrics.precision,
-            "rouge_recall": metrics.recall,
-            "rouge_f1_score": metrics.fmeasure,
+            "rouge_precision": rouge_precision,
+            "rouge_recall": rouge_recall,
+            "rouge_f1_score": rouge_f1_score,
+            "rouge_precision_result": EVALUATION_PASS_FAIL_MAPPING[binary_results["rouge_precision_result"]],
+            "rouge_recall_result": EVALUATION_PASS_FAIL_MAPPING[binary_results["rouge_recall_result"]],
+            "rouge_f1_score_result": EVALUATION_PASS_FAIL_MAPPING[binary_results["rouge_f1_score_result"]],
+            "rouge_precision_threshold": self._threshold["precision"],
+            "rouge_recall_threshold": self._threshold["recall"],
+            "rouge_f1_score_threshold": self._threshold["f1_score"],
         }
 
     @overload  # type: ignore

azure/ai/evaluation/_evaluators/_service_groundedness/_service_groundedness.py

@@ -27,6 +27,8 @@ class GroundednessProEvaluator(RaiServiceEvaluatorBase[Union[str, bool]]):
     :param azure_ai_project: The scope of the Azure AI project.
         It contains subscription id, resource group, and project name.
     :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
+    :param threshold: The threshold for the groundedness pro evaluator. Default is 5.
+    :type threshold: int
     :param kwargs: Additional arguments to pass to the evaluator.
     :type kwargs: Any
 
@@ -39,6 +41,15 @@ class GroundednessProEvaluator(RaiServiceEvaluatorBase[Union[str, bool]]):
             :dedent: 8
             :caption: Initialize and call a GroundednessProEvaluator with a query, response, and context.
 
+    .. admonition:: Example with threshold:
+
+        .. literalinclude:: ../samples/evaluation_samples_threshold.py
+            :start-after: [START threshold_groundedness_pro_evaluator]
+            :end-before: [END threshold_groundedness_pro_evaluator]
+            :language: python
+            :dedent: 8
+            :caption: Initialize with a specified threshold and call GroundednessProEvaluator with a query, response, and context.
+
     .. note::
 
         If this evaluator is supplied to the `evaluate` function, the aggregated metric
@@ -53,14 +64,18 @@ class GroundednessProEvaluator(RaiServiceEvaluatorBase[Union[str, bool]]):
         self,
         credential,
         azure_ai_project,
+        *,
+        threshold: int = 5,
         **kwargs,
     ):
-        self._passing_score = 5  # TODO update once the binarization PR is merged
+        self.threshold = threshold
+        self._higher_is_better = True
         self._output_prefix = "groundedness_pro"
         super().__init__(
             eval_metric=EvaluationMetrics.GROUNDEDNESS,
             azure_ai_project=azure_ai_project,
             credential=credential,
+            threshold=self.threshold,
             **kwargs,
         )
 
@@ -141,8 +156,13 @@ class GroundednessProEvaluator(RaiServiceEvaluatorBase[Union[str, bool]]):
         """
         result = await super()._do_eval(eval_input)
         real_result = {}
+        real_result[self._output_prefix + "_reason"] = result[EvaluationMetrics.GROUNDEDNESS + "_reason"]
         real_result[self._output_prefix + "_label"] = (
-            result[EvaluationMetrics.GROUNDEDNESS + "_score"] >= self._passing_score
+            result[EvaluationMetrics.GROUNDEDNESS + "_score"] >= self.threshold
         )
-        real_result[self._output_prefix + "_reason"] = result[EvaluationMetrics.GROUNDEDNESS + "_reason"]
+        if self._higher_is_better:
+            real_result[self._output_prefix + "_score"] = max(result[EvaluationMetrics.GROUNDEDNESS + "_score"], 0)
+        else:
+            real_result[self._output_prefix + "_score"] = min(result[EvaluationMetrics.GROUNDEDNESS + "_score"], 1)
+
         return real_result