azure-ai-evaluation 1.5.0__py3-none-any.whl → 1.6.0__py3-none-any.whl

azure/ai/evaluation/_evaluators/_document_retrieval/_document_retrieval.py

@@ -0,0 +1,467 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+import math
+import operator
+from itertools import starmap
+from typing import Dict, List, TypedDict, Tuple, Optional
+from azure.ai.evaluation._evaluators._common import EvaluatorBase
+from azure.ai.evaluation._exceptions import EvaluationException
+from typing_extensions import override, overload
+
+
+RetrievalGroundTruthDocument = TypedDict(
+    "RetrievalGroundTruthDocument", {"document_id": str, "query_relevance_label": int}
+)
+
+RetrievedDocument = TypedDict(
+    "RetrievedDocument", {"document_id": str, "relevance_score": float}
+)
+
+
+class DocumentRetrievalEvaluator(EvaluatorBase):
+    """
+    Calculate document retrieval metrics, such as NDCG, XDCG, Fidelity, Top K Relevance and Holes.
+
+    .. admonition:: Example:
+
+        .. literalinclude:: ../samples/evaluation_samples_evaluate.py
+            :start-after: [START document_retrieval_evaluator]
+            :end-before: [END document_retrieval_evaluator]
+            :language: python
+            :dedent: 8
+            :caption: Initialize and call a Document RetrievalEvaluator
+
+    .. admonition:: Example with Threshold:
+        .. literalinclude:: ../samples/evaluation_samples_threshold.py
+            :start-after: [START threshold_document_retrieval_evaluator]
+            :end-before: [END threshold_document_retrieval_evaluator]
+            :language: python
+            :dedent: 8
+            :caption: Initialize with threshold and call a DocumentRetrievalEvaluator.
+    """
+
+    def __init__(
+        self,
+        *,
+        ground_truth_label_min: int = 0,
+        ground_truth_label_max: int = 4,
+        threshold: Optional[dict] = None,
+    ):
+        super().__init__()
+        self.k = 3
+        self.xdcg_discount_factor = 0.6
+
+        if ground_truth_label_min >= ground_truth_label_max:
+            raise EvaluationException(
+                "The ground truth label maximum must be strictly greater than the ground truth label minimum."
+            )
+
+        if not isinstance(ground_truth_label_min, int):
+            raise EvaluationException(
+                "The ground truth label minimum must be an integer value."
+            )
+
+        if not isinstance(ground_truth_label_max, int):
+            raise EvaluationException(
+                "The ground truth label maximum must be an integer value."
+            )
+
+        self.ground_truth_label_min = ground_truth_label_min
+        self.ground_truth_label_max = ground_truth_label_max
+
+        # The default threshold for metrics where higher numbers are better.
+        self._threshold_metrics = {
+            "ndcg@3": 0.5,
+            "xdcg@3": 0.5,
+            "fidelity": 0.5,
+            "top1_relevance": 50,
+            "top3_max_relevance": 50,
+            "total_retrieved_documents": 50,
+            "total_ground_truth_documents": 50,
+        }
+
+        # Ideally, the number of holes should be zero.
+        self._threshold_holes = {"holes": 0, "holes_ratio": 0}
+
+        if threshold and not isinstance(threshold, dict):
+            raise EvaluationException(
+                f"Threshold must be a dictionary, got {type(threshold)}"
+            )
+
+        elif isinstance(threshold, dict):
+            self._threshold_metrics.update(threshold)
+
+    def _compute_holes(self, actual_docs: List[str], labeled_docs: List[str]) -> int:
+        """
+        The number of documents retrieved from a search query which have no provided ground-truth label.
+        This metric is helpful for determining the accuracy of other metrics that are highly sensitive to missing ground-truth knowledge,
+        such as NDCG, XDCG, and Fidelity.
+
+        :param actual_docs: A list of retrieved documents' IDs.
+        :type actual_docs: List[str]
+        :param labeled_docs: A list of ideal documents' IDs.
+        :type labeled: List[str]
+        :return: The holes calculation result.
+        :rtype: int
+        """
+        return len(set(actual_docs).difference(set(labeled_docs)))
+
+    def _compute_ndcg(
+        self,
+        result_docs_groundtruth_labels: List[int],
+        ideal_docs_groundtruth_labels: List[int],
+    ) -> float:
+        """NDCG (Normalized Discounted Cumulative Gain) calculated for the top K documents retrieved from a search query.
+        NDCG measures how well a document ranking compares to an ideal document ranking given a list of ground-truth documents.
+        
+        :param result_docs_groundtruth_labels: A list of retrieved documents' ground truth labels.
+        :type result_docs_groundtruth_labels: List[int]
+        :param ideal_docs_groundtruth_labels: A list of ideal documents' ground truth labels.
+        :type ideal_docs_groundtruth_labels: List[int]
+        :return: The NDCG@K calculation result.
+        :rtype: float
+        """
+
+        # Set the scoring function
+        def calculate_dcg(relevance: float, rank: int):
+            return (math.pow(2, relevance) - 1) / (math.log2(rank + 1))
+
+        ranks = list(range(1, self.k + 1))
+        dcg = sum(starmap(calculate_dcg, zip(result_docs_groundtruth_labels, ranks)))
+        idcg = sum(starmap(calculate_dcg, zip(ideal_docs_groundtruth_labels, ranks)))
+        ndcg = dcg / float(idcg)
+
+        return ndcg
+
+    def _compute_xdcg(self, result_docs_groundtruth_labels: List[int]) -> float:
+        """XDCG calculated for the top K documents retrieved from a search query.
+        XDCG measures how objectively good are the top K documents, discounted by their position in the list.
+        
+        :param result_docs_groundtruth_labels: A list of retrieved documents' ground truth labels.
+        :type result_docs_groundtruth_labels: List[int]
+        :return: The XDCG@K calculation result.
+        :rtype: float
+        """
+
+        def calculate_xdcg_numerator(relevance, rank):
+            return 25 * relevance * math.pow(self.xdcg_discount_factor, rank - 1)
+
+        def calculate_xdcg_denominator(rank):
+            return math.pow(self.xdcg_discount_factor, rank - 1)
+
+        ranks = list(range(1, self.k + 1))
+        xdcg_n = sum(
+            starmap(
+                calculate_xdcg_numerator, zip(result_docs_groundtruth_labels, ranks)
+            )
+        )
+        xdcg_d = sum(map(calculate_xdcg_denominator, ranks))
+
+        return xdcg_n / float(xdcg_d)
+
+    def _compute_fidelity(
+        self,
+        result_docs_groundtruth_labels: List[int],
+        ideal_docs_groundtruth_labels: List[int],
+    ) -> float:
+        """Fidelity calculated over all documents retrieved from a search query.
+        Fidelity measures how objectively good are all of the documents retrieved compared with all known good documents in the underlying data store.
+        
+        :param result_docs_groundtruth_labels: A list of retrieved documents' ground truth labels.
+        :type result_docs_groundtruth_labels: List[int]
+        :param ideal_docs_groundtruth_labels: A list of ideal documents' ground truth labels.
+        :type ideal_docs_groundtruth_labels: List[int]
+        :return: The fidelity calculation result.
+        :rtype: float
+        """
+
+        def calculate_weighted_sum_by_rating(labels: List[int]) -> float:
+            # here we assume that the configured groundtruth label minimum translates to "irrelevant",
+            # so we exclude documents with that label from the calculation.
+            s = self.ground_truth_label_min + 1
+
+            # get a count of each label
+            label_counts = {str(i): 0 for i in range(s, self.ground_truth_label_max + 1)}
+
+            for label in labels:
+                if label >= s:
+                    label_counts[str(label)] += 1
+
+            sorted_label_counts = [
+                x[1] for x in sorted(label_counts.items(), key=lambda x: x[0])
+            ]
+
+            # calculate weights
+            weights = [
+                (math.pow(2, i + 1) - 1)
+                for i in range(s, self.ground_truth_label_max + 1)
+            ]
+
+            # return weighted sum
+            return sum(starmap(operator.mul, zip(sorted_label_counts, weights)))
+
+        weighted_sum_by_rating_results = calculate_weighted_sum_by_rating(
+            result_docs_groundtruth_labels
+        )
+        weighted_sum_by_rating_index = calculate_weighted_sum_by_rating(
+            ideal_docs_groundtruth_labels
+        )
+
+        if weighted_sum_by_rating_index == 0:
+            return math.nan
+
+        return weighted_sum_by_rating_results / float(weighted_sum_by_rating_index)
+
+    def _get_binary_result(self, **metrics) -> Dict[str, float]:
+        result = {}
+
+        for metric_name, metric_value in metrics.items():
+            if metric_name in self._threshold_metrics.keys():
+                result[f"{metric_name}_result"] = (
+                    metric_value >= self._threshold_metrics[metric_name]
+                )
+                result[f"{metric_name}_threshold"] = self._threshold_metrics[
+                    metric_name
+                ]
+                result[f"{metric_name}_higher_is_better"] = True
+
+            elif metric_name in self._threshold_holes.keys():
+                result[f"{metric_name}_result"] = (
+                    metric_value <= self._threshold_holes[metric_name]
+                )
+                result[f"{metric_name}_threshold"] = self._threshold_holes[metric_name]
+                result[f"{metric_name}_higher_is_better"] = False
+
+            else:
+                raise ValueError(f"No threshold set for metric '{metric_name}'")
+
+        return result
+
+    def _validate_eval_input(
+        self, eval_input: Dict
+    ) -> Tuple[List[RetrievalGroundTruthDocument], List[RetrievedDocument]]:
+        """Validate document retrieval evaluator inputs.
+
+        :param eval_input: The input to the evaluation function.
+        :type eval_input: Dict
+        :return: The evaluation result.
+        :rtype: Tuple[List[azure.ai.evaluation.RetrievalGroundTruthDocument], List[azure.ai.evaluation.RetrievedDocument]]
+        """
+        retrieval_ground_truth = eval_input.get("retrieval_ground_truth")
+        retrieved_documents = eval_input.get("retrieved_documents")
+
+        # if the qrels are empty, no meaningful evaluation is possible
+        if not retrieval_ground_truth:
+            raise EvaluationException(
+                ("'retrieval_ground_truth' parameter must contain at least one item. "
+                 "Check your data input to be sure that each input record has ground truth defined.")
+            )
+
+        qrels = []
+
+        # validate the qrels to be sure they are the correct type and are bounded by the given configuration
+        for qrel in retrieval_ground_truth:
+            document_id = qrel.get("document_id")
+            query_relevance_label = qrel.get("query_relevance_label")
+
+            if document_id is None or query_relevance_label is None:
+                raise EvaluationException(
+                    (
+                        "Invalid input data was found in the retrieval ground truth. "
+                        "Ensure that all items in the 'retrieval_ground_truth' array contain "
+                        "'document_id' and 'query_relevance_label' properties."
+                    )
+                )
+
+            if not isinstance(query_relevance_label, int):
+                raise EvaluationException(
+                    "Query relevance labels must be integer values."
+                )
+
+            if query_relevance_label < self.ground_truth_label_min:
+                raise EvaluationException(
+                    (
+                        "A query relevance label less than the configured minimum value was detected in the evaluation input data. "
+                        "Check the range of ground truth label values in the input data and set the value of ground_truth_label_min to "
+                        "the appropriate value for your data."
+                    )
+                )
+
+            if query_relevance_label > self.ground_truth_label_max:
+                raise EvaluationException(
+                    (
+                        "A query relevance label greater than the configured maximum value was detected in the evaluation input data. "
+                        "Check the range of ground truth label values in the input data and set the value of ground_truth_label_max to "
+                        "the appropriate value for your data."
+                    )
+                )
+
+            qrels.append(qrel)
+
+        # validate retrieved documents to be sure they are the correct type
+        results = []
+
+        if isinstance(retrieved_documents, list):
+            for result in retrieved_documents:
+                document_id = result.get("document_id")
+                relevance_score = result.get("relevance_score")
+
+                if document_id is None or relevance_score is None:
+                    raise EvaluationException(
+                        (
+                            "Invalid input data was found in the retrieved documents. "
+                            "Ensure that all items in the 'retrieved_documents' array contain "
+                            "'document_id' and 'relevance_score' properties."
+                        )
+                    )
+
+                if not isinstance(relevance_score, float) and not isinstance(
+                    relevance_score, int
+                ):
+                    raise EvaluationException(
+                        "Retrieved document relevance score must be a numerical value."
+                    )
+
+                results.append(result)
+
+        if len(qrels) > 10000 or len(results) > 10000:
+            raise EvaluationException(
+                "'retrieval_ground_truth' and 'retrieved_documents' inputs should contain no more than 10000 items."
+            )
+
+        return qrels, results
+
+    async def _do_eval(self, eval_input: Dict) -> Dict[str, float]:
+        """Produce a document retrieval evaluation result.
+
+        :param eval_input: The input to the evaluation function.
+        :type eval_input: Dict
+        :return: The evaluation result.
+        :rtype: Dict[str, float]
+        """
+        qrels, results = self._validate_eval_input(eval_input)
+
+        # if the results set is empty, results are all zero
+        if len(results) == 0:
+            metrics = {
+                f"ndcg@{self.k}": 0.0,
+                f"xdcg@{self.k}": 0.0,
+                "fidelity": 0.0,
+                "top1_relevance": 0.0,
+                "top3_max_relevance": 0.0,
+                "holes": 0,
+                "holes_ratio": 0,
+                "total_retrieved_documents": len(results),
+                "total_ground_truth_documents": len(qrels),
+            }
+            binary_result = self._get_binary_result(**metrics)
+            for k, v in binary_result.items():
+                metrics[k] = v
+
+            return metrics
+
+        # flatten qrels and results to normal dictionaries
+        qrels_lookup = {x["document_id"]: x["query_relevance_label"] for x in qrels}
+        results_lookup = {x["document_id"]: x["relevance_score"] for x in results}
+
+        # sort each input set by label to get the ranking
+        qrels_sorted_by_rank = sorted(
+            qrels_lookup.items(), key=lambda x: x[1], reverse=True
+        )
+        results_sorted_by_rank = sorted(
+            results_lookup.items(), key=lambda x: x[1], reverse=True
+        )
+
+        # find ground truth labels for the results set and ideal set
+        result_docs_groundtruth_labels = [
+            qrels_lookup[doc_id] if doc_id in qrels_lookup else 0
+            for (doc_id, _) in results_sorted_by_rank
+        ]
+        ideal_docs_groundtruth_labels = [label for (_, label) in qrels_sorted_by_rank]
+
+        # calculate the proportion of result docs with no ground truth label (holes)
+        holes = self._compute_holes(
+            [x[0] for x in results_sorted_by_rank], [x[0] for x in qrels_sorted_by_rank]
+        )
+        holes_ratio = holes / float(len(results))
+
+        # if none of the retrieved docs are labeled, report holes only
+        if not any(result_docs_groundtruth_labels):
+            metrics = {
+                f"ndcg@{self.k}": 0,
+                f"xdcg@{self.k}": 0,
+                "fidelity": 0,
+                "top1_relevance": 0,
+                "top3_max_relevance": 0,
+                "holes": holes,
+                "holes_ratio": holes_ratio,
+                "total_retrieved_documents": len(results),
+                "total_ground_truth_documents": len(qrels),
+            }
+            binary_result = self._get_binary_result(**metrics)
+            for k, v in binary_result.items():
+                metrics[k] = v
+
+            return metrics
+
+        metrics = {
+            f"ndcg@{self.k}": self._compute_ndcg(
+                result_docs_groundtruth_labels[: self.k],
+                ideal_docs_groundtruth_labels[: self.k],
+            ),
+            f"xdcg@{self.k}": self._compute_xdcg(
+                result_docs_groundtruth_labels[: self.k]
+            ),
+            "fidelity": self._compute_fidelity(
+                result_docs_groundtruth_labels, ideal_docs_groundtruth_labels
+            ),
+            "top1_relevance": result_docs_groundtruth_labels[0],
+            "top3_max_relevance": max(result_docs_groundtruth_labels[: self.k]),
+            "holes": holes,
+            "holes_ratio": holes_ratio,
+            "total_retrieved_documents": len(results),
+            "total_ground_truth_documents": len(qrels),
+        }
+
+        binary_result = self._get_binary_result(**metrics)
+        for k, v in binary_result.items():
+            metrics[k] = v
+
+        return metrics
+
+    @overload
+    def __call__(  # type: ignore
+        self,
+        *,
+        retrieval_ground_truth: List[RetrievalGroundTruthDocument],
+        retrieved_documents: List[RetrievedDocument],
+    ) -> Dict[str, float]:
+        """
+        Compute document retrieval metrics for documents retrieved from a search algorithm against a known set of ground truth documents.
+
+        Evaluation metrics calculated include NDCG@3, XDCG@3, Fidelity, Top K Relevance and Holes.
+
+        :keyword retrieval_ground_truth: a list of ground-truth document judgements for a query, where each item in the list contains a unique document identifier and a query relevance label.
+        :paramtype retrieval_ground_truth: List[azure.ai.evaluation.RetrievalGroundTruthDocument]
+        :keyword retrieved_documents: a list of documents scored by a search algorithm for a query, where each item in the list contains a unique document identifier and a relevance score.
+        :paramtype retrieved_documents: List[azure.ai.evaluation.RetrievedDocument]
+        :return: The document retrieval metrics.
+        :rtype: Dict[str, float]
+        """
+
+    @override
+    def __call__(self, *args, **kwargs):
+        """
+        Compute document retrieval metrics for documents retrieved from a search algorithm against a known set of ground truth documents.
+
+        Evaluation metrics calculated include NDCG@3, XDCG@3, Fidelity, Top K Relevance and Holes.
+
+        :keyword retrieval_ground_truth: a list of ground-truth document judgements for a query, where each item in the list contains a unique document identifier and a query relevance label.
+        :paramtype retrieval_ground_truth: List[azure.ai.evaluation.RetrievalGroundTruthDocument]
+        :keyword retrieved_documents: a list of documents scored by a search algorithm for a query, where each item in the list contains a unique document identifier and a relevance score.
+        :paramtype retrieved_documents: List[azure.ai.evaluation.RetrievedDocument]
+        :return: The document retrieval metrics.
+        :rtype: Dict[str, float]
+        """
+        return super().__call__(*args, **kwargs)

azure/ai/evaluation/_evaluators/_fluency/_fluency.py

@@ -23,7 +23,7 @@ class FluencyEvaluator(PromptyEvaluatorBase[Union[str, float]]):
     :param model_config: Configuration for the Azure OpenAI model.
     :type model_config: Union[~azure.ai.evaluation.AzureOpenAIModelConfiguration,
         ~azure.ai.evaluation.OpenAIModelConfiguration]
-    :param threshold: The threshold for the fluency evaluator. Default is 5.
+    :param threshold: The threshold for the fluency evaluator. Default is 3.
     :type threshold: int
 
     .. admonition:: Example:

azure/ai/evaluation/_evaluators/_groundedness/_groundedness.py

@@ -33,7 +33,7 @@ class GroundednessEvaluator(PromptyEvaluatorBase[Union[str, float]]):
     :param model_config: Configuration for the Azure OpenAI model.
     :type model_config: Union[~azure.ai.evaluation.AzureOpenAIModelConfiguration,
         ~azure.ai.evaluation.OpenAIModelConfiguration]
-    :param threshold: The threshold for the groundedness evaluator. Default is 5.
+    :param threshold: The threshold for the groundedness evaluator. Default is 3.
     :type threshold: int
         
     .. admonition:: Example:

azure/ai/evaluation/_evaluators/_intent_resolution/_intent_resolution.py

@@ -47,11 +47,15 @@ class IntentResolutionEvaluator(PromptyEvaluatorBase[Union[str, float]]):
     """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
 
     @override
-    def __init__(self, model_config, *, threshold = _DEFAULT_INTENT_RESOLUTION_THRESHOLD):
+    def __init__(self, model_config, *,
+                 threshold = _DEFAULT_INTENT_RESOLUTION_THRESHOLD,
+                 **kwargs):
         current_dir = os.path.dirname(__file__)
         prompty_path = os.path.join(current_dir, self._PROMPTY_FILE)
         self.threshold = threshold
-        super().__init__(model_config=model_config, prompty_file=prompty_path, result_key=self._RESULT_KEY)
+        super().__init__(model_config=model_config, prompty_file=prompty_path,
+                         result_key=self._RESULT_KEY,
+                         **kwargs)
 
     @overload
     def __call__(

azure/ai/evaluation/_evaluators/_relevance/_relevance.py

@@ -27,7 +27,7 @@ class RelevanceEvaluator(PromptyEvaluatorBase):
     :param model_config: Configuration for the Azure OpenAI model.
     :type model_config: Union[~azure.ai.evaluation.AzureOpenAIModelConfiguration,
         ~azure.ai.evaluation.OpenAIModelConfiguration]
-    :param threshold: The threshold for the relevance evaluator. Default is 5.
+    :param threshold: The threshold for the relevance evaluator. Default is 3.
     :type threshold: int
 
     .. admonition:: Example:

azure/ai/evaluation/_evaluators/_response_completeness/_response_completeness.py

@@ -60,11 +60,16 @@ class ResponseCompletenessEvaluator(PromptyEvaluatorBase[Union[str, float]]):
     """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
 
     @override
-    def __init__(self, model_config, *, threshold: Optional[float] = _DEFAULT_COMPLETENESS_THRESHOLD):
+    def __init__(self, model_config, *,
+                 threshold: Optional[float] = _DEFAULT_COMPLETENESS_THRESHOLD,
+                 **kwargs):
         current_dir = os.path.dirname(__file__)
         prompty_path = os.path.join(current_dir, self._PROMPTY_FILE)
         self.threshold = threshold
-        super().__init__(model_config=model_config, prompty_file=prompty_path, result_key=self._RESULT_KEY)
+        super().__init__(model_config=model_config,
+                         prompty_file=prompty_path,
+                         result_key=self._RESULT_KEY,
+                         **kwargs)
 
     @overload
     def __call__(

azure/ai/evaluation/_evaluators/_response_completeness/response_completeness.prompty

@@ -22,65 +22,51 @@ inputs:
 ---
 system:
 # Instruction
-## Context
-### You are an expert in evaluating the quality of an answer from an intelligent system based on provided definitions and data. Your goal will involve answering the questions below using the information provided.
-- **Definition**: You are given a definition of the response quality that is being evaluated to help guide your Score.
-- **Data**: Your input data include a response and its ground truth.
-- **Questions**: To complete your evaluation you will be asked to evaluate the Data in different ways.
-
+## Goal
+### You are an expert in evaluating the quality of a Response from an intelligent system based on provided definition and data. Your goal will involve answering the questions below using the information provided.
+- **Definition**: You are given a definition of the communication trait that is being evaluated to help guide your Score.
+- **Data**: Your input data include Response and Ground Truth.
+- **Tasks**: To complete your evaluation you will be asked to evaluate the Data in different ways.
 
+user:
 # Definition
+**Completeness** refers to how accurately and thoroughly a response represents the information provided in the ground truth. It considers both the inclusion of all relevant statements and the correctness of those statements. Each statement in the ground truth should be evaluated individually to determine if it is accurately reflected in the response without missing any key information. The scale ranges from 1 to 5, with higher numbers indicating greater completeness.
 
-**Level 1: Fully incomplete**
-
-**Definition:**
-A response is considered fully incomplete if it does not contain any the necessary and relevant information with respect to the ground truth. In other words, it completely misses all the information - especially claims and statements - established in the ground truth. 
+# Ratings
+## [Completeness: 1] (Fully Incomplete)
+**Definition:** A response that does not contain any of the necessary and relevant information with respect to the ground truth. It completely misses all the information, especially claims and statements, established in the ground truth.
 
 **Examples:**
-1. **Response:** "Flu shot cannot cure cancer. Stay healthy requires sleeping exactly 8 hours a day. A few hours of exercise per week will have little benefits for physical and mental health. Physical and mental health benefits are separate topics. Scientists have not studied any of them."
-   **Ground Truth:** "Flu shot can prevent flu-related illnesses. Staying healthy requires proper hydration and moderate exercise. Even a few hours of exercise per week can have long-term benefits for physical and mental health. This is because physical and mental health benefits have intricate relationships through behavioral changes. Scientists are starting to discover them through rigorous studies."
-
-
-**Level 2: Barely complete**
+  **Response:** "Flu shot cannot cure cancer. Stay healthy requires sleeping exactly 8 hours a day. A few hours of exercise per week will have little benefits for physical and mental health. Physical and mental health benefits are separate topics. Scientists have not studied any of them."
+  **Ground Truth:** "Flu shot can prevent flu-related illnesses. Staying healthy requires proper hydration and moderate exercise. Even a few hours of exercise per week can have long-term benefits for physical and mental health. This is because physical and mental health benefits have intricate relationships through behavioral changes. Scientists are starting to discover them through rigorous studies."
 
-**Definition:**
-A response is considered barely complete if it only contains a small percentage of all the necessary and relevant information with respect to the ground truth. In other words, it misses almost all the information - especially claims and statements - established in the ground truth. 
+## [Completeness: 2] (Barely Complete)
+**Definition:** A response that contains only a small percentage of all the necessary and relevant information with respect to the ground truth. It misses almost all the information, especially claims and statements, established in the ground truth.
 
 **Examples:**
-1. **Response:** "Flu shot can prevent flu-related illnesses. Staying healthy requires 2 meals a day. Exercise per week makes not difference to physical and mental health. This is because physical and mental health benefits have low correlation through scientific studies. Scientists are making this observation in studies."
-   **Ground Truth:** "Flu shot can prevent flu-related illnesses. Stay healthy by proper hydration and moderate exercise. Even a few hours of exercise per week can have long-term benefits for physical and mental health. This is because physical and mental health benefits have intricate relationships through behavioral changes. Scientists are starting to discover them through rigorous studies."
+  **Response:** "Flu shot can prevent flu-related illnesses. Staying healthy requires 2 meals a day. Exercise per week makes no difference to physical and mental health. This is because physical and mental health benefits have low correlation through scientific studies. Scientists are making this observation in studies."
+  **Ground Truth:** "Flu shot can prevent flu-related illnesses. Stay healthy by proper hydration and moderate exercise. Even a few hours of exercise per week can have long-term benefits for physical and mental health. This is because physical and mental health benefits have intricate relationships through behavioral changes. Scientists are starting to discover them through rigorous studies."
 
-
-**Level 3: Moderately complete**
-
-**Definition:**
-A response is considered moderately complete if it contains half of the necessary and relevant information with respect to the ground truth. In other words, it miss half of the information - especially claims and statements - established in the ground truth. 
+## [Completeness: 3] (Moderately Complete)
+**Definition:** A response that contains half of the necessary and relevant information with respect to the ground truth. It misses half of the information, especially claims and statements, established in the ground truth.
 
 **Examples:**
-1. **Response:** "Flu shot can prevent flu-related illnesses. Staying healthy requires a few dollar of investments a day. Even a few dollars of investments per week will not make an impact on physical and mental health. This is because physical and mental health benefits have intricate relationships through behavioral changes. Fiction writers are starting to discover them through their works."
-   **Ground Truth:** "Flu shot can prevent flu-related illnesses. Stay healthy by proper hydration and moderate exercise. Even a few hours of exercise per week can have long-term benefits for physical and mental health. This is because physical and mental health benefits have intricate relationships through behavioral changes. Scientists are starting to discover them through rigorous studies."
-
-
-**Level 4: Mostly complete**
-
-**Definition:**
-A response is considered mostly complete if it contains most of the necessary and relevant information with respect to the ground truth. In other words, it misses some minor information - especially claims and statements - established in the ground truth. 
+  **Response:** "Flu shot can prevent flu-related illnesses. Staying healthy requires a few dollars of investments a day. Even a few dollars of investments per week will not make an impact on physical and mental health. This is because physical and mental health benefits have intricate relationships through behavioral changes. Fiction writers are starting to discover them through their works."
+  **Ground Truth:** "Flu shot can prevent flu-related illnesses. Stay healthy by proper hydration and moderate exercise. Even a few hours of exercise per week can have long-term benefits for physical and mental health. This is because physical and mental health benefits have intricate relationships through behavioral changes. Scientists are starting to discover them through rigorous studies."
 
+## [Completeness: 4] (Mostly Complete)
+**Definition:** A response that contains most of the necessary and relevant information with respect to the ground truth. It misses some minor information, especially claims and statements, established in the ground truth.
 
 **Examples:**
-1. **Response:** "Flu shot can prevent flu-related illnesses. Staying healthy requires keto diet and rigorous athletic training. Even a few hours of exercise per week can have long-term benefits for physical and mental health. This is because physical and mental health benefits have intricate relationships through behavioral changes. Scientists are starting to discover them through rigorous studies."
-   **Ground Truth:** "Flu shot can prevent flu-related illnesses. Stay healthy by proper hydration and moderate exercise. Even a few hours of exercise per week can have long-term benefits for physical and mental health. This is because physical and mental health benefits have intricate relationships through behavioral changes. Scientists are starting to discover them through rigorous studies."
-
+  **Response:** "Flu shot can prevent flu-related illnesses. Staying healthy requires keto diet and rigorous athletic training. Even a few hours of exercise per week can have long-term benefits for physical and mental health. This is because physical and mental health benefits have intricate relationships through behavioral changes. Scientists are starting to discover them through rigorous studies."
+  **Ground Truth:** "Flu shot can prevent flu-related illnesses. Stay healthy by proper hydration and moderate exercise. Even a few hours of exercise per week can have long-term benefits for physical and mental health. This is because physical and mental health benefits have intricate relationships through behavioral changes. Scientists are starting to discover them through rigorous studies."
 
-**Level 5: Fully complete**
-
-**Definition:**
-A response is considered complete if it perfectly contains all the necessary and relevant information with respect to the ground truth. In other words, it does not miss any information from statements and claims in the ground truth. 
+## [Completeness: 5] (Fully Complete)
+**Definition:** A response that perfectly contains all the necessary and relevant information with respect to the ground truth. It does not miss any information from statements and claims in the ground truth.
 
 **Examples:**
-1. **Response:** "Flu shot can prevent flu-related illnesses. Stay healthy by proper hydration and moderate exercise. Even a few hours of exercise per week can have long-term benefits for physical and mental health. This is because physical and mental health benefits have intricate relationships through behavioral changes. Scientists are starting to discover them through rigorous studies."
-   **Ground Truth:** "Flu shot can prevent flu-related illnesses. Stay healthy by proper hydration and moderate exercise. Even a few hours of exercise per week can have long-term benefits for physical and mental health. This is because physical and mental health benefits have intricate relationships through behavioral changes. Scientists are starting to discover them through rigorous studies."
-
+  **Response:** "Flu shot can prevent flu-related illnesses. Stay healthy by proper hydration and moderate exercise. Even a few hours of exercise per week can have long-term benefits for physical and mental health. This is because physical and mental health benefits have intricate relationships through behavioral changes. Scientists are starting to discover them through rigorous studies."
+  **Ground Truth:** "Flu shot can prevent flu-related illnesses. Stay healthy by proper hydration and moderate exercise. Even a few hours of exercise per week can have long-term benefits for physical and mental health. This is because physical and mental health benefits have intricate relationships through behavioral changes. Scientists are starting to discover them through rigorous studies."
 
 
 # Data
@@ -89,11 +75,10 @@ Ground Truth: {{ground_truth}}
 
 
 # Tasks
-## Please provide your assessment Score for the previous answer. Your output should include the following information:
-- **ThoughtChain**: To improve the reasoning process, Think Step by Step and include a step-by-step explanation of your thought process as you analyze the data based on the definitions. Keep it brief and Start your ThoughtChain with "Let's think step by step:".
+## Please provide your assessment Score for the previous RESPONSE in relation to the GROUND TRUTH based on the Definitions above. Your output should include the following information:
+- **ThoughtChain**: To improve the reasoning process, think step by step and include a step-by-step explanation of your thought process as you analyze the data based on the definitions. Keep it brief and start your ThoughtChain with "Let's think step by step:".
 - **Explanation**: a very short explanation of why you think the input data should get that Score.
-- **Score**: based on your previous analysis, provide your Score. The answer you give MUST be a integer score ("1", "2", ...) based on the categories of the definitions.
-
+- **Score**: based on your previous analysis, provide your Score. The Score you give MUST be an integer score (i.e., "1", "2"...) based on the levels of the definitions.
 
 ## Please provide your answers between the tags: <S0>your chain of thoughts</S0>, <S1>your explanation</S1>, <S2>your score</S2>.
 # Output

azure/ai/evaluation/_evaluators/_similarity/_similarity.py

@@ -28,7 +28,7 @@ class SimilarityEvaluator(PromptyEvaluatorBase):
     :param model_config: Configuration for the Azure OpenAI model.
     :type model_config: Union[~azure.ai.evaluation.AzureOpenAIModelConfiguration,
         ~azure.ai.evaluation.OpenAIModelConfiguration]
-    :param threshold: The threshold for the similarity evaluator. Default is 5.
+    :param threshold: The threshold for the similarity evaluator. Default is 3.
     :type threshold: int
 
     .. admonition:: Example: