azure-ai-evaluation 1.0.0b4__py3-none-any.whl → 1.0.0b5__py3-none-any.whl

azure/ai/evaluation/_evaluators/_content_safety/_content_safety.py

@@ -2,9 +2,13 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
 from concurrent.futures import as_completed
-from typing import Callable, Dict, List, Union
+from typing import Callable, Dict, List, Optional, Union
 
 from promptflow.tracing import ThreadPoolExecutorWithContext as ThreadPoolExecutor
+from typing_extensions import override
+
+from azure.ai.evaluation._common._experimental import experimental
+from azure.ai.evaluation._evaluators._common import EvaluatorBase
 
 from ._hate_unfairness import HateUnfairnessEvaluator
 from ._self_harm import SelfHarmEvaluator
@@ -12,7 +16,8 @@ from ._sexual import SexualEvaluator
 from ._violence import ViolenceEvaluator
 
 
-class ContentSafetyEvaluator:
+@experimental
+class ContentSafetyEvaluator(EvaluatorBase):
     """
     Initialize a content safety evaluator configured to evaluate content safetry metrics for QA scenario.
 
@@ -21,8 +26,10 @@ class ContentSafetyEvaluator:
     :param azure_ai_project: The scope of the Azure AI project.
         It contains subscription id, resource group, and project name.
     :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
-    :param parallel: If True, use parallel execution for evaluators. Else, use sequential execution.
-        Default is True.
+    :param eval_last_turn: Whether to evaluate the last turn of a conversation. Default is False.
+    :type eval_last_turn: bool
+    :param kwargs: Additional arguments to pass to the evaluator.
+    :type kwargs: Any
     :return: A function that evaluates content-safety metrics for "question-answering" scenario.
     :rtype: Callable
 
@@ -61,8 +68,10 @@ class ContentSafetyEvaluator:
         }
     """
 
-    def __init__(self, credential, azure_ai_project: dict, parallel: bool = True):
-        self._parallel = parallel
+    # TODO address 3579092 to re-enabled parallel evals.
+    def __init__(self, credential, azure_ai_project, eval_last_turn: bool = False, **kwargs):
+        super().__init__(eval_last_turn=eval_last_turn)
+        self._parallel = kwargs.pop("parallel", False)
         self._evaluators: List[Callable[..., Dict[str, Union[str, float]]]] = [
             ViolenceEvaluator(credential, azure_ai_project),
             SexualEvaluator(credential, azure_ai_project),
@@ -70,24 +79,52 @@ class ContentSafetyEvaluator:
             HateUnfairnessEvaluator(credential, azure_ai_project),
         ]
 
-    def __call__(self, *, query: str, response: str, **kwargs):
+    @override
+    def __call__(
+        self,
+        *,
+        query: Optional[str] = None,
+        response: Optional[str] = None,
+        conversation=None,
+        **kwargs,
+    ):
+        """Evaluate a collection of content safety metrics for the given query/response pair or conversation.
+        This inputs must supply either a query AND response, or a conversation, but not both.
+
+        :keyword query: The query to evaluate.
+        :paramtype query: Optional[str]
+        :keyword response: The response to evaluate.
+        :paramtype response: Optional[str]
+        :keyword conversation: The conversation to evaluate. Expected to contain a list of conversation turns under the
+            key "messages", and potentially a global context under the key "context". Conversation turns are expected
+            to be dictionaries with keys "content", "role", and possibly "context".
+        :paramtype conversation: Optional[~azure.ai.evaluation.Conversation]
+        :return: The evaluation result.
+        :rtype: Union[Dict[str, Union[str, float]], Dict[str, Union[str, float, Dict[str, List[Union[str, float]]]]]]
         """
-        Evaluates content-safety metrics for "question-answering" scenario.
-
-        :keyword query: The query to be evaluated.
-        :paramtype query: str
-        :keyword response: The response to be evaluated.
-        :paramtype response: str
-        :keyword parallel: Whether to evaluate in parallel.
-        :paramtype parallel: bool
-        :return: The scores for content-safety.
-        :rtype: Dict[str, Union[str, float]]
+        return super().__call__(query=query, response=response, conversation=conversation, **kwargs)
+
+    @override
+    async def _do_eval(self, eval_input: Dict) -> Dict[str, Union[str, float]]:
+        """Perform the evaluation using the Azure AI RAI service.
+        The exact evaluation performed is determined by the evaluation metric supplied
+        by the child class initializer.
+
+        :param eval_input: The input to the evaluation function.
+        :type eval_input: Dict
+        :return: The evaluation result.
+        :rtype: Dict
         """
+        query = eval_input.get("query", None)
+        response = eval_input.get("response", None)
+        conversation = eval_input.get("conversation", None)
         results: Dict[str, Union[str, float]] = {}
+        # TODO fix this to not explode on empty optional inputs (PF SKD error)
         if self._parallel:
             with ThreadPoolExecutor() as executor:
+                # pylint: disable=no-value-for-parameter
                 futures = {
-                    executor.submit(evaluator, query=query, response=response, **kwargs): evaluator
+                    executor.submit(query=query, response=response, conversation=conversation): evaluator
                     for evaluator in self._evaluators
                 }
 
@@ -95,7 +132,7 @@ class ContentSafetyEvaluator:
                     results.update(future.result())
         else:
             for evaluator in self._evaluators:
-                result = evaluator(query=query, response=response, **kwargs)
+                result = evaluator(query=query, response=response, conversation=conversation)
                 results.update(result)
 
         return results

azure/ai/evaluation/_evaluators/_content_safety/_content_safety_chat.py

@@ -92,17 +92,17 @@ class ContentSafetyChatEvaluator:
     def __init__(
         self,
         credential,
-        azure_ai_project: dict,
+        azure_ai_project,
         eval_last_turn: bool = False,
         parallel: bool = True,
     ):
         self._eval_last_turn = eval_last_turn
         self._parallel = parallel
         self._evaluators: List[Callable[..., Dict[str, Union[str, float]]]] = [
-            ViolenceEvaluator(azure_ai_project, credential),
-            SexualEvaluator(azure_ai_project, credential),
-            SelfHarmEvaluator(azure_ai_project, credential),
-            HateUnfairnessEvaluator(azure_ai_project, credential),
+            ViolenceEvaluator(credential, azure_ai_project),
+            SexualEvaluator(credential, azure_ai_project),
+            SelfHarmEvaluator(credential, azure_ai_project),
+            HateUnfairnessEvaluator(credential, azure_ai_project),
         ]
 
     def __call__(self, *, conversation: list, **kwargs):

azure/ai/evaluation/_evaluators/_content_safety/_hate_unfairness.py

@@ -1,12 +1,16 @@
 # ---------------------------------------------------------
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
+from typing import Optional
+
 from typing_extensions import override
 
+from azure.ai.evaluation._common._experimental import experimental
 from azure.ai.evaluation._common.constants import EvaluationMetrics
 from azure.ai.evaluation._evaluators._common import RaiServiceEvaluatorBase
 
 
+@experimental
 class HateUnfairnessEvaluator(RaiServiceEvaluatorBase):
     """
     Initialize a hate-unfairness evaluator for hate unfairness score.
@@ -44,7 +48,7 @@ class HateUnfairnessEvaluator(RaiServiceEvaluatorBase):
     def __init__(
         self,
         credential,
-        azure_ai_project: dict,
+        azure_ai_project,
         eval_last_turn: bool = False,
     ):
         super().__init__(
@@ -53,3 +57,28 @@ class HateUnfairnessEvaluator(RaiServiceEvaluatorBase):
             credential=credential,
             eval_last_turn=eval_last_turn,
         )
+
+    @override
+    def __call__(
+        self,
+        *,
+        query: Optional[str] = None,
+        response: Optional[str] = None,
+        conversation=None,
+        **kwargs,
+    ):
+        """
+        Evaluate whether hateful content is present in your AI system's response.
+
+        :keyword query: The query to be evaluated.
+        :paramtype query: str
+        :keyword response: The response to be evaluated.
+        :paramtype response: str
+        :keyword conversation: The conversation to evaluate. Expected to contain a list of conversation turns under the
+            key "messages". Conversation turns are expected
+            to be dictionaries with keys "content" and "role".
+        :paramtype conversation: Optional[~azure.ai.evaluation.Conversation]
+        :return: The fluency score.
+        :rtype: Union[Dict[str, Union[str, float]], Dict[str, Union[str, float, Dict[str, List[Union[str, float]]]]]]
+        """
+        return super().__call__(query=query, response=response, conversation=conversation, **kwargs)

azure/ai/evaluation/_evaluators/_content_safety/_self_harm.py

@@ -1,12 +1,16 @@
 # ---------------------------------------------------------
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
+from typing import Optional
+
 from typing_extensions import override
 
+from azure.ai.evaluation._common._experimental import experimental
 from azure.ai.evaluation._common.constants import EvaluationMetrics
 from azure.ai.evaluation._evaluators._common import RaiServiceEvaluatorBase
 
 
+@experimental
 class SelfHarmEvaluator(RaiServiceEvaluatorBase):
     """
     Initialize a self harm evaluator for self harm score.
@@ -44,7 +48,7 @@ class SelfHarmEvaluator(RaiServiceEvaluatorBase):
     def __init__(
         self,
         credential,
-        azure_ai_project: dict,
+        azure_ai_project,
         eval_last_turn: bool = False,
     ):
         super().__init__(
@@ -53,3 +57,28 @@ class SelfHarmEvaluator(RaiServiceEvaluatorBase):
             credential=credential,
             eval_last_turn=eval_last_turn,
         )
+
+    @override
+    def __call__(
+        self,
+        *,
+        query: Optional[str] = None,
+        response: Optional[str] = None,
+        conversation=None,
+        **kwargs,
+    ):
+        """
+        Evaluate whether self-harm content is present in your AI system's response.
+
+        :keyword query: The query to be evaluated.
+        :paramtype query: str
+        :keyword response: The response to be evaluated.
+        :paramtype response: str
+        :keyword conversation: The conversation to evaluate. Expected to contain a list of conversation turns under the
+            key "messages". Conversation turns are expected
+            to be dictionaries with keys "content" and "role".
+        :paramtype conversation: Optional[~azure.ai.evaluation.Conversation]
+        :return: The fluency score.
+        :rtype: Union[Dict[str, Union[str, float]], Dict[str, Union[str, float, Dict[str, List[Union[str, float]]]]]]
+        """
+        return super().__call__(query=query, response=response, conversation=conversation, **kwargs)

azure/ai/evaluation/_evaluators/_content_safety/_sexual.py

@@ -1,12 +1,16 @@
 # ---------------------------------------------------------
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
+from typing import Optional
+
 from typing_extensions import override
 
+from azure.ai.evaluation._common._experimental import experimental
 from azure.ai.evaluation._common.constants import EvaluationMetrics
 from azure.ai.evaluation._evaluators._common import RaiServiceEvaluatorBase
 
 
+@experimental
 class SexualEvaluator(RaiServiceEvaluatorBase):
     """
     Initialize a sexual evaluator for sexual score.
@@ -44,7 +48,7 @@ class SexualEvaluator(RaiServiceEvaluatorBase):
     def __init__(
         self,
         credential,
-        azure_ai_project: dict,
+        azure_ai_project,
         eval_last_turn: bool = False,
     ):
         super().__init__(
@@ -53,3 +57,28 @@ class SexualEvaluator(RaiServiceEvaluatorBase):
             credential=credential,
             eval_last_turn=eval_last_turn,
         )
+
+    @override
+    def __call__(
+        self,
+        *,
+        query: Optional[str] = None,
+        response: Optional[str] = None,
+        conversation=None,
+        **kwargs,
+    ):
+        """
+        Evaluate whether sexual content is present in your AI system's response.
+
+        :keyword query: The query to be evaluated.
+        :paramtype query: str
+        :keyword response: The response to be evaluated.
+        :paramtype response: str
+        :keyword conversation: The conversation to evaluate. Expected to contain a list of conversation turns under the
+            key "messages". Conversation turns are expected
+            to be dictionaries with keys "content" and "role".
+        :paramtype conversation: Optional[~azure.ai.evaluation.Conversation]
+        :return: The fluency score.
+        :rtype: Union[Dict[str, Union[str, float]], Dict[str, Union[str, float, Dict[str, List[Union[str, float]]]]]]
+        """
+        return super().__call__(query=query, response=response, conversation=conversation, **kwargs)

azure/ai/evaluation/_evaluators/_content_safety/_violence.py

@@ -1,12 +1,16 @@
 # ---------------------------------------------------------
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
+from typing import Optional
+
 from typing_extensions import override
 
+from azure.ai.evaluation._common._experimental import experimental
 from azure.ai.evaluation._common.constants import EvaluationMetrics
 from azure.ai.evaluation._evaluators._common import RaiServiceEvaluatorBase
 
 
+@experimental
 class ViolenceEvaluator(RaiServiceEvaluatorBase):
     """
     Initialize a violence evaluator for violence score.
@@ -44,7 +48,7 @@ class ViolenceEvaluator(RaiServiceEvaluatorBase):
     def __init__(
         self,
         credential,
-        azure_ai_project: dict,
+        azure_ai_project,
         eval_last_turn: bool = False,
     ):
         super().__init__(
@@ -53,3 +57,28 @@ class ViolenceEvaluator(RaiServiceEvaluatorBase):
             credential=credential,
             eval_last_turn=eval_last_turn,
         )
+
+    @override
+    def __call__(
+        self,
+        *,
+        query: Optional[str] = None,
+        response: Optional[str] = None,
+        conversation=None,
+        **kwargs,
+    ):
+        """
+        Evaluate whether violent content is present in your AI system's response.
+
+        :keyword query: The query to be evaluated.
+        :paramtype query: str
+        :keyword response: The response to be evaluated.
+        :paramtype response: str
+        :keyword conversation: The conversation to evaluate. Expected to contain a list of conversation turns under the
+            key "messages". Conversation turns are expected
+            to be dictionaries with keys "content" and "role".
+        :paramtype conversation: Optional[~azure.ai.evaluation.Conversation]
+        :return: The fluency score.
+        :rtype: Union[Dict[str, Union[str, float]], Dict[str, Union[str, float, Dict[str, List[Union[str, float]]]]]]
+        """
+        return super().__call__(query=query, response=response, conversation=conversation, **kwargs)

azure/ai/evaluation/_evaluators/_eci/_eci.py

@@ -3,10 +3,12 @@
 # ---------------------------------------------------------
 from typing_extensions import override
 
+from azure.ai.evaluation._common._experimental import experimental
 from azure.ai.evaluation._common.constants import _InternalEvaluationMetrics
 from azure.ai.evaluation._evaluators._common import RaiServiceEvaluatorBase
 
 
+@experimental
 class ECIEvaluator(RaiServiceEvaluatorBase):
     """
     Initialize an ECI evaluator to evaluate ECI based on the following guidelines:
@@ -51,7 +53,7 @@ class ECIEvaluator(RaiServiceEvaluatorBase):
     def __init__(
         self,
         credential,
-        azure_ai_project: dict,
+        azure_ai_project,
         eval_last_turn: bool = False,
     ):
         super().__init__(

azure/ai/evaluation/_evaluators/_fluency/_fluency.py

@@ -23,51 +23,51 @@ class FluencyEvaluator(PromptyEvaluatorBase):
     .. code-block:: python
 
         eval_fn = FluencyEvaluator(model_config)
-        result = eval_fn(
-            query="What is the capital of Japan?",
-            response="The capital of Japan is Tokyo.")
+        result = eval_fn(response="The capital of Japan is Tokyo.")
 
     **Output format**
 
     .. code-block:: python
 
         {
-            "gpt_fluency": 4.0
+            "fluency": 4.0,
+            "gpt_fluency": 4.0,
         }
+
+    Note: To align with our support of a diverse set of models, a key without the `gpt_` prefix has been added.
+    To maintain backwards compatibility, the old key with the `gpt_` prefix is still be present in the output;
+    however, it is recommended to use the new key moving forward as the old key will be deprecated in the future.
     """
 
-    PROMPTY_FILE = "fluency.prompty"
-    RESULT_KEY = "gpt_fluency"
+    _PROMPTY_FILE = "fluency.prompty"
+    _RESULT_KEY = "fluency"
 
     @override
-    def __init__(self, model_config: dict):
+    def __init__(self, model_config):
         current_dir = os.path.dirname(__file__)
-        prompty_path = os.path.join(current_dir, self.PROMPTY_FILE)
-        super().__init__(model_config=model_config, prompty_file=prompty_path, result_key=self.RESULT_KEY)
+        prompty_path = os.path.join(current_dir, self._PROMPTY_FILE)
+        super().__init__(model_config=model_config, prompty_file=prompty_path, result_key=self._RESULT_KEY)
 
     @override
     def __call__(
         self,
         *,
-        query: Optional[str] = None,
         response: Optional[str] = None,
-        conversation: Optional[dict] = None,
+        conversation=None,
         **kwargs,
     ):
         """
-        Evaluate fluency. Accepts either a query and response for a single evaluation,
+        Evaluate fluency. Accepts either a response for a single evaluation,
         or a conversation for a multi-turn evaluation. If the conversation has more than one turn,
         the evaluator will aggregate the results of each turn.
 
-        :keyword query: The query to be evaluated.
-        :paramtype query: str
-        :keyword response: The response to be evaluated.
+        :keyword response: The response to be evaluated. Mutually exclusive with the "conversation" parameter.
         :paramtype response: str
         :keyword conversation: The conversation to evaluate. Expected to contain a list of conversation turns under the
-            key "messages". Conversation turns are expected
-            to be dictionaries with keys "content" and "role".
-        :paramtype conversation: Optional[Dict]
+            key "messages". Conversation turns are expected to be dictionaries with keys "content" and "role".
+        :paramtype conversation: Optional[~azure.ai.evaluation.Conversation]
         :return: The fluency score.
-        :rtype: Dict[str, float]
+        :rtype: Union[Dict[str, float], Dict[str, Union[float, Dict[str, List[float]]]]]
         """
-        return super().__call__(query=query, response=response, conversation=conversation, **kwargs)
+
+        return super().__call__(response=response, conversation=conversation, **kwargs)

azure/ai/evaluation/_evaluators/_fluency/fluency.prompty

@@ -5,7 +5,7 @@ model:
   api: chat
   parameters:
     temperature: 0.0
-    max_tokens: 1
+    max_tokens: 800
     top_p: 1.0
     presence_penalty: 0
     frequency_penalty: 0
@@ -13,44 +13,74 @@ model:
       type: text
 
 inputs:
-  query:
-    type: string
   response:
     type: string
 
 ---
 system:
-You are an AI assistant. You will be given the definition of an evaluation metric for assessing the quality of an answer in a question-answering task. Your job is to compute an accurate evaluation score using the provided evaluation metric. You should return a single integer value between 1 to 5 representing the evaluation metric. You will include no other text or information.
+# Instruction
+## Goal
+### You are an expert in evaluating the quality of a RESPONSE from an intelligent system based on provided definition and data. Your goal will involve answering the questions below using the information provided.
+- **Definition**: You are given a definition of the communication trait that is being evaluated to help guide your Score.
+- **Data**: Your input data include a RESPONSE.
+- **Tasks**: To complete your evaluation you will be asked to evaluate the Data in different ways.
+
 user:
-Fluency measures the quality of individual sentences in the answer, and whether they are well-written and grammatically correct. Consider the quality of individual sentences when evaluating fluency. Given the question and answer, score the fluency of the answer between one to five stars using the following rating scale:
-One star: the answer completely lacks fluency
-Two stars: the answer mostly lacks fluency
-Three stars: the answer is partially fluent
-Four stars: the answer is mostly fluent
-Five stars: the answer has perfect fluency
-
-This rating value should always be an integer between 1 and 5. So the rating produced should be 1 or 2 or 3 or 4 or 5.
-
-question: What did you have for breakfast today?
-answer: Breakfast today, me eating cereal and orange juice very good.
-stars: 1
-
-question: How do you feel when you travel alone?
-answer: Alone travel, nervous, but excited also. I feel adventure and like its time.
-stars: 2
-
-question: When was the last time you went on a family vacation?
-answer: Last family vacation, it took place in last summer. We traveled to a beach destination, very fun.
-stars: 3
-
-question: What is your favorite thing about your job?
-answer: My favorite aspect of my job is the chance to interact with diverse people. I am constantly learning from their experiences and stories.
-stars: 4
-
-question: Can you describe your morning routine?
-answer: Every morning, I wake up at 6 am, drink a glass of water, and do some light stretching. After that, I take a shower and get dressed for work. Then, I have a healthy breakfast, usually consisting of oatmeal and fruits, before leaving the house around 7:30 am.
-stars: 5
-
-question: {{query}}
-answer: {{response}}
-stars:
+# Definition
+**Fluency** refers to the effectiveness and clarity of written communication, focusing on grammatical accuracy, vocabulary range, sentence complexity, coherence, and overall readability. It assesses how smoothly ideas are conveyed and how easily the text can be understood by the reader.
+
+# Ratings
+## [Fluency: 1] (Emergent Fluency)
+**Definition:** The response shows minimal command of the language. It contains pervasive grammatical errors, extremely limited vocabulary, and fragmented or incoherent sentences. The message is largely incomprehensible, making understanding very difficult.
+
+**Examples:**
+  **Response:** Free time I. Go park. Not fun. Alone.
+
+  **Response:** Like food pizza. Good cheese eat.
+
+## [Fluency: 2] (Basic Fluency)
+**Definition:** The response communicates simple ideas but has frequent grammatical errors and limited vocabulary. Sentences are short and may be improperly constructed, leading to partial understanding. Repetition and awkward phrasing are common.
+
+**Examples:**
+  **Response:** I like play soccer. I watch movie. It fun.
+
+  **Response:** My town small. Many people. We have market.
+
+## [Fluency: 3] (Competent Fluency)
+**Definition:** The response clearly conveys ideas with occasional grammatical errors. Vocabulary is adequate but not extensive. Sentences are generally correct but may lack complexity and variety. The text is coherent, and the message is easily understood with minimal effort.
+
+**Examples:**
+  **Response:** I'm planning to visit friends and maybe see a movie together.
+
+  **Response:** I try to eat healthy food and exercise regularly by jogging.
+
+## [Fluency: 4] (Proficient Fluency)
+**Definition:** The response is well-articulated with good control of grammar and a varied vocabulary. Sentences are complex and well-structured, demonstrating coherence and cohesion. Minor errors may occur but do not affect overall understanding. The text flows smoothly, and ideas are connected logically.
+
+**Examples:**
+  **Response:** My interest in mathematics and problem-solving inspired me to become an engineer, as I enjoy designing solutions that improve people's lives.
+
+  **Response:** Environmental conservation is crucial because it protects ecosystems, preserves biodiversity, and ensures natural resources are available for future generations.
+
+## [Fluency: 5] (Exceptional Fluency)
+**Definition:** The response demonstrates an exceptional command of language with sophisticated vocabulary and complex, varied sentence structures. It is coherent, cohesive, and engaging, with precise and nuanced expression. Grammar is flawless, and the text reflects a high level of eloquence and style.
+
+**Examples:**
+  **Response:** Globalization exerts a profound influence on cultural diversity by facilitating unprecedented cultural exchange while simultaneously risking the homogenization of distinct cultural identities, which can diminish the richness of global heritage.
+
+  **Response:** Technology revolutionizes modern education by providing interactive learning platforms, enabling personalized learning experiences, and connecting students worldwide, thereby transforming how knowledge is acquired and shared.
+
+
+# Data
+RESPONSE: {{response}}
+
+
+# Tasks
+## Please provide your assessment Score for the previous RESPONSE based on the Definitions above. Your output should include the following information:
+- **ThoughtChain**: To improve the reasoning process, think step by step and include a step-by-step explanation of your thought process as you analyze the data based on the definitions. Keep it brief and start your ThoughtChain with "Let's think step by step:".
+- **Explanation**: a very short explanation of why you think the input Data should get that Score.
+- **Score**: based on your previous analysis, provide your Score. The Score you give MUST be a integer score (i.e., "1", "2"...) based on the levels of the definitions.
+
+
+## Please provide your answers between the tags: <S0>your chain of thoughts</S0>, <S1>your explanation</S1>, <S2>your Score</S2>.
+# Output

azure/ai/evaluation/_evaluators/_gleu/_gleu.py

@@ -61,7 +61,7 @@ class GleuScoreEvaluator:
         :keyword ground_truth: The ground truth to be compared against.
         :paramtype ground_truth: str
         :return: The GLEU score.
-        :rtype: dict
+        :rtype: Dict[str, float]
         """
         return async_run_allowing_running_loop(
             self._async_evaluator, ground_truth=ground_truth, response=response, **kwargs