azure-ai-evaluation 0.0.0b0__py3-none-any.whl → 1.0.0__py3-none-any.whl

azure/ai/evaluation/_evaluators/_qa/_qa.py

@@ -0,0 +1,93 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from concurrent.futures import as_completed
+from typing import Callable, Dict, List, Union
+
+from promptflow.tracing import ThreadPoolExecutorWithContext as ThreadPoolExecutor
+
+from .._coherence import CoherenceEvaluator
+from .._f1_score import F1ScoreEvaluator
+from .._fluency import FluencyEvaluator
+from .._groundedness import GroundednessEvaluator
+from .._relevance import RelevanceEvaluator
+from .._similarity import SimilarityEvaluator
+
+
+class QAEvaluator:
+    """
+    Initialize a question-answer evaluator configured for a specific Azure OpenAI model.
+
+    :param model_config: Configuration for the Azure OpenAI model.
+    :type model_config: Union[~azure.ai.evaluation.AzureOpenAIModelConfiguration,
+        ~azure.ai.evaluation.OpenAIModelConfiguration]
+    :return: A callable class that evaluates and generates metrics for "question-answering" scenario.
+    :param kwargs: Additional arguments to pass to the evaluator.
+    :type kwargs: Any
+
+    .. admonition:: Example:
+
+        .. literalinclude:: ../samples/evaluation_samples_evaluate.py
+            :start-after: [START qa_evaluator]
+            :end-before: [END qa_evaluator]
+            :language: python
+            :dedent: 8
+            :caption: Initialize and call a QAEvaluator.
+
+    .. note::
+
+        To align with our support of a diverse set of models, keys without the `gpt_` prefix has been added.
+        To maintain backwards compatibility, the old keys with the `gpt_` prefix are still be present in the output;
+        however, it is recommended to use the new keys moving forward as the old keys will be deprecated in the future.
+    """
+
+    id = "qa"
+    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+
+    def __init__(self, model_config, **kwargs):
+        self._parallel = kwargs.pop("_parallel", False)
+
+        self._evaluators: List[Union[Callable[..., Dict[str, Union[str, float]]], Callable[..., Dict[str, float]]]] = [
+            GroundednessEvaluator(model_config),
+            RelevanceEvaluator(model_config),
+            CoherenceEvaluator(model_config),
+            FluencyEvaluator(model_config),
+            SimilarityEvaluator(model_config),
+            F1ScoreEvaluator(),
+        ]
+
+    def __call__(self, *, query: str, response: str, context: str, ground_truth: str, **kwargs):
+        """
+        Evaluates question-answering scenario.
+
+        :keyword query: The query to be evaluated.
+        :paramtype query: str
+        :keyword response: The response to be evaluated.
+        :paramtype response: str
+        :keyword context: The context to be evaluated.
+        :paramtype context: str
+        :keyword ground_truth: The ground truth to be evaluated.
+        :paramtype ground_truth: str
+        :return: The scores for QA scenario.
+        :rtype: Dict[str, Union[str, float]]
+        """
+        results: Dict[str, Union[str, float]] = {}
+        if self._parallel:
+            with ThreadPoolExecutor() as executor:
+                futures = {
+                    executor.submit(
+                        evaluator, query=query, response=response, context=context, ground_truth=ground_truth, **kwargs
+                    ): evaluator
+                    for evaluator in self._evaluators
+                }
+
+                # Collect results as they complete
+                for future in as_completed(futures):
+                    results.update(future.result())
+        else:
+            for evaluator in self._evaluators:
+                result = evaluator(query=query, response=response, context=context, ground_truth=ground_truth, **kwargs)
+                results.update(result)
+
+        return results

azure/ai/evaluation/_evaluators/_relevance/__init__.py

@@ -0,0 +1,9 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from ._relevance import RelevanceEvaluator
+
+__all__ = [
+    "RelevanceEvaluator",
+]

azure/ai/evaluation/_evaluators/_relevance/_relevance.py

@@ -0,0 +1,114 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+import os
+from typing import Dict, Union, List
+
+from typing_extensions import overload, override
+
+from azure.ai.evaluation._model_configurations import Conversation
+from azure.ai.evaluation._evaluators._common import PromptyEvaluatorBase
+
+
+class RelevanceEvaluator(PromptyEvaluatorBase):
+    """
+    Evaluates relevance score for a given query and response or a multi-turn conversation, including reasoning.
+
+    The relevance measure assesses the ability of answers to capture the key points of the context.
+    High relevance scores signify the AI system's understanding of the input and its capability to produce coherent
+    and contextually appropriate outputs. Conversely, low relevance scores indicate that generated responses might
+    be off-topic, lacking in context, or insufficient in addressing the user's intended queries. Use the relevance
+    metric when evaluating the AI system's performance in understanding the input and generating contextually
+    appropriate responses.
+
+    Relevance scores range from 1 to 5, with 1 being the worst and 5 being the best.
+
+    :param model_config: Configuration for the Azure OpenAI model.
+    :type model_config: Union[~azure.ai.evaluation.AzureOpenAIModelConfiguration,
+        ~azure.ai.evaluation.OpenAIModelConfiguration]
+
+    .. admonition:: Example:
+
+        .. literalinclude:: ../samples/evaluation_samples_evaluate.py
+            :start-after: [START relevance_evaluator]
+            :end-before: [END relevance_evaluator]
+            :language: python
+            :dedent: 8
+            :caption: Initialize and call a RelevanceEvaluator with a query, response, and context.
+
+    .. note::
+
+        To align with our support of a diverse set of models, an output key without the `gpt_` prefix has been added.
+        To maintain backwards compatibility, the old key with the `gpt_` prefix is still be present in the output;
+        however, it is recommended to use the new key moving forward as the old key will be deprecated in the future.
+    """
+
+    # Constants must be defined within eval's directory to be save/loadable
+    _PROMPTY_FILE = "relevance.prompty"
+    _RESULT_KEY = "relevance"
+
+    id = "azureml://registries/azureml/models/Relevance-Evaluator/versions/4"
+    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+
+    @override
+    def __init__(self, model_config):
+        current_dir = os.path.dirname(__file__)
+        prompty_path = os.path.join(current_dir, self._PROMPTY_FILE)
+        super().__init__(model_config=model_config, prompty_file=prompty_path, result_key=self._RESULT_KEY)
+
+    @overload
+    def __call__(
+        self,
+        *,
+        query: str,
+        response: str,
+    ) -> Dict[str, Union[str, float]]:
+        """Evaluate groundedness for given input of query, response, context
+
+        :keyword query: The query to be evaluated.
+        :paramtype query: str
+        :keyword response: The response to be evaluated.
+        :paramtype response: str
+        :return: The relevance score.
+        :rtype: Dict[str, float]
+        """
+
+    @overload
+    def __call__(
+        self,
+        *,
+        conversation: Conversation,
+    ) -> Dict[str, Union[float, Dict[str, List[Union[str, float]]]]]:
+        """Evaluate relevance for a conversation
+
+        :keyword conversation: The conversation to evaluate. Expected to contain a list of conversation turns under the
+            key "messages", and potentially a global context under the key "context". Conversation turns are expected
+            to be dictionaries with keys "content", "role", and possibly "context".
+        :paramtype conversation: Optional[~azure.ai.evaluation.Conversation]
+        :return: The relevance score.
+        :rtype: Dict[str, Union[float, Dict[str, List[float]]]]
+        """
+
+    @override
+    def __call__(  # pylint: disable=docstring-missing-param
+        self,
+        *args,
+        **kwargs,
+    ):
+        """Evaluate relevance. Accepts either a query and response for a single evaluation,
+        or a conversation for a multi-turn evaluation. If the conversation has more than one turn,
+        the evaluator will aggregate the results of each turn.
+
+        :keyword query: The query to be evaluated. Mutually exclusive with the `conversation` parameter.
+        :paramtype query: Optional[str]
+        :keyword response: The response to be evaluated. Mutually exclusive with the `conversation` parameter.
+        :paramtype response: Optional[str]
+        :keyword conversation: The conversation to evaluate. Expected to contain a list of conversation turns under the
+            key "messages", and potentially a global context under the key "context". Conversation turns are expected
+            to be dictionaries with keys "content", "role", and possibly "context".
+        :paramtype conversation: Optional[~azure.ai.evaluation.Conversation]
+        :return: The relevance score.
+        :rtype: Union[Dict[str, Union[str, float]], Dict[str, Union[float, Dict[str, List[Union[str, float]]]]]]
+        """
+        return super().__call__(*args, **kwargs)

azure/ai/evaluation/_evaluators/_relevance/relevance.prompty

@@ -0,0 +1,100 @@
+---
+name: Relevance
+description: Evaluates relevance score for QA scenario
+model:
+  api: chat
+  parameters:
+    temperature: 0.0
+    max_tokens: 800
+    top_p: 1.0
+    presence_penalty: 0
+    frequency_penalty: 0
+    response_format:
+      type: text
+
+inputs:
+  query:
+    type: string
+  response:
+    type: string
+
+---
+system:
+# Instruction
+## Goal
+### You are an expert in evaluating the quality of a RESPONSE from an intelligent system based on provided definition and data. Your goal will involve answering the questions below using the information provided.
+- **Definition**: You are given a definition of the communication trait that is being evaluated to help guide your Score.
+- **Data**: Your input data include QUERY and RESPONSE.
+- **Tasks**: To complete your evaluation you will be asked to evaluate the Data in different ways.
+
+user:
+# Definition
+**Relevance** refers to how effectively a response addresses a question. It assesses the accuracy, completeness, and direct relevance of the response based solely on the given information.
+
+# Ratings
+## [Relevance: 1] (Irrelevant Response)
+**Definition:** The response is unrelated to the question. It provides information that is off-topic and does not attempt to address the question posed.
+
+**Examples:**
+  **Query:** What is the team preparing for?
+  **Response:** I went grocery shopping yesterday evening.
+
+  **Query:** When will the company's new product line launch?
+  **Response:** International travel can be very rewarding and educational.
+
+## [Relevance: 2] (Incorrect Response)
+**Definition:** The response attempts to address the question but includes incorrect information. It provides a response that is factually wrong based on the provided information.
+
+**Examples:**
+  **Query:** When was the merger between the two firms finalized?
+  **Response:** The merger was finalized on April 10th.     
+
+  **Query:** Where and when will the solar eclipse be visible?
+  **Response:** The solar eclipse will be visible in Asia on December 14th.
+
+## [Relevance: 3] (Incomplete Response)
+**Definition:** The response addresses the question but omits key details necessary for a full understanding. It provides a partial response that lacks essential information.      
+
+**Examples:**
+  **Query:** What type of food does the new restaurant offer?
+  **Response:** The restaurant offers Italian food like pasta.
+
+  **Query:** What topics will the conference cover?
+  **Response:** The conference will cover renewable energy and climate change.
+
+## [Relevance: 4] (Complete Response)
+**Definition:** The response fully addresses the question with accurate and complete information. It includes all essential details required for a comprehensive understanding, without adding any extraneous information.
+
+**Examples:**
+  **Query:** What type of food does the new restaurant offer?
+  **Response:** The new restaurant offers Italian cuisine, featuring dishes like pasta, pizza, and risotto.
+
+  **Query:** What topics will the conference cover?
+  **Response:** The conference will cover renewable energy, climate change, and sustainability practices.
+
+## [Relevance: 5] (Comprehensive Response with Insights)    
+**Definition:** The response not only fully and accurately addresses the question but also includes additional relevant insights or elaboration. It may explain the significance, implications, or provide minor inferences that enhance understanding.
+
+**Examples:**
+  **Query:** What type of food does the new restaurant offer?
+  **Response:** The new restaurant offers Italian cuisine, featuring dishes like pasta, pizza, and risotto, aiming to provide customers with an authentic Italian dining experience.
+
+  **Query:** What topics will the conference cover?
+  **Response:** The conference will cover renewable energy, climate change, and sustainability practices, bringing together global experts to discuss these critical issues. 
+
+
+
+# Data
+QUERY: {{query}}
+RESPONSE: {{response}}
+
+
+# Tasks
+## Please provide your assessment Score for the previous RESPONSE in relation to the QUERY based on the Definitions above. Your output should include the following information:
+- **ThoughtChain**: To improve the reasoning process, think step by step and include a step-by-step explanation of your thought process as you analyze the data based on the definitions. Keep it brief and start your ThoughtChain with "Let's think step by step:".
+- **Explanation**: a very short explanation of why you think the input Data should get that Score.
+- **Score**: based on your previous analysis, provide your Score. The Score you give MUST be a integer score (i.e., "1", "2"...) based on the levels of the definitions.
+
+
+## Please provide your answers between the tags: <S0>your chain of thoughts</S0>, <S1>your explanation</S1>, <S2>your Score</S2>.
+# Output

azure/ai/evaluation/_evaluators/_retrieval/__init__.py

@@ -0,0 +1,9 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from ._retrieval import RetrievalEvaluator
+
+__all__ = [
+    "RetrievalEvaluator",
+]

azure/ai/evaluation/_evaluators/_retrieval/_retrieval.py

@@ -0,0 +1,112 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+import logging
+import os
+from typing import Dict, List, Union
+from typing_extensions import overload, override
+
+from azure.ai.evaluation._evaluators._common._base_prompty_eval import PromptyEvaluatorBase
+from azure.ai.evaluation._model_configurations import Conversation
+
+logger = logging.getLogger(__name__)
+
+
+class RetrievalEvaluator(PromptyEvaluatorBase[Union[str, float]]):
+    """
+    Evaluates retrieval score for a given query and context or a multi-turn conversation, including reasoning.
+
+    The retrieval measure assesses the AI system's performance in retrieving information
+    for additional context (e.g. a RAG scenario).
+
+    Retrieval scores range from 1 to 5, with 1 being the worst and 5 being the best.
+
+    High retrieval scores indicate that the AI system has successfully extracted and ranked
+    the most relevant information at the top, without introducing bias from external knowledge
+    and ignoring factual correctness. Conversely, low retrieval scores suggest that the AI system
+    has failed to surface the most relevant context chunks at the top of the list
+    and/or introduced bias and ignored factual correctness.
+
+    :param model_config: Configuration for the Azure OpenAI model.
+    :type model_config: Union[~azure.ai.evaluation.AzureOpenAIModelConfiguration,
+        ~azure.ai.evaluation.OpenAIModelConfiguration]
+    :return: A function that evaluates and generates metrics for "chat" scenario.
+    :rtype: Callable
+
+    .. admonition:: Example:
+
+        .. literalinclude:: ../samples/evaluation_samples_evaluate.py
+            :start-after: [START retrieval_evaluator]
+            :end-before: [END retrieval_evaluator]
+            :language: python
+            :dedent: 8
+            :caption: Initialize and call a RetrievalEvaluator.
+
+    .. note::
+
+        To align with our support of a diverse set of models, an output key without the `gpt_` prefix has been added.
+        To maintain backwards compatibility, the old key with the `gpt_` prefix is still be present in the output;
+        however, it is recommended to use the new key moving forward as the old key will be deprecated in the future.
+    """
+
+    _PROMPTY_FILE = "retrieval.prompty"
+    _RESULT_KEY = "retrieval"
+
+    id = "azureml://registries/azureml/models/Retrieval-Evaluator/versions/1"
+    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+
+    @override
+    def __init__(self, model_config):  # pylint: disable=super-init-not-called
+        current_dir = os.path.dirname(__file__)
+        prompty_path = os.path.join(current_dir, self._PROMPTY_FILE)
+        super().__init__(model_config=model_config, prompty_file=prompty_path, result_key=self._RESULT_KEY)
+
+    @overload
+    def __call__(
+        self,
+        *,
+        query: str,
+        context: str,
+    ) -> Dict[str, Union[str, float]]:
+        """Evaluates retrieval for a given a query and context
+
+        :keyword query: The query to be evaluated. Mutually exclusive with `conversation` parameter.
+        :paramtype query: Optional[str]
+        :keyword context: The context to be evaluated. Mutually exclusive with `conversation` parameter.
+        :paramtype context: Optional[str]
+        :return: The scores for Chat scenario.
+        :rtype: Dict[str, Union[str, float]]
+        """
+
+    @overload
+    def __call__(
+        self,
+        *,
+        conversation: Conversation,
+    ) -> Dict[str, Union[float, Dict[str, List[Union[str, float]]]]]:
+        """Evaluates retrieval for a for a multi-turn evaluation. If the conversation has more than one turn,
+        the evaluator will aggregate the results of each turn.
+
+        :keyword conversation: The conversation to be evaluated.
+        :paramtype conversation: Optional[~azure.ai.evaluation.Conversation]
+        :return: The scores for Chat scenario.
+        :rtype: Dict[str, Union[float, Dict[str, List[float]]]]
+        """
+
+    @override
+    def __call__(self, *args, **kwargs):  # pylint: disable=docstring-missing-param
+        """Evaluates retrieval score chat scenario. Accepts either a query and context for a single evaluation,
+        or a conversation for a multi-turn evaluation. If the conversation has more than one turn,
+        the evaluator will aggregate the results of each turn.
+
+        :keyword query: The query to be evaluated. Mutually exclusive with `conversation` parameter.
+        :paramtype query: Optional[str]
+        :keyword context: The context to be evaluated. Mutually exclusive with `conversation` parameter.
+        :paramtype context: Optional[str]
+        :keyword conversation: The conversation to be evaluated.
+        :paramtype conversation: Optional[~azure.ai.evaluation.Conversation]
+        :return: The scores for Chat scenario.
+        :rtype: :rtype: Dict[str, Union[float, Dict[str, List[str, float]]]]
+        """
+        return super().__call__(*args, **kwargs)

azure/ai/evaluation/_evaluators/_retrieval/retrieval.prompty

@@ -0,0 +1,93 @@
+---
+name: Retrieval
+description: Evaluates retrieval quality score for RAG scenario
+model:
+  api: chat
+  parameters:
+    temperature: 0.0
+    max_tokens: 1600
+    top_p: 1.0
+    presence_penalty: 0
+    frequency_penalty: 0
+    response_format:
+      type: text
+
+inputs:
+  query:
+    type: string
+  context:
+    type: string
+
+---
+system:
+# Instruction
+## Goal
+### You are an expert in evaluating the quality of a list of CONTEXT chunks from a query based on provided definition and data. Your goal will involve answering the questions below using the information provided.
+- **Definition**: You are given a definition of the retrieval quality that is being evaluated to help guide your Score.
+- **Data**: Your input data include QUERY and CONTEXT.
+- **Tasks**: To complete your evaluation you will be asked to evaluate the Data in different ways.
+
+user:
+# Definition
+**Retrieval** refers to measuring how relevant the context chunks are to address a query and how the most relevant context chunks are surfaced at the top of the list. It emphasizes the extraction and ranking of the most relevant information at the top, without introducing bias from external knowledge and ignoring factual correctness. It assesses the relevance and effectiveness of the retrieved context chunks with respect to the query.
+
+# Ratings
+## [Retrieval: 1] (Irrelevant Context, External Knowledge Bias)
+**Definition:** The retrieved context chunks are not relevant to the query despite any conceptual similarities. There is no overlap between the query and the retrieved information, and no useful chunks appear in the results. They introduce external knowledge that isn't part of the retrieval documents.
+
+**Examples:**
+  **Query:** what is kuchen?
+  **Context:** ["There's nothing like the taste of a cake you made in your own kitchen. Baking a cake is as simple as measuring ingredients, mixing them in the right order, and remembering to take the cake out of the oven before it burns.", "A steady 325-350 degrees is ideal when it comes to baking pound cake. Position the pan in the middle of the oven, and rotate it once, halfway through the baking time, as it bakes to account for any hot spots. "CHOCOLATE POUND CAKE. Cream butter, sugar ... and floured bundt pan, 10 inch pan or 2 (9x5x3 inch) loaf pans. Bake at ... pans. Bake until cake tester inserted in ... to drizzle down sides. 4. BUTTERMILK LEMON POUND CAKE."", "Pour batter into your pan(s) and place in the oven. Cook for 75 minutes, checking periodically. Some ovens cook unevenly or quickly -- if this describes yours, keep an eye on it. 1 If to be used for fancy ornamented cakes, bake 30 to 35 minutes in a dripping-pan. 2 Insert a skewer or toothpick to see if it's finished.", "As a general rule of thumb you can bake most cakes at 375 degrees Fahrenheit (which is 180 degrees Celsius) and check them after about 30 minutes and expect it to take at least 45 minutes.", "Till a toothpick inserted in the center of the cake comes out clean. Depends on the heat of your oven but start checking at about 45 minutes and when the cake is golden brown. sonnyboy · 8 years ago. Thumbs up.", "1 This results in a pound cake with maximum volume. 2 Be patient. Beat softened butter (and cream cheese or vegetable shortening) at medium speed with an electric mixer until creamy. 3 This can take from 1 to 7 minutes, depending on the power of your mixer."]
+
+  **Query:** What are the main economic impacts of global warming?
+  **Context:** ["Economic theories such as supply and demand explain how prices fluctuate in a free market.", "Global warming is caused by increased carbon dioxide levels, which affect the environment and the atmosphere.", "Political factors also play a role in economic decisions across nations."]
+
+## [Retrieval: 2] (Partially Relevant Context, Poor Ranking, External Knowledge Bias)
+**Definition:** The context chunks are partially relevant to address the query but are mostly irrelevant, and external knowledge or LLM bias starts influencing the context chunks. The most relevant chunks are either missing or placed at the bottom.
+
+**Examples:**
+  **Query:** what is rappelling
+  **Context:** ["5. Cancel. Rappelling is the process of coming down from a mountain that is usually done with two pieces of rope. Use a natural anchor or a set of bolts to rappel from with help from an experienced rock climber in this free video on rappelling techniques. Part of the Video Series: Rappelling & Rock Climbing.", "Abseiling (/ˈaebseɪl/ ˈæbseɪl /or/ ; ˈɑːpzaɪl From german, abseilen meaning to rope), down also called, rappelling is the controlled descent of a vertical, drop such as a rock, face using a. Rope climbers use this technique when a cliff or slope is too steep/and or dangerous to descend without. protection", "1. rappel - (mountaineering) a descent of a vertical cliff or wall made by using a doubled rope that is fixed to a higher point and wrapped around the body. abseil. mountain climbing, mountaineering-the activity of climbing a mountain. descent-the act of changing your location in a downward direction."]
+
+  **Query:** Describe the causes of the French Revolution.
+  **Context:** ["The French Revolution started due to economic disparity, leading to unrest among the lower classes.", "The Industrial Revolution also contributed to changes in society during the 18th century.", "Philosophers like Rousseau inspired revolutionary thinking, but the taxation system played a role as well."]
+
+## [Retrieval: 3] (Relevant Context Ranked Bottom)
+**Definition:** The context chunks contain relevant information to address the query, but the most pertinent chunks are located at the bottom of the list.
+
+**Examples:**
+  **Query:** what are monocytes
+  **Context:** ["Monocytes are produced by the bone marrow from precursors called monoblasts, bipotent cells that differentiated from hematopoietic stem cells. Monocytes circulate in the bloodstream for about one to three days and then typically move into tissues throughout the body. Monocytes which migrate from the bloodstream to other tissues will then differentiate into tissue resident macrophages or dendritic cells. Macrophages are responsible for protecting tissues from foreign substances, but are also suspected to be important in the formation of important organs like the heart and brain.", "Report Abuse. A high level of monocytes could mean a number of things. They're a type of phagocyte-a type of cell found in your blood that 'eats' many types of attacking bacteria and other microorganisms when it matures. High levels could mean that you have an infection as more develop to fight it.", "Our immune system has a key component called the white blood cells, of which there are several different kinds. Monocytes are a type of white blood cell that fights off bacteria, viruses and fungi. Monocytes are the biggest type of white blood cell in the immune system. Originally formed in the bone marrow, they are released into our blood and tissues. When certain germs enter the body, they quickly rush to the site for attack.", "Monocyte. Monocytes are produced by the bone marrow from stem cell precursors called monoblasts. Monocytes circulate in the bloodstream for about one to three days and then typically move into tissues throughout the body. They make up three to eight percent of the leukocytes in the blood. Monocyte under a light microscope (40x) from a peripheral blood smear surrounded by red blood cells. Monocytes are a type of white blood cell, part of the human body's immune system. They are usually identified in stained smears by their large two-lobed nucleus.", "A monocyte (pictured below) is a large type of white blood cell with one large, smooth, well-defined, indented, slightly folded, oval, kidney-shaped, or notched nucleus (the cell's control center). White blood cells help protect the body against diseases and fight infections.", "Monocytes are white blood cells that are common to the blood of all vertebrates and they help the immune system to function properly. There are a number of reasons for a high monocyte count, which can also be called monocytosis. Some of the reasons can include stress, viral fevers, inflammation and organ necrosis. A physician may order a monocyte blood count test to check for raised levels of monocytes. There are a number of reasons for this test, from a simple health check up to people suffering from heart attacks and leukemia. Complications with the blood and cancer are two other reasons that this test may be performed.", "Monocytes are considered the largest white blood cell. These cells are part of the innate immune system. Monocytes also play important roles in the immune function of the body. These cells are often found when doing a stained smear and appear large kidney shaped. Many of these are found in the spleen area.", "This is taken directly from-http://www.wisegeek.com/what-are-monocytes.htm#. Monocytes are a type of leukocyte or white blood cell which play a role in immune system function. Depending on a patient's level of health, monocytes make up between one and three percent of the total white blood cells in the body. For example, if monocytes are elevated because of an inflammation caused by a viral infection, the patient would be given medication to kill the virus and bring down the inflammation. Typically, when a monocyte count is requested, the lab will also run other tests on the blood to generate a complete picture.", "3D Rendering of a Monocyte. Monocytes are a type of white blood cells (leukocytes). They are the largest of all leukocytes. They are part of the innate immune system of vertebrates including all mammals (humans included), birds, reptiles, and fish. Monocytes which migrate from the bloodstream to other tissues will then differentiate into tissue resident macrophages or dendritic cells. Macrophages are responsible for protecting tissues from foreign substances, but are also suspected to be important in the formation of important organs like the heart and brain."]
+
+  **Query:** What were the key features of the Magna Carta?
+  **Context:** ["The Magna Carta influenced the legal system in Europe, especially in constitutional law.", "It was signed in 1215 by King John of England to limit the powers of the monarchy.", "The Magna Carta introduced principles like due process and habeas corpus, which are key features of modern legal systems."]
+
+## [Retrieval: 4] (Relevant Context Ranked Middle, No External Knowledge Bias and Factual Accuracy Ignored)
+**Definition:** The context chunks fully address the query, but the most relevant chunk is ranked in the middle of the list. No external knowledge is used to influence the ranking of the chunks; the system only relies on the provided context. Factual accuracy remains out of scope for evaluation.
+
+**Examples:**
+  **Query:** do game shows pay their contestants
+  **Context:** ["So, in the end, game show winners get some of the money that TV advertisers pay to the networks, who pay the show producers, who then pay the game show winners. Just in the same way that the actors, and crew of a show get paid. Game shows, like other programs, have costs to produce the programs—they have to pay for sets, cameras, talent (the hosts), and also prizes to contestants.", "(Valerie Macon/Getty Images). Oh, happy day! You're a contestant on a popular game show—The Price Is Right, let's say. You spin the wheel, you make the winning bid, and suddenly—ka-ching!—you've won the Lexus or the dishwasher or the lifetime supply of nail clippers.", "1 If you can use most of the prizes the show offers, such as a new car or trip, you may be content to appear on a game show that features material prizes. 2 If not, you should probably try out for a show where cash is the main prize. 3 In the United States, game show contestants must pay taxes on any prizes they win. 2. Meet the eligibility requirements. All game shows have certain eligibility requirements for their contestants. Generally, you must be at least 18 years of age, except for those shows that use child or teenage contestants, and you are allowed to appear on no more than 1 game show per year.", "Rating Newest Oldest. Best Answer: You don't always win the money amount on the front of your lectern when you are on a game show. As someone else said, 2nd place earns $2000 and 3rd place earns $1000 in Jeopardy! In any case, the prize money is paid out from the ad revenue that the show receives from sponsors. I think in this case Who Wants to be a Millionaire or Deal or No Deal is the best example of how shows can be successful while still paying the prize money. I feel this way because these shows have a potential, however small it may be, to pay out 1 million dollars to every contestant on the show. Here is the reality. Regardless of the show whether it be a game show or a drama, a network will receive money from commercial advertising based on the viewership. With this in mind a game show costs very little to actually air compared to a full production drama series, that's where the prize money comes from"]
+
+## [Retrieval: 5] (Highly Relevant, Well Ranked, No Bias Introduced)
+**Definition:** The context chunks not only fully address the query, but also surface the most relevant chunks at the top of the list. The retrieval respects the internal context, avoids relying on any outside knowledge, and focuses solely on pulling the most useful content to the forefront, irrespective of the factual correctness of the information.
+
+**Examples:**
+  **Query:** The smallest blood vessels in your body, where gas exchange occurs are called
+  **Context:** ["Gas exchange is the delivery of oxygen from the lungs to the bloodstream, and the elimination of carbon dioxide from the bloodstream to the lungs. It occurs in the lungs between the alveoli and a network of tiny blood vessels called capillaries, which are located in the walls of the alveoli. The walls of the alveoli actually share a membrane with the capillaries in which oxygen and carbon dioxide move freely between the respiratory system and the bloodstream.", "Arterioles branch into capillaries, the smallest of all blood vessels. Capillaries are the sites of nutrient and waste exchange between the blood and body cells. Capillaries are microscopic vessels that join the arterial system with the venous system.", "Arterioles are the smallest arteries and regulate blood flow into capillary beds through vasoconstriction and vasodilation. Capillaries are the smallest vessels and allow for exchange of substances between the blood and interstitial fluid. Continuous capillaries are most common and allow passage of fluids and small solutes. Fenestrated capillaries are more permeable to fluids and solutes than continuous capillaries.", "Tweet. The smallest blood vessels in the human body are capillaries. They are responsible for the absorption of oxygen into the blood stream and for removing the deoxygenated red blood cells for return to the heart and lungs for reoxygenation.", "2. Capillaries—these are the sites of gas exchange between the tissues. 3. Veins—these return oxygen poor blood to the heart, except for the vein that carries blood from the lungs. On the right is a diagram showing how the three connect. Notice the artery and vein are much larger than the capillaries.", "Gas exchange occurs in the capillaries which are the smallest blood vessels in the body. Each artery that comes from the heart is surrounded by capillaries so that they can take it to the various parts of the body."]
+
+
+# Data
+QUERY: {{query}}
+CONTEXT: {{context}}
+
+
+# Tasks
+## Please provide your assessment Score for the previous CONTEXT in relation to the QUERY based on the Definitions above. Your output should include the following information:
+- **ThoughtChain**: To improve the reasoning process, think step by step and include a step-by-step explanation of your thought process as you analyze the data based on the definitions. Keep it brief and start your ThoughtChain with "Let's think step by step:".
+- **Explanation**: a very short explanation of why you think the input Data should get that Score.
+- **Score**: based on your previous analysis, provide your Score. The Score you give MUST be a integer score (i.e., "1", "2"...) based on the levels of the definitions.
+
+
+## Please provide your answers between the tags: <S0>your chain of thoughts</S0>, <S1>your explanation</S1>, <S2>your Score</S2>.
+# Output

azure/ai/evaluation/_evaluators/_rouge/__init__.py

@@ -0,0 +1,10 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from ._rouge import RougeScoreEvaluator, RougeType
+
+__all__ = [
+    "RougeScoreEvaluator",
+    "RougeType",
+]

azure/ai/evaluation/_evaluators/_rouge/_rouge.py

@@ -0,0 +1,98 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+from enum import Enum
+
+from promptflow._utils.async_utils import async_run_allowing_running_loop
+
+from azure.ai.evaluation._vendor.rouge_score import rouge_scorer
+
+
+class RougeType(Enum):
+    """
+    Enumeration of ROUGE (Recall-Oriented Understudy for Gisting Evaluation) types.
+    """
+
+    ROUGE_1 = "rouge1"
+    """Overlap of unigrams (single words) between generated and reference text."""
+
+    ROUGE_2 = "rouge2"
+    """Overlap of bigrams (two consecutive words) between generated and reference text."""
+
+    ROUGE_3 = "rouge3"
+    """Overlap of trigrams (three consecutive words) between generated and reference text."""
+
+    ROUGE_4 = "rouge4"
+    """Overlap of four-grams (four consecutive words) between generated and reference text."""
+
+    ROUGE_5 = "rouge5"
+    """Overlap of five-grams (five consecutive words) between generated and reference text."""
+
+    ROUGE_L = "rougeL"
+    """Overlap of L-grams (L consecutive words) between generated and reference text."""
+
+
+class _AsyncRougeScoreEvaluator:
+    def __init__(self, rouge_type: RougeType):
+        self._rouge_type = rouge_type
+
+    async def __call__(self, *, ground_truth: str, response: str, **kwargs):
+        scorer = rouge_scorer.RougeScorer(rouge_types=[self._rouge_type.value])
+        metrics = scorer.score(ground_truth, response)[self._rouge_type.value]
+        return {
+            "rouge_precision": metrics.precision,
+            "rouge_recall": metrics.recall,
+            "rouge_f1_score": metrics.fmeasure,
+        }
+
+
+class RougeScoreEvaluator:
+    """
+    Calculates the ROUGE score for a given response and ground truth.
+
+    The ROUGE score (Recall-Oriented Understudy for Gisting Evaluation) evaluates the similarity between the
+    generated text and reference text based on n-gram overlap, including ROUGE-N (unigram, bigram, etc.), and
+    ROUGE-L (longest common subsequence). It calculates precision, recall, and F1 scores to capture how well
+    the generated text matches the reference text. Rouge type options are "rouge1" (Unigram overlap), "rouge2"
+    (Bigram overlap), "rouge3" (Trigram overlap), "rouge4" (4-gram overlap), "rouge5" (5-gram overlap), "rougeL"
+    (L-graph overlap)
+
+    Use the ROUGE score when you need a robust evaluation metric for text summarization, machine translation, and
+    other natural language processing tasks, especially when focusing on recall and the ability to capture relevant
+    information from the reference text.
+
+    ROUGE scores range from 0 to 1, with higher scores indicating better quality.
+
+    .. admonition:: Example:
+
+        .. literalinclude:: ../samples/evaluation_samples_evaluate.py
+            :start-after: [START rouge_score_evaluator]
+            :end-before: [END rouge_score_evaluator]
+            :language: python
+            :dedent: 8
+            :caption: Initialize and call a RougeScoreEvaluator with a four-gram rouge type.
+    """
+
+    id = "azureml://registries/azureml/models/Rouge-Score-Evaluator/versions/3"
+    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+
+    def __init__(self, rouge_type: RougeType):
+        self._async_evaluator = _AsyncRougeScoreEvaluator(rouge_type)
+
+    def __call__(self, *, ground_truth: str, response: str, **kwargs):
+        """
+        Evaluate the ROUGE score between the response and the ground truth.
+
+        :keyword response: The response to be evaluated.
+        :paramtype response: str
+        :keyword ground_truth: The ground truth to be compared against.
+        :paramtype ground_truth: str
+        :return: The ROUGE score.
+        :rtype: Dict[str, float]
+        """
+        return async_run_allowing_running_loop(
+            self._async_evaluator, ground_truth=ground_truth, response=response, **kwargs
+        )
+
+    def _to_async(self):
+        return self._async_evaluator