azure-ai-evaluation 0.0.0b0__py3-none-any.whl → 1.0.0__py3-none-any.whl

azure/ai/evaluation/_evaluators/_eci/_eci.py

@@ -0,0 +1,89 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+from typing_extensions import overload, override
+
+from azure.ai.evaluation._common._experimental import experimental
+from azure.ai.evaluation._common.constants import _InternalEvaluationMetrics
+from azure.ai.evaluation._evaluators._common import RaiServiceEvaluatorBase
+from azure.ai.evaluation._model_configurations import Conversation
+
+
+@experimental
+class ECIEvaluator(RaiServiceEvaluatorBase):
+    """
+    Initialize an ECI evaluator to evaluate ECI based on the following guidelines:
+    Detects whether ECI is present without a disclaimer in the AI system’s response.
+    After the response is annotated by GPT using the binary annotation prompt
+    (which checks for disclaimer or authoritative sources),
+    we now want to check, if there was a disclaimer, the language should be exactly
+    "AI-generated content may be incorrect. If you are seeking ECI-related information, please go to Bing Search."
+    Outputs True or False with AI-generated reasoning.
+
+    :param credential: The credential for connecting to Azure AI project. Required
+    :type credential: ~azure.core.credentials.TokenCredential
+    :param azure_ai_project: The scope of the Azure AI project.
+        It contains subscription id, resource group, and project name.
+    :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
+    :return: Whether or not ECI was found in the response without a disclaimer, with AI-generated reasoning
+    :rtype: Dict[str, str]
+
+    **Usage**
+
+    .. code-block:: python
+
+        azure_ai_project = {
+            "subscription_id": "<subscription_id>",
+            "resource_group_name": "<resource_group_name>",
+            "project_name": "<project_name>",
+        }
+        eval_fn = ECIEvaluator(azure_ai_project)
+        result = eval_fn(query="What is the capital of France?", response="Paris.")
+
+    **Output format**
+
+    .. code-block:: python
+
+        {
+            "eci_label": "False",
+            "eci_reason": "Some reason."
+        }
+    """
+
+    id = "eci"
+    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+
+    @override
+    def __init__(
+        self,
+        credential,
+        azure_ai_project,
+    ):
+        super().__init__(
+            eval_metric=_InternalEvaluationMetrics.ECI,
+            azure_ai_project=azure_ai_project,
+            credential=credential,
+        )
+
+    @overload
+    def __call__(
+        self,
+        *,
+        query: str,
+        response: str,
+    ): ...
+
+    @overload
+    def __call__(
+        self,
+        *,
+        conversation: Conversation,
+    ): ...
+
+    @override
+    def __call__(  # pylint: disable=docstring-missing-param
+        self,
+        *args,
+        **kwargs,
+    ):
+        return super().__call__(*args, **kwargs)

azure/ai/evaluation/_evaluators/_f1_score/__init__.py

@@ -0,0 +1,9 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from ._f1_score import F1ScoreEvaluator
+
+__all__ = [
+    "F1ScoreEvaluator",
+]

azure/ai/evaluation/_evaluators/_f1_score/_f1_score.py

@@ -0,0 +1,157 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from collections import Counter
+from typing import List
+
+from promptflow._utils.async_utils import async_run_allowing_running_loop
+
+from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
+
+
+class _AsyncF1ScoreEvaluator:
+    def __init__(self):
+        pass
+
+    async def __call__(self, *, response: str, ground_truth: str, **kwargs):
+        """
+        Evaluate F1 score.
+
+        :keyword response: The response to be evaluated.
+        :paramtype response: str
+        :keyword ground_truth: The ground truth to be evaluated.
+        :paramtype ground_truth: str
+        :return: The F1 score.
+        :rtype: Dict[str, float]
+        """
+        # Validate inputs
+        if not (response and response.strip() and response != "None") or not (
+            ground_truth and ground_truth.strip() and ground_truth != "None"
+        ):
+            msg = "Both 'response' and 'ground_truth' must be non-empty strings."
+            raise EvaluationException(
+                message=msg,
+                internal_message=msg,
+                error_category=ErrorCategory.MISSING_FIELD,
+                error_blame=ErrorBlame.USER_ERROR,
+                error_target=ErrorTarget.F1_EVALUATOR,
+            )
+
+        # Run f1 score computation.
+        f1_result = self._compute_f1_score(response=response, ground_truth=ground_truth)
+
+        return {"f1_score": f1_result}
+
+    @classmethod
+    def _compute_f1_score(cls, response: str, ground_truth: str) -> float:
+        import re
+        import string
+
+        class QASplitTokenizer:
+            """Quality assurance tokenizer that splits text on whitespace."""
+
+            def __call__(self, line) -> List[str]:
+                """Tokenizes an input line using split() on whitespace
+
+                :param line: The input segment to be tokenized
+                :type line: str
+                :return: The tokenized segment
+                :rtype: List[str]
+                """
+
+                return line.split()
+
+        def normalize_text(text: str) -> str:
+            """Lower text and remove punctuation, articles and extra whitespace.
+
+            :param text: The text to be normalized
+            :type text: str
+            :return: The normalized text
+            :rtype: str
+            """
+
+            def remove_articles(text):
+                return re.sub(r"\b(a|an|the)\b", " ", text)
+
+            def white_space_fix(text):
+                return " ".join(text.split())
+
+            def remove_punctuation(text):
+                exclude = set(string.punctuation)
+                return "".join(ch for ch in text if ch not in exclude)
+
+            def lower(text):
+                return text.lower()
+
+            return white_space_fix(remove_articles(remove_punctuation(lower(text))))
+
+        tokenizer = QASplitTokenizer()
+        prediction_tokens = tokenizer(normalize_text(response))
+        reference_tokens = tokenizer(normalize_text(ground_truth))
+
+        common_tokens = Counter(prediction_tokens) & Counter(reference_tokens)
+        num_common_tokens = sum(common_tokens.values())
+
+        if num_common_tokens == 0:
+            f1 = 0.0
+        else:
+            precision = 1.0 * num_common_tokens / len(prediction_tokens)
+            recall = 1.0 * num_common_tokens / len(reference_tokens)
+
+            f1 = (2.0 * precision * recall) / (precision + recall)
+
+        return f1
+
+
+class F1ScoreEvaluator:
+    """
+    Calculates the F1 score for a given response and ground truth or a multi-turn conversation.
+
+    F1 Scores range from 0 to 1, with 1 being the best possible score.
+
+    The F1-score computes the ratio of the number of shared words between the model generation and
+    the ground truth. Ratio is computed over the individual words in the generated response against those in the ground
+    truth answer. The number of shared words between the generation and the truth is the basis of the F1 score:
+    precision is the ratio of the number of shared words to the total number of words in the generation, and recall
+    is the ratio of the number of shared words to the total number of words in the ground truth.
+
+    Use the F1 score when you want a single comprehensive metric that combines both recall and precision in your
+    model's responses. It provides a balanced evaluation of your model's performance in terms of capturing accurate
+    information in the response.
+
+
+    .. admonition:: Example:
+
+        .. literalinclude:: ../samples/evaluation_samples_evaluate.py
+            :start-after: [START f1_score_evaluator]
+            :end-before: [END f1_score_evaluator]
+            :language: python
+            :dedent: 8
+            :caption: Initialize and call an F1ScoreEvaluator.
+    """
+
+    id = "azureml://registries/azureml/models/F1Score-Evaluator/versions/3"
+    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+
+    def __init__(self):
+        self._async_evaluator = _AsyncF1ScoreEvaluator()
+
+    def __call__(self, *, response: str, ground_truth: str, **kwargs):
+        """
+        Evaluate F1 score.
+
+        :keyword response: The response to be evaluated.
+        :paramtype response: str
+        :keyword ground_truth: The ground truth to be evaluated.
+        :paramtype ground_truth: str
+        :return: The F1 score.
+        :rtype: Dict[str, float]
+        """
+
+        return async_run_allowing_running_loop(
+            self._async_evaluator, response=response, ground_truth=ground_truth, **kwargs
+        )
+
+    def _to_async(self):
+        return self._async_evaluator

azure/ai/evaluation/_evaluators/_fluency/__init__.py

@@ -0,0 +1,9 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from ._fluency import FluencyEvaluator
+
+__all__ = [
+    "FluencyEvaluator",
+]

azure/ai/evaluation/_evaluators/_fluency/_fluency.py

@@ -0,0 +1,104 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+import os
+from typing import Dict, List, Union
+
+from typing_extensions import overload, override
+
+from azure.ai.evaluation._evaluators._common import PromptyEvaluatorBase
+from azure.ai.evaluation._model_configurations import Conversation
+
+
+class FluencyEvaluator(PromptyEvaluatorBase[Union[str, float]]):
+    """
+    Evaluates the fluency of a given response or a multi-turn conversation, including reasoning.
+
+    The fluency measure assesses the extent to which the generated text conforms to grammatical rules, syntactic
+    structures, and appropriate vocabulary usage, resulting in linguistically correct responses.
+
+    Fluency scores range from 1 to 5, with 1 being the least fluent and 5 being the most fluent.
+
+    :param model_config: Configuration for the Azure OpenAI model.
+    :type model_config: Union[~azure.ai.evaluation.AzureOpenAIModelConfiguration,
+        ~azure.ai.evaluation.OpenAIModelConfiguration]
+
+    .. admonition:: Example:
+
+        .. literalinclude:: ../samples/evaluation_samples_evaluate.py
+            :start-after: [START fluency_evaluator]
+            :end-before: [END fluency_evaluator]
+            :language: python
+            :dedent: 8
+            :caption: Initialize and call a FluencyEvaluator.
+
+    .. note::
+
+        To align with our support of a diverse set of models, an output key without the `gpt_` prefix has been added.
+        To maintain backwards compatibility, the old key with the `gpt_` prefix is still be present in the output;
+        however, it is recommended to use the new key moving forward as the old key will be deprecated in the future.
+    """
+
+    _PROMPTY_FILE = "fluency.prompty"
+    _RESULT_KEY = "fluency"
+
+    id = "azureml://registries/azureml/models/Fluency-Evaluator/versions/4"
+    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+
+    @override
+    def __init__(self, model_config):
+        current_dir = os.path.dirname(__file__)
+        prompty_path = os.path.join(current_dir, self._PROMPTY_FILE)
+        super().__init__(model_config=model_config, prompty_file=prompty_path, result_key=self._RESULT_KEY)
+
+    @overload
+    def __call__(
+        self,
+        *,
+        response: str,
+    ) -> Dict[str, Union[str, float]]:
+        """Evaluate fluency in given response
+
+        :keyword response: The response to be evaluated.
+        :paramtype response: str
+        :return: The fluency score
+        :rtype: Dict[str, float]
+        """
+
+    @overload
+    def __call__(
+        self,
+        *,
+        conversation: Conversation,
+    ) -> Dict[str, Union[float, Dict[str, List[Union[str, float]]]]]:
+        """Evaluate fluency for a conversation
+
+        :keyword conversation: The conversation to evaluate. Expected to contain a list of conversation turns under the
+            key "messages", and potentially a global context under the key "context". Conversation turns are expected
+            to be dictionaries with keys "content", "role", and possibly "context".
+        :paramtype conversation: Optional[~azure.ai.evaluation.Conversation]
+        :return: The fluency score
+        :rtype: Dict[str, Union[float, Dict[str, List[float]]]]
+        """
+
+    @override
+    def __call__(  # pylint: disable=docstring-missing-param
+        self,
+        *args,
+        **kwargs,
+    ):
+        """
+        Evaluate fluency. Accepts either a response for a single evaluation,
+        or a conversation for a multi-turn evaluation. If the conversation has more than one turn,
+        the evaluator will aggregate the results of each turn.
+
+        :keyword response: The response to be evaluated. Mutually exclusive with the "conversation" parameter.
+        :paramtype response: Optional[str]
+        :keyword conversation: The conversation to evaluate. Expected to contain a list of conversation turns under the
+            key "messages". Conversation turns are expected to be dictionaries with keys "content" and "role".
+        :paramtype conversation: Optional[~azure.ai.evaluation.Conversation]
+        :return: The fluency score.
+        :rtype: Union[Dict[str, float], Dict[str, Union[float, Dict[str, List[float]]]]]
+        """
+        return super().__call__(*args, **kwargs)

azure/ai/evaluation/_evaluators/_fluency/fluency.prompty

@@ -0,0 +1,86 @@
+---
+name: Fluency
+description: Evaluates fluency score for QA scenario
+model:
+  api: chat
+  parameters:
+    temperature: 0.0
+    max_tokens: 800
+    top_p: 1.0
+    presence_penalty: 0
+    frequency_penalty: 0
+    response_format:
+      type: text
+
+inputs:
+  response:
+    type: string
+
+---
+system:
+# Instruction
+## Goal
+### You are an expert in evaluating the quality of a RESPONSE from an intelligent system based on provided definition and data. Your goal will involve answering the questions below using the information provided.
+- **Definition**: You are given a definition of the communication trait that is being evaluated to help guide your Score.
+- **Data**: Your input data include a RESPONSE.
+- **Tasks**: To complete your evaluation you will be asked to evaluate the Data in different ways.
+
+user:
+# Definition
+**Fluency** refers to the effectiveness and clarity of written communication, focusing on grammatical accuracy, vocabulary range, sentence complexity, coherence, and overall readability. It assesses how smoothly ideas are conveyed and how easily the text can be understood by the reader.
+
+# Ratings
+## [Fluency: 1] (Emergent Fluency)
+**Definition:** The response shows minimal command of the language. It contains pervasive grammatical errors, extremely limited vocabulary, and fragmented or incoherent sentences. The message is largely incomprehensible, making understanding very difficult.
+
+**Examples:**
+  **Response:** Free time I. Go park. Not fun. Alone.
+
+  **Response:** Like food pizza. Good cheese eat.
+
+## [Fluency: 2] (Basic Fluency)
+**Definition:** The response communicates simple ideas but has frequent grammatical errors and limited vocabulary. Sentences are short and may be improperly constructed, leading to partial understanding. Repetition and awkward phrasing are common.
+
+**Examples:**
+  **Response:** I like play soccer. I watch movie. It fun.
+
+  **Response:** My town small. Many people. We have market.
+
+## [Fluency: 3] (Competent Fluency)
+**Definition:** The response clearly conveys ideas with occasional grammatical errors. Vocabulary is adequate but not extensive. Sentences are generally correct but may lack complexity and variety. The text is coherent, and the message is easily understood with minimal effort.
+
+**Examples:**
+  **Response:** I'm planning to visit friends and maybe see a movie together.
+
+  **Response:** I try to eat healthy food and exercise regularly by jogging.
+
+## [Fluency: 4] (Proficient Fluency)
+**Definition:** The response is well-articulated with good control of grammar and a varied vocabulary. Sentences are complex and well-structured, demonstrating coherence and cohesion. Minor errors may occur but do not affect overall understanding. The text flows smoothly, and ideas are connected logically.
+
+**Examples:**
+  **Response:** My interest in mathematics and problem-solving inspired me to become an engineer, as I enjoy designing solutions that improve people's lives.
+
+  **Response:** Environmental conservation is crucial because it protects ecosystems, preserves biodiversity, and ensures natural resources are available for future generations.
+
+## [Fluency: 5] (Exceptional Fluency)
+**Definition:** The response demonstrates an exceptional command of language with sophisticated vocabulary and complex, varied sentence structures. It is coherent, cohesive, and engaging, with precise and nuanced expression. Grammar is flawless, and the text reflects a high level of eloquence and style.
+
+**Examples:**
+  **Response:** Globalization exerts a profound influence on cultural diversity by facilitating unprecedented cultural exchange while simultaneously risking the homogenization of distinct cultural identities, which can diminish the richness of global heritage.
+
+  **Response:** Technology revolutionizes modern education by providing interactive learning platforms, enabling personalized learning experiences, and connecting students worldwide, thereby transforming how knowledge is acquired and shared.
+
+
+# Data
+RESPONSE: {{response}}
+
+
+# Tasks
+## Please provide your assessment Score for the previous RESPONSE based on the Definitions above. Your output should include the following information:
+- **ThoughtChain**: To improve the reasoning process, think step by step and include a step-by-step explanation of your thought process as you analyze the data based on the definitions. Keep it brief and start your ThoughtChain with "Let's think step by step:".
+- **Explanation**: a very short explanation of why you think the input Data should get that Score.
+- **Score**: based on your previous analysis, provide your Score. The Score you give MUST be a integer score (i.e., "1", "2"...) based on the levels of the definitions.
+
+
+## Please provide your answers between the tags: <S0>your chain of thoughts</S0>, <S1>your explanation</S1>, <S2>your Score</S2>.
+# Output

azure/ai/evaluation/_evaluators/_gleu/__init__.py

@@ -0,0 +1,9 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from ._gleu import GleuScoreEvaluator
+
+__all__ = [
+    "GleuScoreEvaluator",
+]

azure/ai/evaluation/_evaluators/_gleu/_gleu.py

@@ -0,0 +1,69 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+from nltk.translate.gleu_score import sentence_gleu
+from promptflow._utils.async_utils import async_run_allowing_running_loop
+
+from azure.ai.evaluation._common.utils import nltk_tokenize
+
+
+class _AsyncGleuScoreEvaluator:
+    def __init__(self):
+        pass
+
+    async def __call__(self, *, ground_truth: str, response: str, **kwargs):
+        reference_tokens = nltk_tokenize(ground_truth)
+        hypothesis_tokens = nltk_tokenize(response)
+
+        score = sentence_gleu([reference_tokens], hypothesis_tokens)
+
+        return {
+            "gleu_score": score,
+        }
+
+
+class GleuScoreEvaluator:
+    """
+    Calculates the GLEU (Google-BLEU) score between a response and the ground truth.
+
+    The GLEU (Google-BLEU) score evaluator measures the similarity between generated and reference texts by
+    evaluating n-gram overlap, considering both precision and recall. This balanced evaluation, designed for
+    sentence-level assessment, makes it ideal for detailed analysis of translation quality. GLEU is well-suited for
+    use cases such as machine translation, text summarization, and text generation.
+
+    GLEU scores range from 0 to 1, where a value of 1 represents perfect overlap between the response and
+    the ground truth and a value of 0 indicates no overlap.
+
+    .. admonition:: Example:
+
+        .. literalinclude:: ../samples/evaluation_samples_evaluate.py
+            :start-after: [START gleu_score_evaluator]
+            :end-before: [END gleu_score_evaluator]
+            :language: python
+            :dedent: 8
+            :caption: Initialize and call a GleuScoreEvaluator.
+    """
+
+    id = "azureml://registries/azureml/models/Gleu-Score-Evaluator/versions/3"
+    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+
+    def __init__(self):
+        self._async_evaluator = _AsyncGleuScoreEvaluator()
+
+    def __call__(self, *, ground_truth: str, response: str, **kwargs):
+        """
+        Evaluate the GLEU score between the response and the ground truth.
+
+        :keyword response: The response to be evaluated.
+        :paramtype response: str
+        :keyword ground_truth: The ground truth to be compared against.
+        :paramtype ground_truth: str
+        :return: The GLEU score.
+        :rtype: Dict[str, float]
+        """
+        return async_run_allowing_running_loop(
+            self._async_evaluator, ground_truth=ground_truth, response=response, **kwargs
+        )
+
+    def _to_async(self):
+        return self._async_evaluator

azure/ai/evaluation/_evaluators/_groundedness/__init__.py

@@ -0,0 +1,9 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from ._groundedness import GroundednessEvaluator
+
+__all__ = [
+    "GroundednessEvaluator",
+]