azure-ai-evaluation 0.0.0b0__py3-none-any.whl → 1.0.0b1__py3-none-any.whl

azure/ai/evaluation/_evaluators/_content_safety/_hate_unfairness.py

@@ -0,0 +1,78 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+from promptflow._utils.async_utils import async_run_allowing_running_loop
+from azure.ai.evaluation._common.constants import EvaluationMetrics
+from azure.ai.evaluation._model_configurations import AzureAIProject
+
+try:
+    from ._content_safety_base import ContentSafetyEvaluatorBase
+except ImportError:
+    from _content_safety_base import ContentSafetyEvaluatorBase
+
+
+class _AsyncHateUnfairnessEvaluator(ContentSafetyEvaluatorBase):
+    def __init__(self, azure_ai_project: dict, credential=None):
+        # Hate_fairness is the actual backend metric name. Which, uh, doesn't sound great.
+        # so invert the name.
+        super().__init__(
+            metric=EvaluationMetrics.HATE_FAIRNESS,
+            azure_ai_project=azure_ai_project,
+            credential=credential,
+        )
+
+    async def __call__(self, *, query: str, response: str, **kwargs):
+        return await super().__call__(query=query, response=response, **kwargs)
+
+
+class HateUnfairnessEvaluator:
+    """
+    Initialize a hate-unfairness evaluator for hate unfairness score.
+
+    :param azure_ai_project: The scope of the Azure AI project.
+        It contains subscription id, resource group, and project name.
+    :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
+    :param credential: The credential for connecting to Azure AI project.
+    :type credential: ~azure.core.credentials.TokenCredential
+
+    **Usage**
+
+    .. code-block:: python
+
+        azure_ai_project = {
+            "subscription_id": "<subscription_id>",
+            "resource_group_name": "<resource_group_name>",
+            "project_name": "<project_name>",
+        }
+        eval_fn = HateUnfairnessEvaluator(azure_ai_project)
+        result = eval_fn(query="What is the capital of France?", response="Paris.")
+
+    **Output format**
+
+    .. code-block:: python
+
+        {
+            "hate_unfairness": "High",
+            "hate_unfairness_score": 6.5,
+            "hate_unfairness_reason": "Some reason"
+        }
+    """
+
+    def __init__(self, azure_ai_project: dict, credential=None) -> None:
+        self._async_evaluator = _AsyncHateUnfairnessEvaluator(azure_ai_project, credential)
+
+    def __call__(self, *, query: str, response: str, **kwargs):
+        """
+        Evaluates hate unfairness content.
+
+        :keyword query: The query to be evaluated.
+        :paramtype query: str
+        :keyword response: The response to be evaluated.
+        :paramtype response: str
+        :return: The hate unfairness score.
+        :rtype: dict
+        """
+        return async_run_allowing_running_loop(self._async_evaluator, query=query, response=response, **kwargs)
+
+    def _to_async(self):
+        return self._async_evaluator

azure/ai/evaluation/_evaluators/_content_safety/_self_harm.py

@@ -0,0 +1,76 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+from promptflow._utils.async_utils import async_run_allowing_running_loop
+from azure.ai.evaluation._common.constants import EvaluationMetrics
+from azure.ai.evaluation._model_configurations import AzureAIProject
+
+try:
+    from ._content_safety_base import ContentSafetyEvaluatorBase
+except ImportError:
+    from _content_safety_base import ContentSafetyEvaluatorBase
+
+
+class _AsyncSelfHarmEvaluator(ContentSafetyEvaluatorBase):
+    def __init__(self, azure_ai_project: dict, credential=None):
+        super().__init__(
+            metric=EvaluationMetrics.SELF_HARM,
+            azure_ai_project=azure_ai_project,
+            credential=credential,
+        )
+
+    async def __call__(self, *, query: str, response: str, **kwargs):
+        return await super().__call__(query=query, response=response, **kwargs)
+
+
+class SelfHarmEvaluator:
+    """
+    Initialize a self harm evaluator for self harm score.
+
+    :param azure_ai_project: The scope of the Azure AI project.
+        It contains subscription id, resource group, and project name.
+    :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
+    :param credential: The credential for connecting to Azure AI project.
+    :type credential: ~azure.core.credentials.TokenCredential
+
+    **Usage**
+
+    .. code-block:: python
+
+        azure_ai_project = {
+            "subscription_id": "<subscription_id>",
+            "resource_group_name": "<resource_group_name>",
+            "project_name": "<project_name>",
+        }
+        eval_fn = SelfHarmEvaluator(azure_ai_project)
+        result = eval_fn(query="What is the capital of France?", response="Paris.")
+
+    **Output format**
+
+    .. code-block:: python
+
+        {
+            "self_harm": "High",
+            "self_harm_score": 6.5,
+            "self_harm_reason": "Some reason"
+        }
+    """
+
+    def __init__(self, azure_ai_project: dict, credential=None):
+        self._async_evaluator = _AsyncSelfHarmEvaluator(azure_ai_project, credential)
+
+    def __call__(self, *, query: str, response: str, **kwargs):
+        """
+        Evaluates self harm content.
+
+        :keyword query: The query to be evaluated.
+        :paramtype query: str
+        :keyword response: The response to be evaluated.
+        :paramtype response: str
+        :return: The self harm score.
+        :rtype: dict
+        """
+        return async_run_allowing_running_loop(self._async_evaluator, query=query, response=response, **kwargs)
+
+    def _to_async(self):
+        return self._async_evaluator

azure/ai/evaluation/_evaluators/_content_safety/_sexual.py

@@ -0,0 +1,76 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+from promptflow._utils.async_utils import async_run_allowing_running_loop
+from azure.ai.evaluation._common.constants import EvaluationMetrics
+from azure.ai.evaluation._model_configurations import AzureAIProject
+
+try:
+    from ._content_safety_base import ContentSafetyEvaluatorBase
+except ImportError:
+    from _content_safety_base import ContentSafetyEvaluatorBase
+
+
+class _AsyncSexualEvaluator(ContentSafetyEvaluatorBase):
+    def __init__(self, azure_ai_project: dict, credential=None):
+        super().__init__(
+            metric=EvaluationMetrics.SEXUAL,
+            azure_ai_project=azure_ai_project,
+            credential=credential,
+        )
+
+    async def __call__(self, *, query: str, response: str, **kwargs):
+        return await super().__call__(query=query, response=response, **kwargs)
+
+
+class SexualEvaluator:
+    """
+    Initialize a sexual evaluator for sexual score.
+
+    :param azure_ai_project: The scope of the Azure AI project.
+        It contains subscription id, resource group, and project name.
+    :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
+    :param credential: The credential for connecting to Azure AI project.
+    :type credential: ~azure.core.credentials.TokenCredential
+
+    **Usage**
+
+    .. code-block:: python
+
+        azure_ai_project = {
+            "subscription_id": "<subscription_id>",
+            "resource_group_name": "<resource_group_name>",
+            "project_name": "<project_name>",
+        }
+        eval_fn = SexualEvaluator(azure_ai_project)
+        result = eval_fn(query="What is the capital of France?", response="Paris.")
+
+    **Output format**
+
+    .. code-block:: python
+
+        {
+            "sexual": "High",
+            "sexual_score": 6.5,
+            "sexual_reason": "Some reason"
+        }
+    """
+
+    def __init__(self, azure_ai_project: dict, credential=None):
+        self._async_evaluator = _AsyncSexualEvaluator(azure_ai_project, credential)
+
+    def __call__(self, *, query: str, response: str, **kwargs):
+        """
+        Evaluates sexual content.
+
+        :keyword query: The query to be evaluated.
+        :paramtype query: str
+        :keyword response: The response to be evaluated.
+        :paramtype response: str
+        :return: The sexual score.
+        :rtype: dict
+        """
+        return async_run_allowing_running_loop(self._async_evaluator, query=query, response=response, **kwargs)
+
+    def _to_async(self):
+        return self._async_evaluator

azure/ai/evaluation/_evaluators/_content_safety/_violence.py

@@ -0,0 +1,76 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+from promptflow._utils.async_utils import async_run_allowing_running_loop
+from azure.ai.evaluation._common.constants import EvaluationMetrics
+from azure.ai.evaluation._model_configurations import AzureAIProject
+
+try:
+    from ._content_safety_base import ContentSafetyEvaluatorBase
+except ImportError:
+    from _content_safety_base import ContentSafetyEvaluatorBase
+
+
+class _AsyncViolenceEvaluator(ContentSafetyEvaluatorBase):
+    def __init__(self, azure_ai_project: dict, credential=None):
+        super().__init__(
+            metric=EvaluationMetrics.VIOLENCE,
+            azure_ai_project=azure_ai_project,
+            credential=credential,
+        )
+
+    async def __call__(self, *, query: str, response: str, **kwargs):
+        return await super().__call__(query=query, response=response, **kwargs)
+
+
+class ViolenceEvaluator:
+    """
+    Initialize a violence evaluator for violence score.
+
+    :param azure_ai_project: The scope of the Azure AI project.
+        It contains subscription id, resource group, and project name.
+    :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
+    :param credential: The credential for connecting to Azure AI project.
+    :type credential: ~azure.core.credentials.TokenCredential
+
+    **Usage**
+
+    .. code-block:: python
+
+        azure_ai_project = {
+            "subscription_id": "<subscription_id>",
+            "resource_group_name": "<resource_group_name>",
+            "project_name": "<project_name>",
+        }
+        eval_fn = ViolenceEvaluator(azure_ai_project)
+        result = eval_fn(query="What is the capital of France?", response="Paris.")
+
+    **Output format**
+
+    .. code-block:: python
+
+        {
+            "violence": "High",
+            "violence_score": 6.5,
+            "violence_reason": "Some reason"
+        }
+    """
+
+    def __init__(self, azure_ai_project: dict, credential=None):
+        self._async_evaluator = _AsyncViolenceEvaluator(azure_ai_project, credential)
+
+    def __call__(self, *, query: str, response: str, **kwargs):
+        """
+        Evaluates violence content.
+
+        :keyword query: The query to be evaluated.
+        :paramtype query: str
+        :keyword response: The response to be evaluated.
+        :paramtype response: str
+        :return: The violence score.
+        :rtype: dict
+        """
+        return async_run_allowing_running_loop(self._async_evaluator, query=query, response=response, **kwargs)
+
+    def _to_async(self):
+        return self._async_evaluator

azure/ai/evaluation/_evaluators/_eci/_eci.py

@@ -0,0 +1,99 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+from promptflow._utils.async_utils import async_run_allowing_running_loop
+from azure.ai.evaluation._common.constants import _InternalEvaluationMetrics
+from azure.ai.evaluation._common.rai_service import evaluate_with_rai_service
+from azure.ai.evaluation._exceptions import EvaluationException, ErrorBlame, ErrorCategory, ErrorTarget
+from azure.ai.evaluation._model_configurations import AzureAIProject
+
+
+class _AsyncECIEvaluator:
+    def __init__(self, azure_ai_project: dict, credential=None):
+        self._azure_ai_project = azure_ai_project
+        self._credential = credential
+
+    async def __call__(self, *, query: str, response: str, **kwargs):
+        # Validate inputs
+        # Raises value error if failed, so execution alone signifies success.
+        if not (query and query.strip() and query != "None") or not (
+            response and response.strip() and response != "None"
+        ):
+            msg = "Both 'query' and 'response' must be non-empty strings."
+            raise EvaluationException(
+                message=msg,
+                internal_message=msg,
+                error_category=ErrorCategory.MISSING_FIELD,
+                error_blame=ErrorBlame.USER_ERROR,
+                error_target=ErrorTarget.ECI_EVALUATOR,
+            )
+
+        # Run score computation based on supplied metric.
+        result = await evaluate_with_rai_service(
+            metric_name=_InternalEvaluationMetrics.ECI,
+            query=query,
+            response=response,
+            project_scope=self._azure_ai_project,
+            credential=self._credential,
+        )
+        return result
+
+
+class ECIEvaluator:
+    """
+    Initialize an ECI evaluator to evaluate ECI based on the following guidelines:
+    Detects whether ECI is present without a disclaimer in the AI system’s response.
+    After the response is annotated by GPT using the binary annotation prompt
+    (which checks for disclaimer or authoritative sources),
+    we now want to check, if there was a disclaimer, the language should be exactly
+    "AI-generated content may be incorrect. If you are seeking ECI-related information, please go to Bing Search."
+    Outputs True or False with AI-generated reasoning.
+
+    :param azure_ai_project: The scope of the Azure AI project.
+        It contains subscription id, resource group, and project name.
+    :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
+    :param credential: The credential for connecting to Azure AI project.
+    :type credential: ~azure.core.credentials.TokenCredential
+    :return: Whether or not ECI was found in the response without a disclaimer, with AI-generated reasoning
+    :rtype: Dict[str, str]
+
+    **Usage**
+
+    .. code-block:: python
+
+        azure_ai_project = {
+            "subscription_id": "<subscription_id>",
+            "resource_group_name": "<resource_group_name>",
+            "project_name": "<project_name>",
+        }
+        eval_fn = ECIEvaluator(azure_ai_project)
+        result = eval_fn(query="What is the capital of France?", response="Paris.")
+
+    **Output format**
+
+    .. code-block:: python
+
+        {
+            "eci_label": "False",
+            "eci_reason": "Some reason."
+        }
+    """
+
+    def __init__(self, azure_ai_project: dict, credential=None) -> None:
+        self._async_evaluator = _AsyncECIEvaluator(azure_ai_project, credential)
+
+    def __call__(self, *, query: str, response: str, **kwargs):
+        """
+        Evaluates ECI content.
+
+        :keyword query: The query to be evaluated.
+        :paramtype query: str
+        :keyword response: The response to be evaluated.
+        :paramtype response: str
+        :return: The ECI result.
+        :rtype: dict
+        """
+        return async_run_allowing_running_loop(self._async_evaluator, query=query, response=response, **kwargs)
+
+    def _to_async(self):
+        return self._async_evaluator

azure/ai/evaluation/_evaluators/_f1_score/__init__.py

@@ -0,0 +1,9 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from ._f1_score import F1ScoreEvaluator
+
+__all__ = [
+    "F1ScoreEvaluator",
+]

azure/ai/evaluation/_evaluators/_f1_score/_f1_score.py

@@ -0,0 +1,141 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from collections import Counter
+from typing import List
+
+from promptflow._utils.async_utils import async_run_allowing_running_loop
+from azure.ai.evaluation._exceptions import EvaluationException, ErrorBlame, ErrorCategory, ErrorTarget
+
+
+class _AsyncF1ScoreEvaluator:
+    def __init__(self):
+        pass
+
+    async def __call__(self, *, response: str, ground_truth: str, **kwargs):
+        # Validate inputs
+        if not (response and response.strip() and response != "None") or not (
+            ground_truth and ground_truth.strip() and ground_truth != "None"
+        ):
+            msg = "Both 'response' and 'ground_truth' must be non-empty strings."
+            raise EvaluationException(
+                message=msg,
+                internal_message=msg,
+                error_category=ErrorCategory.MISSING_FIELD,
+                error_blame=ErrorBlame.USER_ERROR,
+                error_target=ErrorTarget.F1_EVALUATOR,
+            )
+
+        # Run f1 score computation.
+        f1_result = self._compute_f1_score(response=response, ground_truth=ground_truth)
+
+        return {"f1_score": f1_result}
+
+    @classmethod
+    def _compute_f1_score(cls, response: str, ground_truth: str) -> str:
+        import re
+        import string
+
+        class QASplitTokenizer:
+            """Quality assurance tokenizer that splits text on whitespace."""
+
+            def __call__(self, line) -> List[str]:
+                """Tokenizes an input line using split() on whitespace
+
+                :param line: The input segment to be tokenized
+                :type line: str
+                :return: The tokenized segment
+                :rtype: List[str]
+                """
+
+                return line.split()
+
+        def normalize_text(text: str) -> str:
+            """Lower text and remove punctuation, articles and extra whitespace.
+
+            :param text: The text to be normalized
+            :type text: str
+            :return: The normalized text
+            :rtype: str
+            """
+
+            def remove_articles(text):
+                return re.sub(r"\b(a|an|the)\b", " ", text)
+
+            def white_space_fix(text):
+                return " ".join(text.split())
+
+            def remove_punctuation(text):
+                exclude = set(string.punctuation)
+                return "".join(ch for ch in text if ch not in exclude)
+
+            def lower(text):
+                return text.lower()
+
+            return white_space_fix(remove_articles(remove_punctuation(lower(text))))
+
+        prediction_tokens = normalize_text(response)
+        reference_tokens = normalize_text(ground_truth)
+        tokenizer = QASplitTokenizer()
+        prediction_tokens = tokenizer(prediction_tokens)
+        reference_tokens = tokenizer(reference_tokens)
+
+        common_tokens = Counter(prediction_tokens) & Counter(reference_tokens)
+        num_common_tokens = sum(common_tokens.values())
+
+        if num_common_tokens == 0:
+            f1 = 0.0
+        else:
+            precision = 1.0 * num_common_tokens / len(prediction_tokens)
+            recall = 1.0 * num_common_tokens / len(reference_tokens)
+
+            f1 = (2.0 * precision * recall) / (precision + recall)
+
+        return f1
+
+
+class F1ScoreEvaluator:
+    """
+    Initialize a f1 score evaluator for calculating F1 score.
+
+    **Usage**
+
+    .. code-block:: python
+
+        eval_fn = F1ScoreEvaluator()
+        result = eval_fn(
+            response="The capital of Japan is Tokyo.",
+            ground_truth="Tokyo is Japan's capital, known for its blend of traditional culture \
+                and technological advancements.")
+
+    **Output format**
+
+    .. code-block:: python
+
+        {
+            "f1_score": 0.42
+        }
+    """
+
+    def __init__(self):
+        self._async_evaluator = _AsyncF1ScoreEvaluator()
+
+    def __call__(self, *, response: str, ground_truth: str, **kwargs):
+        """
+        Evaluate F1 score.
+
+        :keyword response: The response to be evaluated.
+        :paramtype response: str
+        :keyword ground_truth: The ground truth to be evaluated.
+        :paramtype ground_truth: str
+        :return: The F1 score.
+        :rtype: dict
+        """
+
+        return async_run_allowing_running_loop(
+            self._async_evaluator, response=response, ground_truth=ground_truth, **kwargs
+        )
+
+    def _to_async(self):
+        return self._async_evaluator

azure/ai/evaluation/_evaluators/_fluency/__init__.py

@@ -0,0 +1,9 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from ._fluency import FluencyEvaluator
+
+__all__ = [
+    "FluencyEvaluator",
+]

azure/ai/evaluation/_evaluators/_fluency/_fluency.py

@@ -0,0 +1,122 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+import os
+import re
+from typing import Union
+
+import numpy as np
+
+from promptflow._utils.async_utils import async_run_allowing_running_loop
+from azure.ai.evaluation._exceptions import EvaluationException, ErrorBlame, ErrorCategory, ErrorTarget
+from promptflow.core import AsyncPrompty
+
+from ..._model_configurations import AzureOpenAIModelConfiguration, OpenAIModelConfiguration
+from ..._common.utils import (
+    check_and_add_api_version_for_aoai_model_config,
+    check_and_add_user_agent_for_aoai_model_config,
+)
+
+try:
+    from ..._user_agent import USER_AGENT
+except ImportError:
+    USER_AGENT = None
+
+
+class _AsyncFluencyEvaluator:
+    # Constants must be defined within eval's directory to be save/loadable
+    PROMPTY_FILE = "fluency.prompty"
+    LLM_CALL_TIMEOUT = 600
+    DEFAULT_OPEN_API_VERSION = "2024-02-15-preview"
+
+    def __init__(self, model_config: dict):
+        check_and_add_api_version_for_aoai_model_config(model_config, self.DEFAULT_OPEN_API_VERSION)
+
+        prompty_model_config = {"configuration": model_config, "parameters": {"extra_headers": {}}}
+
+        # Handle "RuntimeError: Event loop is closed" from httpx AsyncClient
+        # https://github.com/encode/httpx/discussions/2959
+        prompty_model_config["parameters"]["extra_headers"].update({"Connection": "close"})
+
+        check_and_add_user_agent_for_aoai_model_config(
+            model_config,
+            prompty_model_config,
+            USER_AGENT,
+        )
+
+        current_dir = os.path.dirname(__file__)
+        prompty_path = os.path.join(current_dir, self.PROMPTY_FILE)
+        self._flow = AsyncPrompty.load(source=prompty_path, model=prompty_model_config)
+
+    async def __call__(self, *, query: str, response: str, **kwargs):
+        # Validate input parameters
+        query = str(query or "")
+        response = str(response or "")
+
+        if not (query.strip() and response.strip()):
+            msg = "Both 'query' and 'response' must be non-empty strings."
+            raise EvaluationException(
+                message=msg,
+                internal_message=msg,
+                error_category=ErrorCategory.MISSING_FIELD,
+                error_blame=ErrorBlame.USER_ERROR,
+                error_target=ErrorTarget.F1_EVALUATOR,
+            )
+
+        # Run the evaluation flow
+        llm_output = await self._flow(query=query, response=response, timeout=self.LLM_CALL_TIMEOUT, **kwargs)
+
+        score = np.nan
+        if llm_output:
+            match = re.search(r"\d", llm_output)
+            if match:
+                score = float(match.group())
+
+        return {"gpt_fluency": float(score)}
+
+
+class FluencyEvaluator:
+    """
+    Initialize a fluency evaluator configured for a specific Azure OpenAI model.
+
+    :param model_config: Configuration for the Azure OpenAI model.
+    :type model_config: Union[~azure.ai.evaluation.AzureOpenAIModelConfiguration,
+        ~azure.ai.evaluation.OpenAIModelConfiguration]
+
+    **Usage**
+
+    .. code-block:: python
+
+        eval_fn = FluencyEvaluator(model_config)
+        result = eval_fn(
+            query="What is the capital of Japan?",
+            response="The capital of Japan is Tokyo.")
+
+    **Output format**
+
+    .. code-block:: python
+
+        {
+            "gpt_fluency": 4.0
+        }
+    """
+
+    def __init__(self, model_config: dict):
+        self._async_evaluator = _AsyncFluencyEvaluator(model_config)
+
+    def __call__(self, *, query: str, response: str, **kwargs):
+        """
+        Evaluate fluency.
+
+        :keyword query: The query to be evaluated.
+        :paramtype query: str
+        :keyword response: The response to be evaluated.
+        :paramtype response: str
+        :return: The fluency score.
+        :rtype: dict
+        """
+        return async_run_allowing_running_loop(self._async_evaluator, query=query, response=response, **kwargs)
+
+    def _to_async(self):
+        return self._async_evaluator