azure-ai-evaluation 1.0.0b1__py3-none-any.whl → 1.0.0b3__py3-none-any.whl

azure/ai/evaluation/_evaluators/_content_safety/_content_safety_chat.py

@@ -7,9 +7,8 @@ from typing import Dict, List
 
 import numpy as np
 from promptflow.tracing import ThreadPoolExecutorWithContext as ThreadPoolExecutor
-from azure.ai.evaluation._exceptions import EvaluationException, ErrorBlame, ErrorCategory, ErrorTarget
 
-from azure.ai.evaluation._model_configurations import AzureAIProject 
+from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
 
 try:
     from ._hate_unfairness import HateUnfairnessEvaluator
@@ -165,7 +164,10 @@ class ContentSafetyChatEvaluator:
             return score
         except Exception as e:  # pylint: disable=broad-exception-caught
             logger.warning(
-                f"Evaluator {evaluator.__class__.__name__} failed for turn {turn_num + 1} with exception: {e}"
+                "Evaluator %s failed for turn %s with exception: %s",
+                evaluator.__class__.__name__,
+                turn_num + 1,
+                e,
             )
             return {}
 
@@ -235,7 +237,10 @@ class ContentSafetyChatEvaluator:
                 )
 
             if "role" not in turn or "content" not in turn:
-                msg = f"Each turn in 'conversation' must have 'role' and 'content' keys. Turn number: {one_based_turn_num}"
+                msg = (
+                    "Each turn in 'conversation' must have 'role' and 'content' keys. "
+                    + f"Turn number: {one_based_turn_num}"
+                )
                 raise EvaluationException(
                     message=msg,
                     internal_message=msg,

azure/ai/evaluation/_evaluators/_content_safety/_hate_unfairness.py

@@ -1,31 +1,13 @@
 # ---------------------------------------------------------
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
-from promptflow._utils.async_utils import async_run_allowing_running_loop
+from typing import Optional
+from typing_extensions import override
 from azure.ai.evaluation._common.constants import EvaluationMetrics
-from azure.ai.evaluation._model_configurations import AzureAIProject
+from azure.ai.evaluation._evaluators._common import RaiServiceEvaluatorBase
 
-try:
-    from ._content_safety_base import ContentSafetyEvaluatorBase
-except ImportError:
-    from _content_safety_base import ContentSafetyEvaluatorBase
 
-
-class _AsyncHateUnfairnessEvaluator(ContentSafetyEvaluatorBase):
-    def __init__(self, azure_ai_project: dict, credential=None):
-        # Hate_fairness is the actual backend metric name. Which, uh, doesn't sound great.
-        # so invert the name.
-        super().__init__(
-            metric=EvaluationMetrics.HATE_FAIRNESS,
-            azure_ai_project=azure_ai_project,
-            credential=credential,
-        )
-
-    async def __call__(self, *, query: str, response: str, **kwargs):
-        return await super().__call__(query=query, response=response, **kwargs)
-
-
-class HateUnfairnessEvaluator:
+class HateUnfairnessEvaluator(RaiServiceEvaluatorBase):
     """
     Initialize a hate-unfairness evaluator for hate unfairness score.
 
@@ -33,7 +15,7 @@ class HateUnfairnessEvaluator:
         It contains subscription id, resource group, and project name.
     :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
     :param credential: The credential for connecting to Azure AI project.
-    :type credential: ~azure.core.credentials.TokenCredential
+    :type credential: Optional[~azure.core.credentials.TokenCredential]
 
     **Usage**
 
@@ -58,21 +40,16 @@ class HateUnfairnessEvaluator:
         }
     """
 
-    def __init__(self, azure_ai_project: dict, credential=None) -> None:
-        self._async_evaluator = _AsyncHateUnfairnessEvaluator(azure_ai_project, credential)
-
-    def __call__(self, *, query: str, response: str, **kwargs):
-        """
-        Evaluates hate unfairness content.
-
-        :keyword query: The query to be evaluated.
-        :paramtype query: str
-        :keyword response: The response to be evaluated.
-        :paramtype response: str
-        :return: The hate unfairness score.
-        :rtype: dict
-        """
-        return async_run_allowing_running_loop(self._async_evaluator, query=query, response=response, **kwargs)
-
-    def _to_async(self):
-        return self._async_evaluator
+    @override
+    def __init__(
+        self,
+        azure_ai_project: dict,
+        credential: Optional[dict] = None,
+        eval_last_turn: bool = False,
+    ):
+        super().__init__(
+            eval_metric=EvaluationMetrics.HATE_FAIRNESS,
+            azure_ai_project=azure_ai_project,
+            credential=credential,
+            eval_last_turn=eval_last_turn,
+        )

azure/ai/evaluation/_evaluators/_content_safety/_self_harm.py

@@ -1,29 +1,13 @@
 # ---------------------------------------------------------
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
-from promptflow._utils.async_utils import async_run_allowing_running_loop
+from typing import Optional
+from typing_extensions import override
 from azure.ai.evaluation._common.constants import EvaluationMetrics
-from azure.ai.evaluation._model_configurations import AzureAIProject
+from azure.ai.evaluation._evaluators._common import RaiServiceEvaluatorBase
 
-try:
-    from ._content_safety_base import ContentSafetyEvaluatorBase
-except ImportError:
-    from _content_safety_base import ContentSafetyEvaluatorBase
 
-
-class _AsyncSelfHarmEvaluator(ContentSafetyEvaluatorBase):
-    def __init__(self, azure_ai_project: dict, credential=None):
-        super().__init__(
-            metric=EvaluationMetrics.SELF_HARM,
-            azure_ai_project=azure_ai_project,
-            credential=credential,
-        )
-
-    async def __call__(self, *, query: str, response: str, **kwargs):
-        return await super().__call__(query=query, response=response, **kwargs)
-
-
-class SelfHarmEvaluator:
+class SelfHarmEvaluator(RaiServiceEvaluatorBase):
     """
     Initialize a self harm evaluator for self harm score.
 
@@ -31,7 +15,7 @@ class SelfHarmEvaluator:
         It contains subscription id, resource group, and project name.
     :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
     :param credential: The credential for connecting to Azure AI project.
-    :type credential: ~azure.core.credentials.TokenCredential
+    :type credential: Optional[~azure.core.credentials.TokenCredential]
 
     **Usage**
 
@@ -56,21 +40,16 @@ class SelfHarmEvaluator:
         }
     """
 
-    def __init__(self, azure_ai_project: dict, credential=None):
-        self._async_evaluator = _AsyncSelfHarmEvaluator(azure_ai_project, credential)
-
-    def __call__(self, *, query: str, response: str, **kwargs):
-        """
-        Evaluates self harm content.
-
-        :keyword query: The query to be evaluated.
-        :paramtype query: str
-        :keyword response: The response to be evaluated.
-        :paramtype response: str
-        :return: The self harm score.
-        :rtype: dict
-        """
-        return async_run_allowing_running_loop(self._async_evaluator, query=query, response=response, **kwargs)
-
-    def _to_async(self):
-        return self._async_evaluator
+    @override
+    def __init__(
+        self,
+        azure_ai_project: dict,
+        credential: Optional[dict] = None,
+        eval_last_turn: bool = False,
+    ):
+        super().__init__(
+            eval_metric=EvaluationMetrics.SELF_HARM,
+            azure_ai_project=azure_ai_project,
+            credential=credential,
+            eval_last_turn=eval_last_turn,
+        )

azure/ai/evaluation/_evaluators/_content_safety/_sexual.py

@@ -1,29 +1,13 @@
 # ---------------------------------------------------------
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
-from promptflow._utils.async_utils import async_run_allowing_running_loop
+from typing import Optional
+from typing_extensions import override
 from azure.ai.evaluation._common.constants import EvaluationMetrics
-from azure.ai.evaluation._model_configurations import AzureAIProject
+from azure.ai.evaluation._evaluators._common import RaiServiceEvaluatorBase
 
-try:
-    from ._content_safety_base import ContentSafetyEvaluatorBase
-except ImportError:
-    from _content_safety_base import ContentSafetyEvaluatorBase
 
-
-class _AsyncSexualEvaluator(ContentSafetyEvaluatorBase):
-    def __init__(self, azure_ai_project: dict, credential=None):
-        super().__init__(
-            metric=EvaluationMetrics.SEXUAL,
-            azure_ai_project=azure_ai_project,
-            credential=credential,
-        )
-
-    async def __call__(self, *, query: str, response: str, **kwargs):
-        return await super().__call__(query=query, response=response, **kwargs)
-
-
-class SexualEvaluator:
+class SexualEvaluator(RaiServiceEvaluatorBase):
     """
     Initialize a sexual evaluator for sexual score.
 
@@ -31,7 +15,7 @@ class SexualEvaluator:
         It contains subscription id, resource group, and project name.
     :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
     :param credential: The credential for connecting to Azure AI project.
-    :type credential: ~azure.core.credentials.TokenCredential
+    :type credential: Optional[~azure.core.credentials.TokenCredential]
 
     **Usage**
 
@@ -56,21 +40,16 @@ class SexualEvaluator:
         }
     """
 
-    def __init__(self, azure_ai_project: dict, credential=None):
-        self._async_evaluator = _AsyncSexualEvaluator(azure_ai_project, credential)
-
-    def __call__(self, *, query: str, response: str, **kwargs):
-        """
-        Evaluates sexual content.
-
-        :keyword query: The query to be evaluated.
-        :paramtype query: str
-        :keyword response: The response to be evaluated.
-        :paramtype response: str
-        :return: The sexual score.
-        :rtype: dict
-        """
-        return async_run_allowing_running_loop(self._async_evaluator, query=query, response=response, **kwargs)
-
-    def _to_async(self):
-        return self._async_evaluator
+    @override
+    def __init__(
+        self,
+        azure_ai_project: dict,
+        credential: Optional[dict] = None,
+        eval_last_turn: bool = False,
+    ):
+        super().__init__(
+            eval_metric=EvaluationMetrics.SEXUAL,
+            azure_ai_project=azure_ai_project,
+            credential=credential,
+            eval_last_turn=eval_last_turn,
+        )

azure/ai/evaluation/_evaluators/_content_safety/_violence.py

@@ -1,29 +1,13 @@
 # ---------------------------------------------------------
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
-from promptflow._utils.async_utils import async_run_allowing_running_loop
+from typing import Optional
+from typing_extensions import override
 from azure.ai.evaluation._common.constants import EvaluationMetrics
-from azure.ai.evaluation._model_configurations import AzureAIProject
+from azure.ai.evaluation._evaluators._common import RaiServiceEvaluatorBase
 
-try:
-    from ._content_safety_base import ContentSafetyEvaluatorBase
-except ImportError:
-    from _content_safety_base import ContentSafetyEvaluatorBase
 
-
-class _AsyncViolenceEvaluator(ContentSafetyEvaluatorBase):
-    def __init__(self, azure_ai_project: dict, credential=None):
-        super().__init__(
-            metric=EvaluationMetrics.VIOLENCE,
-            azure_ai_project=azure_ai_project,
-            credential=credential,
-        )
-
-    async def __call__(self, *, query: str, response: str, **kwargs):
-        return await super().__call__(query=query, response=response, **kwargs)
-
-
-class ViolenceEvaluator:
+class ViolenceEvaluator(RaiServiceEvaluatorBase):
     """
     Initialize a violence evaluator for violence score.
 
@@ -31,7 +15,7 @@ class ViolenceEvaluator:
         It contains subscription id, resource group, and project name.
     :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
     :param credential: The credential for connecting to Azure AI project.
-    :type credential: ~azure.core.credentials.TokenCredential
+    :type credential: Optional[~azure.core.credentials.TokenCredential]
 
     **Usage**
 
@@ -56,21 +40,16 @@ class ViolenceEvaluator:
         }
     """
 
-    def __init__(self, azure_ai_project: dict, credential=None):
-        self._async_evaluator = _AsyncViolenceEvaluator(azure_ai_project, credential)
-
-    def __call__(self, *, query: str, response: str, **kwargs):
-        """
-        Evaluates violence content.
-
-        :keyword query: The query to be evaluated.
-        :paramtype query: str
-        :keyword response: The response to be evaluated.
-        :paramtype response: str
-        :return: The violence score.
-        :rtype: dict
-        """
-        return async_run_allowing_running_loop(self._async_evaluator, query=query, response=response, **kwargs)
-
-    def _to_async(self):
-        return self._async_evaluator
+    @override
+    def __init__(
+        self,
+        azure_ai_project: dict,
+        credential: Optional[dict] = None,
+        eval_last_turn: bool = False,
+    ):
+        super().__init__(
+            eval_metric=EvaluationMetrics.VIOLENCE,
+            azure_ai_project=azure_ai_project,
+            credential=credential,
+            eval_last_turn=eval_last_turn,
+        )

azure/ai/evaluation/_evaluators/_eci/_eci.py

@@ -1,45 +1,13 @@
 # ---------------------------------------------------------
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
-from promptflow._utils.async_utils import async_run_allowing_running_loop
+from typing import Optional
+from typing_extensions import override
 from azure.ai.evaluation._common.constants import _InternalEvaluationMetrics
-from azure.ai.evaluation._common.rai_service import evaluate_with_rai_service
-from azure.ai.evaluation._exceptions import EvaluationException, ErrorBlame, ErrorCategory, ErrorTarget
-from azure.ai.evaluation._model_configurations import AzureAIProject
+from azure.ai.evaluation._evaluators._common import RaiServiceEvaluatorBase
 
 
-class _AsyncECIEvaluator:
-    def __init__(self, azure_ai_project: dict, credential=None):
-        self._azure_ai_project = azure_ai_project
-        self._credential = credential
-
-    async def __call__(self, *, query: str, response: str, **kwargs):
-        # Validate inputs
-        # Raises value error if failed, so execution alone signifies success.
-        if not (query and query.strip() and query != "None") or not (
-            response and response.strip() and response != "None"
-        ):
-            msg = "Both 'query' and 'response' must be non-empty strings."
-            raise EvaluationException(
-                message=msg,
-                internal_message=msg,
-                error_category=ErrorCategory.MISSING_FIELD,
-                error_blame=ErrorBlame.USER_ERROR,
-                error_target=ErrorTarget.ECI_EVALUATOR,
-            )
-
-        # Run score computation based on supplied metric.
-        result = await evaluate_with_rai_service(
-            metric_name=_InternalEvaluationMetrics.ECI,
-            query=query,
-            response=response,
-            project_scope=self._azure_ai_project,
-            credential=self._credential,
-        )
-        return result
-
-
-class ECIEvaluator:
+class ECIEvaluator(RaiServiceEvaluatorBase):
     """
     Initialize an ECI evaluator to evaluate ECI based on the following guidelines:
     Detects whether ECI is present without a disclaimer in the AI system’s response.
@@ -53,7 +21,7 @@ class ECIEvaluator:
         It contains subscription id, resource group, and project name.
     :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
     :param credential: The credential for connecting to Azure AI project.
-    :type credential: ~azure.core.credentials.TokenCredential
+    :type credential: Optional[~azure.core.credentials.TokenCredential]
     :return: Whether or not ECI was found in the response without a disclaimer, with AI-generated reasoning
     :rtype: Dict[str, str]
 
@@ -79,21 +47,16 @@ class ECIEvaluator:
         }
     """
 
-    def __init__(self, azure_ai_project: dict, credential=None) -> None:
-        self._async_evaluator = _AsyncECIEvaluator(azure_ai_project, credential)
-
-    def __call__(self, *, query: str, response: str, **kwargs):
-        """
-        Evaluates ECI content.
-
-        :keyword query: The query to be evaluated.
-        :paramtype query: str
-        :keyword response: The response to be evaluated.
-        :paramtype response: str
-        :return: The ECI result.
-        :rtype: dict
-        """
-        return async_run_allowing_running_loop(self._async_evaluator, query=query, response=response, **kwargs)
-
-    def _to_async(self):
-        return self._async_evaluator
+    @override
+    def __init__(
+        self,
+        azure_ai_project: dict,
+        credential: Optional[dict] = None,
+        eval_last_turn: bool = False,
+    ):
+        super().__init__(
+            eval_metric=_InternalEvaluationMetrics.ECI,
+            azure_ai_project=azure_ai_project,
+            credential=credential,
+            eval_last_turn=eval_last_turn,
+        )

azure/ai/evaluation/_evaluators/_f1_score/_f1_score.py

@@ -6,7 +6,8 @@ from collections import Counter
 from typing import List
 
 from promptflow._utils.async_utils import async_run_allowing_running_loop
-from azure.ai.evaluation._exceptions import EvaluationException, ErrorBlame, ErrorCategory, ErrorTarget
+
+from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
 
 
 class _AsyncF1ScoreEvaluator:

azure/ai/evaluation/_evaluators/_fluency/_fluency.py

@@ -3,80 +3,13 @@
 # ---------------------------------------------------------
 
 import os
-import re
-from typing import Union
+from typing import Optional
+from typing_extensions import override
 
-import numpy as np
+from azure.ai.evaluation._evaluators._common import PromptyEvaluatorBase
 
-from promptflow._utils.async_utils import async_run_allowing_running_loop
-from azure.ai.evaluation._exceptions import EvaluationException, ErrorBlame, ErrorCategory, ErrorTarget
-from promptflow.core import AsyncPrompty
 
-from ..._model_configurations import AzureOpenAIModelConfiguration, OpenAIModelConfiguration
-from ..._common.utils import (
-    check_and_add_api_version_for_aoai_model_config,
-    check_and_add_user_agent_for_aoai_model_config,
-)
-
-try:
-    from ..._user_agent import USER_AGENT
-except ImportError:
-    USER_AGENT = None
-
-
-class _AsyncFluencyEvaluator:
-    # Constants must be defined within eval's directory to be save/loadable
-    PROMPTY_FILE = "fluency.prompty"
-    LLM_CALL_TIMEOUT = 600
-    DEFAULT_OPEN_API_VERSION = "2024-02-15-preview"
-
-    def __init__(self, model_config: dict):
-        check_and_add_api_version_for_aoai_model_config(model_config, self.DEFAULT_OPEN_API_VERSION)
-
-        prompty_model_config = {"configuration": model_config, "parameters": {"extra_headers": {}}}
-
-        # Handle "RuntimeError: Event loop is closed" from httpx AsyncClient
-        # https://github.com/encode/httpx/discussions/2959
-        prompty_model_config["parameters"]["extra_headers"].update({"Connection": "close"})
-
-        check_and_add_user_agent_for_aoai_model_config(
-            model_config,
-            prompty_model_config,
-            USER_AGENT,
-        )
-
-        current_dir = os.path.dirname(__file__)
-        prompty_path = os.path.join(current_dir, self.PROMPTY_FILE)
-        self._flow = AsyncPrompty.load(source=prompty_path, model=prompty_model_config)
-
-    async def __call__(self, *, query: str, response: str, **kwargs):
-        # Validate input parameters
-        query = str(query or "")
-        response = str(response or "")
-
-        if not (query.strip() and response.strip()):
-            msg = "Both 'query' and 'response' must be non-empty strings."
-            raise EvaluationException(
-                message=msg,
-                internal_message=msg,
-                error_category=ErrorCategory.MISSING_FIELD,
-                error_blame=ErrorBlame.USER_ERROR,
-                error_target=ErrorTarget.F1_EVALUATOR,
-            )
-
-        # Run the evaluation flow
-        llm_output = await self._flow(query=query, response=response, timeout=self.LLM_CALL_TIMEOUT, **kwargs)
-
-        score = np.nan
-        if llm_output:
-            match = re.search(r"\d", llm_output)
-            if match:
-                score = float(match.group())
-
-        return {"gpt_fluency": float(score)}
-
-
-class FluencyEvaluator:
+class FluencyEvaluator(PromptyEvaluatorBase):
     """
     Initialize a fluency evaluator configured for a specific Azure OpenAI model.
 
@@ -102,21 +35,38 @@ class FluencyEvaluator:
         }
     """
 
-    def __init__(self, model_config: dict):
-        self._async_evaluator = _AsyncFluencyEvaluator(model_config)
+    PROMPTY_FILE = "fluency.prompty"
+    RESULT_KEY = "gpt_fluency"
 
-    def __call__(self, *, query: str, response: str, **kwargs):
+    @override
+    def __init__(self, model_config: dict):
+        current_dir = os.path.dirname(__file__)
+        prompty_path = os.path.join(current_dir, self.PROMPTY_FILE)
+        super().__init__(model_config=model_config, prompty_file=prompty_path, result_key=self.RESULT_KEY)
+
+    @override
+    def __call__(
+        self,
+        *,
+        query: Optional[str] = None,
+        response: Optional[str] = None,
+        conversation: Optional[dict] = None,
+        **kwargs
+    ):
         """
-        Evaluate fluency.
+        Evaluate fluency. Accepts either a query and response for a single evaluation,
+        or a conversation for a multi-turn evaluation. If the conversation has more than one turn,
+        the evaluator will aggregate the results of each turn.
 
         :keyword query: The query to be evaluated.
         :paramtype query: str
         :keyword response: The response to be evaluated.
         :paramtype response: str
+        :keyword conversation: The conversation to evaluate. Expected to contain a list of conversation turns under the
+            key "messages". Conversation turns are expected
+            to be dictionaries with keys "content" and "role".
+        :paramtype conversation: Optional[Dict]
         :return: The fluency score.
         :rtype: dict
         """
-        return async_run_allowing_running_loop(self._async_evaluator, query=query, response=response, **kwargs)
-
-    def _to_async(self):
-        return self._async_evaluator
+        return super().__call__(query=query, response=response, conversation=conversation, **kwargs)

azure/ai/evaluation/_evaluators/_fluency/fluency.prompty

@@ -3,11 +3,6 @@ name: Fluency
 description: Evaluates fluency score for QA scenario
 model:
   api: chat
-  configuration:
-    type: azure_openai
-    azure_deployment: ${env:AZURE_DEPLOYMENT}
-    api_key: ${env:AZURE_OPENAI_API_KEY}
-    azure_endpoint: ${env:AZURE_OPENAI_ENDPOINT}
   parameters:
     temperature: 0.0
     max_tokens: 1

azure/ai/evaluation/_evaluators/_gleu/_gleu.py

@@ -2,8 +2,8 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
 from nltk.translate.gleu_score import sentence_gleu
-
 from promptflow._utils.async_utils import async_run_allowing_running_loop
+
 from azure.ai.evaluation._common.utils import nltk_tokenize