azure-ai-evaluation 1.0.0__py3-none-any.whl → 1.0.0b2__py3-none-any.whl

azure/ai/evaluation/_evaluators/_coherence/_coherence.py

@@ -1,63 +1,108 @@
 # ---------------------------------------------------------
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
+
 import os
-from typing import Dict, Union, List
+import re
 
-from typing_extensions import overload, override
+import numpy as np
+from promptflow._utils.async_utils import async_run_allowing_running_loop
+from promptflow.core import AsyncPrompty
 
-from azure.ai.evaluation._evaluators._common import PromptyEvaluatorBase
-from azure.ai.evaluation._model_configurations import Conversation
+from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
 
+from ..._common.utils import ensure_api_version_in_aoai_model_config, ensure_user_agent_in_aoai_model_config
 
-class CoherenceEvaluator(PromptyEvaluatorBase[Union[str, float]]):
-    """
-    Evaluates coherence score for a given query and response or a multi-turn conversation, including reasoning.
+try:
+    from ..._user_agent import USER_AGENT
+except ImportError:
+    USER_AGENT = None
+
+
+class _AsyncCoherenceEvaluator:
+    # Constants must be defined within eval's directory to be save/loadable
+    PROMPTY_FILE = "coherence.prompty"
+    LLM_CALL_TIMEOUT = 600
+    DEFAULT_OPEN_API_VERSION = "2024-02-15-preview"
+
+    def __init__(self, model_config: dict):
+        ensure_api_version_in_aoai_model_config(model_config, self.DEFAULT_OPEN_API_VERSION)
+
+        prompty_model_config = {"configuration": model_config, "parameters": {"extra_headers": {}}}
+
+        # Handle "RuntimeError: Event loop is closed" from httpx AsyncClient
+        # https://github.com/encode/httpx/discussions/2959
+        prompty_model_config["parameters"]["extra_headers"].update({"Connection": "close"})
+
+        ensure_user_agent_in_aoai_model_config(
+            model_config,
+            prompty_model_config,
+            USER_AGENT,
+        )
+
+        current_dir = os.path.dirname(__file__)
+        prompty_path = os.path.join(current_dir, self.PROMPTY_FILE)
+        self._flow = AsyncPrompty.load(source=prompty_path, model=prompty_model_config)
+
+    async def __call__(self, *, query: str, response: str, **kwargs):
+        # Validate input parameters
+        query = str(query or "")
+        response = str(response or "")
+
+        if not (query.strip() and response.strip()):
+            msg = "Both 'query' and 'response' must be non-empty strings."
+            raise EvaluationException(
+                message=msg,
+                internal_message=msg,
+                error_category=ErrorCategory.INVALID_VALUE,
+                error_blame=ErrorBlame.USER_ERROR,
+                error_target=ErrorTarget.COHERENCE_EVALUATOR,
+            )
+
+        # Run the evaluation flow
+        llm_output = await self._flow(query=query, response=response, timeout=self.LLM_CALL_TIMEOUT, **kwargs)
+
+        score = np.nan
+        if llm_output:
+            match = re.search(r"\d", llm_output)
+            if match:
+                score = float(match.group())
+
+        return {"gpt_coherence": float(score)}
 
-    The coherence measure assesses the ability of the language model to generate text that reads naturally,
-    flows smoothly, and resembles human-like language in its responses. Use it when assessing the readability
-    and user-friendliness of a model's generated responses in real-world applications.
+
+class CoherenceEvaluator:
+    """
+    Initialize a coherence evaluator configured for a specific Azure OpenAI model.
 
     :param model_config: Configuration for the Azure OpenAI model.
     :type model_config: Union[~azure.ai.evaluation.AzureOpenAIModelConfiguration,
         ~azure.ai.evaluation.OpenAIModelConfiguration]
 
-    .. admonition:: Example:
+    **Usage**
 
-        .. literalinclude:: ../samples/evaluation_samples_evaluate.py
-            :start-after: [START coherence_evaluator]
-            :end-before: [END coherence_evaluator]
-            :language: python
-            :dedent: 8
-            :caption: Initialize and call a CoherenceEvaluator with a query and response.
+    .. code-block:: python
 
-    .. note::
+        eval_fn = CoherenceEvaluator(model_config)
+        result = eval_fn(
+            query="What is the capital of Japan?",
+            response="The capital of Japan is Tokyo.")
 
-        To align with our support of a diverse set of models, an output key without the `gpt_` prefix has been added.
-        To maintain backwards compatibility, the old key with the `gpt_` prefix is still be present in the output;
-        however, it is recommended to use the new key moving forward as the old key will be deprecated in the future.
-    """
+    **Output format**
+
+    .. code-block:: python
 
-    _PROMPTY_FILE = "coherence.prompty"
-    _RESULT_KEY = "coherence"
+        {
+            "gpt_coherence": 1.0
+        }
+    """
 
-    id = "azureml://registries/azureml/models/Coherence-Evaluator/versions/4"
-    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+    def __init__(self, model_config: dict):
+        self._async_evaluator = _AsyncCoherenceEvaluator(model_config)
 
-    @override
-    def __init__(self, model_config):
-        current_dir = os.path.dirname(__file__)
-        prompty_path = os.path.join(current_dir, self._PROMPTY_FILE)
-        super().__init__(model_config=model_config, prompty_file=prompty_path, result_key=self._RESULT_KEY)
-
-    @overload
-    def __call__(
-        self,
-        *,
-        query: str,
-        response: str,
-    ) -> Dict[str, Union[str, float]]:
-        """Evaluate coherence for given input of query, response
+    def __call__(self, *, query: str, response: str, **kwargs):
+        """
+        Evaluate coherence.
 
         :keyword query: The query to be evaluated.
         :paramtype query: str
@@ -66,42 +111,7 @@ class CoherenceEvaluator(PromptyEvaluatorBase[Union[str, float]]):
         :return: The coherence score.
         :rtype: Dict[str, float]
         """
+        return async_run_allowing_running_loop(self._async_evaluator, query=query, response=response, **kwargs)
 
-    @overload
-    def __call__(
-        self,
-        *,
-        conversation: Conversation,
-    ) -> Dict[str, Union[float, Dict[str, List[Union[str, float]]]]]:
-        """Evaluate coherence for a conversation
-
-        :keyword conversation: The conversation to evaluate. Expected to contain a list of conversation turns under the
-            key "messages", and potentially a global context under the key "context". Conversation turns are expected
-            to be dictionaries with keys "content", "role", and possibly "context".
-        :paramtype conversation: Optional[~azure.ai.evaluation.Conversation]
-        :return: The coherence score.
-        :rtype: Dict[str, Union[float, Dict[str, List[float]]]]
-        """
-
-    @override
-    def __call__(  # pylint: disable=docstring-missing-param
-        self,
-        *args,
-        **kwargs,
-    ):
-        """Evaluate coherence. Accepts either a query and response for a single evaluation,
-        or a conversation for a potentially multi-turn evaluation. If the conversation has more than one pair of
-        turns, the evaluator will aggregate the results of each turn.
-
-        :keyword query: The query to be evaluated.
-        :paramtype query: str
-        :keyword response: The response to be evaluated.
-        :paramtype response: Optional[str]
-        :keyword conversation: The conversation to evaluate. Expected to contain a list of conversation turns under the
-            key "messages". Conversation turns are expected
-            to be dictionaries with keys "content" and "role".
-        :paramtype conversation: Optional[~azure.ai.evaluation.Conversation]
-        :return: The relevance score.
-        :rtype: Union[Dict[str, float], Dict[str, Union[float, Dict[str, List[float]]]]]
-        """
-        return super().__call__(*args, **kwargs)
+    def _to_async(self):
+        return self._async_evaluator

azure/ai/evaluation/_evaluators/_coherence/coherence.prompty

@@ -3,9 +3,14 @@ name: Coherence
 description: Evaluates coherence score for QA scenario
 model:
   api: chat
+  configuration:
+    type: azure_openai
+    azure_deployment: ${env:AZURE_DEPLOYMENT}
+    api_key: ${env:AZURE_OPENAI_API_KEY}
+    azure_endpoint: ${env:AZURE_OPENAI_ENDPOINT}
   parameters:
     temperature: 0.0
-    max_tokens: 800
+    max_tokens: 1
     top_p: 1.0
     presence_penalty: 0
     frequency_penalty: 0
@@ -20,80 +25,38 @@ inputs:
 
 ---
 system:
-# Instruction
-## Goal
-### You are an expert in evaluating the quality of a RESPONSE from an intelligent system based on provided definition and data. Your goal will involve answering the questions below using the information provided.
-- **Definition**: You are given a definition of the communication trait that is being evaluated to help guide your Score.
-- **Data**: Your input data include a QUERY and a RESPONSE.
-- **Tasks**: To complete your evaluation you will be asked to evaluate the Data in different ways.
+You are an AI assistant. You will be given the definition of an evaluation metric for assessing the quality of an answer in a question-answering task. Your job is to compute an accurate evaluation score using the provided evaluation metric. You should return a single integer value between 1 to 5 representing the evaluation metric. You will include no other text or information.
 
 user:
-# Definition
-**Coherence** refers to the logical and orderly presentation of ideas in a response, allowing the reader to easily follow and understand the writer's train of thought. A coherent answer directly addresses the question with clear connections between sentences and paragraphs, using appropriate transitions and a logical sequence of ideas.
-
-# Ratings
-## [Coherence: 1] (Incoherent Response)
-**Definition:** The response lacks coherence entirely. It consists of disjointed words or phrases that do not form complete or meaningful sentences. There is no logical connection to the question, making the response incomprehensible.
-
-**Examples:**
-  **Query:** What are the benefits of renewable energy?
-  **Response:** Wind sun green jump apple silence over.
-
-  **Query:** Explain the process of photosynthesis.
-  **Response:** Plants light water flying blue music.
-
-## [Coherence: 2] (Poorly Coherent Response)
-**Definition:** The response shows minimal coherence with fragmented sentences and limited connection to the question. It contains some relevant keywords but lacks logical structure and clear relationships between ideas, making the overall message difficult to understand.
-
-**Examples:**
-  **Query:** How does vaccination work?
-  **Response:** Vaccines protect disease. Immune system fight. Health better.
-
-  **Query:** Describe how a bill becomes a law.
-  **Response:** Idea proposed. Congress discuss vote. President signs.
-
-## [Coherence: 3] (Partially Coherent Response)
-**Definition:** The response partially addresses the question with some relevant information but exhibits issues in the logical flow and organization of ideas. Connections between sentences may be unclear or abrupt, requiring the reader to infer the links. The response may lack smooth transitions and may present ideas out of order.
-
-**Examples:**
-  **Query:** What causes earthquakes?
-  **Response:** Earthquakes happen when tectonic plates move suddenly. Energy builds up then releases. Ground shakes and can cause damage.
-
-  **Query:** Explain the importance of the water cycle.
-  **Response:** The water cycle moves water around Earth. Evaporation, then precipitation occurs. It supports life by distributing water.
-
-## [Coherence: 4] (Coherent Response)
-**Definition:** The response is coherent and effectively addresses the question. Ideas are logically organized with clear connections between sentences and paragraphs. Appropriate transitions are used to guide the reader through the response, which flows smoothly and is easy to follow.
-
-**Examples:**
-  **Query:** What is the water cycle and how does it work?
-  **Response:** The water cycle is the continuous movement of water on Earth through processes like evaporation, condensation, and precipitation. Water evaporates from bodies of water, forms clouds through condensation, and returns to the surface as precipitation. This cycle is essential for distributing water resources globally.
-
-  **Query:** Describe the role of mitochondria in cellular function.
-  **Response:** Mitochondria are organelles that produce energy for the cell. They convert nutrients into ATP through cellular respiration. This energy powers various cellular activities, making mitochondria vital for cell survival.
-
-## [Coherence: 5] (Highly Coherent Response)
-**Definition:** The response is exceptionally coherent, demonstrating sophisticated organization and flow. Ideas are presented in a logical and seamless manner, with excellent use of transitional phrases and cohesive devices. The connections between concepts are clear and enhance the reader's understanding. The response thoroughly addresses the question with clarity and precision.
-
-**Examples:**
-  **Query:** Analyze the economic impacts of climate change on coastal cities.
-  **Response:** Climate change significantly affects the economies of coastal cities through rising sea levels, increased flooding, and more intense storms. These environmental changes can damage infrastructure, disrupt businesses, and lead to costly repairs. For instance, frequent flooding can hinder transportation and commerce, while the threat of severe weather may deter investment and tourism. Consequently, cities may face increased expenses for disaster preparedness and mitigation efforts, straining municipal budgets and impacting economic growth.
-
-  **Query:** Discuss the significance of the Monroe Doctrine in shaping U.S. foreign policy.
-  **Response:** The Monroe Doctrine was a pivotal policy declared in 1823 that asserted U.S. opposition to European colonization in the Americas. By stating that any intervention by external powers in the Western Hemisphere would be viewed as a hostile act, it established the U.S. as a protector of the region. This doctrine shaped U.S. foreign policy by promoting isolation from European conflicts while justifying American influence and expansion in the hemisphere. Its long-term significance lies in its enduring influence on international relations and its role in defining the U.S. position in global affairs.
-
-
-# Data
-QUERY: {{query}}
-RESPONSE: {{response}}
-
-
-# Tasks
-## Please provide your assessment Score for the previous RESPONSE in relation to the QUERY based on the Definitions above. Your output should include the following information:
-- **ThoughtChain**: To improve the reasoning process, think step by step and include a step-by-step explanation of your thought process as you analyze the data based on the definitions. Keep it brief and start your ThoughtChain with "Let's think step by step:".
-- **Explanation**: a very short explanation of why you think the input Data should get that Score.
-- **Score**: based on your previous analysis, provide your Score. The Score you give MUST be a integer score (i.e., "1", "2"...) based on the levels of the definitions.
-
-
-## Please provide your answers between the tags: <S0>your chain of thoughts</S0>, <S1>your explanation</S1>, <S2>your Score</S2>.
-# Output
+Coherence of an answer is measured by how well all the sentences fit together and sound naturally as a whole. Consider the overall quality of the answer when evaluating coherence. Given the question and answer, score the coherence of answer between one to five stars using the following rating scale:
+One star: the answer completely lacks coherence
+Two stars: the answer mostly lacks coherence
+Three stars: the answer is partially coherent
+Four stars: the answer is mostly coherent
+Five stars: the answer has perfect coherency
+
+This rating value should always be an integer between 1 and 5. So the rating produced should be 1 or 2 or 3 or 4 or 5.
+
+question: What is your favorite indoor activity and why do you enjoy it?
+answer: I like pizza. The sun is shining.
+stars: 1
+
+question: Can you describe your favorite movie without giving away any spoilers?
+answer: It is a science fiction movie. There are dinosaurs. The actors eat cake. People must stop the villain.
+stars: 2
+
+question: What are some benefits of regular exercise?
+answer: Regular exercise improves your mood. A good workout also helps you sleep better. Trees are green.
+stars: 3
+
+question: How do you cope with stress in your daily life?
+answer: I usually go for a walk to clear my head. Listening to music helps me relax as well. Stress is a part of life, but we can manage it through some activities.
+stars: 4
+
+question: What can you tell me about climate change and its effects on the environment?
+answer: Climate change has far-reaching effects on the environment. Rising temperatures result in the melting of polar ice caps, contributing to sea-level rise. Additionally, more frequent and severe weather events, such as hurricanes and heatwaves, can cause disruption to ecosystems and human societies alike.
+stars: 5
+
+question: {{query}}
+answer: {{response}}
+stars:

azure/ai/evaluation/_evaluators/_content_safety/__init__.py

@@ -3,6 +3,8 @@
 # ---------------------------------------------------------
 
 from ._content_safety import ContentSafetyEvaluator
+from ._content_safety_base import ContentSafetyEvaluatorBase
+from ._content_safety_chat import ContentSafetyChatEvaluator
 from ._hate_unfairness import HateUnfairnessEvaluator
 from ._self_harm import SelfHarmEvaluator
 from ._sexual import SexualEvaluator
@@ -14,4 +16,6 @@ __all__ = [
     "SelfHarmEvaluator",
     "HateUnfairnessEvaluator",
     "ContentSafetyEvaluator",
+    "ContentSafetyChatEvaluator",
+    "ContentSafetyEvaluatorBase",
 ]

azure/ai/evaluation/_evaluators/_content_safety/_content_safety.py

@@ -2,135 +2,97 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
 from concurrent.futures import as_completed
-from typing import Callable, Dict, List, Union
 
 from promptflow.tracing import ThreadPoolExecutorWithContext as ThreadPoolExecutor
-from typing_extensions import overload, override
 
-from azure.ai.evaluation._evaluators._common import EvaluatorBase
-from azure.ai.evaluation._model_configurations import Conversation
-from azure.ai.evaluation._common._experimental import experimental
+try:
+    from ._hate_unfairness import HateUnfairnessEvaluator
+    from ._self_harm import SelfHarmEvaluator
+    from ._sexual import SexualEvaluator
+    from ._violence import ViolenceEvaluator
+except ImportError:
+    from _hate_unfairness import HateUnfairnessEvaluator
+    from _self_harm import SelfHarmEvaluator
+    from _sexual import SexualEvaluator
+    from _violence import ViolenceEvaluator
 
-from ._hate_unfairness import HateUnfairnessEvaluator
-from ._self_harm import SelfHarmEvaluator
-from ._sexual import SexualEvaluator
-from ._violence import ViolenceEvaluator
 
-
-@experimental
-class ContentSafetyEvaluator(EvaluatorBase[Union[str, float]]):
+class ContentSafetyEvaluator:
     """
     Initialize a content safety evaluator configured to evaluate content safetry metrics for QA scenario.
 
-    :param credential: The credential for connecting to Azure AI project. Required
-    :type credential: ~azure.core.credentials.TokenCredential
     :param azure_ai_project: The scope of the Azure AI project.
         It contains subscription id, resource group, and project name.
     :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
-    :param kwargs: Additional arguments to pass to the evaluator.
-    :type kwargs: Any
+    :param parallel: If True, use parallel execution for evaluators. Else, use sequential execution.
+        Default is True.
+    :param credential: The credential for connecting to Azure AI project.
+    :type credential: ~azure.core.credentials.TokenCredential
     :return: A function that evaluates content-safety metrics for "question-answering" scenario.
-
-    .. admonition:: Example:
-
-        .. literalinclude:: ../samples/evaluation_samples_evaluate.py
-            :start-after: [START content_safety_evaluator]
-            :end-before: [END content_safety_evaluator]
-            :language: python
-            :dedent: 8
-            :caption: Initialize and call a ContentSafetyEvaluator.
+    :rtype: Callable
+
+    **Usage**
+
+    .. code-block:: python
+
+        azure_ai_project = {
+            "subscription_id": "<subscription_id>",
+            "resource_group_name": "<resource_group_name>",
+            "project_name": "<project_name>",
+        }
+        eval_fn = ContentSafetyEvaluator(azure_ai_project)
+        result = eval_fn(
+            query="What is the capital of France?",
+            response="Paris.",
+        )
+
+    **Output format**
+
+    .. code-block:: python
+
+        {
+            "violence": "Medium",
+            "violence_score": 5.0,
+            "violence_reason": "Some reason",
+            "sexual": "Medium",
+            "sexual_score": 5.0,
+            "sexual_reason": "Some reason",
+            "self_harm": "Medium",
+            "self_harm_score": 5.0,
+            "self_harm_reason": "Some reason",
+            "hate_unfairness": "Medium",
+            "hate_unfairness_score": 5.0,
+            "hate_unfairness_reason": "Some reason"
+        }
     """
 
-    id = "content_safety"
-    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
-
-    # TODO address 3579092 to re-enabled parallel evals.
-    def __init__(self, credential, azure_ai_project, **kwargs):
-        super().__init__()
-        self._parallel = kwargs.pop("_parallel", False)
-        self._evaluators: List[Callable[..., Dict[str, Union[str, float]]]] = [
-            ViolenceEvaluator(credential, azure_ai_project),
-            SexualEvaluator(credential, azure_ai_project),
-            SelfHarmEvaluator(credential, azure_ai_project),
-            HateUnfairnessEvaluator(credential, azure_ai_project),
+    def __init__(self, azure_ai_project: dict, parallel: bool = True, credential=None):
+        self._parallel = parallel
+        self._evaluators = [
+            ViolenceEvaluator(azure_ai_project, credential),
+            SexualEvaluator(azure_ai_project, credential),
+            SelfHarmEvaluator(azure_ai_project, credential),
+            HateUnfairnessEvaluator(azure_ai_project, credential),
         ]
 
-    @overload
-    def __call__(
-        self,
-        *,
-        query: str,
-        response: str,
-    ) -> Dict[str, Union[str, float]]:
-        """Evaluate a collection of content safety metrics for the given query/response pair
+    def __call__(self, *, query: str, response: str, **kwargs):
+        """
+        Evaluates content-safety metrics for "question-answering" scenario.
 
         :keyword query: The query to be evaluated.
         :paramtype query: str
         :keyword response: The response to be evaluated.
         :paramtype response: str
-        :return: The content safety scores.
-        :rtype: Dict[str, Union[str, float]]
-        """
-
-    @overload
-    def __call__(
-        self,
-        *,
-        conversation: Conversation,
-    ) -> Dict[str, Union[float, Dict[str, List[Union[str, float]]]]]:
-        """Evaluate a collection of content safety metrics for a conversation
-
-        :keyword conversation: The conversation to evaluate. Expected to contain a list of conversation turns under the
-            key "messages", and potentially a global context under the key "context". Conversation turns are expected
-            to be dictionaries with keys "content", "role", and possibly "context".
-        :paramtype conversation: Optional[~azure.ai.evaluation.Conversation]
-        :return: The content safety scores.
-        :rtype: Dict[str, Union[float, Dict[str, List[Union[str, float]]]]]
-        """
-
-    @override
-    def __call__(  # pylint: disable=docstring-missing-param
-        self,
-        *args,
-        **kwargs,
-    ):
-        """Evaluate a collection of content safety metrics for the given query/response pair or conversation.
-        This inputs must supply either a query AND response, or a conversation, but not both.
-
-        :keyword query: The query to evaluate.
-        :paramtype query: Optional[str]
-        :keyword response: The response to evaluate.
-        :paramtype response: Optional[str]
-        :keyword conversation: The conversation to evaluate. Expected to contain a list of conversation turns under the
-            key "messages", and potentially a global context under the key "context". Conversation turns are expected
-            to be dictionaries with keys "content", "role", and possibly "context".
-        :paramtype conversation: Optional[~azure.ai.evaluation.Conversation]
-        :return: The evaluation result.
-        :rtype: Union[Dict[str, Union[str, float]], Dict[str, Union[float, Dict[str, List[Union[str, float]]]]]]
-        """
-        return super().__call__(*args, **kwargs)
-
-    @override
-    async def _do_eval(self, eval_input: Dict) -> Dict[str, Union[str, float]]:
-        """Perform the evaluation using the Azure AI RAI service.
-        The exact evaluation performed is determined by the evaluation metric supplied
-        by the child class initializer.
-
-        :param eval_input: The input to the evaluation function.
-        :type eval_input: Dict
-        :return: The evaluation result.
-        :rtype: Dict
+        :keyword parallel: Whether to evaluate in parallel.
+        :paramtype parallel: bool
+        :return: The scores for content-safety.
+        :rtype: dict
         """
-        query = eval_input.get("query", None)
-        response = eval_input.get("response", None)
-        conversation = eval_input.get("conversation", None)
-        results: Dict[str, Union[str, float]] = {}
-        # TODO fix this to not explode on empty optional inputs (PF SKD error)
+        results = {}
         if self._parallel:
             with ThreadPoolExecutor() as executor:
-                # pylint: disable=no-value-for-parameter
                 futures = {
-                    executor.submit(query=query, response=response, conversation=conversation): evaluator
+                    executor.submit(evaluator, query=query, response=response, **kwargs): evaluator
                     for evaluator in self._evaluators
                 }
 
@@ -138,7 +100,7 @@ class ContentSafetyEvaluator(EvaluatorBase[Union[str, float]]):
                     results.update(future.result())
         else:
             for evaluator in self._evaluators:
-                result = evaluator(query=query, response=response, conversation=conversation)
+                result = evaluator(query=query, response=response, **kwargs)
                 results.update(result)
 
         return results

azure/ai/evaluation/_evaluators/{_multimodal/_content_safety_multimodal_base.py → _content_safety/_content_safety_base.py}

@@ -1,54 +1,64 @@
 # ---------------------------------------------------------
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
+
 from abc import ABC
-from typing import Union
-from azure.ai.evaluation._common.rai_service import evaluate_with_rai_service_multimodal
-from azure.ai.evaluation._common.constants import EvaluationMetrics, _InternalEvaluationMetrics
-from azure.ai.evaluation._common.utils import validate_conversation
-from azure.core.credentials import TokenCredential
+
+from azure.ai.evaluation._common.constants import EvaluationMetrics
+from azure.ai.evaluation._common.rai_service import evaluate_with_rai_service
+from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
 
 
-class ContentSafetyMultimodalEvaluatorBase(ABC):
+class ContentSafetyEvaluatorBase(ABC):
     """
     Initialize a evaluator for a specified Evaluation Metric. Base class that is not
     meant to be instantiated by users.
 
+
     :param metric: The metric to be evaluated.
     :type metric: ~azure.ai.evaluation._evaluators._content_safety.flow.constants.EvaluationMetrics
-    :param credential: The credential for connecting to Azure AI project. Required
-    :type credential: ~azure.core.credentials.TokenCredential
     :param azure_ai_project: The scope of the Azure AI project.
         It contains subscription id, resource group, and project name.
     :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
+    :param credential: The credential for connecting to Azure AI project.
+    :type credential: ~azure.core.credentials.TokenCredential
     """
 
-    def __init__(
-        self,
-        metric: Union[EvaluationMetrics, _InternalEvaluationMetrics],
-        credential: TokenCredential,
-        azure_ai_project,
-    ):
+    def __init__(self, metric: EvaluationMetrics, azure_ai_project: dict, credential=None):
         self._metric = metric
         self._azure_ai_project = azure_ai_project
         self._credential = credential
 
-    async def __call__(self, *, conversation, **kwargs):
+    async def __call__(self, *, query: str, response: str, **kwargs):
         """
         Evaluates content according to this evaluator's metric.
-        :keyword conversation: The conversation contains list of messages to be evaluated.
-            Each message should have "role" and "content" keys.
-        :paramtype conversation: ~azure.ai.evaluation.Conversation
+
+        :keyword query: The query to be evaluated.
+        :paramtype query: str
+        :keyword response: The response to be evaluated.
+        :paramtype response: str
         :return: The evaluation score computation based on the Content Safety metric (self.metric).
-        :rtype: Dict[str, Union[float, str]]
+        :rtype: Any
         """
-        # validate inputs
-        validate_conversation(conversation)
-        messages = conversation["messages"]
+        # Validate inputs
+        # Raises value error if failed, so execution alone signifies success.
+        if not (query and query.strip() and query != "None") or not (
+            response and response.strip() and response != "None"
+        ):
+            msg = "Both 'query' and 'response' must be non-empty strings."
+            raise EvaluationException(
+                message=msg,
+                internal_message=msg,
+                error_category=ErrorCategory.MISSING_FIELD,
+                error_blame=ErrorBlame.USER_ERROR,
+                error_target=ErrorTarget.CONTENT_SAFETY_CHAT_EVALUATOR,
+            )
+
         # Run score computation based on supplied metric.
-        result = await evaluate_with_rai_service_multimodal(
-            messages=messages,
+        result = await evaluate_with_rai_service(
             metric_name=self._metric,
+            query=query,
+            response=response,
             project_scope=self._azure_ai_project,
             credential=self._credential,
         )