azure-ai-evaluation 1.0.0b2__py3-none-any.whl → 1.13.3__py3-none-any.whl

azure/ai/evaluation/_evaluators/_ungrounded_attributes/_ungrounded_attributes.py

@@ -0,0 +1,102 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+from typing_extensions import overload, override
+from typing import Dict, Union
+
+from azure.ai.evaluation._common._experimental import experimental
+from azure.ai.evaluation._common.constants import EvaluationMetrics
+from azure.ai.evaluation._evaluators._common import RaiServiceEvaluatorBase
+
+
+@experimental
+class UngroundedAttributesEvaluator(RaiServiceEvaluatorBase[Union[str, bool]]):
+    """
+    Evaluates ungrounded inference of human attributes for a given query, response, and context for a single-turn evaluation only,
+    where query represents the user query and response represents the AI system response given the provided context.
+
+    Ungrounded Attributes checks for whether a response is first, ungrounded, and checks if it contains information about protected class or
+    emotional state of a person.
+
+
+    It identifies the following attributes:
+
+    - emotional_state
+    - protected_class
+    - groundedness
+
+    :param credential: The credential for connecting to Azure AI project. Required
+    :type credential: ~azure.core.credentials.TokenCredential
+    :param azure_ai_project: The Azure AI project, which can either be a string representing the project endpoint
+        or an instance of AzureAIProject. It contains subscription id, resource group, and project name.
+    :type azure_ai_project: Union[str, ~azure.ai.evaluation.AzureAIProject]
+    :param kwargs: Additional arguments to pass to the evaluator.
+    :type kwargs: Any
+
+    .. note::
+
+        If this evaluator is supplied to the `evaluate` function, the metric
+        for the ungrounded attributes will be "ungrounded_attributes_label".
+    """
+
+    id = "azureai://built-in/evaluators/ungrounded_attributes"
+    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+    _OPTIONAL_PARAMS = ["query"]
+
+    @override
+    def __init__(
+        self,
+        credential,
+        azure_ai_project,
+        **kwargs,
+    ):
+        # Set default for evaluate_query if not provided
+        if "evaluate_query" not in kwargs:
+            kwargs["evaluate_query"] = True
+
+        super().__init__(
+            eval_metric=EvaluationMetrics.UNGROUNDED_ATTRIBUTES,
+            azure_ai_project=azure_ai_project,
+            credential=credential,
+            **kwargs,
+        )
+
+    @overload
+    def __call__(
+        self,
+        *,
+        query: str,
+        response: str,
+        context: str,
+    ) -> Dict[str, Union[str, float]]:
+        """Evaluate a given query/response pair and context for ungrounded attributes
+
+        :keyword query: The query to be evaluated.
+        :paramtype query: str
+        :keyword response: The response to be evaluated.
+        :paramtype response: str
+        :keyword context: The context to be used for evaluation.
+        :paramtype context: str
+        :return: The ungrounded attributes label.
+        :rtype: Dict[str, Union[str, bool]]
+        """
+
+    @override
+    def __call__(  # pylint: disable=docstring-missing-param
+        self,
+        *args,
+        **kwargs,
+    ):
+        """Evaluate a given query/response pair and context for ungrounded attributes
+
+        :keyword query: The query to be evaluated.
+        :paramtype query: str
+        :keyword response: The response to be evaluated.
+        :paramtype response: str
+        :keyword context: The context to be used for evaluation.
+        :paramtype context: str
+        :return: The ungrounded attributes label.
+        :rtype: Dict[str, Union[str, bool]]
+        """
+
+        return super().__call__(*args, **kwargs)

azure/ai/evaluation/_evaluators/_xpia/xpia.py

@@ -2,138 +2,140 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
 import logging
-from typing import Optional
+from typing import Dict, List, Union
 
-from promptflow._utils.async_utils import async_run_allowing_running_loop
+from typing_extensions import overload, override
 
+from azure.ai.evaluation._common._experimental import experimental
 from azure.ai.evaluation._common.constants import EvaluationMetrics
-from azure.ai.evaluation._common.rai_service import evaluate_with_rai_service
-from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
+from azure.ai.evaluation._evaluators._common import RaiServiceEvaluatorBase
+from azure.ai.evaluation._model_configurations import Conversation
 
 logger = logging.getLogger(__name__)
 
 
-class IndirectAttackEvaluator:
-    """A Cross-Domain Prompt Injection Attack (XPIA) jailbreak evaluator.
-
-    Detect whether cross domain injected attacks are present in your AI system's response.
-
-    :param azure_ai_project: The scope of the Azure AI project. It contains subscription id, resource group, and project
-        name.
-    :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
-    :param eval_last_turn: Set to True to evaluate only the most recent exchange in the dialogue,
-        focusing on the latest user inquiry and the assistant's corresponding response. Defaults to False
-    :type eval_last_turn: bool
-    :param credential: The credential for connecting to Azure AI project.
+@experimental
+class IndirectAttackEvaluator(RaiServiceEvaluatorBase[Union[str, bool]]):
+    """
+    Evaluates the indirect attack score for a given query and response or a multi-turn conversation, with reasoning.
+
+    Indirect attacks, also known as cross-domain prompt injected attacks (XPIA), are when jailbreak attacks are
+    injected into the context of a document or source that may result in an altered, unexpected behavior.
+
+    Indirect attacks evaluations are broken down into three subcategories:
+        - Manipulated Content: This category involves commands that aim to alter or fabricate information, often
+            to mislead or deceive. It includes actions like spreading false information, altering language or
+            formatting, and hiding or emphasizing specific details. The goal is often to manipulate perceptions
+            or behaviors by controlling the flow and presentation of information.
+        - Intrusion: This category encompasses commands that attempt to breach systems, gain unauthorized access,
+            or elevate privileges illicitly. It includes creating backdoors, exploiting vulnerabilities, and
+            traditional jailbreaks to bypass security measures. The intent is often to gain control or access
+            sensitive data without detection.
+        - Information Gathering: This category pertains to accessing, deleting, or modifying data without
+            authorization, often for malicious purposes. It includes exfiltrating sensitive data, tampering with
+            system records, and removing or altering existing information. The focus is on acquiring or
+            manipulating data to exploit or compromise systems and individuals.
+
+    Indirect attack scores are boolean values, where True indicates that the response contains an indirect attack.
+
+    :param credential: The credential for connecting to Azure AI project. Required
     :type credential: ~azure.core.credentials.TokenCredential
-    :return: A function that evaluates and generates metrics for XPIA chat scenario. Metrics include the overall
-        evaluation label and reason for the Q/A Pair, as well as sub-labels for manipulated content, intrusion, and
-        information.
-    :rtype: Callable
-        **Usage**
-
-        .. code-block:: python
-
-            eval_fn = IndirectAttackEvaluator(model_config)
-            result = eval_fn(query="What is the capital of France?", response="Paris.")
+    :param azure_ai_project: The Azure AI project, which can either be a string representing the project endpoint
+        or an instance of AzureAIProject. It contains subscription id, resource group, and project name.
+    :type azure_ai_project: Union[str, ~azure.ai.evaluation.AzureAIProject]
+    :param threshold: The threshold for the IndirectAttack evaluator. Default is 0.
+    :type threshold: int
+
+    .. admonition:: Example:
+
+        .. literalinclude:: ../samples/evaluation_samples_evaluate.py
+            :start-after: [START indirect_attack_evaluator]
+            :end-before: [END indirect_attack_evaluator]
+            :language: python
+            :dedent: 8
+            :caption: Initialize and call an IndirectAttackEvaluator.
+
+    .. admonition:: Example using Azure AI Project URL:
+
+        .. literalinclude:: ../samples/evaluation_samples_evaluate_fdp.py
+            :start-after: [START indirect_attack_evaluator]
+            :end-before: [END indirect_attack_evaluator]
+            :language: python
+            :dedent: 8
+            :caption: Initialize and call IndirectAttackEvaluator using Azure AI Project URL in the following format
+                https://{resource_name}.services.ai.azure.com/api/projects/{project_name}
 
-        **Output format for question-answer pair**
-
-        .. code-block:: python
-
-            {
-                'xpia_label': False,
-                'xpia_reason': 'The conversation does not contain any manipulated content, intrusion or
-                              information gathering.'
-                'xpia_information_gathering': False,
-                'xpia_intrusion': False
-                'xpia_manipulated_content': False
-            }
     """
 
-    def __init__(self, azure_ai_project: dict, eval_last_turn: bool = False, credential=None):
-        self._evaluator = _IndirectAttackEvaluator(azure_ai_project, credential)
-        self._eval_last_turn = eval_last_turn
+    id = "azureai://built-in/evaluators/indirect_attack"
+    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+    _OPTIONAL_PARAMS = ["query"]
 
-    def __call__(
+    @override
+    def __init__(
         self,
-        *,
-        query: Optional[str],
-        response: Optional[str],
+        credential,
+        azure_ai_project,
         **kwargs,
     ):
-        """
-        Evaluates content according to the presence of attacks injected into the conversation context to
-        interrupt normal expected functionality by eliciting manipulated content, intrusion and attempting
-        to gather information outside the scope of your AI system.
-        :keyword query: The query to be evaluated. Mutually exclusive with 'conversation'.
-        :paramtype query: Optional[str]
-        :keyword response: The response to be evaluated. Mutually exclusive with 'conversation'.
-        :paramtype response: Optional[str]
-        :return: The evaluation scores and reasoning.
-        :rtype: dict
-        """
-
-        return self._evaluator(query=query, response=response, **kwargs)
-
+        super().__init__(
+            eval_metric=EvaluationMetrics.XPIA,
+            azure_ai_project=azure_ai_project,
+            credential=credential,
+            **kwargs,
+        )
 
-class _AsyncIndirectAttackEvaluator:
-    def __init__(self, azure_ai_project: dict, credential=None):
-        self._azure_ai_project = azure_ai_project
-        self._credential = credential
+    @overload
+    def __call__(
+        self,
+        *,
+        query: str,
+        response: str,
+    ) -> Dict[str, Union[str, bool]]:
+        """Evaluate whether cross domain injected attacks are present in given query/response
 
-    async def __call__(self, *, query: str, response: str, **kwargs):
-        """
-        Evaluates content according to this evaluator's metric.
         :keyword query: The query to be evaluated.
         :paramtype query: str
         :keyword response: The response to be evaluated.
         :paramtype response: str
-        :return: The evaluation score computation based on the metric (self.metric).
-        :rtype: Any
+        :return: The cross domain injection attack score
+        :rtype: Dict[str, Union[str, bool]]
         """
-        # Validate inputs
-        # Raises value error if failed, so execution alone signifies success.
-        if not (query and query.strip() and query != "None") or not (
-            response and response.strip() and response != "None"
-        ):
-            msg = "Both 'query' and 'response' must be non-empty strings."
-            raise EvaluationException(
-                message=msg,
-                internal_message=msg,
-                error_category=ErrorCategory.MISSING_FIELD,
-                error_blame=ErrorBlame.USER_ERROR,
-                error_target=ErrorTarget.INDIRECT_ATTACK_EVALUATOR,
-            )
-
-        # Run score computation based on supplied metric.
-        result = await evaluate_with_rai_service(
-            metric_name=EvaluationMetrics.XPIA,
-            query=query,
-            response=response,
-            project_scope=self._azure_ai_project,
-            credential=self._credential,
-        )
-        return result
 
+    @overload
+    def __call__(
+        self,
+        *,
+        conversation: Conversation,
+    ) -> Dict[str, Union[float, Dict[str, List[Union[str, bool]]]]]:
+        """Evaluate cross domain injected attacks are present in a conversation
+
+        :keyword conversation: The conversation to evaluate. Expected to contain a list of conversation turns under the
+            key "messages", and potentially a global context under the key "context". Conversation turns are expected
+            to be dictionaries with keys "content", "role", and possibly "context".
+        :paramtype conversation: Optional[~azure.ai.evaluation.Conversation]
+        :return: The cross domain injection attack score
+        :rtype: Dict[str, Union[str, bool, Dict[str, List[Union[str, bool]]]]]
+        """
 
-class _IndirectAttackEvaluator:
-    def __init__(self, azure_ai_project: dict, credential=None):
-        self._async_evaluator = _AsyncIndirectAttackEvaluator(azure_ai_project, credential)
-
-    def __call__(self, *, query: str, response: str, **kwargs):
+    @override
+    def __call__(  # pylint: disable=docstring-missing-param
+        self,
+        *args,
+        **kwargs,
+    ):
         """
-        Evaluates XPIA content.
+        Evaluate whether cross domain injected attacks are present in your AI system's response.
+
         :keyword query: The query to be evaluated.
-        :paramtype query: str
+        :paramtype query: Optional[str]
         :keyword response: The response to be evaluated.
-        :paramtype response: str
-        :keyword context: The context to be evaluated.
-        :paramtype context: str
-        :return: The XPIA score.
-        :rtype: dict
+        :paramtype response: Optional[str]
+        :keyword conversation: The conversation to evaluate. Expected to contain a list of conversation turns under the
+            key "messages". Conversation turns are expected
+            to be dictionaries with keys "content" and "role".
+        :paramtype conversation: Optional[~azure.ai.evaluation.Conversation]
+        :return: The cross domain injection attack score
+        :rtype: Union[Dict[str, Union[str, bool]], Dict[str, Union[float, Dict[str, List[Union[str, bool]]]]]]
         """
-        return async_run_allowing_running_loop(self._async_evaluator, query=query, response=response, **kwargs)
-
-    def _to_async(self):
-        return self._async_evaluator
+        return super().__call__(*args, **kwargs)

azure/ai/evaluation/_exceptions.py

@@ -4,10 +4,20 @@
 """This includes enums and classes for exceptions for use in azure-ai-evaluation."""
 
 from enum import Enum
+from typing import Optional
 
 from azure.core.exceptions import AzureError
 
 
+class ErrorMessage(Enum):
+    """Error messages to be used when raising EvaluationException.
+
+    These messages are used to provide a consistent error message format across the SDK.
+    """
+
+    MALFORMED_CONVERSATION_HISTORY = "Malformed Conversation History: Query parameter representing conversation history should have exactly one more user query than agent responses"
+
+
 class ErrorCategory(Enum):
     """Error category to be specified when using EvaluationException class.
 
@@ -20,6 +30,9 @@ class ErrorCategory(Enum):
     * RESOURCE_NOT_FOUND -> Resource could not be found
     * FAILED_EXECUTION -> Execution failed
     * SERVICE_UNAVAILABLE -> Service is unavailable
+    * MISSING_PACKAGE -> Required package is missing
+    * FAILED_REMOTE_TRACKING -> Remote tracking failed
+    * PROJECT_ACCESS_ERROR -> Access to project failed
     * UNKNOWN -> Undefined placeholder. Avoid using.
     """
 
@@ -30,7 +43,12 @@ class ErrorCategory(Enum):
     RESOURCE_NOT_FOUND = "RESOURCE NOT FOUND"
     FAILED_EXECUTION = "FAILED_EXECUTION"
     SERVICE_UNAVAILABLE = "SERVICE UNAVAILABLE"
+    MISSING_PACKAGE = "MISSING PACKAGE"
+    FAILED_REMOTE_TRACKING = "FAILED REMOTE TRACKING"
+    PROJECT_ACCESS_ERROR = "PROJECT ACCESS ERROR"
     UNKNOWN = "UNKNOWN"
+    UPLOAD_ERROR = "UPLOAD ERROR"
+    NOT_APPLICABLE = "NOT APPLICABLE"
 
 
 class ErrorBlame(Enum):
@@ -54,15 +72,22 @@ class ErrorTarget(Enum):
     EVAL_RUN = "EvalRun"
     CODE_CLIENT = "CodeClient"
     RAI_CLIENT = "RAIClient"
-    CHAT_EVALUATOR = "ChatEvaluator"
     COHERENCE_EVALUATOR = "CoherenceEvaluator"
+    COMPLETENESS_EVALUATOR = "CompletenessEvaluator"
     CONTENT_SAFETY_CHAT_EVALUATOR = "ContentSafetyEvaluator"
     ECI_EVALUATOR = "ECIEvaluator"
     F1_EVALUATOR = "F1Evaluator"
     GROUNDEDNESS_EVALUATOR = "GroundednessEvaluator"
+    TASK_NAVIGATION_EFFICIENCY_EVALUATOR = "_TaskNavigationEfficiencyEvaluator"
     PROTECTED_MATERIAL_EVALUATOR = "ProtectedMaterialEvaluator"
+    INTENT_RESOLUTION_EVALUATOR = "IntentResolutionEvaluator"
     RELEVANCE_EVALUATOR = "RelevanceEvaluator"
     SIMILARITY_EVALUATOR = "SimilarityEvaluator"
+    FLUENCY_EVALUATOR = "FluencyEvaluator"
+    RETRIEVAL_EVALUATOR = "RetrievalEvaluator"
+    TOOL_SUCCESS_EVALUATOR = "_ToolSuccessEvaluator"
+    TASK_ADHERENCE_EVALUATOR = "TaskAdherenceEvaluator"
+    TASK_COMPLETION_EVALUATOR = "_TaskCompletionEvaluator"
     INDIRECT_ATTACK_EVALUATOR = "IndirectAttackEvaluator"
     INDIRECT_ATTACK_SIMULATOR = "IndirectAttackSimulator"
     ADVERSARIAL_SIMULATOR = "AdversarialSimulator"
@@ -72,10 +97,17 @@ class ErrorTarget(Enum):
     MODELS = "Models"
     UNKNOWN = "Unknown"
     CONVERSATION = "Conversation"
+    TOOL_CALL_ACCURACY_EVALUATOR = "ToolCallAccuracyEvaluator"
+    TOOL_SELECTION_EVALUATOR = "_ToolSelectionEvaluator"
+    TOOL_INPUT_ACCURACY_EVALUATOR = "_ToolInputAccuracyEvaluator"
+    RED_TEAM = "RedTeam"
+    AOAI_GRADER = "AoaiGrader"
+    CONVERSATION_HISTORY_PARSING = "_get_conversation_history"
+    TOOL_OUTPUT_UTILIZATION_EVALUATOR = "ToolOutputUtilizationEvaluator"
 
 
 class EvaluationException(AzureError):
-    """The base class for all exceptions raised in pazure-ai-evaluation. If there is a need to define a custom
+    """The base class for all exceptions raised in azure-ai-evaluation. If there is a need to define a custom
     exception type, that custom exception type should extend from this class.
 
     :param message: A message describing the error. This is the error message the user will see.
@@ -84,24 +116,36 @@ class EvaluationException(AzureError):
     :type internal_message: str
     :param target: The name of the element that caused the exception to be thrown.
     :type target: ~azure.ai.evaluation._exceptions.ErrorTarget
-    :param error_category: The error category, defaults to Unknown.
-    :type error_category: ~azure.ai.evaluation._exceptionsErrorCategory
-    :param error: The original exception if any.
-    :type error: Exception
+    :param category: The error category, defaults to Unknown.
+    :type category: ~azure.ai.evaluation._exceptions.ErrorCategory
+    :param blame: The source of blame for the error, defaults to Unknown.
+    :type balance: ~azure.ai.evaluation._exceptions.ErrorBlame
+    :param tsg_link: A link to the TSG page for troubleshooting the error.
+    :type tsg_link: str
     """
 
     def __init__(
         self,
         message: str,
-        internal_message: str,
         *args,
+        internal_message: Optional[str] = None,
         target: ErrorTarget = ErrorTarget.UNKNOWN,
         category: ErrorCategory = ErrorCategory.UNKNOWN,
         blame: ErrorBlame = ErrorBlame.UNKNOWN,
+        tsg_link: Optional[str] = None,
         **kwargs,
     ) -> None:
         self.category = category
         self.target = target
         self.blame = blame
         self.internal_message = internal_message
+        self.tsg_link = tsg_link
         super().__init__(message, *args, **kwargs)
+
+    def __str__(self):
+        error_blame = "InternalError" if self.blame != ErrorBlame.USER_ERROR else "UserError"
+        msg = f"({error_blame}) {super().__str__()}"
+        if self.tsg_link:
+            msg += f"\nVisit {self.tsg_link} to troubleshoot this issue."
+
+        return msg