azure-ai-evaluation 1.0.0b2__py3-none-any.whl → 1.0.0b4__py3-none-any.whl

azure/ai/evaluation/_evaluators/_xpia/xpia.py

@@ -2,30 +2,28 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
 import logging
-from typing import Optional
 
-from promptflow._utils.async_utils import async_run_allowing_running_loop
+from typing_extensions import override
 
 from azure.ai.evaluation._common.constants import EvaluationMetrics
-from azure.ai.evaluation._common.rai_service import evaluate_with_rai_service
-from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
+from azure.ai.evaluation._evaluators._common import RaiServiceEvaluatorBase
 
 logger = logging.getLogger(__name__)
 
 
-class IndirectAttackEvaluator:
+class IndirectAttackEvaluator(RaiServiceEvaluatorBase):
     """A Cross-Domain Prompt Injection Attack (XPIA) jailbreak evaluator.
 
     Detect whether cross domain injected attacks are present in your AI system's response.
 
+    :param credential: The credential for connecting to Azure AI project. Required
+    :type credential: ~azure.core.credentials.TokenCredential
     :param azure_ai_project: The scope of the Azure AI project. It contains subscription id, resource group, and project
         name.
     :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
     :param eval_last_turn: Set to True to evaluate only the most recent exchange in the dialogue,
         focusing on the latest user inquiry and the assistant's corresponding response. Defaults to False
     :type eval_last_turn: bool
-    :param credential: The credential for connecting to Azure AI project.
-    :type credential: ~azure.core.credentials.TokenCredential
     :return: A function that evaluates and generates metrics for XPIA chat scenario. Metrics include the overall
         evaluation label and reason for the Q/A Pair, as well as sub-labels for manipulated content, intrusion, and
         information.
@@ -51,89 +49,16 @@ class IndirectAttackEvaluator:
             }
     """
 
-    def __init__(self, azure_ai_project: dict, eval_last_turn: bool = False, credential=None):
-        self._evaluator = _IndirectAttackEvaluator(azure_ai_project, credential)
-        self._eval_last_turn = eval_last_turn
-
-    def __call__(
+    @override
+    def __init__(
         self,
-        *,
-        query: Optional[str],
-        response: Optional[str],
-        **kwargs,
+        credential,
+        azure_ai_project: dict,
+        eval_last_turn: bool = False,
     ):
-        """
-        Evaluates content according to the presence of attacks injected into the conversation context to
-        interrupt normal expected functionality by eliciting manipulated content, intrusion and attempting
-        to gather information outside the scope of your AI system.
-        :keyword query: The query to be evaluated. Mutually exclusive with 'conversation'.
-        :paramtype query: Optional[str]
-        :keyword response: The response to be evaluated. Mutually exclusive with 'conversation'.
-        :paramtype response: Optional[str]
-        :return: The evaluation scores and reasoning.
-        :rtype: dict
-        """
-
-        return self._evaluator(query=query, response=response, **kwargs)
-
-
-class _AsyncIndirectAttackEvaluator:
-    def __init__(self, azure_ai_project: dict, credential=None):
-        self._azure_ai_project = azure_ai_project
-        self._credential = credential
-
-    async def __call__(self, *, query: str, response: str, **kwargs):
-        """
-        Evaluates content according to this evaluator's metric.
-        :keyword query: The query to be evaluated.
-        :paramtype query: str
-        :keyword response: The response to be evaluated.
-        :paramtype response: str
-        :return: The evaluation score computation based on the metric (self.metric).
-        :rtype: Any
-        """
-        # Validate inputs
-        # Raises value error if failed, so execution alone signifies success.
-        if not (query and query.strip() and query != "None") or not (
-            response and response.strip() and response != "None"
-        ):
-            msg = "Both 'query' and 'response' must be non-empty strings."
-            raise EvaluationException(
-                message=msg,
-                internal_message=msg,
-                error_category=ErrorCategory.MISSING_FIELD,
-                error_blame=ErrorBlame.USER_ERROR,
-                error_target=ErrorTarget.INDIRECT_ATTACK_EVALUATOR,
-            )
-
-        # Run score computation based on supplied metric.
-        result = await evaluate_with_rai_service(
-            metric_name=EvaluationMetrics.XPIA,
-            query=query,
-            response=response,
-            project_scope=self._azure_ai_project,
-            credential=self._credential,
+        super().__init__(
+            eval_metric=EvaluationMetrics.XPIA,
+            azure_ai_project=azure_ai_project,
+            credential=credential,
+            eval_last_turn=eval_last_turn,
         )
-        return result
-
-
-class _IndirectAttackEvaluator:
-    def __init__(self, azure_ai_project: dict, credential=None):
-        self._async_evaluator = _AsyncIndirectAttackEvaluator(azure_ai_project, credential)
-
-    def __call__(self, *, query: str, response: str, **kwargs):
-        """
-        Evaluates XPIA content.
-        :keyword query: The query to be evaluated.
-        :paramtype query: str
-        :keyword response: The response to be evaluated.
-        :paramtype response: str
-        :keyword context: The context to be evaluated.
-        :paramtype context: str
-        :return: The XPIA score.
-        :rtype: dict
-        """
-        return async_run_allowing_running_loop(self._async_evaluator, query=query, response=response, **kwargs)
-
-    def _to_async(self):
-        return self._async_evaluator

azure/ai/evaluation/_exceptions.py

@@ -4,6 +4,7 @@
 """This includes enums and classes for exceptions for use in azure-ai-evaluation."""
 
 from enum import Enum
+from typing import Optional
 
 from azure.core.exceptions import AzureError
 
@@ -20,6 +21,7 @@ class ErrorCategory(Enum):
     * RESOURCE_NOT_FOUND -> Resource could not be found
     * FAILED_EXECUTION -> Execution failed
     * SERVICE_UNAVAILABLE -> Service is unavailable
+    * MISSING_PACKAGE -> Required package is missing
     * UNKNOWN -> Undefined placeholder. Avoid using.
     """
 
@@ -30,6 +32,7 @@ class ErrorCategory(Enum):
     RESOURCE_NOT_FOUND = "RESOURCE NOT FOUND"
     FAILED_EXECUTION = "FAILED_EXECUTION"
     SERVICE_UNAVAILABLE = "SERVICE UNAVAILABLE"
+    MISSING_PACKAGE = "MISSING PACKAGE"
     UNKNOWN = "UNKNOWN"
 
 
@@ -54,7 +57,6 @@ class ErrorTarget(Enum):
     EVAL_RUN = "EvalRun"
     CODE_CLIENT = "CodeClient"
     RAI_CLIENT = "RAIClient"
-    CHAT_EVALUATOR = "ChatEvaluator"
     COHERENCE_EVALUATOR = "CoherenceEvaluator"
     CONTENT_SAFETY_CHAT_EVALUATOR = "ContentSafetyEvaluator"
     ECI_EVALUATOR = "ECIEvaluator"
@@ -75,7 +77,7 @@ class ErrorTarget(Enum):
 
 
 class EvaluationException(AzureError):
-    """The base class for all exceptions raised in pazure-ai-evaluation. If there is a need to define a custom
+    """The base class for all exceptions raised in azure-ai-evaluation. If there is a need to define a custom
     exception type, that custom exception type should extend from this class.
 
     :param message: A message describing the error. This is the error message the user will see.
@@ -84,17 +86,17 @@ class EvaluationException(AzureError):
     :type internal_message: str
     :param target: The name of the element that caused the exception to be thrown.
     :type target: ~azure.ai.evaluation._exceptions.ErrorTarget
-    :param error_category: The error category, defaults to Unknown.
-    :type error_category: ~azure.ai.evaluation._exceptionsErrorCategory
-    :param error: The original exception if any.
-    :type error: Exception
+    :param category: The error category, defaults to Unknown.
+    :type category: ~azure.ai.evaluation._exceptions.ErrorCategory
+    :param blame: The source of blame for the error, defaults to Unknown.
+    :type balance: ~azure.ai.evaluation._exceptions.ErrorBlame
     """
 
     def __init__(
         self,
         message: str,
-        internal_message: str,
         *args,
+        internal_message: Optional[str] = None,
         target: ErrorTarget = ErrorTarget.UNKNOWN,
         category: ErrorCategory = ErrorCategory.UNKNOWN,
         blame: ErrorBlame = ErrorBlame.UNKNOWN,

azure/ai/evaluation/_http_utils.py

@@ -3,10 +3,9 @@
 # ---------------------------------------------------------
 
 
-from functools import wraps
-from typing import Any, Awaitable, Callable, Dict, MutableMapping, Optional
+from typing import Any, Dict, MutableMapping, Optional, TypedDict, cast
 
-from typing_extensions import Self
+from typing_extensions import Self, Unpack
 
 from azure.ai.evaluation._user_agent import USER_AGENT
 from azure.core.configuration import Configuration
@@ -33,78 +32,25 @@ from azure.core.rest import AsyncHttpResponse, HttpRequest, HttpResponse
 from azure.core.rest._rest_py3 import ContentType, FilesType, ParamsType
 
 
-def _request_fn(f: Callable[["HttpPipeline"], None]):
-    """Decorator to generate convenience methods for HTTP method.
+class RequestKwargs(TypedDict, total=False):
+    """Keyword arguments for request-style http request functions
 
-    :param Callable[["HttpPipeline"],None] f: A HttpPipeline classmethod to wrap.
-        The f.__name__ is the HTTP method used
-    :return: A wrapped callable that sends a `f.__name__` request
-    :rtype: Callable
-    """
+    .. note::
 
-    @wraps(f)
-    def request_fn(
-        self: "HttpPipeline",
-        url: str,
-        *,
-        params: Optional[ParamsType] = None,
-        headers: Optional[MutableMapping[str, str]] = None,
-        json: Any = None,
-        content: Optional[ContentType] = None,
-        data: Optional[Dict[str, Any]] = None,
-        files: Optional[FilesType] = None,
-        **kwargs,
-    ) -> HttpResponse:
-        return self.request(
-            f.__name__.upper(),
-            url,
-            params=params,
-            headers=headers,
-            json=json,
-            content=content,
-            data=data,
-            files=files,
-            **kwargs,
-        )
+        Ideally, we'd be able to express that these are the known subset of kwargs, but it's possible to provide
+        others. But that currently isn't possible; there's no way currently to express a TypedDict that expects
+        a known set of keys and an unknown set of keys.
 
-    return request_fn
-
-
-def _async_request_fn(f: Callable[["AsyncHttpPipeline"], Awaitable[None]]):
-    """Decorator to generate convenience methods for HTTP method.
-
-    :param Callable[["HttpPipeline"],None] f: A HttpPipeline classmethod to wrap.
-        The f.__name__ is the HTTP method used
-    :return: A wrapped callable that sends a `f.__name__` request
-    :rtype: Callable
+        PEP 728 - TypedDict with Typed Extra Items (https://peps.python.org/pep-0728/) would rectify this but it's
+        still in Draft status.
     """
 
-    @wraps(f)
-    async def request_fn(
-        self: "AsyncHttpPipeline",
-        url: str,
-        *,
-        params: Optional[ParamsType] = None,
-        headers: Optional[MutableMapping[str, str]] = None,
-        json: Any = None,
-        content: Optional[ContentType] = None,
-        data: Optional[Dict[str, Any]] = None,
-        files: Optional[FilesType] = None,
-        **kwargs,
-    ) -> AsyncHttpResponse:
-        return await self.request(
-            f.__name__.upper(),
-            url,
-            params=params,
-            headers=headers,
-            json=json,
-            content=content,
-            data=data,
-            files=files,
-            **kwargs,
-        )
-
-    return request_fn
+    params: ParamsType
+    headers: MutableMapping[str, str]
+    json: Any
+    content: ContentType
+    data: Dict[str, Any]
+    files: FilesType
 
 
 class HttpPipeline(Pipeline):
@@ -145,14 +91,32 @@ class HttpPipeline(Pipeline):
         :param RedirectPolicy redirect_policy:
         """
         config = config or Configuration()
-        config.headers_policy = headers_policy or config.headers_policy or HeadersPolicy(**kwargs)
-        config.proxy_policy = proxy_policy or config.proxy_policy or ProxyPolicy(**kwargs)
-        config.redirect_policy = redirect_policy or config.redirect_policy or RedirectPolicy(**kwargs)
-        config.retry_policy = retry_policy or config.retry_policy or RetryPolicy(**kwargs)
-        config.custom_hook_policy = custom_hook_policy or config.custom_hook_policy or CustomHookPolicy(**kwargs)
-        config.logging_policy = logging_policy or config.logging_policy or NetworkTraceLoggingPolicy(**kwargs)
-        config.http_logging_policy = http_logging_policy or config.http_logging_policy or HttpLoggingPolicy(**kwargs)
-        config.user_agent_policy = user_agent_policy or config.user_agent_policy or UserAgentPolicy(**kwargs)
+        config.headers_policy = (
+            headers_policy or cast(Optional[HeadersPolicy], config.headers_policy) or HeadersPolicy(**kwargs)
+        )
+        config.proxy_policy = proxy_policy or cast(Optional[ProxyPolicy], config.proxy_policy) or ProxyPolicy(**kwargs)
+        config.redirect_policy = (
+            redirect_policy or cast(Optional[RedirectPolicy], config.redirect_policy) or RedirectPolicy(**kwargs)
+        )
+        config.retry_policy = retry_policy or cast(Optional[RetryPolicy], config.retry_policy) or RetryPolicy(**kwargs)
+        config.custom_hook_policy = (
+            custom_hook_policy
+            or cast(Optional[CustomHookPolicy], config.custom_hook_policy)
+            or CustomHookPolicy(**kwargs)
+        )
+        config.logging_policy = (
+            logging_policy
+            or cast(Optional[NetworkTraceLoggingPolicy], config.logging_policy)
+            or NetworkTraceLoggingPolicy(**kwargs)
+        )
+        config.http_logging_policy = (
+            http_logging_policy
+            or cast(Optional[HttpLoggingPolicy], config.http_logging_policy)
+            or HttpLoggingPolicy(**kwargs)
+        )
+        config.user_agent_policy = (
+            user_agent_policy or cast(Optional[UserAgentPolicy], config.user_agent_policy) or UserAgentPolicy(**kwargs)
+        )
         config.polling_interval = kwargs.get("polling_interval", 30)
 
         super().__init__(
@@ -166,7 +130,6 @@ class HttpPipeline(Pipeline):
                 config.proxy_policy,
                 config.redirect_policy,
                 config.retry_policy,
-                config.authentication_policy,
                 config.custom_hook_policy,
                 config.logging_policy,
             ],
@@ -199,7 +162,6 @@ class HttpPipeline(Pipeline):
         files: Optional[FilesType] = None,
         **kwargs,
     ) -> HttpResponse:
-
         request = HttpRequest(
             method,
             url,
@@ -213,33 +175,78 @@ class HttpPipeline(Pipeline):
 
         return self.run(request, **kwargs).http_response
 
-    @_request_fn
-    def delete(self) -> None:
-        """Send a DELETE request."""
+    def delete(self: "HttpPipeline", url: str, **kwargs: Unpack[RequestKwargs]) -> HttpResponse:
+        """Send a DELETE request.
 
-    @_request_fn
-    def put(self) -> None:
-        """Send a PUT request."""
+        :param str url: The request url
+        :returns: The request response
+        :rtype: HttpResponse
+        """
 
-    @_request_fn
-    def get(self) -> None:
-        """Send a GET request."""
+        return self.request(self.delete.__name__.upper(), url, **kwargs)
 
-    @_request_fn
-    def post(self) -> None:
-        """Send a POST request."""
+    def put(self: "HttpPipeline", url: str, **kwargs: Unpack[RequestKwargs]) -> HttpResponse:
+        """Send a PUT request.
 
-    @_request_fn
-    def head(self) -> None:
-        """Send a HEAD request."""
+        :param str url: The request url
+        :returns: The request response
+        :rtype: HttpResponse
+        """
 
-    @_request_fn
-    def options(self) -> None:
-        """Send a OPTIONS request."""
+        return self.request(self.put.__name__.upper(), url, **kwargs)
 
-    @_request_fn
-    def patch(self) -> None:
-        """Send a PATCH request."""
+    def get(self: "HttpPipeline", url: str, **kwargs: Unpack[RequestKwargs]) -> HttpResponse:
+        """Send a GET request.
+
+        :param str url: The request url
+        :returns: The request response
+        :rtype: HttpResponse
+        """
+
+        return self.request(self.get.__name__.upper(), url, **kwargs)
+
+    def post(self: "HttpPipeline", url: str, **kwargs: Unpack[RequestKwargs]) -> HttpResponse:
+        """Send a POST request.
+
+        :param str url: The request url
+        :returns: The request response
+        :rtype: HttpResponse
+        """
+
+        return self.request(self.post.__name__.upper(), url, **kwargs)
+
+    def head(self: "HttpPipeline", url: str, **kwargs: Unpack[RequestKwargs]) -> HttpResponse:
+        """Send a HEAD request.
+
+        :param str url: The request url
+        :returns: The request response
+        :rtype: HttpResponse
+        """
+
+        return self.request(self.head.__name__.upper(), url, **kwargs)
+
+    def options(self: "HttpPipeline", url: str, **kwargs: Unpack[RequestKwargs]) -> HttpResponse:
+        """Send a OPTIONS request.
+
+        :param str url: The request url
+        :returns: The request response
+        :rtype: HttpResponse
+        """
+
+        return self.request(self.options.__name__.upper(), url, **kwargs)
+
+    def patch(self: "HttpPipeline", url: str, **kwargs: Unpack[RequestKwargs]) -> HttpResponse:
+        """Send a PATCH request.
+
+        :param str url: The request url
+        :returns: The request response
+        :rtype: HttpResponse
+        """
+
+        return self.request(self.patch.__name__.upper(), url, **kwargs)
+
+    def __enter__(self) -> Self:
+        return cast(Self, super().__enter__())
 
 
 class AsyncHttpPipeline(AsyncPipeline):
@@ -280,14 +287,36 @@ class AsyncHttpPipeline(AsyncPipeline):
         :param AsyncRedirectPolicy redirect_policy:
         """
         config = config or Configuration()
-        config.headers_policy = headers_policy or config.headers_policy or HeadersPolicy(**kwargs)
-        config.proxy_policy = proxy_policy or config.proxy_policy or ProxyPolicy(**kwargs)
-        config.redirect_policy = redirect_policy or config.redirect_policy or AsyncRedirectPolicy(**kwargs)
-        config.retry_policy = retry_policy or config.retry_policy or AsyncRetryPolicy(**kwargs)
-        config.custom_hook_policy = custom_hook_policy or config.custom_hook_policy or CustomHookPolicy(**kwargs)
-        config.logging_policy = logging_policy or config.logging_policy or NetworkTraceLoggingPolicy(**kwargs)
-        config.http_logging_policy = http_logging_policy or config.http_logging_policy or HttpLoggingPolicy(**kwargs)
-        config.user_agent_policy = user_agent_policy or config.user_agent_policy or UserAgentPolicy(**kwargs)
+        config.headers_policy = (
+            headers_policy or cast(Optional[HeadersPolicy], config.headers_policy) or HeadersPolicy(**kwargs)
+        )
+        config.proxy_policy = proxy_policy or cast(Optional[ProxyPolicy], config.proxy_policy) or ProxyPolicy(**kwargs)
+        config.redirect_policy = (
+            redirect_policy
+            or cast(Optional[AsyncRedirectPolicy], config.redirect_policy)
+            or AsyncRedirectPolicy(**kwargs)
+        )
+        config.retry_policy = (
+            retry_policy or cast(Optional[AsyncRetryPolicy], config.retry_policy) or AsyncRetryPolicy(**kwargs)
+        )
+        config.custom_hook_policy = (
+            custom_hook_policy
+            or cast(Optional[CustomHookPolicy], config.custom_hook_policy)
+            or CustomHookPolicy(**kwargs)
+        )
+        config.logging_policy = (
+            logging_policy
+            or cast(Optional[NetworkTraceLoggingPolicy], config.logging_policy)
+            or NetworkTraceLoggingPolicy(**kwargs)
+        )
+        config.http_logging_policy = (
+            http_logging_policy
+            or cast(Optional[HttpLoggingPolicy], config.http_logging_policy)
+            or HttpLoggingPolicy(**kwargs)
+        )
+        config.user_agent_policy = (
+            user_agent_policy or cast(Optional[UserAgentPolicy], config.user_agent_policy) or UserAgentPolicy(**kwargs)
+        )
         config.polling_interval = kwargs.get("polling_interval", 30)
 
         super().__init__(
@@ -301,7 +330,6 @@ class AsyncHttpPipeline(AsyncPipeline):
                 config.proxy_policy,
                 config.redirect_policy,
                 config.retry_policy,
-                config.authentication_policy,
                 config.custom_hook_policy,
                 config.logging_policy,
             ],
@@ -334,7 +362,6 @@ class AsyncHttpPipeline(AsyncPipeline):
         files: Optional[FilesType] = None,
         **kwargs,
     ) -> AsyncHttpResponse:
-
         request = HttpRequest(
             method,
             url,
@@ -348,33 +375,77 @@ class AsyncHttpPipeline(AsyncPipeline):
 
         return (await self.run(request, **kwargs)).http_response
 
-    @_async_request_fn
-    async def delete(self) -> None:
-        """Send a DELETE request."""
+    async def delete(self: "AsyncHttpPipeline", url: str, **kwargs: Unpack[RequestKwargs]) -> AsyncHttpResponse:
+        """Send a DELETE request.
+
+        :param str url: The request url
+        :returns: The request response
+        :rtype: AsyncHttpResponse
+        """
+        return await self.request(self.delete.__name__.upper(), url, **kwargs)
+
+    async def put(self: "AsyncHttpPipeline", url: str, **kwargs: Unpack[RequestKwargs]) -> AsyncHttpResponse:
+        """Send a PUT request.
+
+        :param str url: The request url
+        :returns: The request response
+        :rtype: AsyncHttpResponse
+        """
+
+        return await self.request(self.put.__name__.upper(), url, **kwargs)
+
+    async def get(self: "AsyncHttpPipeline", url: str, **kwargs: Unpack[RequestKwargs]) -> AsyncHttpResponse:
+        """Send a GET request.
+
+        :param str url: The request url
+        :returns: The request response
+        :rtype: AsyncHttpResponse
+        """
+
+        return await self.request(self.get.__name__.upper(), url, **kwargs)
+
+    async def post(self: "AsyncHttpPipeline", url: str, **kwargs: Unpack[RequestKwargs]) -> AsyncHttpResponse:
+        """Send a POST request.
+
+        :param str url: The request url
+        :returns: The request response
+        :rtype: AsyncHttpResponse
+        """
+
+        return await self.request(self.post.__name__.upper(), url, **kwargs)
+
+    async def head(self: "AsyncHttpPipeline", url: str, **kwargs: Unpack[RequestKwargs]) -> AsyncHttpResponse:
+        """Send a HEAD request.
 
-    @_async_request_fn
-    async def put(self) -> None:
-        """Send a PUT request."""
+        :param str url: The request url
+        :returns: The request response
+        :rtype: AsyncHttpResponse
+        """
+
+        return await self.request(self.head.__name__.upper(), url, **kwargs)
+
+    async def options(self: "AsyncHttpPipeline", url: str, **kwargs: Unpack[RequestKwargs]) -> AsyncHttpResponse:
+        """Send a OPTIONS request.
 
-    @_async_request_fn
-    async def get(self) -> None:
-        """Send a GET request."""
+        :param str url: The request url
+        :returns: The request response
+        :rtype: AsyncHttpResponse
+        """
+
+        return await self.request(self.options.__name__.upper(), url, **kwargs)
 
-    @_async_request_fn
-    async def post(self) -> None:
-        """Send a POST request."""
+    async def patch(self: "AsyncHttpPipeline", url: str, **kwargs: Unpack[RequestKwargs]) -> AsyncHttpResponse:
+        """Send a PATCH request.
 
-    @_async_request_fn
-    async def head(self) -> None:
-        """Send a HEAD request."""
+        :param str url: The request url
+        :returns: The request response
+        :rtype: AsyncHttpResponse
+        """
 
-    @_async_request_fn
-    async def options(self) -> None:
-        """Send a OPTIONS request."""
+        return await self.request(self.patch.__name__.upper(), url, **kwargs)
 
-    @_async_request_fn
-    async def patch(self) -> None:
-        """Send a PATCH request."""
+    async def __aenter__(self) -> Self:
+        return cast(Self, await super().__aenter__())
 
 
 def get_http_client() -> HttpPipeline: