azure-ai-evaluation 1.0.0__py3-none-any.whl → 1.0.0b1__py3-none-any.whl

azure/ai/evaluation/_http_utils.py

@@ -3,11 +3,9 @@
 # ---------------------------------------------------------
 
 
-from typing import Any, Dict, MutableMapping, Optional, TypedDict, cast
+from functools import wraps
+from typing import Any, Awaitable, Callable, Dict, MutableMapping, Optional
 
-from typing_extensions import Self, Unpack
-
-from azure.ai.evaluation._user_agent import USER_AGENT
 from azure.core.configuration import Configuration
 from azure.core.pipeline import AsyncPipeline, Pipeline
 from azure.core.pipeline.policies import (
@@ -30,27 +28,83 @@ from azure.core.pipeline.transport import (  # pylint: disable=non-abstract-tran
 )
 from azure.core.rest import AsyncHttpResponse, HttpRequest, HttpResponse
 from azure.core.rest._rest_py3 import ContentType, FilesType, ParamsType
+from typing_extensions import Self
+
+from azure.ai.evaluation._user_agent import USER_AGENT
 
 
-class RequestKwargs(TypedDict, total=False):
-    """Keyword arguments for request-style http request functions
+def _request_fn(f: Callable[["HttpPipeline"], None]):
+    """Decorator to generate convenience methods for HTTP method.
+
+    :param Callable[["HttpPipeline"],None] f: A HttpPipeline classmethod to wrap.
+        The f.__name__ is the HTTP method used
+    :return: A wrapped callable that sends a `f.__name__` request
+    :rtype: Callable
+    """
+
+    @wraps(f)
+    def request_fn(
+        self: "HttpPipeline",
+        url: str,
+        *,
+        params: Optional[ParamsType] = None,
+        headers: Optional[MutableMapping[str, str]] = None,
+        json: Any = None,
+        content: Optional[ContentType] = None,
+        data: Optional[Dict[str, Any]] = None,
+        files: Optional[FilesType] = None,
+        **kwargs,
+    ) -> HttpResponse:
+        return self.request(
+            f.__name__.upper(),
+            url,
+            params=params,
+            headers=headers,
+            json=json,
+            content=content,
+            data=data,
+            files=files,
+            **kwargs,
+        )
+
+    return request_fn
 
-    .. note::
 
-        Ideally, we'd be able to express that these are the known subset of kwargs, but it's possible to provide
-        others. But that currently isn't possible; there's no way currently to express a TypedDict that expects
-        a known set of keys and an unknown set of keys.
+def _async_request_fn(f: Callable[["AsyncHttpPipeline"], Awaitable[None]]):
+    """Decorator to generate convenience methods for HTTP method.
 
-        PEP 728 - TypedDict with Typed Extra Items (https://peps.python.org/pep-0728/) would rectify this but it's
-        still in Draft status.
+    :param Callable[["HttpPipeline"],None] f: A HttpPipeline classmethod to wrap.
+        The f.__name__ is the HTTP method used
+    :return: A wrapped callable that sends a `f.__name__` request
+    :rtype: Callable
     """
 
-    params: ParamsType
-    headers: MutableMapping[str, str]
-    json: Any
-    content: ContentType
-    data: Dict[str, Any]
-    files: FilesType
+    @wraps(f)
+    async def request_fn(
+        self: "AsyncHttpPipeline",
+        url: str,
+        *,
+        params: Optional[ParamsType] = None,
+        headers: Optional[MutableMapping[str, str]] = None,
+        json: Any = None,
+        content: Optional[ContentType] = None,
+        data: Optional[Dict[str, Any]] = None,
+        files: Optional[FilesType] = None,
+        **kwargs,
+    ) -> AsyncHttpResponse:
+        return await self.request(
+            f.__name__.upper(),
+            url,
+            params=params,
+            headers=headers,
+            json=json,
+            content=content,
+            data=data,
+            files=files,
+            **kwargs,
+        )
+
+    return request_fn
 
 
 class HttpPipeline(Pipeline):
@@ -91,32 +145,14 @@ class HttpPipeline(Pipeline):
         :param RedirectPolicy redirect_policy:
         """
         config = config or Configuration()
-        config.headers_policy = (
-            headers_policy or cast(Optional[HeadersPolicy], config.headers_policy) or HeadersPolicy(**kwargs)
-        )
-        config.proxy_policy = proxy_policy or cast(Optional[ProxyPolicy], config.proxy_policy) or ProxyPolicy(**kwargs)
-        config.redirect_policy = (
-            redirect_policy or cast(Optional[RedirectPolicy], config.redirect_policy) or RedirectPolicy(**kwargs)
-        )
-        config.retry_policy = retry_policy or cast(Optional[RetryPolicy], config.retry_policy) or RetryPolicy(**kwargs)
-        config.custom_hook_policy = (
-            custom_hook_policy
-            or cast(Optional[CustomHookPolicy], config.custom_hook_policy)
-            or CustomHookPolicy(**kwargs)
-        )
-        config.logging_policy = (
-            logging_policy
-            or cast(Optional[NetworkTraceLoggingPolicy], config.logging_policy)
-            or NetworkTraceLoggingPolicy(**kwargs)
-        )
-        config.http_logging_policy = (
-            http_logging_policy
-            or cast(Optional[HttpLoggingPolicy], config.http_logging_policy)
-            or HttpLoggingPolicy(**kwargs)
-        )
-        config.user_agent_policy = (
-            user_agent_policy or cast(Optional[UserAgentPolicy], config.user_agent_policy) or UserAgentPolicy(**kwargs)
-        )
+        config.headers_policy = headers_policy or config.headers_policy or HeadersPolicy(**kwargs)
+        config.proxy_policy = proxy_policy or config.proxy_policy or ProxyPolicy(**kwargs)
+        config.redirect_policy = redirect_policy or config.redirect_policy or RedirectPolicy(**kwargs)
+        config.retry_policy = retry_policy or config.retry_policy or RetryPolicy(**kwargs)
+        config.custom_hook_policy = custom_hook_policy or config.custom_hook_policy or CustomHookPolicy(**kwargs)
+        config.logging_policy = logging_policy or config.logging_policy or NetworkTraceLoggingPolicy(**kwargs)
+        config.http_logging_policy = http_logging_policy or config.http_logging_policy or HttpLoggingPolicy(**kwargs)
+        config.user_agent_policy = user_agent_policy or config.user_agent_policy or UserAgentPolicy(**kwargs)
         config.polling_interval = kwargs.get("polling_interval", 30)
 
         super().__init__(
@@ -130,6 +166,7 @@ class HttpPipeline(Pipeline):
                 config.proxy_policy,
                 config.redirect_policy,
                 config.retry_policy,
+                config.authentication_policy,
                 config.custom_hook_policy,
                 config.logging_policy,
             ],
@@ -162,6 +199,7 @@ class HttpPipeline(Pipeline):
         files: Optional[FilesType] = None,
         **kwargs,
     ) -> HttpResponse:
+
         request = HttpRequest(
             method,
             url,
@@ -175,78 +213,33 @@ class HttpPipeline(Pipeline):
 
         return self.run(request, **kwargs).http_response
 
-    def delete(self: "HttpPipeline", url: str, **kwargs: Unpack[RequestKwargs]) -> HttpResponse:
-        """Send a DELETE request.
+    @_request_fn
+    def delete(self) -> None:
+        """Send a DELETE request."""
 
-        :param str url: The request url
-        :returns: The request response
-        :rtype: HttpResponse
-        """
-
-        return self.request(self.delete.__name__.upper(), url, **kwargs)
-
-    def put(self: "HttpPipeline", url: str, **kwargs: Unpack[RequestKwargs]) -> HttpResponse:
-        """Send a PUT request.
-
-        :param str url: The request url
-        :returns: The request response
-        :rtype: HttpResponse
-        """
-
-        return self.request(self.put.__name__.upper(), url, **kwargs)
-
-    def get(self: "HttpPipeline", url: str, **kwargs: Unpack[RequestKwargs]) -> HttpResponse:
-        """Send a GET request.
-
-        :param str url: The request url
-        :returns: The request response
-        :rtype: HttpResponse
-        """
-
-        return self.request(self.get.__name__.upper(), url, **kwargs)
-
-    def post(self: "HttpPipeline", url: str, **kwargs: Unpack[RequestKwargs]) -> HttpResponse:
-        """Send a POST request.
-
-        :param str url: The request url
-        :returns: The request response
-        :rtype: HttpResponse
-        """
+    @_request_fn
+    def put(self) -> None:
+        """Send a PUT request."""
 
-        return self.request(self.post.__name__.upper(), url, **kwargs)
+    @_request_fn
+    def get(self) -> None:
+        """Send a GET request."""
 
-    def head(self: "HttpPipeline", url: str, **kwargs: Unpack[RequestKwargs]) -> HttpResponse:
-        """Send a HEAD request.
+    @_request_fn
+    def post(self) -> None:
+        """Send a POST request."""
 
-        :param str url: The request url
-        :returns: The request response
-        :rtype: HttpResponse
-        """
-
-        return self.request(self.head.__name__.upper(), url, **kwargs)
+    @_request_fn
+    def head(self) -> None:
+        """Send a HEAD request."""
 
-    def options(self: "HttpPipeline", url: str, **kwargs: Unpack[RequestKwargs]) -> HttpResponse:
-        """Send a OPTIONS request.
-
-        :param str url: The request url
-        :returns: The request response
-        :rtype: HttpResponse
-        """
+    @_request_fn
+    def options(self) -> None:
+        """Send a OPTIONS request."""
 
-        return self.request(self.options.__name__.upper(), url, **kwargs)
-
-    def patch(self: "HttpPipeline", url: str, **kwargs: Unpack[RequestKwargs]) -> HttpResponse:
-        """Send a PATCH request.
-
-        :param str url: The request url
-        :returns: The request response
-        :rtype: HttpResponse
-        """
-
-        return self.request(self.patch.__name__.upper(), url, **kwargs)
-
-    def __enter__(self) -> Self:
-        return cast(Self, super().__enter__())
+    @_request_fn
+    def patch(self) -> None:
+        """Send a PATCH request."""
 
 
 class AsyncHttpPipeline(AsyncPipeline):
@@ -287,36 +280,14 @@ class AsyncHttpPipeline(AsyncPipeline):
         :param AsyncRedirectPolicy redirect_policy:
         """
         config = config or Configuration()
-        config.headers_policy = (
-            headers_policy or cast(Optional[HeadersPolicy], config.headers_policy) or HeadersPolicy(**kwargs)
-        )
-        config.proxy_policy = proxy_policy or cast(Optional[ProxyPolicy], config.proxy_policy) or ProxyPolicy(**kwargs)
-        config.redirect_policy = (
-            redirect_policy
-            or cast(Optional[AsyncRedirectPolicy], config.redirect_policy)
-            or AsyncRedirectPolicy(**kwargs)
-        )
-        config.retry_policy = (
-            retry_policy or cast(Optional[AsyncRetryPolicy], config.retry_policy) or AsyncRetryPolicy(**kwargs)
-        )
-        config.custom_hook_policy = (
-            custom_hook_policy
-            or cast(Optional[CustomHookPolicy], config.custom_hook_policy)
-            or CustomHookPolicy(**kwargs)
-        )
-        config.logging_policy = (
-            logging_policy
-            or cast(Optional[NetworkTraceLoggingPolicy], config.logging_policy)
-            or NetworkTraceLoggingPolicy(**kwargs)
-        )
-        config.http_logging_policy = (
-            http_logging_policy
-            or cast(Optional[HttpLoggingPolicy], config.http_logging_policy)
-            or HttpLoggingPolicy(**kwargs)
-        )
-        config.user_agent_policy = (
-            user_agent_policy or cast(Optional[UserAgentPolicy], config.user_agent_policy) or UserAgentPolicy(**kwargs)
-        )
+        config.headers_policy = headers_policy or config.headers_policy or HeadersPolicy(**kwargs)
+        config.proxy_policy = proxy_policy or config.proxy_policy or ProxyPolicy(**kwargs)
+        config.redirect_policy = redirect_policy or config.redirect_policy or AsyncRedirectPolicy(**kwargs)
+        config.retry_policy = retry_policy or config.retry_policy or AsyncRetryPolicy(**kwargs)
+        config.custom_hook_policy = custom_hook_policy or config.custom_hook_policy or CustomHookPolicy(**kwargs)
+        config.logging_policy = logging_policy or config.logging_policy or NetworkTraceLoggingPolicy(**kwargs)
+        config.http_logging_policy = http_logging_policy or config.http_logging_policy or HttpLoggingPolicy(**kwargs)
+        config.user_agent_policy = user_agent_policy or config.user_agent_policy or UserAgentPolicy(**kwargs)
         config.polling_interval = kwargs.get("polling_interval", 30)
 
         super().__init__(
@@ -330,6 +301,7 @@ class AsyncHttpPipeline(AsyncPipeline):
                 config.proxy_policy,
                 config.redirect_policy,
                 config.retry_policy,
+                config.authentication_policy,
                 config.custom_hook_policy,
                 config.logging_policy,
             ],
@@ -362,6 +334,7 @@ class AsyncHttpPipeline(AsyncPipeline):
         files: Optional[FilesType] = None,
         **kwargs,
     ) -> AsyncHttpResponse:
+
         request = HttpRequest(
             method,
             url,
@@ -375,77 +348,33 @@ class AsyncHttpPipeline(AsyncPipeline):
 
         return (await self.run(request, **kwargs)).http_response
 
-    async def delete(self: "AsyncHttpPipeline", url: str, **kwargs: Unpack[RequestKwargs]) -> AsyncHttpResponse:
-        """Send a DELETE request.
+    @_async_request_fn
+    async def delete(self) -> None:
+        """Send a DELETE request."""
 
-        :param str url: The request url
-        :returns: The request response
-        :rtype: AsyncHttpResponse
-        """
-        return await self.request(self.delete.__name__.upper(), url, **kwargs)
-
-    async def put(self: "AsyncHttpPipeline", url: str, **kwargs: Unpack[RequestKwargs]) -> AsyncHttpResponse:
-        """Send a PUT request.
+    @_async_request_fn
+    async def put(self) -> None:
+        """Send a PUT request."""
 
-        :param str url: The request url
-        :returns: The request response
-        :rtype: AsyncHttpResponse
-        """
+    @_async_request_fn
+    async def get(self) -> None:
+        """Send a GET request."""
 
-        return await self.request(self.put.__name__.upper(), url, **kwargs)
+    @_async_request_fn
+    async def post(self) -> None:
+        """Send a POST request."""
 
-    async def get(self: "AsyncHttpPipeline", url: str, **kwargs: Unpack[RequestKwargs]) -> AsyncHttpResponse:
-        """Send a GET request.
-
-        :param str url: The request url
-        :returns: The request response
-        :rtype: AsyncHttpResponse
-        """
-
-        return await self.request(self.get.__name__.upper(), url, **kwargs)
-
-    async def post(self: "AsyncHttpPipeline", url: str, **kwargs: Unpack[RequestKwargs]) -> AsyncHttpResponse:
-        """Send a POST request.
-
-        :param str url: The request url
-        :returns: The request response
-        :rtype: AsyncHttpResponse
-        """
-
-        return await self.request(self.post.__name__.upper(), url, **kwargs)
-
-    async def head(self: "AsyncHttpPipeline", url: str, **kwargs: Unpack[RequestKwargs]) -> AsyncHttpResponse:
-        """Send a HEAD request.
-
-        :param str url: The request url
-        :returns: The request response
-        :rtype: AsyncHttpResponse
-        """
-
-        return await self.request(self.head.__name__.upper(), url, **kwargs)
-
-    async def options(self: "AsyncHttpPipeline", url: str, **kwargs: Unpack[RequestKwargs]) -> AsyncHttpResponse:
-        """Send a OPTIONS request.
-
-        :param str url: The request url
-        :returns: The request response
-        :rtype: AsyncHttpResponse
-        """
-
-        return await self.request(self.options.__name__.upper(), url, **kwargs)
-
-    async def patch(self: "AsyncHttpPipeline", url: str, **kwargs: Unpack[RequestKwargs]) -> AsyncHttpResponse:
-        """Send a PATCH request.
-
-        :param str url: The request url
-        :returns: The request response
-        :rtype: AsyncHttpResponse
-        """
+    @_async_request_fn
+    async def head(self) -> None:
+        """Send a HEAD request."""
 
-        return await self.request(self.patch.__name__.upper(), url, **kwargs)
+    @_async_request_fn
+    async def options(self) -> None:
+        """Send a OPTIONS request."""
 
-    async def __aenter__(self) -> Self:
-        return cast(Self, await super().__aenter__())
+    @_async_request_fn
+    async def patch(self) -> None:
+        """Send a PATCH request."""
 
 
 def get_http_client() -> HttpPipeline:

azure/ai/evaluation/_model_configurations.py

@@ -2,122 +2,26 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
 
-from typing import Any, Dict, List, Literal, TypedDict, Union
+from typing import TypedDict
 
-from typing_extensions import NotRequired
 
-
-class AzureOpenAIModelConfiguration(TypedDict):
-    """Model configuration for Azure OpenAI models
-
-    :param type: The type of the model configuration. Should be 'azure_openai' for AzureOpenAIModelConfiguration
-    :type type: NotRequired[Literal["azure_openai"]]
-    :param azure_deployment: Name of Azure OpenAI deployment to make requests to
-    :type azure_deployment: str
-    :param azure_endpoint: Endpoint of Azure OpenAI resource to make requests to
-    :type azure_endpoint: str
-    :param api_key: API key of Azure OpenAI resource
-    :type api_key: str
-    :param api_version: API version to use in request to Azure OpenAI deployment. Optional.
-    :type api_version: NotRequired[str]
-
-    .. admonition:: Example:
-
-        .. literalinclude:: ../samples/evaluation_samples_common.py
-            :start-after: [START create_AOAI_model_config]
-            :end-before: [END create_AOAI_model_config]
-            :language: python
-            :dedent: 8
-            :caption: Creating an AzureOpenAIModelConfiguration object.
-
-    """
-
-    type: NotRequired[Literal["azure_openai"]]
+class AzureOpenAIModelConfigurationBase(TypedDict):
     azure_deployment: str
     azure_endpoint: str
-    """Endpoint of Azure OpenAI resource to make request to"""
-    api_key: NotRequired[str]
-    """API key of Azure OpenAI resource"""
-    api_version: NotRequired[str]
-
-
-class OpenAIModelConfiguration(TypedDict):
-    """Model configuration for OpenAI models
-
-    :param type: The type of the model configuration. Should be 'openai' for OpenAIModelConfiguration
-    :type type: NotRequired[Literal["openai"]]
-    :param api_key: API key needed to make requests to model
-    :type api_key: str
-    :param model: Name of model to be used in OpenAI request
-    :type model: str
-    :param base_url: Base URL to be used in OpenAI request. Optional.
-    :type base_url: NotRequired[str]
-    :param organization: OpenAI organization. Optional.
-    :type organization: NotRequired[str]
+    api_key: str
 
-    .. admonition:: Example:
 
-        .. literalinclude:: ../samples/evaluation_samples_common.py
-            :start-after: [START create_OAI_model_config]
-            :end-before: [END create_OAI_model_config]
-            :language: python
-            :dedent: 8
-            :caption: Creating an OpenAIModelConfiguration object.
+class AzureOpenAIModelConfiguration(AzureOpenAIModelConfigurationBase, total=False):
+    api_version: str
 
-    """
 
-    type: NotRequired[Literal["openai"]]
+class OpenAIModelConfiguration(TypedDict):
     api_key: str
-    model: str
-    base_url: NotRequired[str]
-    organization: NotRequired[str]
+    base_url: str
+    organization: str
 
 
 class AzureAIProject(TypedDict):
-    """Information about the Azure AI project
-
-    :param subscription_id: ID of the Azure subscription the project is in
-    :type subscription_id: str
-    :param resource_group_name: Name of the Azure resource group the project is in
-    :type resource_group_name: str
-    :param project_name: Name of the Azure project
-    :type project_name: str
-
-    .. admonition:: Example:
-
-        .. literalinclude:: ../samples/evaluation_samples_common.py
-            :start-after: [START create_azure_ai_project_object]
-            :end-before: [END create_azure_ai_project_object]
-            :language: python
-            :dedent: 8
-            :caption: Creating an AzureAIProject object.
-
-    """
-
     subscription_id: str
     resource_group_name: str
     project_name: str
-
-
-class EvaluatorConfig(TypedDict, total=False):
-    """Configuration for an evaluator"""
-
-    column_mapping: Dict[str, str]
-    """Dictionary mapping evaluator input name to column in data"""
-
-
-class Message(TypedDict):
-    role: str
-    content: Union[str, List[Dict]]
-    context: NotRequired[Dict[str, Any]]
-
-
-class Conversation(TypedDict):
-    messages: Union[List[Message], List[Dict]]
-    context: NotRequired[Dict[str, Any]]
-
-
-class EvaluationResult(TypedDict):
-    metrics: Dict
-    studio_url: NotRequired[str]
-    rows: List[Dict]

azure/ai/evaluation/_version.py

@@ -2,4 +2,4 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
 
-VERSION = "1.0.0"
+VERSION = "1.0.0b1"

azure/ai/evaluation/simulator/__init__.py

@@ -1,14 +1,13 @@
-from ._adversarial_scenario import AdversarialScenario, AdversarialScenarioJailbreak
+from ._adversarial_scenario import AdversarialScenario
 from ._adversarial_simulator import AdversarialSimulator
 from ._constants import SupportedLanguages
 from ._direct_attack_simulator import DirectAttackSimulator
 from ._indirect_attack_simulator import IndirectAttackSimulator
-from ._simulator import Simulator
+from .simulator import Simulator
 
 __all__ = [
     "AdversarialSimulator",
     "AdversarialScenario",
-    "AdversarialScenarioJailbreak",
     "DirectAttackSimulator",
     "IndirectAttackSimulator",
     "SupportedLanguages",

azure/ai/evaluation/simulator/_adversarial_scenario.py

@@ -3,22 +3,10 @@
 # ---------------------------------------------------------
 
 from enum import Enum
-from azure.ai.evaluation._common._experimental import experimental
 
 
-@experimental
 class AdversarialScenario(Enum):
-    """Adversarial scenario types
-
-    .. admonition:: Example:
-
-        .. literalinclude:: ../samples/evaluation_samples_simulate.py
-            :start-after: [START adversarial_scenario]
-            :end-before: [END adversarial_scenario]
-            :language: python
-            :dedent: 8
-            :caption: Configure an AdversarialSimulator with an Adversarial Conversation scenario.
-    """
+    """Adversarial scenario types"""
 
     ADVERSARIAL_QA = "adv_qa"
     ADVERSARIAL_CONVERSATION = "adv_conversation"
@@ -28,16 +16,9 @@ class AdversarialScenario(Enum):
     ADVERSARIAL_CONTENT_GEN_UNGROUNDED = "adv_content_gen_ungrounded"
     ADVERSARIAL_CONTENT_GEN_GROUNDED = "adv_content_gen_grounded"
     ADVERSARIAL_CONTENT_PROTECTED_MATERIAL = "adv_content_protected_material"
-
-
-@experimental
-class AdversarialScenarioJailbreak(Enum):
-    """Adversarial scenario types for XPIA Jailbreak"""
-
     ADVERSARIAL_INDIRECT_JAILBREAK = "adv_xpia"
 
 
-@experimental
 class _UnstableAdversarialScenario(Enum):
     """Adversarial scenario types that we haven't published, but still want available for internal use
     Values listed here are subject to potential change, and/or migration to the main enum over time.