azure-ai-evaluation 1.0.0b3__py3-none-any.whl → 1.0.0b5__py3-none-any.whl

azure/ai/evaluation/_http_utils.py

@@ -3,10 +3,9 @@
 # ---------------------------------------------------------
 
 
-from functools import wraps
-from typing import Any, Awaitable, Callable, Dict, MutableMapping, Optional
+from typing import Any, Dict, MutableMapping, Optional, TypedDict, cast
 
-from typing_extensions import Self
+from typing_extensions import Self, Unpack
 
 from azure.ai.evaluation._user_agent import USER_AGENT
 from azure.core.configuration import Configuration
@@ -33,78 +32,25 @@ from azure.core.rest import AsyncHttpResponse, HttpRequest, HttpResponse
 from azure.core.rest._rest_py3 import ContentType, FilesType, ParamsType
 
 
-def _request_fn(f: Callable[["HttpPipeline"], None]):
-    """Decorator to generate convenience methods for HTTP method.
+class RequestKwargs(TypedDict, total=False):
+    """Keyword arguments for request-style http request functions
 
-    :param Callable[["HttpPipeline"],None] f: A HttpPipeline classmethod to wrap.
-        The f.__name__ is the HTTP method used
-    :return: A wrapped callable that sends a `f.__name__` request
-    :rtype: Callable
-    """
+    .. note::
 
-    @wraps(f)
-    def request_fn(
-        self: "HttpPipeline",
-        url: str,
-        *,
-        params: Optional[ParamsType] = None,
-        headers: Optional[MutableMapping[str, str]] = None,
-        json: Any = None,
-        content: Optional[ContentType] = None,
-        data: Optional[Dict[str, Any]] = None,
-        files: Optional[FilesType] = None,
-        **kwargs,
-    ) -> HttpResponse:
-        return self.request(
-            f.__name__.upper(),
-            url,
-            params=params,
-            headers=headers,
-            json=json,
-            content=content,
-            data=data,
-            files=files,
-            **kwargs,
-        )
+        Ideally, we'd be able to express that these are the known subset of kwargs, but it's possible to provide
+        others. But that currently isn't possible; there's no way currently to express a TypedDict that expects
+        a known set of keys and an unknown set of keys.
 
-    return request_fn
-
-
-def _async_request_fn(f: Callable[["AsyncHttpPipeline"], Awaitable[None]]):
-    """Decorator to generate convenience methods for HTTP method.
-
-    :param Callable[["HttpPipeline"],None] f: A HttpPipeline classmethod to wrap.
-        The f.__name__ is the HTTP method used
-    :return: A wrapped callable that sends a `f.__name__` request
-    :rtype: Callable
+        PEP 728 - TypedDict with Typed Extra Items (https://peps.python.org/pep-0728/) would rectify this but it's
+        still in Draft status.
     """
 
-    @wraps(f)
-    async def request_fn(
-        self: "AsyncHttpPipeline",
-        url: str,
-        *,
-        params: Optional[ParamsType] = None,
-        headers: Optional[MutableMapping[str, str]] = None,
-        json: Any = None,
-        content: Optional[ContentType] = None,
-        data: Optional[Dict[str, Any]] = None,
-        files: Optional[FilesType] = None,
-        **kwargs,
-    ) -> AsyncHttpResponse:
-        return await self.request(
-            f.__name__.upper(),
-            url,
-            params=params,
-            headers=headers,
-            json=json,
-            content=content,
-            data=data,
-            files=files,
-            **kwargs,
-        )
-
-    return request_fn
+    params: ParamsType
+    headers: MutableMapping[str, str]
+    json: Any
+    content: ContentType
+    data: Dict[str, Any]
+    files: FilesType
 
 
 class HttpPipeline(Pipeline):
@@ -145,14 +91,32 @@ class HttpPipeline(Pipeline):
         :param RedirectPolicy redirect_policy:
         """
         config = config or Configuration()
-        config.headers_policy = headers_policy or config.headers_policy or HeadersPolicy(**kwargs)
-        config.proxy_policy = proxy_policy or config.proxy_policy or ProxyPolicy(**kwargs)
-        config.redirect_policy = redirect_policy or config.redirect_policy or RedirectPolicy(**kwargs)
-        config.retry_policy = retry_policy or config.retry_policy or RetryPolicy(**kwargs)
-        config.custom_hook_policy = custom_hook_policy or config.custom_hook_policy or CustomHookPolicy(**kwargs)
-        config.logging_policy = logging_policy or config.logging_policy or NetworkTraceLoggingPolicy(**kwargs)
-        config.http_logging_policy = http_logging_policy or config.http_logging_policy or HttpLoggingPolicy(**kwargs)
-        config.user_agent_policy = user_agent_policy or config.user_agent_policy or UserAgentPolicy(**kwargs)
+        config.headers_policy = (
+            headers_policy or cast(Optional[HeadersPolicy], config.headers_policy) or HeadersPolicy(**kwargs)
+        )
+        config.proxy_policy = proxy_policy or cast(Optional[ProxyPolicy], config.proxy_policy) or ProxyPolicy(**kwargs)
+        config.redirect_policy = (
+            redirect_policy or cast(Optional[RedirectPolicy], config.redirect_policy) or RedirectPolicy(**kwargs)
+        )
+        config.retry_policy = retry_policy or cast(Optional[RetryPolicy], config.retry_policy) or RetryPolicy(**kwargs)
+        config.custom_hook_policy = (
+            custom_hook_policy
+            or cast(Optional[CustomHookPolicy], config.custom_hook_policy)
+            or CustomHookPolicy(**kwargs)
+        )
+        config.logging_policy = (
+            logging_policy
+            or cast(Optional[NetworkTraceLoggingPolicy], config.logging_policy)
+            or NetworkTraceLoggingPolicy(**kwargs)
+        )
+        config.http_logging_policy = (
+            http_logging_policy
+            or cast(Optional[HttpLoggingPolicy], config.http_logging_policy)
+            or HttpLoggingPolicy(**kwargs)
+        )
+        config.user_agent_policy = (
+            user_agent_policy or cast(Optional[UserAgentPolicy], config.user_agent_policy) or UserAgentPolicy(**kwargs)
+        )
         config.polling_interval = kwargs.get("polling_interval", 30)
 
         super().__init__(
@@ -166,7 +130,6 @@ class HttpPipeline(Pipeline):
                 config.proxy_policy,
                 config.redirect_policy,
                 config.retry_policy,
-                config.authentication_policy,
                 config.custom_hook_policy,
                 config.logging_policy,
             ],
@@ -199,7 +162,6 @@ class HttpPipeline(Pipeline):
         files: Optional[FilesType] = None,
         **kwargs,
     ) -> HttpResponse:
-
         request = HttpRequest(
             method,
             url,
@@ -213,33 +175,78 @@ class HttpPipeline(Pipeline):
 
         return self.run(request, **kwargs).http_response
 
-    @_request_fn
-    def delete(self) -> None:
-        """Send a DELETE request."""
+    def delete(self: "HttpPipeline", url: str, **kwargs: Unpack[RequestKwargs]) -> HttpResponse:
+        """Send a DELETE request.
 
-    @_request_fn
-    def put(self) -> None:
-        """Send a PUT request."""
+        :param str url: The request url
+        :returns: The request response
+        :rtype: HttpResponse
+        """
 
-    @_request_fn
-    def get(self) -> None:
-        """Send a GET request."""
+        return self.request(self.delete.__name__.upper(), url, **kwargs)
 
-    @_request_fn
-    def post(self) -> None:
-        """Send a POST request."""
+    def put(self: "HttpPipeline", url: str, **kwargs: Unpack[RequestKwargs]) -> HttpResponse:
+        """Send a PUT request.
 
-    @_request_fn
-    def head(self) -> None:
-        """Send a HEAD request."""
+        :param str url: The request url
+        :returns: The request response
+        :rtype: HttpResponse
+        """
 
-    @_request_fn
-    def options(self) -> None:
-        """Send a OPTIONS request."""
+        return self.request(self.put.__name__.upper(), url, **kwargs)
 
-    @_request_fn
-    def patch(self) -> None:
-        """Send a PATCH request."""
+    def get(self: "HttpPipeline", url: str, **kwargs: Unpack[RequestKwargs]) -> HttpResponse:
+        """Send a GET request.
+
+        :param str url: The request url
+        :returns: The request response
+        :rtype: HttpResponse
+        """
+
+        return self.request(self.get.__name__.upper(), url, **kwargs)
+
+    def post(self: "HttpPipeline", url: str, **kwargs: Unpack[RequestKwargs]) -> HttpResponse:
+        """Send a POST request.
+
+        :param str url: The request url
+        :returns: The request response
+        :rtype: HttpResponse
+        """
+
+        return self.request(self.post.__name__.upper(), url, **kwargs)
+
+    def head(self: "HttpPipeline", url: str, **kwargs: Unpack[RequestKwargs]) -> HttpResponse:
+        """Send a HEAD request.
+
+        :param str url: The request url
+        :returns: The request response
+        :rtype: HttpResponse
+        """
+
+        return self.request(self.head.__name__.upper(), url, **kwargs)
+
+    def options(self: "HttpPipeline", url: str, **kwargs: Unpack[RequestKwargs]) -> HttpResponse:
+        """Send a OPTIONS request.
+
+        :param str url: The request url
+        :returns: The request response
+        :rtype: HttpResponse
+        """
+
+        return self.request(self.options.__name__.upper(), url, **kwargs)
+
+    def patch(self: "HttpPipeline", url: str, **kwargs: Unpack[RequestKwargs]) -> HttpResponse:
+        """Send a PATCH request.
+
+        :param str url: The request url
+        :returns: The request response
+        :rtype: HttpResponse
+        """
+
+        return self.request(self.patch.__name__.upper(), url, **kwargs)
+
+    def __enter__(self) -> Self:
+        return cast(Self, super().__enter__())
 
 
 class AsyncHttpPipeline(AsyncPipeline):
@@ -280,14 +287,36 @@ class AsyncHttpPipeline(AsyncPipeline):
         :param AsyncRedirectPolicy redirect_policy:
         """
         config = config or Configuration()
-        config.headers_policy = headers_policy or config.headers_policy or HeadersPolicy(**kwargs)
-        config.proxy_policy = proxy_policy or config.proxy_policy or ProxyPolicy(**kwargs)
-        config.redirect_policy = redirect_policy or config.redirect_policy or AsyncRedirectPolicy(**kwargs)
-        config.retry_policy = retry_policy or config.retry_policy or AsyncRetryPolicy(**kwargs)
-        config.custom_hook_policy = custom_hook_policy or config.custom_hook_policy or CustomHookPolicy(**kwargs)
-        config.logging_policy = logging_policy or config.logging_policy or NetworkTraceLoggingPolicy(**kwargs)
-        config.http_logging_policy = http_logging_policy or config.http_logging_policy or HttpLoggingPolicy(**kwargs)
-        config.user_agent_policy = user_agent_policy or config.user_agent_policy or UserAgentPolicy(**kwargs)
+        config.headers_policy = (
+            headers_policy or cast(Optional[HeadersPolicy], config.headers_policy) or HeadersPolicy(**kwargs)
+        )
+        config.proxy_policy = proxy_policy or cast(Optional[ProxyPolicy], config.proxy_policy) or ProxyPolicy(**kwargs)
+        config.redirect_policy = (
+            redirect_policy
+            or cast(Optional[AsyncRedirectPolicy], config.redirect_policy)
+            or AsyncRedirectPolicy(**kwargs)
+        )
+        config.retry_policy = (
+            retry_policy or cast(Optional[AsyncRetryPolicy], config.retry_policy) or AsyncRetryPolicy(**kwargs)
+        )
+        config.custom_hook_policy = (
+            custom_hook_policy
+            or cast(Optional[CustomHookPolicy], config.custom_hook_policy)
+            or CustomHookPolicy(**kwargs)
+        )
+        config.logging_policy = (
+            logging_policy
+            or cast(Optional[NetworkTraceLoggingPolicy], config.logging_policy)
+            or NetworkTraceLoggingPolicy(**kwargs)
+        )
+        config.http_logging_policy = (
+            http_logging_policy
+            or cast(Optional[HttpLoggingPolicy], config.http_logging_policy)
+            or HttpLoggingPolicy(**kwargs)
+        )
+        config.user_agent_policy = (
+            user_agent_policy or cast(Optional[UserAgentPolicy], config.user_agent_policy) or UserAgentPolicy(**kwargs)
+        )
         config.polling_interval = kwargs.get("polling_interval", 30)
 
         super().__init__(
@@ -301,7 +330,6 @@ class AsyncHttpPipeline(AsyncPipeline):
                 config.proxy_policy,
                 config.redirect_policy,
                 config.retry_policy,
-                config.authentication_policy,
                 config.custom_hook_policy,
                 config.logging_policy,
             ],
@@ -334,7 +362,6 @@ class AsyncHttpPipeline(AsyncPipeline):
         files: Optional[FilesType] = None,
         **kwargs,
     ) -> AsyncHttpResponse:
-
         request = HttpRequest(
             method,
             url,
@@ -348,33 +375,77 @@ class AsyncHttpPipeline(AsyncPipeline):
 
         return (await self.run(request, **kwargs)).http_response
 
-    @_async_request_fn
-    async def delete(self) -> None:
-        """Send a DELETE request."""
+    async def delete(self: "AsyncHttpPipeline", url: str, **kwargs: Unpack[RequestKwargs]) -> AsyncHttpResponse:
+        """Send a DELETE request.
+
+        :param str url: The request url
+        :returns: The request response
+        :rtype: AsyncHttpResponse
+        """
+        return await self.request(self.delete.__name__.upper(), url, **kwargs)
+
+    async def put(self: "AsyncHttpPipeline", url: str, **kwargs: Unpack[RequestKwargs]) -> AsyncHttpResponse:
+        """Send a PUT request.
+
+        :param str url: The request url
+        :returns: The request response
+        :rtype: AsyncHttpResponse
+        """
+
+        return await self.request(self.put.__name__.upper(), url, **kwargs)
+
+    async def get(self: "AsyncHttpPipeline", url: str, **kwargs: Unpack[RequestKwargs]) -> AsyncHttpResponse:
+        """Send a GET request.
+
+        :param str url: The request url
+        :returns: The request response
+        :rtype: AsyncHttpResponse
+        """
+
+        return await self.request(self.get.__name__.upper(), url, **kwargs)
+
+    async def post(self: "AsyncHttpPipeline", url: str, **kwargs: Unpack[RequestKwargs]) -> AsyncHttpResponse:
+        """Send a POST request.
+
+        :param str url: The request url
+        :returns: The request response
+        :rtype: AsyncHttpResponse
+        """
+
+        return await self.request(self.post.__name__.upper(), url, **kwargs)
+
+    async def head(self: "AsyncHttpPipeline", url: str, **kwargs: Unpack[RequestKwargs]) -> AsyncHttpResponse:
+        """Send a HEAD request.
 
-    @_async_request_fn
-    async def put(self) -> None:
-        """Send a PUT request."""
+        :param str url: The request url
+        :returns: The request response
+        :rtype: AsyncHttpResponse
+        """
+
+        return await self.request(self.head.__name__.upper(), url, **kwargs)
+
+    async def options(self: "AsyncHttpPipeline", url: str, **kwargs: Unpack[RequestKwargs]) -> AsyncHttpResponse:
+        """Send a OPTIONS request.
 
-    @_async_request_fn
-    async def get(self) -> None:
-        """Send a GET request."""
+        :param str url: The request url
+        :returns: The request response
+        :rtype: AsyncHttpResponse
+        """
+
+        return await self.request(self.options.__name__.upper(), url, **kwargs)
 
-    @_async_request_fn
-    async def post(self) -> None:
-        """Send a POST request."""
+    async def patch(self: "AsyncHttpPipeline", url: str, **kwargs: Unpack[RequestKwargs]) -> AsyncHttpResponse:
+        """Send a PATCH request.
 
-    @_async_request_fn
-    async def head(self) -> None:
-        """Send a HEAD request."""
+        :param str url: The request url
+        :returns: The request response
+        :rtype: AsyncHttpResponse
+        """
 
-    @_async_request_fn
-    async def options(self) -> None:
-        """Send a OPTIONS request."""
+        return await self.request(self.patch.__name__.upper(), url, **kwargs)
 
-    @_async_request_fn
-    async def patch(self) -> None:
-        """Send a PATCH request."""
+    async def __aenter__(self) -> Self:
+        return cast(Self, await super().__aenter__())
 
 
 def get_http_client() -> HttpPipeline:

azure/ai/evaluation/_model_configurations.py

@@ -2,30 +2,30 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
 
-from typing import Dict, Literal, TypedDict
+from typing import Any, Dict, List, Literal, TypedDict, Union
 
 from typing_extensions import NotRequired
 
 
-class AzureOpenAIModelConfiguration(TypedDict, total=False):
+class AzureOpenAIModelConfiguration(TypedDict):
     """Model Configuration for Azure OpenAI Model"""
 
-    type: Literal["azure_openai"]
+    type: NotRequired[Literal["azure_openai"]]
     """The type of the model configuration. Should be 'azure_openai' for AzureOpenAIModelConfiguration"""
     azure_deployment: str
     """Name of Azure OpenAI deployment to make request to"""
     azure_endpoint: str
     """Endpoint of Azure OpenAI resource to make request to"""
-    api_key: str
+    api_key: NotRequired[str]
     """API key of Azure OpenAI resource"""
     api_version: NotRequired[str]
     """(Optional) API version to use in request to Azure OpenAI deployment"""
 
 
-class OpenAIModelConfiguration(TypedDict, total=False):
+class OpenAIModelConfiguration(TypedDict):
     """Model Configuration for OpenAI Model"""
 
-    type: Literal["openai"]
+    type: NotRequired[Literal["openai"]]
     """The type of the model configuration. Should be 'openai' for OpenAIModelConfiguration"""
     api_key: str
     "API key needed to make request to model"
@@ -53,3 +53,20 @@ class EvaluatorConfig(TypedDict, total=False):
 
     column_mapping: Dict[str, str]
     """Dictionary mapping evaluator input name to column in data"""
+
+
+class Message(TypedDict):
+    role: str
+    content: Union[str, List[Dict]]
+    context: NotRequired[Dict[str, Any]]
+
+
+class Conversation(TypedDict):
+    messages: Union[List[Message], List[Dict]]
+    context: NotRequired[Dict[str, Any]]
+
+
+class EvaluationResult(TypedDict):
+    metrics: Dict
+    studio_url: NotRequired[str]
+    rows: List[Dict]

azure/ai/evaluation/_vendor/__init__.py

@@ -0,0 +1,3 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------

azure/ai/evaluation/_vendor/rouge_score/__init__.py

@@ -0,0 +1,14 @@
+# coding=utf-8
+# Copyright 2024 The Google Research Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.