aidial-sdk 0.41.0.dev4__tar.gz → 0.42.0.dev1__tar.gz

{aidial_sdk-0.41.0.dev4 → aidial_sdk-0.42.0.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aidial-sdk
-Version: 0.41.0.dev4
+Version: 0.42.0.dev1
 Summary: Framework to create applications and model adapters for AI DIAL
 License-Expression: Apache-2.0
 License-File: LICENSE
@@ -61,6 +61,7 @@ Description-Content-Type: text/markdown
   - [Echo application example](#echo-application-example)
     - [Run](#run)
     - [Check](#check)
+  - [Endpoints](#endpoints)
 - [Development](#development)
   - [Development Environment](#development-environment)
   - [Setup](#setup)
@@ -177,6 +178,34 @@ You will see the JSON response as:
 
 ---
 
+### Endpoints
+
+Every deployment is exposed under two base paths:
+
+|Base path|Description|
+|---|---|
+|`/openai/deployments/{deployment_name}`|The DIAL API path. The deployment name is a part of the path.|
+|`/openai/v1`|The Azure OpenAI v1 API path. The deployment name comes from the request headers.|
+
+The following endpoints are served under each of them:
+
+- `POST <base path>/chat/completions`
+- `POST <base path>/rate`
+- `POST <base path>/tokenize`
+- `POST <base path>/truncate_prompt`
+- `GET <base path>/configuration`
+- `POST <base path>/embeddings`
+
+`request.deployment_id` is resolved from the first of the following sources that is set:
+
+1. the `X-DIAL-OVERRIDE-NAME` header,
+2. the deployment name from the path, for the `/openai/deployments/{deployment_name}` requests,
+3. the `X-DIAL-DEPLOYMENT-ID` header, which DIAL Core sets when it calls the `/openai/v1` endpoints.
+
+An `/openai/v1` request with none of the headers set fails with `500 Internal Server Error`.
+
+---
+
 ## Development
 
 ### Development Environment

{aidial_sdk-0.41.0.dev4 → aidial_sdk-0.42.0.dev1}/README.md

@@ -22,6 +22,7 @@
   - [Echo application example](#echo-application-example)
     - [Run](#run)
     - [Check](#check)
+  - [Endpoints](#endpoints)
 - [Development](#development)
   - [Development Environment](#development-environment)
   - [Setup](#setup)
@@ -138,6 +139,34 @@ You will see the JSON response as:
 
 ---
 
+### Endpoints
+
+Every deployment is exposed under two base paths:
+
+|Base path|Description|
+|---|---|
+|`/openai/deployments/{deployment_name}`|The DIAL API path. The deployment name is a part of the path.|
+|`/openai/v1`|The Azure OpenAI v1 API path. The deployment name comes from the request headers.|
+
+The following endpoints are served under each of them:
+
+- `POST <base path>/chat/completions`
+- `POST <base path>/rate`
+- `POST <base path>/tokenize`
+- `POST <base path>/truncate_prompt`
+- `GET <base path>/configuration`
+- `POST <base path>/embeddings`
+
+`request.deployment_id` is resolved from the first of the following sources that is set:
+
+1. the `X-DIAL-OVERRIDE-NAME` header,
+2. the deployment name from the path, for the `/openai/deployments/{deployment_name}` requests,
+3. the `X-DIAL-DEPLOYMENT-ID` header, which DIAL Core sets when it calls the `/openai/v1` endpoints.
+
+An `/openai/v1` request with none of the headers set fails with `500 Internal Server Error`.
+
+---
+
 ## Development
 
 ### Development Environment

{aidial_sdk-0.41.0.dev4 → aidial_sdk-0.42.0.dev1}/aidial_sdk/application.py

@@ -1,7 +1,7 @@
 import logging
 import re
 import warnings
-from collections.abc import Callable, Coroutine
+from collections.abc import Callable, Coroutine, Iterator
 from typing import Any, Literal, TypeVar
 
 from fastapi import FastAPI, HTTPException, Request
@@ -20,12 +20,16 @@ from aidial_sdk.chat_completion.response import (
     Response as ChatCompletionResponse,
 )
 from aidial_sdk.deployment.configuration import ConfigurationRequest
-from aidial_sdk.deployment.from_request_mixin import FromRequestMixin
+from aidial_sdk.deployment.from_request_mixin import (
+    FromRequestMixin,
+    resolve_deployment_id,
+)
 from aidial_sdk.deployment.rate import RateRequest
 from aidial_sdk.deployment.tokenize import TokenizeRequest
 from aidial_sdk.deployment.truncate_prompt import TruncatePromptRequest
 from aidial_sdk.embeddings.base import Embeddings
 from aidial_sdk.embeddings.request import Request as EmbeddingsRequest
+from aidial_sdk.exceptions import DeploymentNotFoundError
 from aidial_sdk.exceptions import HTTPException as DIALException
 from aidial_sdk.header_propagator import HeaderPropagator
 from aidial_sdk.telemetry.types import TelemetryConfig
@@ -45,6 +49,8 @@ configure_sdk_logger()
 
 RequestType = TypeVar("RequestType", bound=FromRequestMixin)
 
+Handler = Callable[[Request], Coroutine[Any, Any, Response]]
+
 
 def _interpolate_deployment_id(deployment_id: str, path_params: dict) -> str:
     result = deployment_id
@@ -70,6 +76,7 @@ class PathFilter(logging.Filter):
 class DIALApp(FastAPI):
     _allow_extra_request_fields: bool
     _dial_url: str | None
+    _v1_handlers: dict[str, dict[str, Handler]]
 
     def __init__(
         self,
@@ -95,6 +102,7 @@ class DIALApp(FastAPI):
 
         self._allow_extra_request_fields = allow_extra_request_fields
         self._dial_url = dial_url
+        self._v1_handlers = {}
 
         self.configure_telemetry(telemetry_config)
 
@@ -152,6 +160,13 @@ class DIALApp(FastAPI):
             methods=["POST"],
         )
 
+        self._add_v1_route(
+            deployment_name,
+            "embeddings",
+            self._embeddings(None, impl),
+            methods=["POST"],
+        )
+
         return self
 
     def add_chat_completion(
@@ -161,60 +176,115 @@ class DIALApp(FastAPI):
         *,
         heartbeat_interval: float | None = None,
     ) -> "DIALApp":
-        self.add_api_route(
-            f"/openai/deployments/{deployment_name}/chat/completions",
+        for endpoint, handler, methods in self._chat_completion_handlers(
+            deployment_name, impl, heartbeat_interval
+        ):
+            self.add_api_route(
+                f"/openai/deployments/{deployment_name}/{endpoint}",
+                handler,
+                methods=methods,
+            )
+
+        for endpoint, handler, methods in self._chat_completion_handlers(
+            None, impl, heartbeat_interval
+        ):
+            self._add_v1_route(
+                deployment_name, endpoint, handler, methods=methods
+            )
+
+        return self
+
+    def _chat_completion_handlers(
+        self,
+        deployment_id: str | None,
+        impl: ChatCompletion,
+        heartbeat_interval: float | None,
+    ) -> Iterator[tuple[str, Handler, list[str]]]:
+        yield (
+            "chat/completions",
             self._chat_completion(
-                deployment_name,
-                impl,
-                heartbeat_interval=heartbeat_interval,
+                deployment_id, impl, heartbeat_interval=heartbeat_interval
             ),
-            methods=["POST"],
+            ["POST"],
         )
 
-        self.add_api_route(
-            f"/openai/deployments/{deployment_name}/rate",
-            self._rate_response(deployment_name, impl),
-            methods=["POST"],
-        )
+        yield "rate", self._rate_response(deployment_id, impl), ["POST"]
 
         if endpoint_impl := get_method_implementation(impl, "tokenize"):
-            self.add_api_route(
-                f"/openai/deployments/{deployment_name}/tokenize",
+            yield (
+                "tokenize",
                 self._endpoint_factory(
-                    deployment_name, endpoint_impl, "tokenize", TokenizeRequest
+                    deployment_id, endpoint_impl, "tokenize", TokenizeRequest
                 ),
-                methods=["POST"],
+                ["POST"],
             )
 
         if endpoint_impl := get_method_implementation(impl, "truncate_prompt"):
-            self.add_api_route(
-                f"/openai/deployments/{deployment_name}/truncate_prompt",
+            yield (
+                "truncate_prompt",
                 self._endpoint_factory(
-                    deployment_name,
+                    deployment_id,
                     endpoint_impl,
                     "truncate_prompt",
                     TruncatePromptRequest,
                 ),
-                methods=["POST"],
+                ["POST"],
             )
 
         if endpoint_impl := get_method_implementation(impl, "configuration"):
-            self.add_api_route(
-                f"/openai/deployments/{deployment_name}/configuration",
+            yield (
+                "configuration",
                 self._endpoint_factory(
-                    deployment_name,
+                    deployment_id,
                     endpoint_impl,
                     "configuration",
                     ConfigurationRequest,
                 ),
-                methods=["GET"],
+                ["GET"],
             )
 
-        return self
+    def _add_v1_route(
+        self,
+        deployment_name: str,
+        endpoint: str,
+        handler: Handler,
+        *,
+        methods: list[str],
+    ) -> None:
+        """The /openai/v1 endpoints are shared by all the deployments of
+        the application, so they are dispatched by the deployment id
+        resolved from the request headers."""
+
+        if endpoint not in self._v1_handlers:
+            self._v1_handlers[endpoint] = {}
+            self.add_api_route(
+                f"/openai/v1/{endpoint}",
+                self._v1_dispatcher(endpoint),
+                methods=methods,
+            )
+
+        self._v1_handlers[endpoint][deployment_name] = handler
+
+    def _v1_dispatcher(self, endpoint: str) -> Handler:
+        async def _handler(original_request: Request) -> Response:
+            deployment_id = resolve_deployment_id(
+                original_request.headers, None
+            )
+            set_log_deployment(deployment_id)
+
+            handler = self._v1_handlers[endpoint].get(deployment_id)
+            if handler is None:
+                raise DeploymentNotFoundError(
+                    f"The deployment {deployment_id!r} doesn't provide the {endpoint!r} endpoint"
+                )
+
+            return await handler(original_request)
+
+        return _handler
 
     def _endpoint_factory(
         self,
-        deployment_id: str,
+        deployment_id: str | None,
         endpoint_impl: Callable[[RequestType], Coroutine[Any, Any, Any]],
         endpoint: Literal["tokenize", "truncate_prompt", "configuration"],
         request_type: type["RequestType"],
@@ -242,7 +312,7 @@ class DIALApp(FastAPI):
 
         return _handler
 
-    def _rate_response(self, deployment_id: str, impl: ChatCompletion):
+    def _rate_response(self, deployment_id: str | None, impl: ChatCompletion):
         async def _handler(original_request: Request):
             request = await self._parse_request(
                 RateRequest, original_request, deployment_id
@@ -257,15 +327,16 @@ class DIALApp(FastAPI):
         self,
         request: type[RequestType],
         original_request: Request,
-        deployment_id: str,
+        deployment_id: str | None,
     ) -> RequestType:
-        interpolated_deployment_id = _interpolate_deployment_id(
-            deployment_id, original_request.path_params
-        )
-        set_log_deployment(interpolated_deployment_id)
+        if deployment_id is not None:
+            deployment_id = _interpolate_deployment_id(
+                deployment_id, original_request.path_params
+            )
+            set_log_deployment(deployment_id)
 
         ret = await request.from_request(
-            original_request, interpolated_deployment_id, self._dial_url
+            original_request, deployment_id, self._dial_url
         )
         if not self._allow_extra_request_fields:
             model_validate_extra_fields(ret)
@@ -273,7 +344,7 @@ class DIALApp(FastAPI):
 
     def _chat_completion(
         self,
-        deployment_id: str,
+        deployment_id: str | None,
         impl: ChatCompletion,
         *,
         heartbeat_interval: float | None,
@@ -316,7 +387,7 @@ class DIALApp(FastAPI):
 
         return _handler
 
-    def _embeddings(self, deployment_id: str, impl: Embeddings):
+    def _embeddings(self, deployment_id: str | None, impl: Embeddings):
         async def _handler(original_request: Request):
             request = await self._parse_request(
                 EmbeddingsRequest, original_request, deployment_id

{aidial_sdk-0.41.0.dev4 → aidial_sdk-0.42.0.dev1}/aidial_sdk/deployment/_headers.py

@@ -1,6 +1,9 @@
 DIAL_CONVERSATION_ID = "X-CONVERSATION-ID"
 DIAL_JOB_TITLE = "X-JOB-TITLE"
 
+DIAL_OVERRIDE_NAME = "X-DIAL-OVERRIDE-NAME"
+DIAL_DEPLOYMENT_ID = "X-DIAL-DEPLOYMENT-ID"
+
 DIAL_UPSTREAM_ENDPOINT = "X-UPSTREAM-ENDPOINT"
 DIAL_UPSTREAM_KEY = "X-UPSTREAM-KEY"
 DIAL_UPSTREAM_EXTRA_DATA = "X-UPSTREAM-EXTRA-DATA"

{aidial_sdk-0.41.0.dev4 → aidial_sdk-0.42.0.dev1}/aidial_sdk/deployment/from_request_mixin.py

@@ -1,6 +1,7 @@
 import json
 import warnings
 from abc import ABC, abstractmethod
+from collections.abc import Mapping
 from typing import Any, TypeVar
 from urllib.parse import urljoin
 
@@ -21,7 +22,9 @@ from aidial_sdk.deployment._headers import (
     DIAL_CACHE_BREAKPOINT_PATH,
     DIAL_CACHE_EXTRA_METADATA,
     DIAL_CONVERSATION_ID,
+    DIAL_DEPLOYMENT_ID,
     DIAL_JOB_TITLE,
+    DIAL_OVERRIDE_NAME,
     DIAL_UPSTREAM_ENDPOINT,
     DIAL_UPSTREAM_EXTRA_DATA,
     DIAL_UPSTREAM_KEY,
@@ -33,13 +36,33 @@ from aidial_sdk.utils.pydantic import ExtraAllowModel
 T = TypeVar("T", bound="FromRequestMixin")
 
 
+def resolve_deployment_id(
+    headers: Mapping[str, str], deployment_id: str | None
+) -> str:
+    """The deployment id of a request comes either from the path it was
+    sent to or, for the deployment-agnostic paths, from its headers."""
+
+    effective_deployment_id = (
+        headers.get(DIAL_OVERRIDE_NAME)
+        or deployment_id
+        or headers.get(DIAL_DEPLOYMENT_ID)
+    )
+
+    if effective_deployment_id is None:
+        raise InternalServerError(
+            f"The request headers are missing {DIAL_DEPLOYMENT_ID} header."
+        )
+
+    return effective_deployment_id
+
+
 class FromRequestMixin(ABC, ExtraAllowModel):
     @classmethod
     @abstractmethod
     async def from_request(
         cls: type[T],
         request: fastapi.Request,
-        deployment_id: str,
+        deployment_id: str | None,
         base_url: str | None,
     ) -> T:
         pass
@@ -176,7 +199,7 @@ class FromRequestDeploymentMixin(FromRequestMixin):
     async def from_request(
         cls,
         request: fastapi.Request,
-        deployment_id: StrictStr,
+        deployment_id: StrictStr | None,
         base_url: str | None,
     ):
         headers = request.headers.mutablecopy()
@@ -212,7 +235,7 @@ class FromRequestDeploymentMixin(FromRequestMixin):
             bearer_token_secret=(
                 SecretStr(bearer_token) if bearer_token else None
             ),
-            deployment_id=deployment_id,
+            deployment_id=resolve_deployment_id(headers, deployment_id),
             api_version=request.query_params.get("api-version"),
             headers=headers,
             original_request=request,

{aidial_sdk-0.41.0.dev4 → aidial_sdk-0.42.0.dev1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "aidial-sdk"
-version = "0.41.0.dev4"
+version = "0.42.0.dev1"
 description = "Framework to create applications and model adapters for AI DIAL"
 authors = [{ name = "EPAM RAIL", email = "SpecialEPM-DIALDevTeam@epam.com" }]
 license = "Apache-2.0"