aidial-sdk 0.48.0.dev1__tar.gz → 0.48.0.dev3__tar.gz

{aidial_sdk-0.48.0.dev1 → aidial_sdk-0.48.0.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aidial-sdk
-Version: 0.48.0.dev1
+Version: 0.48.0.dev3
 Summary: Framework to create applications and model adapters for AI DIAL
 License-Expression: Apache-2.0
 License-File: LICENSE

{aidial_sdk-0.48.0.dev1 → aidial_sdk-0.48.0.dev3}/aidial_sdk/application.py

@@ -77,7 +77,20 @@ class PathFilter(logging.Filter):
 class DIALApp(FastAPI):
     _allow_extra_request_fields: bool
     _dial_url: str | None
-    _v1_handlers: dict[str, dict[str, Handler]]
+
+    _generic_handlers: dict[str, dict[str, Handler]]
+    """
+    Request handlers for endpoints that do not allow to distinguish deployments in the path.
+    A dictionary from such a deployment-id-less endpoint to the handlers for each deployment:
+
+    /openai/v1/chat/completions:
+        gpt-5.4: handler1
+        gpt-5.5: handler2
+    /openai/v1/embeddings:
+        ada-embeddings-2024-10-10: handler3
+    ...
+    """
+
     _reset_otel_context: Callable[[], AbstractContextManager[None]]
 
     def __init__(
@@ -104,7 +117,7 @@ class DIALApp(FastAPI):
 
         self._allow_extra_request_fields = allow_extra_request_fields
         self._dial_url = dial_url
-        self._v1_handlers = {}
+        self._generic_handlers = {}
         self._reset_otel_context = nullcontext
 
         self.configure_telemetry(telemetry_config)
@@ -175,9 +188,9 @@ class DIALApp(FastAPI):
             methods=["POST"],
         )
 
-        self._add_v1_route(
+        self._add_generic_route(
             deployment_name,
-            "embeddings",
+            "/openai/v1/embeddings",
             self._embeddings(None, impl),
             methods=["POST"],
         )
@@ -203,8 +216,11 @@ class DIALApp(FastAPI):
         for endpoint, handler, methods in self._chat_completion_handlers(
             None, impl, heartbeat_interval
         ):
-            self._add_v1_route(
-                deployment_name, endpoint, handler, methods=methods
+            self._add_generic_route(
+                deployment_name,
+                f"/openai/v1/{endpoint}",
+                handler,
+                methods=methods,
             )
 
         return self
@@ -258,7 +274,7 @@ class DIALApp(FastAPI):
                 ["GET"],
             )
 
-    def _add_v1_route(
+    def _add_generic_route(
         self,
         deployment_name: str,
         endpoint: str,
@@ -266,21 +282,16 @@ class DIALApp(FastAPI):
         *,
         methods: list[str],
     ) -> None:
-        """The /openai/v1 endpoints are shared by all the deployments of
-        the application, so they are dispatched by the deployment id
-        resolved from the request headers."""
-
-        if endpoint not in self._v1_handlers:
-            self._v1_handlers[endpoint] = {}
+        if endpoint not in self._generic_handlers:
+            self._generic_handlers[endpoint] = {}
             self.add_api_route(
-                f"/openai/v1/{endpoint}",
-                self._v1_dispatcher(endpoint),
+                endpoint,
+                self._generic_dispatcher(endpoint),
                 methods=methods,
             )
+        self._generic_handlers[endpoint][deployment_name] = handler
 
-        self._v1_handlers[endpoint][deployment_name] = handler
-
-    def _v1_dispatcher(self, endpoint: str) -> Handler:
+    def _generic_dispatcher(self, endpoint: str) -> Handler:
         async def _handler(original_request: Request) -> Response:
             deployment_id = get_upstream_deployment_id(
                 original_request.headers, None
@@ -302,7 +313,7 @@ class DIALApp(FastAPI):
     def _get_handler(
         self, endpoint: str, deployment_id: str, request: Request
     ) -> Handler | None:
-        handlers = self._v1_handlers[endpoint]
+        handlers = self._generic_handlers[endpoint]
 
         if (handler := handlers.get(deployment_id)) is not None:
             return handler

{aidial_sdk-0.48.0.dev1 → aidial_sdk-0.48.0.dev3}/aidial_sdk/chat_completion/response.py

@@ -70,7 +70,9 @@ class Response:
                 if self.request.stream
                 else "chat.completion"
             ),
-            model=get_client_deployment_id(request.headers),
+            model=get_client_deployment_id(
+                request.headers, request.deployment_id
+            ),
         )
 
         self._headers = []

{aidial_sdk-0.48.0.dev1 → aidial_sdk-0.48.0.dev3}/aidial_sdk/deployment/from_request_mixin.py

@@ -44,14 +44,24 @@ def get_upstream_deployment_id(
     return (
         headers.get(DIAL_OVERRIDE_NAME)
         or deployment_id
-        or get_client_deployment_id(headers)
+        or _get_dial_deployment_id(headers)
     )
 
 
-def get_client_deployment_id(headers: Mapping[str, str]) -> str:
+def get_client_deployment_id(
+    headers: Mapping[str, str], deployment_id: str | None
+) -> str:
     """The client-facing deployment id that a DIAL application is expected
     to return as a model parameter in the response."""
 
+    # NOTE: Once https://github.com/epam/ai-dial-core/issues/2243 is fixed,
+    # this could be replaced with just _get_dial_deployment_id.
+    return headers.get(DIAL_DEPLOYMENT_ID) or get_upstream_deployment_id(
+        headers, deployment_id
+    )
+
+
+def _get_dial_deployment_id(headers: Mapping[str, str]) -> str:
     if (deployment_id := headers.get(DIAL_DEPLOYMENT_ID)) is None:
         raise InternalServerError(
             f"The request headers are missing {DIAL_DEPLOYMENT_ID} header."

{aidial_sdk-0.48.0.dev1 → aidial_sdk-0.48.0.dev3}/aidial_sdk/translators/messages_to_chat_completions/__init__.py

@@ -1,4 +1,3 @@
-import httpx
 from fastapi import FastAPI
 
 from aidial_sdk._pydantic import PYDANTIC_V2
@@ -9,10 +8,18 @@ if not PYDANTIC_V2:
         "install pydantic>=2 and set PYDANTIC_V2=True"
     )
 
-from aidial_sdk.translators.messages_to_chat_completions.chat_completions.app import (  # noqa: E402
-    ClientFactory,
-    create_anthropic_api_app,
-)
+try:
+    import httpx
+
+    from aidial_sdk.translators.messages_to_chat_completions.chat_completions.app import (
+        ClientFactory,
+        create_anthropic_api_app,
+    )
+except ModuleNotFoundError as e:
+    raise ImportError(
+        "The Anthropic Messages API translator requires the translators extra: "
+        'pip install "aidial-sdk[translators]"'
+    ) from e
 
 __all__ = ["ClientFactory", "create_anthropic_api_app", "mount_translator"]
 

aidial_sdk-0.48.0.dev1/aidial_sdk/translators/messages_to_chat_completions/core_client.py → aidial_sdk-0.48.0.dev3/aidial_sdk/translators/messages_to_chat_completions/backend/http.py

@@ -1,8 +1,10 @@
+from dataclasses import dataclass
 from urllib.parse import quote
 
 import httpx
 from fastapi import Request
-from openai import AsyncOpenAI, Omit, omit
+from openai import AsyncOpenAI, AsyncStream, Omit, omit
+from openai.types.chat import ChatCompletion, ChatCompletionChunk
 
 from aidial_sdk.translators.messages_to_chat_completions.settings import (
     get_dial_api_version,
@@ -62,3 +64,29 @@ def core_headers(request: Request) -> dict[str, str | Omit]:
         }
     )
     return headers
+
+
+@dataclass
+class ChatClientHttp:
+    client: AsyncOpenAI
+    core_headers: dict[str, str | Omit]
+
+    async def create_stream(
+        self, body: dict
+    ) -> AsyncStream[ChatCompletionChunk]:
+        return await self.client.post(
+            "/chat/completions",
+            cast_to=ChatCompletion,
+            body=body,
+            options={"headers": self.core_headers},
+            stream=True,
+            stream_cls=AsyncStream[ChatCompletionChunk],
+        )
+
+    async def create_block(self, body: dict) -> ChatCompletion:
+        return await self.client.post(
+            "/chat/completions",
+            cast_to=ChatCompletion,
+            body=body,
+            options={"headers": self.core_headers},
+        )

{aidial_sdk-0.48.0.dev1 → aidial_sdk-0.48.0.dev3}/aidial_sdk/translators/messages_to_chat_completions/chat_completions/app.py

@@ -7,9 +7,14 @@ import httpx
 from anthropic.types.beta.message_create_params import MessageCreateParams
 from fastapi import FastAPI, Request
 from fastapi.responses import JSONResponse, Response, StreamingResponse
-from openai import AsyncStream
-from openai.types.chat import ChatCompletion, ChatCompletionChunk
 
+from aidial_sdk.deployment.from_request_mixin import get_client_deployment_id
+from aidial_sdk.translators.messages_to_chat_completions.backend.http import (
+    ChatClientHttp,
+    core_headers,
+    core_legacy_client,
+    core_v1_client,
+)
 from aidial_sdk.translators.messages_to_chat_completions.chat_completions.from_chat_completions import (
     to_anthropic_response,
 )
@@ -23,11 +28,6 @@ from aidial_sdk.translators.messages_to_chat_completions.common import (
     not_found,
     with_debug_logging,
 )
-from aidial_sdk.translators.messages_to_chat_completions.core_client import (
-    core_headers,
-    core_legacy_client,
-    core_v1_client,
-)
 from aidial_sdk.translators.messages_to_chat_completions.errors import (
     AnthropicErrorType,
     AnthropicHTTPError,
@@ -45,13 +45,10 @@ ClientT = TypeVar("ClientT")
 ClientFactory = Callable[[Request], Awaitable[ClientT]] | ClientT
 
 
-async def _handle_messages(
-    request: Request,
-    get_client: ClientFactory[httpx.AsyncClient],
-    legacy: bool,
+async def handle_anthropic_messages(
+    request: Request, client: ChatClientHttp
 ) -> Response:
-    dial_url = get_dial_url()
-    translation_state = TranslationState()
+    deployment = get_client_deployment_id(request.headers, None)
 
     try:
         req: MessageCreateParams = await request.json()
@@ -61,7 +58,7 @@ async def _handle_messages(
             f"Request body is not valid JSON: {e}",
         ) from e
 
-    deployment = request.headers.get("x-dial-deployment-id") or req["model"]
+    translation_state = TranslationState()
     chat_request = to_chat_completions_request(
         req, deployment, translation_state
     )
@@ -70,38 +67,14 @@ async def _handle_messages(
     if log.isEnabledFor(logging.DEBUG):
         log.debug(f"Translated request to Core: {json.dumps(body)}")
 
-    http_client = (
-        get_client
-        if isinstance(get_client, httpx.AsyncClient)
-        else await get_client(request)
-    )
-    client = (
-        core_legacy_client(dial_url, deployment, http_client)
-        if legacy
-        else core_v1_client(dial_url, http_client)
-    )
-    headers = core_headers(request)
-
     if chat_request.stream:
-        events = await client.post(
-            "/chat/completions",
-            cast_to=ChatCompletion,
-            body=body,
-            options={"headers": headers},
-            stream=True,
-            stream_cls=AsyncStream[ChatCompletionChunk],
-        )
+        events = await client.create_stream(body)
         return StreamingResponse(
             translate_stream(events, deployment, translation_state),
             media_type="text/event-stream",
         )
 
-    response = await client.post(
-        "/chat/completions",
-        cast_to=ChatCompletion,
-        body=body,
-        options={"headers": headers},
-    )
+    response = await client.create_block(body)
     message = to_anthropic_response(response, deployment, translation_state)
     return JSONResponse(
         content=message.model_dump(mode="json", exclude_unset=True)
@@ -113,17 +86,34 @@ def create_anthropic_api_app(
 ) -> FastAPI:
     app = FastAPI()
 
+    async def _chat_client(request: Request, *, legacy: bool) -> ChatClientHttp:
+        http_client = (
+            await get_client(request) if callable(get_client) else get_client
+        )
+        dial_url = get_dial_url()
+        if legacy:
+            deployment_id = get_client_deployment_id(request.headers, None)
+            client = core_legacy_client(dial_url, deployment_id, http_client)
+        else:
+            client = core_v1_client(dial_url, http_client)
+
+        return ChatClientHttp(client, core_headers(request))
+
     @app.post("/anthropic/v1/messages")
     @with_debug_logging
     @translator_error_handler
     async def _handle_v1_messages(request: Request) -> Response:
-        return await _handle_messages(request, get_client, legacy=False)
+        return await handle_anthropic_messages(
+            request, await _chat_client(request, legacy=False)
+        )
 
     @app.post("/legacy/anthropic/v1/messages")
     @with_debug_logging
     @translator_error_handler
     async def _handle_legacy_messages(request: Request) -> Response:
-        return await _handle_messages(request, get_client, legacy=True)
+        return await handle_anthropic_messages(
+            request, await _chat_client(request, legacy=True)
+        )
 
     app.add_api_route(
         "/{full_path:path}",

{aidial_sdk-0.48.0.dev1 → aidial_sdk-0.48.0.dev3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "aidial-sdk"
-version = "0.48.0.dev1"
+version = "0.48.0.dev3"
 description = "Framework to create applications and model adapters for AI DIAL"
 authors = [{ name = "EPAM RAIL", email = "SpecialEPM-DIALDevTeam@epam.com" }]
 license = "Apache-2.0"
@@ -52,7 +52,6 @@ aiointercept = "^0.1.7"
 requests = "^2.33"
 responses = "^0.25.3"
 httpx2 = "^2.12.0"        # required for Starlette.TestClient
-openai = "^2.46.0"        # third-party logger for log-takeover tests
 
 [tool.poetry.group.lint.dependencies]
 pyright = "1.1.385"