aidial-sdk 0.47.0.dev10__tar.gz → 0.48.0.dev2__tar.gz

{aidial_sdk-0.47.0.dev10 → aidial_sdk-0.48.0.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aidial-sdk
-Version: 0.47.0.dev10
+Version: 0.48.0.dev2
 Summary: Framework to create applications and model adapters for AI DIAL
 License-Expression: Apache-2.0
 License-File: LICENSE

{aidial_sdk-0.47.0.dev10 → aidial_sdk-0.48.0.dev2}/aidial_sdk/application.py

@@ -37,6 +37,10 @@ from aidial_sdk.header_propagator import HeaderPropagator
 from aidial_sdk.response_headers import NonForwardableHeadersMiddleware
 from aidial_sdk.telemetry.types import TelemetryConfig
 from aidial_sdk.utils._disconnect_middleware import DisconnectMiddleware
+from aidial_sdk.utils._interpolation import (
+    interpolate_deployment_id,
+    match_deployment_id,
+)
 from aidial_sdk.utils._reflection import get_method_implementation
 from aidial_sdk.utils.env import env_float, env_var_list
 from aidial_sdk.utils.log_config import configure_sdk_logger
@@ -59,16 +63,6 @@ RequestType = TypeVar("RequestType", bound=FromRequestMixin)
 Handler = Callable[[Request], Coroutine[Any, Any, Response]]
 
 
-def _interpolate_deployment_id(deployment_id: str, path_params: dict) -> str:
-    result = deployment_id
-    for key, value in path_params.items():
-        # Replace {key} or {key:type} patterns with actual value
-        # Pattern matches {key} or {key:int}, {key:path}, etc.
-        pattern = r"\{" + re.escape(key) + r"(?::[^}]*)?\}"
-        result = re.sub(pattern, str(value), result)
-    return result
-
-
 class PathFilter(logging.Filter):
     path: str
 
@@ -83,7 +77,20 @@ class PathFilter(logging.Filter):
 class DIALApp(FastAPI):
     _allow_extra_request_fields: bool
     _dial_url: str | None
-    _v1_handlers: dict[str, dict[str, Handler]]
+
+    _generic_handlers: dict[str, dict[str, Handler]]
+    """
+    Request handlers for endpoints that do not allow to distinguish deployments in the path.
+    A dictionary from such a deployment-id-less endpoint to the handlers for each deployment:
+
+    /openai/v1/chat/completions:
+        gpt-5.4: handler1
+        gpt-5.5: handler2
+    /openai/v1/embeddings:
+        ada-embeddings-2024-10-10: handler3
+    ...
+    """
+
     _reset_otel_context: Callable[[], AbstractContextManager[None]]
 
     def __init__(
@@ -110,7 +117,7 @@ class DIALApp(FastAPI):
 
         self._allow_extra_request_fields = allow_extra_request_fields
         self._dial_url = dial_url
-        self._v1_handlers = {}
+        self._generic_handlers = {}
         self._reset_otel_context = nullcontext
 
         self.configure_telemetry(telemetry_config)
@@ -181,9 +188,9 @@ class DIALApp(FastAPI):
             methods=["POST"],
         )
 
-        self._add_v1_route(
+        self._add_generic_route(
             deployment_name,
-            "embeddings",
+            "/openai/v1/embeddings",
             self._embeddings(None, impl),
             methods=["POST"],
         )
@@ -209,8 +216,11 @@ class DIALApp(FastAPI):
         for endpoint, handler, methods in self._chat_completion_handlers(
             None, impl, heartbeat_interval
         ):
-            self._add_v1_route(
-                deployment_name, endpoint, handler, methods=methods
+            self._add_generic_route(
+                deployment_name,
+                f"/openai/v1/{endpoint}",
+                handler,
+                methods=methods,
             )
 
         return self
@@ -264,7 +274,7 @@ class DIALApp(FastAPI):
                 ["GET"],
             )
 
-    def _add_v1_route(
+    def _add_generic_route(
         self,
         deployment_name: str,
         endpoint: str,
@@ -272,28 +282,25 @@ class DIALApp(FastAPI):
         *,
         methods: list[str],
     ) -> None:
-        """The /openai/v1 endpoints are shared by all the deployments of
-        the application, so they are dispatched by the deployment id
-        resolved from the request headers."""
-
-        if endpoint not in self._v1_handlers:
-            self._v1_handlers[endpoint] = {}
+        if endpoint not in self._generic_handlers:
+            self._generic_handlers[endpoint] = {}
             self.add_api_route(
-                f"/openai/v1/{endpoint}",
-                self._v1_dispatcher(endpoint),
+                endpoint,
+                self._generic_dispatcher(endpoint),
                 methods=methods,
             )
+        self._generic_handlers[endpoint][deployment_name] = handler
 
-        self._v1_handlers[endpoint][deployment_name] = handler
-
-    def _v1_dispatcher(self, endpoint: str) -> Handler:
+    def _generic_dispatcher(self, endpoint: str) -> Handler:
         async def _handler(original_request: Request) -> Response:
             deployment_id = get_upstream_deployment_id(
                 original_request.headers, None
             )
             set_log_deployment(deployment_id)
 
-            handler = self._v1_handlers[endpoint].get(deployment_id)
+            handler = self._get_handler(
+                endpoint, deployment_id, original_request
+            )
             if handler is None:
                 raise DeploymentNotFoundError(
                     f"The deployment {deployment_id!r} doesn't provide the {endpoint!r} endpoint"
@@ -303,6 +310,22 @@ class DIALApp(FastAPI):
 
         return _handler
 
+    def _get_handler(
+        self, endpoint: str, deployment_id: str, request: Request
+    ) -> Handler | None:
+        handlers = self._generic_handlers[endpoint]
+
+        if (handler := handlers.get(deployment_id)) is not None:
+            return handler
+
+        for name, handler in handlers.items():
+            params = match_deployment_id(name, deployment_id)
+            if params is not None:
+                request.scope["path_params"] = params
+                return handler
+
+        return None
+
     def _endpoint_factory(
         self,
         deployment_id: str | None,
@@ -351,7 +374,7 @@ class DIALApp(FastAPI):
         deployment_id: str | None,
     ) -> RequestType:
         if deployment_id is not None:
-            deployment_id = _interpolate_deployment_id(
+            deployment_id = interpolate_deployment_id(
                 deployment_id, original_request.path_params
             )
             set_log_deployment(deployment_id)

{aidial_sdk-0.47.0.dev10 → aidial_sdk-0.48.0.dev2}/aidial_sdk/translators/messages_to_chat_completions/__init__.py

@@ -1,4 +1,3 @@
-import httpx
 from fastapi import FastAPI
 
 from aidial_sdk._pydantic import PYDANTIC_V2
@@ -9,10 +8,18 @@ if not PYDANTIC_V2:
         "install pydantic>=2 and set PYDANTIC_V2=True"
     )
 
-from aidial_sdk.translators.messages_to_chat_completions.chat_completions.app import (  # noqa: E402
-    ClientFactory,
-    create_anthropic_api_app,
-)
+try:
+    import httpx
+
+    from aidial_sdk.translators.messages_to_chat_completions.chat_completions.app import (
+        ClientFactory,
+        create_anthropic_api_app,
+    )
+except ModuleNotFoundError as e:
+    raise ImportError(
+        "The Anthropic Messages API translator requires the translators extra: "
+        'pip install "aidial-sdk[translators]"'
+    ) from e
 
 __all__ = ["ClientFactory", "create_anthropic_api_app", "mount_translator"]
 

aidial_sdk-0.47.0.dev10/aidial_sdk/translators/messages_to_chat_completions/core_client.py → aidial_sdk-0.48.0.dev2/aidial_sdk/translators/messages_to_chat_completions/backend/http.py

@@ -1,8 +1,10 @@
+from dataclasses import dataclass
 from urllib.parse import quote
 
 import httpx
 from fastapi import Request
-from openai import AsyncOpenAI, Omit, omit
+from openai import AsyncOpenAI, AsyncStream, Omit, omit
+from openai.types.chat import ChatCompletion, ChatCompletionChunk
 
 from aidial_sdk.translators.messages_to_chat_completions.settings import (
     get_dial_api_version,
@@ -62,3 +64,29 @@ def core_headers(request: Request) -> dict[str, str | Omit]:
         }
     )
     return headers
+
+
+@dataclass
+class ChatClientHttp:
+    client: AsyncOpenAI
+    core_headers: dict[str, str | Omit]
+
+    async def create_stream(
+        self, body: dict
+    ) -> AsyncStream[ChatCompletionChunk]:
+        return await self.client.post(
+            "/chat/completions",
+            cast_to=ChatCompletion,
+            body=body,
+            options={"headers": self.core_headers},
+            stream=True,
+            stream_cls=AsyncStream[ChatCompletionChunk],
+        )
+
+    async def create_block(self, body: dict) -> ChatCompletion:
+        return await self.client.post(
+            "/chat/completions",
+            cast_to=ChatCompletion,
+            body=body,
+            options={"headers": self.core_headers},
+        )

{aidial_sdk-0.47.0.dev10 → aidial_sdk-0.48.0.dev2}/aidial_sdk/translators/messages_to_chat_completions/chat_completions/app.py

@@ -7,9 +7,14 @@ import httpx
 from anthropic.types.beta.message_create_params import MessageCreateParams
 from fastapi import FastAPI, Request
 from fastapi.responses import JSONResponse, Response, StreamingResponse
-from openai import AsyncStream
-from openai.types.chat import ChatCompletion, ChatCompletionChunk
 
+from aidial_sdk.deployment.from_request_mixin import get_client_deployment_id
+from aidial_sdk.translators.messages_to_chat_completions.backend.http import (
+    ChatClientHttp,
+    core_headers,
+    core_legacy_client,
+    core_v1_client,
+)
 from aidial_sdk.translators.messages_to_chat_completions.chat_completions.from_chat_completions import (
     to_anthropic_response,
 )
@@ -23,11 +28,6 @@ from aidial_sdk.translators.messages_to_chat_completions.common import (
     not_found,
     with_debug_logging,
 )
-from aidial_sdk.translators.messages_to_chat_completions.core_client import (
-    core_headers,
-    core_legacy_client,
-    core_v1_client,
-)
 from aidial_sdk.translators.messages_to_chat_completions.errors import (
     AnthropicErrorType,
     AnthropicHTTPError,
@@ -45,13 +45,10 @@ ClientT = TypeVar("ClientT")
 ClientFactory = Callable[[Request], Awaitable[ClientT]] | ClientT
 
 
-async def _handle_messages(
-    request: Request,
-    get_client: ClientFactory[httpx.AsyncClient],
-    legacy: bool,
+async def handle_anthropic_messages(
+    request: Request, client: ChatClientHttp
 ) -> Response:
-    dial_url = get_dial_url()
-    translation_state = TranslationState()
+    deployment = get_client_deployment_id(request.headers)
 
     try:
         req: MessageCreateParams = await request.json()
@@ -61,7 +58,7 @@ async def _handle_messages(
             f"Request body is not valid JSON: {e}",
         ) from e
 
-    deployment = request.headers.get("x-dial-deployment-id") or req["model"]
+    translation_state = TranslationState()
     chat_request = to_chat_completions_request(
         req, deployment, translation_state
     )
@@ -70,38 +67,14 @@ async def _handle_messages(
     if log.isEnabledFor(logging.DEBUG):
         log.debug(f"Translated request to Core: {json.dumps(body)}")
 
-    http_client = (
-        get_client
-        if isinstance(get_client, httpx.AsyncClient)
-        else await get_client(request)
-    )
-    client = (
-        core_legacy_client(dial_url, deployment, http_client)
-        if legacy
-        else core_v1_client(dial_url, http_client)
-    )
-    headers = core_headers(request)
-
     if chat_request.stream:
-        events = await client.post(
-            "/chat/completions",
-            cast_to=ChatCompletion,
-            body=body,
-            options={"headers": headers},
-            stream=True,
-            stream_cls=AsyncStream[ChatCompletionChunk],
-        )
+        events = await client.create_stream(body)
         return StreamingResponse(
             translate_stream(events, deployment, translation_state),
             media_type="text/event-stream",
         )
 
-    response = await client.post(
-        "/chat/completions",
-        cast_to=ChatCompletion,
-        body=body,
-        options={"headers": headers},
-    )
+    response = await client.create_block(body)
     message = to_anthropic_response(response, deployment, translation_state)
     return JSONResponse(
         content=message.model_dump(mode="json", exclude_unset=True)
@@ -113,17 +86,34 @@ def create_anthropic_api_app(
 ) -> FastAPI:
     app = FastAPI()
 
+    async def _chat_client(request: Request, *, legacy: bool) -> ChatClientHttp:
+        http_client = (
+            await get_client(request) if callable(get_client) else get_client
+        )
+        dial_url = get_dial_url()
+        if legacy:
+            deployment_id = get_client_deployment_id(request.headers)
+            client = core_legacy_client(dial_url, deployment_id, http_client)
+        else:
+            client = core_v1_client(dial_url, http_client)
+
+        return ChatClientHttp(client, core_headers(request))
+
     @app.post("/anthropic/v1/messages")
     @with_debug_logging
     @translator_error_handler
     async def _handle_v1_messages(request: Request) -> Response:
-        return await _handle_messages(request, get_client, legacy=False)
+        return await handle_anthropic_messages(
+            request, await _chat_client(request, legacy=False)
+        )
 
     @app.post("/legacy/anthropic/v1/messages")
     @with_debug_logging
     @translator_error_handler
     async def _handle_legacy_messages(request: Request) -> Response:
-        return await _handle_messages(request, get_client, legacy=True)
+        return await handle_anthropic_messages(
+            request, await _chat_client(request, legacy=True)
+        )
 
     app.add_api_route(
         "/{full_path:path}",

aidial_sdk-0.48.0.dev2/aidial_sdk/utils/_interpolation.py

@@ -0,0 +1,55 @@
+"""Deployment names may be templates, e.g. "app-{idx}" or "app-{idx:int}",
+in the same syntax Starlette uses for the path params."""
+
+from collections.abc import Callable
+from functools import cache
+from re import Match
+from typing import Any
+
+from starlette.convertors import Convertor
+from starlette.routing import compile_path
+
+
+@cache
+def _compile_path_template(
+    name: str,
+) -> tuple[Callable[[str], Match[str] | None], str, dict[str, Convertor[Any]]]:
+    # compile_path reads a pattern with no leading slash as a *host* pattern,
+    # which it truncates at ":", so the name is compiled as a path instead.
+    regex, path_format, convertors = compile_path("/" + name)
+
+    def matcher(deployment_id: str) -> Match[str] | None:
+        return regex.match("/" + deployment_id)
+
+    return matcher, path_format.removeprefix("/"), convertors
+
+
+def match_deployment_id(
+    template: str, deployment_id: str
+) -> dict[str, Any] | None:
+    """Matches a deployment id against a deployment name template
+    (e.g. "app-{idx}"), returning the path params it resolves to,
+    or None if the name isn't a template or doesn't match."""
+
+    if "{" not in template:
+        return None
+
+    matcher, _path_format, convertors = _compile_path_template(template)
+
+    if (match := matcher(deployment_id)) is None:
+        return None
+
+    return {
+        key: convertors[key].convert(value)
+        for key, value in match.groupdict().items()
+    }
+
+
+def interpolate_deployment_id(deployment_id: str, path_params: dict) -> str:
+    """Fills a deployment name template (e.g. "app-{idx:int}") with the
+    path params the request it came in on was routed by."""
+
+    _matcher, path_format, convertors = _compile_path_template(deployment_id)
+
+    # the format drops the convertor types, so "app-{idx:int}" -> "app-{idx}"
+    return path_format.format(**path_params) if convertors else deployment_id

{aidial_sdk-0.47.0.dev10 → aidial_sdk-0.48.0.dev2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "aidial-sdk"
-version = "0.47.0.dev10"
+version = "0.48.0.dev2"
 description = "Framework to create applications and model adapters for AI DIAL"
 authors = [{ name = "EPAM RAIL", email = "SpecialEPM-DIALDevTeam@epam.com" }]
 license = "Apache-2.0"
@@ -52,7 +52,6 @@ aiointercept = "^0.1.7"
 requests = "^2.33"
 responses = "^0.25.3"
 httpx2 = "^2.12.0"        # required for Starlette.TestClient
-openai = "^2.46.0"        # third-party logger for log-takeover tests
 
 [tool.poetry.group.lint.dependencies]
 pyright = "1.1.385"