llama-stack 0.4.3__py3-none-any.whl → 0.5.0rc1__py3-none-any.whl

llama_stack/providers/utils/inference/http_client.py

@@ -0,0 +1,239 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+import ssl
+from pathlib import Path
+from typing import Any
+
+import httpx
+from openai._base_client import DefaultAsyncHttpxClient
+
+from llama_stack.log import get_logger
+from llama_stack.providers.utils.inference.model_registry import (
+    NetworkConfig,
+    ProxyConfig,
+    TimeoutConfig,
+    TLSConfig,
+)
+
+logger = get_logger(name=__name__, category="providers::utils")
+
+
+def _build_ssl_context(tls_config: TLSConfig) -> ssl.SSLContext | bool | Path:
+    """
+    Build an SSL context from TLS configuration.
+
+    Returns:
+        - ssl.SSLContext if advanced options (min_version, ciphers, or mTLS) are configured
+        - Path if only a CA bundle path is specified
+        - bool if only verify is specified as boolean
+    """
+    has_advanced_options = (
+        tls_config.min_version is not None or tls_config.ciphers is not None or tls_config.client_cert is not None
+    )
+
+    if not has_advanced_options:
+        return tls_config.verify
+
+    ctx = ssl.create_default_context()
+
+    if isinstance(tls_config.verify, Path):
+        ctx.load_verify_locations(str(tls_config.verify))
+    elif not tls_config.verify:
+        ctx.check_hostname = False
+        ctx.verify_mode = ssl.CERT_NONE
+
+    if tls_config.min_version:
+        if tls_config.min_version == "TLSv1.2":
+            ctx.minimum_version = ssl.TLSVersion.TLSv1_2
+        elif tls_config.min_version == "TLSv1.3":
+            ctx.minimum_version = ssl.TLSVersion.TLSv1_3
+
+    if tls_config.ciphers:
+        ctx.set_ciphers(":".join(tls_config.ciphers))
+
+    if tls_config.client_cert and tls_config.client_key:
+        ctx.load_cert_chain(certfile=str(tls_config.client_cert), keyfile=str(tls_config.client_key))
+
+    return ctx
+
+
+def _build_proxy_mounts(proxy_config: ProxyConfig) -> dict[str, httpx.AsyncHTTPTransport] | None:
+    """
+    Build httpx proxy mounts from proxy configuration.
+
+    Returns:
+        Dictionary of proxy mounts for httpx, or None if no proxies configured
+    """
+    transport_kwargs: dict[str, Any] = {}
+    if proxy_config.cacert:
+        # Convert Path to string for httpx
+        transport_kwargs["verify"] = str(proxy_config.cacert)
+
+    if proxy_config.url:
+        # Convert HttpUrl to string for httpx
+        proxy_url = str(proxy_config.url)
+        return {
+            "http://": httpx.AsyncHTTPTransport(proxy=proxy_url, **transport_kwargs),
+            "https://": httpx.AsyncHTTPTransport(proxy=proxy_url, **transport_kwargs),
+        }
+
+    mounts = {}
+    if proxy_config.http:
+        mounts["http://"] = httpx.AsyncHTTPTransport(proxy=str(proxy_config.http), **transport_kwargs)
+    if proxy_config.https:
+        mounts["https://"] = httpx.AsyncHTTPTransport(proxy=str(proxy_config.https), **transport_kwargs)
+
+    return mounts if mounts else None
+
+
+def _build_network_client_kwargs(network_config: NetworkConfig | None) -> dict[str, Any]:
+    """
+    Build httpx.AsyncClient kwargs from network configuration.
+
+    This function creates the appropriate kwargs to pass to httpx.AsyncClient
+    based on the provided NetworkConfig, without creating the client itself.
+
+    Args:
+        network_config: Network configuration including TLS, proxy, and timeout settings
+
+    Returns:
+        Dictionary of kwargs to pass to httpx.AsyncClient constructor
+    """
+    if network_config is None:
+        return {}
+
+    client_kwargs: dict[str, Any] = {}
+
+    if network_config.tls:
+        ssl_context = _build_ssl_context(network_config.tls)
+        client_kwargs["verify"] = ssl_context
+
+    if network_config.proxy:
+        mounts = _build_proxy_mounts(network_config.proxy)
+        if mounts:
+            client_kwargs["mounts"] = mounts
+
+    if network_config.timeout is not None:
+        if isinstance(network_config.timeout, TimeoutConfig):
+            # httpx.Timeout requires all four parameters (connect, read, write, pool)
+            # to be set explicitly, or a default timeout value
+            timeout_kwargs: dict[str, float | None] = {
+                "connect": network_config.timeout.connect,
+                "read": network_config.timeout.read,
+                "write": None,
+                "pool": None,
+            }
+            client_kwargs["timeout"] = httpx.Timeout(**timeout_kwargs)
+        else:
+            client_kwargs["timeout"] = httpx.Timeout(network_config.timeout)
+
+    if network_config.headers:
+        client_kwargs["headers"] = network_config.headers
+
+    return client_kwargs
+
+
+def _extract_client_config(existing_client: httpx.AsyncClient | DefaultAsyncHttpxClient) -> dict[str, Any]:
+    """
+    Extract configuration (auth, headers) from an existing http_client.
+
+    Args:
+        existing_client: Existing httpx client (may be DefaultAsyncHttpxClient)
+
+    Returns:
+        Dictionary with extracted auth and headers, if available
+    """
+    config: dict[str, Any] = {}
+
+    # Extract from DefaultAsyncHttpxClient
+    if isinstance(existing_client, DefaultAsyncHttpxClient):
+        underlying_client = existing_client._client  # type: ignore[union-attr,attr-defined]
+        if hasattr(underlying_client, "_auth"):
+            config["auth"] = underlying_client._auth  # type: ignore[attr-defined]
+        if hasattr(existing_client, "_headers"):
+            config["headers"] = existing_client._headers  # type: ignore[attr-defined]
+    else:
+        # Extract from plain httpx.AsyncClient
+        if hasattr(existing_client, "_auth"):
+            config["auth"] = existing_client._auth  # type: ignore[attr-defined]
+        if hasattr(existing_client, "_headers"):
+            config["headers"] = existing_client._headers  # type: ignore[attr-defined]
+
+    return config
+
+
+def _merge_network_config_into_client(
+    existing_client: httpx.AsyncClient | DefaultAsyncHttpxClient, network_config: NetworkConfig | None
+) -> httpx.AsyncClient | DefaultAsyncHttpxClient:
+    """
+    Merge network configuration into an existing http_client.
+
+    Extracts auth and headers from the existing client, merges with network config,
+    and creates a new client with all settings combined.
+
+    Args:
+        existing_client: Existing httpx client (may be DefaultAsyncHttpxClient)
+        network_config: Network configuration to apply
+
+    Returns:
+        New client with network config applied, or original client if merge fails
+    """
+    if network_config is None:
+        return existing_client
+
+    network_kwargs = _build_network_client_kwargs(network_config)
+    if not network_kwargs:
+        return existing_client
+
+    try:
+        # Extract existing client config (auth, headers)
+        existing_config = _extract_client_config(existing_client)
+
+        # Merge headers: existing headers first, then network config (network takes precedence)
+        if existing_config.get("headers") and network_kwargs.get("headers"):
+            merged_headers = dict(existing_config["headers"])
+            merged_headers.update(network_kwargs["headers"])
+            network_kwargs["headers"] = merged_headers
+        elif existing_config.get("headers"):
+            network_kwargs["headers"] = existing_config["headers"]
+
+        # Preserve auth from existing client
+        if existing_config.get("auth"):
+            network_kwargs["auth"] = existing_config["auth"]
+
+        # Create new client with merged config
+        new_client = httpx.AsyncClient(**network_kwargs)
+
+        # If original was DefaultAsyncHttpxClient, wrap the new client
+        if isinstance(existing_client, DefaultAsyncHttpxClient):
+            return DefaultAsyncHttpxClient(client=new_client, headers=network_kwargs.get("headers"))  # type: ignore[call-arg]
+
+        return new_client
+    except Exception as e:
+        logger.debug(f"Could not merge network config into existing http_client: {e}. Using original client.")
+        return existing_client
+
+
+def build_http_client(network_config: NetworkConfig | None) -> dict[str, Any]:
+    """
+    Build httpx.AsyncClient parameters from network configuration.
+
+    This function creates the appropriate kwargs to pass to httpx.AsyncClient
+    based on the provided NetworkConfig.
+
+    Args:
+        network_config: Network configuration including TLS, proxy, and timeout settings
+
+    Returns:
+        Dictionary of kwargs to pass to httpx.AsyncClient constructor,
+        wrapped in {"http_client": AsyncClient(...)} for use with AsyncOpenAI
+    """
+    network_kwargs = _build_network_client_kwargs(network_config)
+    if not network_kwargs:
+        return {}
+
+    return {"http_client": httpx.AsyncClient(**network_kwargs)}

llama_stack/providers/utils/inference/litellm_openai_mixin.py

@@ -30,6 +30,7 @@ from llama_stack_api import (
     OpenAIEmbeddingsRequestWithExtraBody,
     OpenAIEmbeddingsResponse,
     OpenAIEmbeddingUsage,
+    validate_embeddings_input_is_text,
 )
 
 logger = get_logger(name=__name__, category="providers::utils")
@@ -146,6 +147,9 @@ class LiteLLMOpenAIMixin(
         self,
         params: OpenAIEmbeddingsRequestWithExtraBody,
     ) -> OpenAIEmbeddingsResponse:
+        # Validate that input contains only text, not token arrays
+        validate_embeddings_input_is_text(params)
+
         if not self.model_store:
             raise ValueError("Model store is not initialized")
 
@@ -270,6 +274,7 @@ class LiteLLMOpenAIMixin(
             top_logprobs=params.top_logprobs,
             top_p=params.top_p,
             user=params.user,
+            reasoning_effort=params.reasoning_effort,
             api_key=self.get_api_key(),
             api_base=self.api_base,
             **self._litellm_extra_request_params(params),

llama_stack/providers/utils/inference/model_registry.py

@@ -4,9 +4,10 @@
 # This source code is licensed under the terms described in the LICENSE file in
 # the root directory of this source tree.
 
-from typing import Any
+from pathlib import Path
+from typing import Any, Literal
 
-from pydantic import BaseModel, Field, SecretStr
+from pydantic import BaseModel, Field, HttpUrl, SecretStr, field_validator, model_validator
 
 from llama_stack.log import get_logger
 from llama_stack.providers.utils.inference import (
@@ -17,6 +18,147 @@ from llama_stack_api import Model, ModelsProtocolPrivate, ModelType, Unsupported
 logger = get_logger(name=__name__, category="providers::utils")
 
 
+class TLSConfig(BaseModel):
+    """TLS/SSL configuration for secure connections."""
+
+    verify: bool | Path = Field(
+        default=True,
+        description="Whether to verify TLS certificates. Can be a boolean or a path to a CA certificate file.",
+    )
+    min_version: Literal["TLSv1.2", "TLSv1.3"] | None = Field(
+        default=None,
+        description="Minimum TLS version to use. Defaults to system default if not specified.",
+    )
+    ciphers: list[str] | None = Field(
+        default=None,
+        description="List of allowed cipher suites (e.g., ['ECDHE+AESGCM', 'DHE+AESGCM']).",
+    )
+    client_cert: Path | None = Field(
+        default=None,
+        description="Path to client certificate file for mTLS authentication.",
+    )
+    client_key: Path | None = Field(
+        default=None,
+        description="Path to client private key file for mTLS authentication.",
+    )
+
+    @field_validator("verify", mode="before")
+    @classmethod
+    def validate_verify(cls, v: bool | str | Path) -> bool | Path:
+        if isinstance(v, bool):
+            return v
+        if isinstance(v, str):
+            cert_path = Path(v).expanduser().resolve()
+        else:
+            cert_path = v.expanduser().resolve()
+        if not cert_path.exists():
+            raise ValueError(f"TLS certificate file does not exist: {v}")
+        if not cert_path.is_file():
+            raise ValueError(f"TLS certificate path is not a file: {v}")
+        return cert_path
+
+    @field_validator("client_cert", "client_key", mode="before")
+    @classmethod
+    def validate_cert_paths(cls, v: str | Path | None) -> Path | None:
+        if v is None:
+            return None
+        if isinstance(v, str):
+            cert_path = Path(v).expanduser().resolve()
+        else:
+            cert_path = v.expanduser().resolve()
+        if not cert_path.exists():
+            raise ValueError(f"Certificate/key file does not exist: {v}")
+        if not cert_path.is_file():
+            raise ValueError(f"Certificate/key path is not a file: {v}")
+        return cert_path
+
+    @model_validator(mode="after")
+    def validate_mtls_pair(self) -> "TLSConfig":
+        if (self.client_cert is None) != (self.client_key is None):
+            raise ValueError("Both client_cert and client_key must be provided together for mTLS")
+        return self
+
+
+class ProxyConfig(BaseModel):
+    """Proxy configuration for HTTP connections."""
+
+    url: HttpUrl | None = Field(
+        default=None,
+        description="Single proxy URL for all connections (e.g., 'http://proxy.example.com:8080').",
+    )
+    http: HttpUrl | None = Field(
+        default=None,
+        description="Proxy URL for HTTP connections.",
+    )
+    https: HttpUrl | None = Field(
+        default=None,
+        description="Proxy URL for HTTPS connections.",
+    )
+    cacert: Path | None = Field(
+        default=None,
+        description="Path to CA certificate file for verifying the proxy's certificate. Required for proxies in interception mode.",
+    )
+    no_proxy: list[str] | None = Field(
+        default=None,
+        description="List of hosts that should bypass the proxy (e.g., ['localhost', '127.0.0.1', '.internal.corp']).",
+    )
+
+    @field_validator("cacert", mode="before")
+    @classmethod
+    def validate_cacert(cls, v: str | Path | None) -> Path | None:
+        if v is None:
+            return None
+        if isinstance(v, str):
+            cert_path = Path(v).expanduser().resolve()
+        else:
+            cert_path = v.expanduser().resolve()
+        if not cert_path.exists():
+            raise ValueError(f"Proxy CA certificate file does not exist: {v}")
+        if not cert_path.is_file():
+            raise ValueError(f"Proxy CA certificate path is not a file: {v}")
+        return cert_path
+
+    @model_validator(mode="after")
+    def validate_proxy_config(self) -> "ProxyConfig":
+        if self.url and (self.http or self.https):
+            raise ValueError("Cannot specify both 'url' and 'http'/'https' proxy settings")
+        return self
+
+
+class TimeoutConfig(BaseModel):
+    """Timeout configuration for HTTP connections."""
+
+    connect: float | None = Field(
+        default=None,
+        description="Connection timeout in seconds.",
+    )
+    read: float | None = Field(
+        default=None,
+        description="Read timeout in seconds.",
+    )
+
+
+class NetworkConfig(BaseModel):
+    """Network configuration for remote provider connections."""
+
+    tls: TLSConfig | None = Field(
+        default=None,
+        description="TLS/SSL configuration for secure connections.",
+    )
+    proxy: ProxyConfig | None = Field(
+        default=None,
+        description="Proxy configuration for HTTP connections.",
+    )
+    timeout: float | TimeoutConfig | None = Field(
+        default=None,
+        description="Timeout configuration. Can be a float (for both connect and read) or a TimeoutConfig object with separate connect and read timeouts.",
+    )
+    headers: dict[str, str] | None = Field(
+        default=None,
+        description="Additional HTTP headers to include in all requests.",
+    )
+
+
 class RemoteInferenceProviderConfig(BaseModel):
     allowed_models: list[str] | None = Field(
         default=None,
@@ -31,6 +173,10 @@ class RemoteInferenceProviderConfig(BaseModel):
         description="Authentication credential for the provider",
         alias="api_key",
     )
+    network: NetworkConfig | None = Field(
+        default=None,
+        description="Network configuration including TLS, proxy, and timeout settings.",
+    )
 
 
 # TODO: this class is more confusing than useful right now. We need to make it

llama_stack/providers/utils/inference/openai_compat.py

@@ -19,6 +19,7 @@ from llama_stack.models.llama.datatypes import (
     ToolCall,
     ToolDefinition,
 )
+from llama_stack_api import OpenAIFinishReason
 
 logger = get_logger(name=__name__, category="providers::utils")
 
@@ -38,7 +39,7 @@ class OpenAICompatLogprobs(BaseModel):
 
 
 class OpenAICompatCompletionChoice(BaseModel):
-    finish_reason: str | None = None
+    finish_reason: OpenAIFinishReason | None = None
     text: str | None = None
     delta: OpenAICompatCompletionChoiceDelta | None = None
     logprobs: OpenAICompatLogprobs | None = None

llama_stack/providers/utils/inference/openai_mixin.py

@@ -10,11 +10,16 @@ from abc import ABC, abstractmethod
 from collections.abc import AsyncIterator, Iterable
 from typing import Any
 
+import httpx
 from openai import AsyncOpenAI
 from pydantic import BaseModel, ConfigDict
 
 from llama_stack.core.request_headers import NeedsRequestProviderData
 from llama_stack.log import get_logger
+from llama_stack.providers.utils.inference.http_client import (
+    _build_network_client_kwargs,
+    _merge_network_config_into_client,
+)
 from llama_stack.providers.utils.inference.model_registry import RemoteInferenceProviderConfig
 from llama_stack.providers.utils.inference.openai_compat import (
     get_stream_options_for_telemetry,
@@ -34,6 +39,7 @@ from llama_stack_api import (
     OpenAIEmbeddingsResponse,
     OpenAIEmbeddingUsage,
     OpenAIMessageParam,
+    validate_embeddings_input_is_text,
 )
 
 logger = get_logger(name=__name__, category="providers::utils")
@@ -82,6 +88,10 @@ class OpenAIMixin(NeedsRequestProviderData, ABC, BaseModel):
     # Set to False for providers that don't support stream_options (e.g., Ollama, vLLM)
     supports_stream_options: bool = True
 
+    # Allow subclasses to control whether the provider supports tokenized embeddings input
+    # Set to True for providers that support pre-tokenized input (list[int] and list[list[int]])
+    supports_tokenized_embeddings_input: bool = False
+
     # Embedding model metadata for this provider
     # Can be set by subclasses or instances to provide embedding models
     # Format: {"model_id": {"embedding_dimension": 1536, "context_length": 8192}}
@@ -121,7 +131,10 @@ class OpenAIMixin(NeedsRequestProviderData, ABC, BaseModel):
         Get any extra parameters to pass to the AsyncOpenAI client.
 
         Child classes can override this method to provide additional parameters
-        such as timeout settings, proxies, etc.
+        such as custom http_client, timeout settings, proxies, etc.
+
+        Note: Network configuration from config.network is automatically applied
+        in the client property. This method is for provider-specific customizations.
 
         :return: A dictionary of extra parameters
         """
@@ -194,6 +207,7 @@ class OpenAIMixin(NeedsRequestProviderData, ABC, BaseModel):
         Uses the abstract methods get_api_key() and get_base_url() which must be
         implemented by child classes.
 
+        Network configuration from config.network is automatically applied.
         Users can also provide the API key via the provider data header, which
         is used instead of any config API key.
         """
@@ -205,10 +219,30 @@ class OpenAIMixin(NeedsRequestProviderData, ABC, BaseModel):
                 message += f' Please provide a valid API key in the provider data header, e.g. x-llamastack-provider-data: {{"{self.provider_data_api_key_field}": "<API_KEY>"}}.'
             raise ValueError(message)
 
+        extra_params = self.get_extra_client_params()
+        network_kwargs = _build_network_client_kwargs(self.config.network)
+
+        # Handle http_client creation/merging:
+        # - If get_extra_client_params() provides an http_client (e.g., OCI with custom auth),
+        #   merge network config into it. The merge behavior:
+        #   * Preserves auth from get_extra_client_params() (provider-specific auth like OCI signer)
+        #   * Preserves headers from get_extra_client_params() as base
+        #   * Applies network config (TLS, proxy, timeout, headers) on top
+        #   * Network config headers take precedence over provider headers (allows override)
+        # - Otherwise, if network config exists, create http_client from it
+        # This allows providers with custom auth to still use standard network settings
+        if "http_client" in extra_params:
+            if network_kwargs:
+                extra_params["http_client"] = _merge_network_config_into_client(
+                    extra_params["http_client"], self.config.network
+                )
+        elif network_kwargs:
+            extra_params["http_client"] = httpx.AsyncClient(**network_kwargs)
+
         return AsyncOpenAI(
             api_key=api_key,
             base_url=self.get_base_url(),
-            **self.get_extra_client_params(),
+            **extra_params,
         )
 
     def _get_api_key_from_config_or_provider_data(self) -> str | None:
@@ -371,6 +405,7 @@ class OpenAIMixin(NeedsRequestProviderData, ABC, BaseModel):
             top_logprobs=params.top_logprobs,
             top_p=params.top_p,
             user=params.user,
+            reasoning_effort=params.reasoning_effort,
         )
 
         if extra_body := params.model_extra:
@@ -386,6 +421,10 @@ class OpenAIMixin(NeedsRequestProviderData, ABC, BaseModel):
         """
         Direct OpenAI embeddings API call.
         """
+        # Validate token array support if provider doesn't support it
+        if not self.supports_tokenized_embeddings_input:
+            validate_embeddings_input_is_text(params)
+
         provider_model_id = await self._get_provider_model_id(params.model)
         self._validate_model_allowed(provider_model_id)