llama-stack 0.4.3__py3-none-any.whl → 0.5.0__py3-none-any.whl

llama_stack/providers/utils/inference/http_client.py

@@ -0,0 +1,239 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+import ssl
+from pathlib import Path
+from typing import Any
+
+import httpx
+from openai._base_client import DefaultAsyncHttpxClient
+
+from llama_stack.log import get_logger
+from llama_stack.providers.utils.inference.model_registry import (
+    NetworkConfig,
+    ProxyConfig,
+    TimeoutConfig,
+    TLSConfig,
+)
+
+logger = get_logger(name=__name__, category="providers::utils")
+
+
+def _build_ssl_context(tls_config: TLSConfig) -> ssl.SSLContext | bool | Path:
+    """
+    Build an SSL context from TLS configuration.
+
+    Returns:
+        - ssl.SSLContext if advanced options (min_version, ciphers, or mTLS) are configured
+        - Path if only a CA bundle path is specified
+        - bool if only verify is specified as boolean
+    """
+    has_advanced_options = (
+        tls_config.min_version is not None or tls_config.ciphers is not None or tls_config.client_cert is not None
+    )
+
+    if not has_advanced_options:
+        return tls_config.verify
+
+    ctx = ssl.create_default_context()
+
+    if isinstance(tls_config.verify, Path):
+        ctx.load_verify_locations(str(tls_config.verify))
+    elif not tls_config.verify:
+        ctx.check_hostname = False
+        ctx.verify_mode = ssl.CERT_NONE
+
+    if tls_config.min_version:
+        if tls_config.min_version == "TLSv1.2":
+            ctx.minimum_version = ssl.TLSVersion.TLSv1_2
+        elif tls_config.min_version == "TLSv1.3":
+            ctx.minimum_version = ssl.TLSVersion.TLSv1_3
+
+    if tls_config.ciphers:
+        ctx.set_ciphers(":".join(tls_config.ciphers))
+
+    if tls_config.client_cert and tls_config.client_key:
+        ctx.load_cert_chain(certfile=str(tls_config.client_cert), keyfile=str(tls_config.client_key))
+
+    return ctx
+
+
+def _build_proxy_mounts(proxy_config: ProxyConfig) -> dict[str, httpx.AsyncHTTPTransport] | None:
+    """
+    Build httpx proxy mounts from proxy configuration.
+
+    Returns:
+        Dictionary of proxy mounts for httpx, or None if no proxies configured
+    """
+    transport_kwargs: dict[str, Any] = {}
+    if proxy_config.cacert:
+        # Convert Path to string for httpx
+        transport_kwargs["verify"] = str(proxy_config.cacert)
+
+    if proxy_config.url:
+        # Convert HttpUrl to string for httpx
+        proxy_url = str(proxy_config.url)
+        return {
+            "http://": httpx.AsyncHTTPTransport(proxy=proxy_url, **transport_kwargs),
+            "https://": httpx.AsyncHTTPTransport(proxy=proxy_url, **transport_kwargs),
+        }
+
+    mounts = {}
+    if proxy_config.http:
+        mounts["http://"] = httpx.AsyncHTTPTransport(proxy=str(proxy_config.http), **transport_kwargs)
+    if proxy_config.https:
+        mounts["https://"] = httpx.AsyncHTTPTransport(proxy=str(proxy_config.https), **transport_kwargs)
+
+    return mounts if mounts else None
+
+
+def _build_network_client_kwargs(network_config: NetworkConfig | None) -> dict[str, Any]:
+    """
+    Build httpx.AsyncClient kwargs from network configuration.
+
+    This function creates the appropriate kwargs to pass to httpx.AsyncClient
+    based on the provided NetworkConfig, without creating the client itself.
+
+    Args:
+        network_config: Network configuration including TLS, proxy, and timeout settings
+
+    Returns:
+        Dictionary of kwargs to pass to httpx.AsyncClient constructor
+    """
+    if network_config is None:
+        return {}
+
+    client_kwargs: dict[str, Any] = {}
+
+    if network_config.tls:
+        ssl_context = _build_ssl_context(network_config.tls)
+        client_kwargs["verify"] = ssl_context
+
+    if network_config.proxy:
+        mounts = _build_proxy_mounts(network_config.proxy)
+        if mounts:
+            client_kwargs["mounts"] = mounts
+
+    if network_config.timeout is not None:
+        if isinstance(network_config.timeout, TimeoutConfig):
+            # httpx.Timeout requires all four parameters (connect, read, write, pool)
+            # to be set explicitly, or a default timeout value
+            timeout_kwargs: dict[str, float | None] = {
+                "connect": network_config.timeout.connect,
+                "read": network_config.timeout.read,
+                "write": None,
+                "pool": None,
+            }
+            client_kwargs["timeout"] = httpx.Timeout(**timeout_kwargs)
+        else:
+            client_kwargs["timeout"] = httpx.Timeout(network_config.timeout)
+
+    if network_config.headers:
+        client_kwargs["headers"] = network_config.headers
+
+    return client_kwargs
+
+
+def _extract_client_config(existing_client: httpx.AsyncClient | DefaultAsyncHttpxClient) -> dict[str, Any]:
+    """
+    Extract configuration (auth, headers) from an existing http_client.
+
+    Args:
+        existing_client: Existing httpx client (may be DefaultAsyncHttpxClient)
+
+    Returns:
+        Dictionary with extracted auth and headers, if available
+    """
+    config: dict[str, Any] = {}
+
+    # Extract from DefaultAsyncHttpxClient
+    if isinstance(existing_client, DefaultAsyncHttpxClient):
+        underlying_client = existing_client._client  # type: ignore[union-attr,attr-defined]
+        if hasattr(underlying_client, "_auth"):
+            config["auth"] = underlying_client._auth  # type: ignore[attr-defined]
+        if hasattr(existing_client, "_headers"):
+            config["headers"] = existing_client._headers  # type: ignore[attr-defined]
+    else:
+        # Extract from plain httpx.AsyncClient
+        if hasattr(existing_client, "_auth"):
+            config["auth"] = existing_client._auth  # type: ignore[attr-defined]
+        if hasattr(existing_client, "_headers"):
+            config["headers"] = existing_client._headers  # type: ignore[attr-defined]
+
+    return config
+
+
+def _merge_network_config_into_client(
+    existing_client: httpx.AsyncClient | DefaultAsyncHttpxClient, network_config: NetworkConfig | None
+) -> httpx.AsyncClient | DefaultAsyncHttpxClient:
+    """
+    Merge network configuration into an existing http_client.
+
+    Extracts auth and headers from the existing client, merges with network config,
+    and creates a new client with all settings combined.
+
+    Args:
+        existing_client: Existing httpx client (may be DefaultAsyncHttpxClient)
+        network_config: Network configuration to apply
+
+    Returns:
+        New client with network config applied, or original client if merge fails
+    """
+    if network_config is None:
+        return existing_client
+
+    network_kwargs = _build_network_client_kwargs(network_config)
+    if not network_kwargs:
+        return existing_client
+
+    try:
+        # Extract existing client config (auth, headers)
+        existing_config = _extract_client_config(existing_client)
+
+        # Merge headers: existing headers first, then network config (network takes precedence)
+        if existing_config.get("headers") and network_kwargs.get("headers"):
+            merged_headers = dict(existing_config["headers"])
+            merged_headers.update(network_kwargs["headers"])
+            network_kwargs["headers"] = merged_headers
+        elif existing_config.get("headers"):
+            network_kwargs["headers"] = existing_config["headers"]
+
+        # Preserve auth from existing client
+        if existing_config.get("auth"):
+            network_kwargs["auth"] = existing_config["auth"]
+
+        # Create new client with merged config
+        new_client = httpx.AsyncClient(**network_kwargs)
+
+        # If original was DefaultAsyncHttpxClient, wrap the new client
+        if isinstance(existing_client, DefaultAsyncHttpxClient):
+            return DefaultAsyncHttpxClient(client=new_client, headers=network_kwargs.get("headers"))  # type: ignore[call-arg]
+
+        return new_client
+    except Exception as e:
+        logger.debug(f"Could not merge network config into existing http_client: {e}. Using original client.")
+        return existing_client
+
+
+def build_http_client(network_config: NetworkConfig | None) -> dict[str, Any]:
+    """
+    Build httpx.AsyncClient parameters from network configuration.
+
+    This function creates the appropriate kwargs to pass to httpx.AsyncClient
+    based on the provided NetworkConfig.
+
+    Args:
+        network_config: Network configuration including TLS, proxy, and timeout settings
+
+    Returns:
+        Dictionary of kwargs to pass to httpx.AsyncClient constructor,
+        wrapped in {"http_client": AsyncClient(...)} for use with AsyncOpenAI
+    """
+    network_kwargs = _build_network_client_kwargs(network_config)
+    if not network_kwargs:
+        return {}
+
+    return {"http_client": httpx.AsyncClient(**network_kwargs)}

llama_stack/providers/utils/inference/litellm_openai_mixin.py

@@ -30,6 +30,7 @@ from llama_stack_api import (
     OpenAIEmbeddingsRequestWithExtraBody,
     OpenAIEmbeddingsResponse,
     OpenAIEmbeddingUsage,
+    validate_embeddings_input_is_text,
 )
 
 logger = get_logger(name=__name__, category="providers::utils")
@@ -146,6 +147,9 @@ class LiteLLMOpenAIMixin(
         self,
         params: OpenAIEmbeddingsRequestWithExtraBody,
     ) -> OpenAIEmbeddingsResponse:
+        # Validate that input contains only text, not token arrays
+        validate_embeddings_input_is_text(params)
+
         if not self.model_store:
             raise ValueError("Model store is not initialized")
 
@@ -270,6 +274,8 @@ class LiteLLMOpenAIMixin(
             top_logprobs=params.top_logprobs,
             top_p=params.top_p,
             user=params.user,
+            safety_identifier=params.safety_identifier,
+            reasoning_effort=params.reasoning_effort,
             api_key=self.get_api_key(),
             api_base=self.api_base,
             **self._litellm_extra_request_params(params),

llama_stack/providers/utils/inference/model_registry.py

@@ -4,9 +4,10 @@
 # This source code is licensed under the terms described in the LICENSE file in
 # the root directory of this source tree.
 
-from typing import Any
+from pathlib import Path
+from typing import Any, Literal
 
-from pydantic import BaseModel, Field, SecretStr
+from pydantic import BaseModel, Field, HttpUrl, SecretStr, field_validator, model_validator
 
 from llama_stack.log import get_logger
 from llama_stack.providers.utils.inference import (
@@ -17,6 +18,147 @@ from llama_stack_api import Model, ModelsProtocolPrivate, ModelType, Unsupported
 logger = get_logger(name=__name__, category="providers::utils")
 
 
+class TLSConfig(BaseModel):
+    """TLS/SSL configuration for secure connections."""
+
+    verify: bool | Path = Field(
+        default=True,
+        description="Whether to verify TLS certificates. Can be a boolean or a path to a CA certificate file.",
+    )
+    min_version: Literal["TLSv1.2", "TLSv1.3"] | None = Field(
+        default=None,
+        description="Minimum TLS version to use. Defaults to system default if not specified.",
+    )
+    ciphers: list[str] | None = Field(
+        default=None,
+        description="List of allowed cipher suites (e.g., ['ECDHE+AESGCM', 'DHE+AESGCM']).",
+    )
+    client_cert: Path | None = Field(
+        default=None,
+        description="Path to client certificate file for mTLS authentication.",
+    )
+    client_key: Path | None = Field(
+        default=None,
+        description="Path to client private key file for mTLS authentication.",
+    )
+
+    @field_validator("verify", mode="before")
+    @classmethod
+    def validate_verify(cls, v: bool | str | Path) -> bool | Path:
+        if isinstance(v, bool):
+            return v
+        if isinstance(v, str):
+            cert_path = Path(v).expanduser().resolve()
+        else:
+            cert_path = v.expanduser().resolve()
+        if not cert_path.exists():
+            raise ValueError(f"TLS certificate file does not exist: {v}")
+        if not cert_path.is_file():
+            raise ValueError(f"TLS certificate path is not a file: {v}")
+        return cert_path
+
+    @field_validator("client_cert", "client_key", mode="before")
+    @classmethod
+    def validate_cert_paths(cls, v: str | Path | None) -> Path | None:
+        if v is None:
+            return None
+        if isinstance(v, str):
+            cert_path = Path(v).expanduser().resolve()
+        else:
+            cert_path = v.expanduser().resolve()
+        if not cert_path.exists():
+            raise ValueError(f"Certificate/key file does not exist: {v}")
+        if not cert_path.is_file():
+            raise ValueError(f"Certificate/key path is not a file: {v}")
+        return cert_path
+
+    @model_validator(mode="after")
+    def validate_mtls_pair(self) -> "TLSConfig":
+        if (self.client_cert is None) != (self.client_key is None):
+            raise ValueError("Both client_cert and client_key must be provided together for mTLS")
+        return self
+
+
+class ProxyConfig(BaseModel):
+    """Proxy configuration for HTTP connections."""
+
+    url: HttpUrl | None = Field(
+        default=None,
+        description="Single proxy URL for all connections (e.g., 'http://proxy.example.com:8080').",
+    )
+    http: HttpUrl | None = Field(
+        default=None,
+        description="Proxy URL for HTTP connections.",
+    )
+    https: HttpUrl | None = Field(
+        default=None,
+        description="Proxy URL for HTTPS connections.",
+    )
+    cacert: Path | None = Field(
+        default=None,
+        description="Path to CA certificate file for verifying the proxy's certificate. Required for proxies in interception mode.",
+    )
+    no_proxy: list[str] | None = Field(
+        default=None,
+        description="List of hosts that should bypass the proxy (e.g., ['localhost', '127.0.0.1', '.internal.corp']).",
+    )
+
+    @field_validator("cacert", mode="before")
+    @classmethod
+    def validate_cacert(cls, v: str | Path | None) -> Path | None:
+        if v is None:
+            return None
+        if isinstance(v, str):
+            cert_path = Path(v).expanduser().resolve()
+        else:
+            cert_path = v.expanduser().resolve()
+        if not cert_path.exists():
+            raise ValueError(f"Proxy CA certificate file does not exist: {v}")
+        if not cert_path.is_file():
+            raise ValueError(f"Proxy CA certificate path is not a file: {v}")
+        return cert_path
+
+    @model_validator(mode="after")
+    def validate_proxy_config(self) -> "ProxyConfig":
+        if self.url and (self.http or self.https):
+            raise ValueError("Cannot specify both 'url' and 'http'/'https' proxy settings")
+        return self
+
+
+class TimeoutConfig(BaseModel):
+    """Timeout configuration for HTTP connections."""
+
+    connect: float | None = Field(
+        default=None,
+        description="Connection timeout in seconds.",
+    )
+    read: float | None = Field(
+        default=None,
+        description="Read timeout in seconds.",
+    )
+
+
+class NetworkConfig(BaseModel):
+    """Network configuration for remote provider connections."""
+
+    tls: TLSConfig | None = Field(
+        default=None,
+        description="TLS/SSL configuration for secure connections.",
+    )
+    proxy: ProxyConfig | None = Field(
+        default=None,
+        description="Proxy configuration for HTTP connections.",
+    )
+    timeout: float | TimeoutConfig | None = Field(
+        default=None,
+        description="Timeout configuration. Can be a float (for both connect and read) or a TimeoutConfig object with separate connect and read timeouts.",
+    )
+    headers: dict[str, str] | None = Field(
+        default=None,
+        description="Additional HTTP headers to include in all requests.",
+    )
+
+
 class RemoteInferenceProviderConfig(BaseModel):
     allowed_models: list[str] | None = Field(
         default=None,
@@ -31,6 +173,10 @@ class RemoteInferenceProviderConfig(BaseModel):
         description="Authentication credential for the provider",
         alias="api_key",
     )
+    network: NetworkConfig | None = Field(
+        default=None,
+        description="Network configuration including TLS, proxy, and timeout settings.",
+    )
 
 
 # TODO: this class is more confusing than useful right now. We need to make it

llama_stack/providers/utils/inference/openai_compat.py

@@ -7,144 +7,14 @@ from typing import (
     Any,
 )
 
-from openai.types.chat import (
-    ChatCompletionMessageToolCall,
-)
 from pydantic import BaseModel
 
 from llama_stack.log import get_logger
-from llama_stack.models.llama.datatypes import (
-    BuiltinTool,
-    StopReason,
-    ToolCall,
-    ToolDefinition,
-)
+from llama_stack.models.llama.datatypes import BuiltinTool, ToolDefinition
 
 logger = get_logger(name=__name__, category="providers::utils")
 
 
-class OpenAICompatCompletionChoiceDelta(BaseModel):
-    content: str
-
-
-class OpenAICompatLogprobs(BaseModel):
-    text_offset: list[int] | None = None
-
-    token_logprobs: list[float] | None = None
-
-    tokens: list[str] | None = None
-
-    top_logprobs: list[dict[str, float]] | None = None
-
-
-class OpenAICompatCompletionChoice(BaseModel):
-    finish_reason: str | None = None
-    text: str | None = None
-    delta: OpenAICompatCompletionChoiceDelta | None = None
-    logprobs: OpenAICompatLogprobs | None = None
-
-
-class OpenAICompatCompletionResponse(BaseModel):
-    choices: list[OpenAICompatCompletionChoice]
-
-
-def text_from_choice(choice) -> str:
-    if hasattr(choice, "delta") and choice.delta:
-        return choice.delta.content  # type: ignore[no-any-return]  # external OpenAI types lack precise annotations
-
-    if hasattr(choice, "message"):
-        return choice.message.content  # type: ignore[no-any-return]  # external OpenAI types lack precise annotations
-
-    return choice.text  # type: ignore[no-any-return]  # external OpenAI types lack precise annotations
-
-
-def get_stop_reason(finish_reason: str) -> StopReason:
-    if finish_reason in ["stop", "eos"]:
-        return StopReason.end_of_turn
-    elif finish_reason == "eom":
-        return StopReason.end_of_message
-    elif finish_reason == "length":
-        return StopReason.out_of_tokens
-
-    return StopReason.out_of_tokens
-
-
-class UnparseableToolCall(BaseModel):
-    """
-    A ToolCall with arguments that are not valid JSON.
-    Mirrors the ToolCall schema, but with arguments as a string.
-    """
-
-    call_id: str = ""
-    tool_name: str = ""
-    arguments: str = ""
-
-
-def convert_tool_call(
-    tool_call: ChatCompletionMessageToolCall,
-) -> ToolCall | UnparseableToolCall:
-    """
-    Convert a ChatCompletionMessageToolCall tool call to either a
-    ToolCall or UnparseableToolCall. Returns an UnparseableToolCall
-    if the tool call is not valid ToolCall.
-    """
-    try:
-        valid_tool_call = ToolCall(
-            call_id=tool_call.id,
-            tool_name=tool_call.function.name,
-            arguments=tool_call.function.arguments,
-        )
-    except Exception:
-        return UnparseableToolCall(
-            call_id=tool_call.id or "",
-            tool_name=tool_call.function.name or "",
-            arguments=tool_call.function.arguments or "",
-        )
-
-    return valid_tool_call
-
-
-PYTHON_TYPE_TO_LITELLM_TYPE = {
-    "int": "integer",
-    "float": "number",
-    "bool": "boolean",
-    "str": "string",
-}
-
-
-def to_openai_param_type(param_type: str) -> dict:
-    """
-    Convert Python type hints to OpenAI parameter type format.
-
-    Examples:
-        'str' -> {'type': 'string'}
-        'int' -> {'type': 'integer'}
-        'list[str]' -> {'type': 'array', 'items': {'type': 'string'}}
-        'list[int]' -> {'type': 'array', 'items': {'type': 'integer'}}
-    """
-    # Handle basic types first
-    basic_types = {
-        "str": "string",
-        "int": "integer",
-        "float": "number",
-        "bool": "boolean",
-    }
-
-    if param_type in basic_types:
-        return {"type": basic_types[param_type]}
-
-    # Handle list/array types
-    if param_type.startswith("list[") and param_type.endswith("]"):
-        inner_type = param_type[5:-1]
-        if inner_type in basic_types:
-            return {
-                "type": "array",
-                "items": {"type": basic_types.get(inner_type, inner_type)},
-            }
-
-    return {"type": param_type}
-
-
 def convert_tooldef_to_openai_tool(tool: ToolDefinition) -> dict:
     """
     Convert a ToolDefinition to an OpenAI API-compatible dictionary.
@@ -210,33 +80,6 @@ async def prepare_openai_completion_params(**params):
     return completion_params
 
 
-def prepare_openai_embeddings_params(
-    model: str,
-    input: str | list[str],
-    encoding_format: str | None = "float",
-    dimensions: int | None = None,
-    user: str | None = None,
-):
-    if model is None:
-        raise ValueError("Model must be provided for embeddings")
-
-    input_list = [input] if isinstance(input, str) else input
-
-    params: dict[str, Any] = {
-        "model": model,
-        "input": input_list,
-    }
-
-    if encoding_format is not None:
-        params["encoding_format"] = encoding_format
-    if dimensions is not None:
-        params["dimensions"] = dimensions
-    if user is not None:
-        params["user"] = user
-
-    return params
-
-
 def get_stream_options_for_telemetry(
     stream_options: dict[str, Any] | None,
     is_streaming: bool,