rasa-pro 3.11.0a4.dev3__py3-none-any.whl → 3.11.0rc1__py3-none-any.whl

rasa/shared/utils/health_check.py

@@ -0,0 +1,533 @@
+import os
+from typing import Optional, Dict, Any
+
+from rasa.shared.constants import (
+    LLM_API_HEALTH_CHECK_ENV_VAR,
+    MODELS_CONFIG_KEY,
+    MODEL_GROUP_ID_CONFIG_KEY,
+    LLM_API_HEALTH_CHECK_DEFAULT_VALUE,
+)
+from rasa.shared.exceptions import ProviderClientValidationError
+from rasa.shared.providers.embedding.embedding_client import EmbeddingClient
+from rasa.shared.providers.llm.llm_client import LLMClient
+from rasa.shared.utils.cli import print_error_and_exit
+from rasa.shared.utils.llm import llm_factory, structlogger, embedder_factory
+
+
+def try_instantiate_llm_client(
+    custom_llm_config: Optional[Dict],
+    default_llm_config: Optional[Dict],
+    log_source_function: str,
+    log_source_component: str,
+) -> LLMClient:
+    """Validate llm configuration."""
+    try:
+        return llm_factory(custom_llm_config, default_llm_config)
+    except (ProviderClientValidationError, ValueError) as e:
+        structlogger.error(
+            f"{log_source_function}.llm_instantiation_failed",
+            message="Unable to instantiate LLM client.",
+            error=e,
+        )
+        print_error_and_exit(
+            f"Unable to create the LLM client for component - {log_source_component}. "
+            f"Please make sure you specified the required environment variables "
+            f"and configuration keys. "
+            f"Error: {e}"
+        )
+
+
+def try_instantiate_embedder(
+    custom_embeddings_config: Optional[Dict],
+    default_embeddings_config: Optional[Dict],
+    log_source_function: str,
+    log_source_component: str,
+) -> EmbeddingClient:
+    """Validate embeddings configuration."""
+    try:
+        return embedder_factory(custom_embeddings_config, default_embeddings_config)
+    except (ProviderClientValidationError, ValueError) as e:
+        structlogger.error(
+            f"{log_source_function}.embedder_instantiation_failed",
+            message="Unable to instantiate Embedding client.",
+            error=e,
+        )
+        print_error_and_exit(
+            f"Unable to create the Embedding client for component - "
+            f"{log_source_component}. Please make sure you specified the required "
+            f"environment variables and configuration keys. Error: {e}"
+        )
+
+
+def perform_training_time_llm_health_check(
+    custom_config: Optional[Dict[str, Any]],
+    default_config: Dict[str, Any],
+    log_source_function: str,
+    log_source_component: str,
+) -> Optional[str]:
+    """Try to instantiate the LLM Client to validate the provided config.
+    If the LLM_API_HEALTH_CHECK environment variable is true, perform a test call
+    to the LLM API. If config contains multiple models, perform a consistency
+    check for the model group.
+
+    This method supports both single model configurations and model group configurations
+    (configs that have the `models` key).
+
+    Returns:
+        model name from the API response or `None` if LLM_API_HEALTH_CHECK is false
+    """
+    llm_client = try_instantiate_llm_client(
+        custom_config, default_config, log_source_function, log_source_component
+    )
+
+    if (
+        os.getenv(
+            LLM_API_HEALTH_CHECK_ENV_VAR, LLM_API_HEALTH_CHECK_DEFAULT_VALUE
+        ).lower()
+        == "true"
+    ):
+        train_model_name: Optional[str] = None
+        if (
+            custom_config
+            and MODELS_CONFIG_KEY in custom_config
+            and len(custom_config[MODELS_CONFIG_KEY]) > 1
+        ):
+            train_model_name = perform_llm_model_group_consistency_check(
+                custom_config,
+                default_config,
+                log_source_function,
+                log_source_component,
+            )
+        else:
+            train_model_name = send_test_llm_api_request(
+                llm_client,
+                log_source_function,
+                log_source_component,
+            )
+        return train_model_name
+    else:
+        structlogger.warning(
+            f"{log_source_function}.perform_training_time_llm_health_check.disabled",
+            event_info=(
+                f"The {LLM_API_HEALTH_CHECK_ENV_VAR} environment variable is set "
+                f"to false, which will disable model consistency check. "
+                f"It is recommended to set this variable to true in production "
+                f"environments."
+            ),
+        )
+        return None
+
+
+def perform_training_time_embeddings_health_check(
+    custom_config: Optional[Dict[str, Any]],
+    default_config: Dict[str, Any],
+    log_source_function: str,
+    log_source_component: str,
+) -> Optional[str]:
+    """Try to instantiate the Embedder to validate the provided config.
+    If the LLM_API_HEALTH_CHECK environment variable is true, perform a test call
+    to the Embeddings API. If config contains multiple models, perform a consistency
+    check for the model group.
+
+    This method supports both single model configurations and model group configurations
+    (configs that have the `models` key).
+
+    Returns:
+        model name from the API response or `None` if LLM_API_HEALTH_CHECK is false
+    """
+    if (
+        os.getenv(
+            LLM_API_HEALTH_CHECK_ENV_VAR, LLM_API_HEALTH_CHECK_DEFAULT_VALUE
+        ).lower()
+        == "true"
+    ):
+        train_model_name: Optional[str] = None
+        if (
+            custom_config
+            and MODELS_CONFIG_KEY in custom_config
+            and len(custom_config[MODELS_CONFIG_KEY]) > 1
+        ):
+            train_model_name = perform_embeddings_model_group_consistency_check(
+                custom_config,
+                default_config,
+                log_source_function,
+                log_source_component,
+            )
+        else:
+            embedder = try_instantiate_embedder(
+                custom_config, default_config, log_source_function, log_source_component
+            )
+            train_model_name = send_test_embeddings_api_request(
+                embedder, log_source_function, log_source_component
+            )
+        return train_model_name
+    else:
+        structlogger.warning(
+            f"{log_source_function}"
+            f".perform_training_time_embeddings_health_check.disabled",
+            event_info=(
+                f"The {LLM_API_HEALTH_CHECK_ENV_VAR} environment variable is set "
+                f"to false, which will disable model consistency check. "
+                f"It is recommended to set this variable to true in production "
+                f"environments."
+            ),
+        )
+        return None
+
+
+def perform_inference_time_llm_health_check(
+    custom_config: Optional[Dict[str, Any]],
+    default_config: Dict[str, Any],
+    train_model_name: Optional[str],
+    log_source_function: str,
+    log_source_component: str,
+) -> None:
+    """If the LLM_API_HEALTH_CHECK environment variable is true, perform a test call
+    to the LLM API. If config contains multiple models, perform a consistency
+    check for the model group.
+    Compare the model name from the API response with the model name used for training
+    (if available) and raise an exception if they are different.
+
+    This method supports both single model configurations and model group configurations
+    (configs that have the `models` key).
+    """
+    if (
+        os.getenv(
+            LLM_API_HEALTH_CHECK_ENV_VAR, LLM_API_HEALTH_CHECK_DEFAULT_VALUE
+        ).lower()
+        == "true"
+    ):
+        structlogger.info(
+            f"{log_source_function}.perform_inference_time_llm_health_check",
+            event_info=(
+                f"Performing an inference-time health check on the LLM API for "
+                f"the component - {log_source_component}."
+            ),
+            config=custom_config,
+        )
+
+        inference_model_name: Optional[str] = None
+        if (
+            custom_config
+            and MODELS_CONFIG_KEY in custom_config
+            and len(custom_config[MODELS_CONFIG_KEY]) > 1
+        ):
+            inference_model_name = perform_llm_model_group_consistency_check(
+                custom_config,
+                default_config,
+                log_source_function,
+                log_source_component,
+            )
+        else:
+            llm_client = try_instantiate_llm_client(
+                custom_config,
+                default_config,
+                log_source_function,
+                log_source_component,
+            )
+            inference_model_name = send_test_llm_api_request(
+                llm_client,
+                log_source_function,
+                log_source_component,
+            )
+
+        if not inference_model_name:
+            structlogger.warning(
+                f"{log_source_function}"
+                f".perform_inference_time_llm_health_check.no_inference_model",
+                event_info=(
+                    "Failed to perform model consistency check: "
+                    "the API response does not contain a model name."
+                ),
+            )
+        elif not train_model_name:
+            structlogger.warning(
+                f"{log_source_function}"
+                f".perform_inference_time_llm_health_check.no_train_model",
+                event_info=(
+                    f"The model was trained with {LLM_API_HEALTH_CHECK_ENV_VAR} "
+                    f"environment variable set to false, so the model "
+                    f"consistency check is not available."
+                ),
+            )
+        elif inference_model_name != train_model_name:
+            error_message = (
+                f"The LLM used to train the {log_source_component} "
+                f"({train_model_name}) is not the same as the LLM used for inference "
+                f"({inference_model_name}). Please verify your configuration."
+            )
+            structlogger.error(
+                f"{log_source_function}.train_inference_llm_model_mismatch",
+                event_info=error_message,
+            )
+            print_error_and_exit(error_message)
+    else:
+        structlogger.warning(
+            f"{log_source_function}.perform_inference_time_llm_health_check.disabled",
+            event_info=(
+                f"The {LLM_API_HEALTH_CHECK_ENV_VAR} environment variable is set "
+                f"to false, which will disable model consistency check. "
+                f"It is recommended to set this variable to true in production "
+                f"environments."
+            ),
+        )
+
+
+def perform_inference_time_embeddings_health_check(
+    custom_config: Optional[Dict[str, Any]],
+    default_config: Dict[str, Any],
+    train_model_name: Optional[str],
+    log_source_function: str,
+    log_source_component: str,
+) -> None:
+    """If the LLM_API_HEALTH_CHECK environment variable is true, perform a test call
+    to the Embeddings API. If config contains multiple models, perform a consistency
+    check for the model group.
+    Compare the model name from the API response with the model name used for training
+    (if available) and raise an exception if they are different.
+
+    This method supports both single model configurations and model group configurations
+    (configs that have the `models` key).
+    """
+    if (
+        os.getenv(
+            LLM_API_HEALTH_CHECK_ENV_VAR, LLM_API_HEALTH_CHECK_DEFAULT_VALUE
+        ).lower()
+        == "true"
+    ):
+        structlogger.info(
+            f"{log_source_function}.perform_inference_time_embeddings_health_check",
+            event_info=(
+                f"Performing an inference-time health check on the Embeddings API for "
+                f"the component - {log_source_component}."
+            ),
+            config=custom_config,
+        )
+
+        inference_model_name: Optional[str] = None
+        if (
+            custom_config
+            and MODELS_CONFIG_KEY in custom_config
+            and len(custom_config[MODELS_CONFIG_KEY]) > 1
+        ):
+            inference_model_name = perform_embeddings_model_group_consistency_check(
+                custom_config,
+                default_config,
+                log_source_function,
+                log_source_component,
+            )
+        else:
+            embedder = try_instantiate_embedder(
+                custom_config, default_config, log_source_function, log_source_component
+            )
+            inference_model_name = send_test_embeddings_api_request(
+                embedder, log_source_function, log_source_component
+            )
+
+        if not inference_model_name:
+            structlogger.warning(
+                f"{log_source_function}"
+                f".perform_inference_time_embeddings_health_check.no_inference_model",
+                event_info=(
+                    "Failed to perform embeddings model consistency check: "
+                    "the API response does not contain a model name."
+                ),
+            )
+        elif not train_model_name:
+            structlogger.warning(
+                f"{log_source_function}"
+                f".perform_inference_time_embeddings_health_check.no_train_model",
+                event_info=(
+                    f"The model was trained with {LLM_API_HEALTH_CHECK_ENV_VAR} "
+                    f"environment variable set to false, so the model "
+                    f"consistency check is not available."
+                ),
+            )
+        elif inference_model_name != train_model_name:
+            error_message = (
+                f"The Embeddings model used to train the {log_source_component} "
+                f"({train_model_name}) is not the same as the model used for inference "
+                f"({inference_model_name}). Please verify your configuration."
+            )
+            structlogger.error(
+                f"{log_source_function}.train_inference_embeddings_model_mismatch",
+                event_info=error_message,
+            )
+            print_error_and_exit(error_message)
+    else:
+        structlogger.warning(
+            f"{log_source_function}"
+            f".perform_inference_time_embeddings_health_check.disabled",
+            event_info=(
+                f"The {LLM_API_HEALTH_CHECK_ENV_VAR} environment variable is set "
+                f"to false, which will disable model consistency check. "
+                f"It is recommended to set this variable to true in production "
+                f"environments."
+            ),
+        )
+
+
+def perform_llm_model_group_consistency_check(
+    custom_config: Dict[str, Any],
+    default_config: Dict[str, Any],
+    log_source_function: str,
+    log_source_component: str,
+) -> Optional[str]:
+    """Perform a consistency check for multiple models inside a model group.
+
+    This function checks if all models within a model group are consistent by verifying
+    that they all return the same model name from the LLM API health check.
+
+    This method supports only model group configuration (config that have the `models`
+    key) and will ignore single model configuration.
+
+    Returns:
+        The model name if all models are consistent, otherwise raises
+        an InvalidConfigException.
+
+    Raises:
+        InvalidConfigException: If the model group contains different models.
+    """
+    if not custom_config or MODELS_CONFIG_KEY not in custom_config:
+        return None
+
+    model_names = set()
+    for model_config in custom_config[MODELS_CONFIG_KEY]:
+        llm_client = try_instantiate_llm_client(
+            model_config, default_config, log_source_function, log_source_component
+        )
+        model_name = send_test_llm_api_request(
+            llm_client, log_source_function, log_source_component
+        )
+        model_names.add(model_name)
+
+    if len(model_names) > 1:
+        error_message = (
+            f"The model group {custom_config.get(MODEL_GROUP_ID_CONFIG_KEY)} used by "
+            f"{log_source_component} component is inconsistent. "
+            f"It contains different models: {model_names}. "
+            f"Please verify your configuration."
+        )
+        structlogger.error(
+            f"{log_source_function}.inconsistent_model_group", event_info=error_message
+        )
+        print_error_and_exit(error_message)
+
+    return model_names.pop() if len(model_names) > 0 else None
+
+
+def send_test_llm_api_request(
+    llm_client: LLMClient, log_source_function: str, log_source_component: str
+) -> Optional[str]:
+    """Sends a test request to the LLM API to perform a health check.
+
+    Returns:
+        The model name from the API response.
+
+    Raises:
+        Exception: If the API call fails.
+    """
+    structlogger.info(
+        f"{log_source_function}.send_test_llm_api_request",
+        event_info=(
+            f"Sending a test LLM API request for the component - "
+            f"{log_source_component}."
+        ),
+        config=llm_client.config,
+    )
+    try:
+        response = llm_client.completion("hello")
+        return response.model
+    except Exception as e:
+        structlogger.error(
+            f"{log_source_function}.send_test_llm_api_request_failed",
+            event_info="Test call to the LLM API failed.",
+            error=e,
+        )
+        print_error_and_exit(
+            f"Test call to the LLM API failed for component - {log_source_component}. "
+            f"Error: {e}"
+        )
+
+
+def send_test_embeddings_api_request(
+    embedder: EmbeddingClient, log_source_function: str, log_source_component: str
+) -> Optional[str]:
+    """Sends a test request to the Embeddings API to perform a health check.
+
+    Returns:
+        The model name from the API response.
+
+    Raises:
+        Exception: If the API call fails.
+    """
+    structlogger.info(
+        f"{log_source_function}.send_test_embeddings_api_request",
+        event_info=(
+            f"Sending a test Embeddings API request for the component - "
+            f"{log_source_component}."
+        ),
+        config=embedder.config,
+    )
+    try:
+        response = embedder.embed(["hello"])
+        return response.model
+    except Exception as e:
+        structlogger.error(
+            f"{log_source_function}.send_test_llm_api_request_failed",
+            event_info="Test call to the Embeddings API failed.",
+            error=e,
+        )
+        print_error_and_exit(
+            f"Test call to the Embeddings API failed for component - "
+            f"{log_source_component}. Error: {e}"
+        )
+
+
+def perform_embeddings_model_group_consistency_check(
+    custom_config: Dict[str, Any],
+    default_config: Dict[str, Any],
+    log_source_function: str,
+    log_source_component: str,
+) -> Optional[str]:
+    """Perform a consistency check for multiple embeddings models inside a model group.
+
+    This function checks if all models within a model group are consistent by verifying
+    that they all return the same model name from the Embeddings API health check.
+
+    This method supports only model group configuration (config that have the `models`
+    key) and will ignore single model configuration.
+
+    Returns:
+        The model name if all models are consistent, otherwise raises
+        an InvalidConfigException.
+
+    Raises:
+        InvalidConfigException: If the model group contains different models.
+    """
+    if not custom_config or MODELS_CONFIG_KEY not in custom_config:
+        return None
+
+    model_names = set()
+    for model_config in custom_config[MODELS_CONFIG_KEY]:
+        embedder = try_instantiate_embedder(
+            custom_config, default_config, log_source_function, log_source_component
+        )
+        model_name = send_test_embeddings_api_request(
+            embedder, log_source_function, log_source_component
+        )
+        model_names.add(model_name)
+
+    if len(model_names) > 1:
+        error_message = (
+            f"The embeddings model group {custom_config.get(MODEL_GROUP_ID_CONFIG_KEY)}"
+            f" used by {log_source_component} component is inconsistent. "
+            f"It contains different models: {model_names}. "
+            f"Please verify your configuration."
+        )
+        structlogger.error(
+            f"{log_source_function}.inconsistent_embeddings", event_info=error_message
+        )
+        print_error_and_exit(error_message)
+
+    return model_names.pop() if len(model_names) > 0 else None

rasa/shared/utils/io.py

@@ -1,18 +1,19 @@
-from collections import OrderedDict
-from functools import wraps
-from hashlib import md5
 import asyncio
 import errno
 import glob
 import json
 import logging
 import os
+import random
+import string
 import sys
+import warnings
+from collections import OrderedDict
+from functools import wraps
+from hashlib import md5
 from pathlib import Path
 from typing import Any, cast, Callable, Dict, List, Optional, Text, Type, TypeVar, Union
-import warnings
-import random
-import string
+
 import portalocker
 
 from rasa.shared.constants import (
@@ -475,3 +476,24 @@ def suppress_logs(log_level: int = logging.WARNING) -> Callable[[F], F]:
             return cast(F, sync_wrapper)
 
     return decorator
+
+
+def resolve_environment_variables(
+    value: Union[str, List[Any], Dict[str, Any]],
+) -> Union[str, List[Any], Dict[str, Any]]:
+    """Resolve environment variables in a string, list, or dictionary.
+
+    Args:
+        value: The value to resolve environment variables in.
+
+    Returns:
+        The value with environment variables resolved.
+    """
+    if isinstance(value, str):
+        return os.path.expandvars(value)
+    elif isinstance(value, list):
+        return [resolve_environment_variables(item) for item in value]
+    elif isinstance(value, dict):
+        return {key: resolve_environment_variables(val) for key, val in value.items()}
+    else:
+        return value