rasa-pro 3.10.15__py3-none-any.whl → 3.11.0__py3-none-any.whl

rasa/engine/validation.py

@@ -1,7 +1,7 @@
 import dataclasses
 import inspect
-import re
 import logging
+import re
 import sys
 import typing
 from typing import (
@@ -16,14 +16,12 @@ from typing import (
     Union,
     TypeVar,
     List,
-    Literal,
 )
 
+import rasa.utils.common
 import structlog
 import typing_utils
-
-import rasa.utils.common
-from rasa.core import IntentlessPolicy
+from rasa.core import IntentlessPolicy, ContextualResponseRephraser
 from rasa.core.policies.policy import PolicyPrediction
 from rasa.core.utils import AvailableEndpoints
 from rasa.dialogue_understanding.coexistence.constants import (
@@ -32,10 +30,17 @@ from rasa.dialogue_understanding.coexistence.constants import (
     STICKY,
     NON_STICKY,
 )
+from rasa.dialogue_understanding.coexistence.intent_based_router import (
+    IntentBasedRouter,
+)
+from rasa.dialogue_understanding.coexistence.llm_based_router import LLMBasedRouter
 from rasa.dialogue_understanding.generator import (
     LLMBasedCommandGenerator,
 )
-from rasa.dialogue_understanding.generator.constants import FLOW_RETRIEVAL_KEY
+from rasa.dialogue_understanding.generator.constants import (
+    LLM_CONFIG_KEY,
+    FLOW_RETRIEVAL_KEY,
+)
 from rasa.dialogue_understanding.patterns.chitchat import FLOW_PATTERN_CHITCHAT
 from rasa.engine.constants import RESERVED_PLACEHOLDERS
 from rasa.engine.exceptions import GraphSchemaValidationException
@@ -47,16 +52,31 @@ from rasa.engine.graph import (
     GraphModelConfiguration,
 )
 from rasa.engine.storage.resource import Resource
-from rasa.engine.storage.storage import ModelStorage
+from rasa.engine.storage.storage import ModelStorage, ModelMetadata
 from rasa.engine.training.fingerprinting import Fingerprintable
 from rasa.shared.constants import (
     DOCS_URL_GRAPH_COMPONENTS,
     ROUTE_TO_CALM_SLOT,
-    API_TYPE_CONFIG_KEY,
-    VALID_PROVIDERS_FOR_API_TYPE_CONFIG_KEY,
-    PROVIDER_CONFIG_KEY,
-    LLM_CONFIG_KEY,
     EMBEDDINGS_CONFIG_KEY,
+    API_BASE_CONFIG_KEY,
+    DEPLOYMENT_CONFIG_KEY,
+    API_VERSION_CONFIG_KEY,
+    API_KEY,
+    AWS_REGION_NAME_CONFIG_KEY,
+    MODEL_GROUP_ID_CONFIG_KEY,
+    ROUTER_CONFIG_KEY,
+    MODELS_CONFIG_KEY,
+    MODEL_GROUP_CONFIG_KEY,
+    ROUTING_STRATEGY_CONFIG_KEY,
+    VALID_ROUTING_STRATEGIES,
+    ROUTING_STRATEGIES_REQUIRING_REDIS_CACHE,
+    ROUTING_STRATEGIES_NOT_REQUIRING_CACHE,
+    REDIS_HOST_CONFIG_KEY,
+    AWS_ACCESS_KEY_ID_CONFIG_KEY,
+    AWS_SECRET_ACCESS_KEY_CONFIG_KEY,
+    AWS_SESSION_TOKEN_CONFIG_KEY,
+    SENSITIVE_DATA,
+    USE_CHAT_COMPLETIONS_ENDPOINT_CONFIG_KEY,
 )
 from rasa.shared.core.constants import ACTION_RESET_ROUTING, ACTION_TRIGGER_CHITCHAT
 from rasa.shared.core.domain import Domain
@@ -65,11 +85,6 @@ from rasa.shared.core.slots import Slot
 from rasa.shared.exceptions import RasaException
 from rasa.shared.nlu.training_data.message import Message
 
-from rasa.dialogue_understanding.coexistence.intent_based_router import (
-    IntentBasedRouter,
-)
-from rasa.dialogue_understanding.coexistence.llm_based_router import LLMBasedRouter
-
 TypeAnnotation = Union[TypeVar, Text, Type, Optional[AvailableEndpoints]]
 
 structlogger = structlog.get_logger()
@@ -855,96 +870,534 @@ def validate_coexistance_routing_setup(
     )
 
 
-def validate_command_generator_exclusivity(schema: GraphSchema) -> None:
-    """Validate that multiple command generators are not defined at same time."""
-    from rasa.dialogue_understanding.generator import (
-        LLMBasedCommandGenerator,
-    )
+def _validate_component_model_client_config(
+    component_config: Dict[str, Any],
+    key: str,
+    model_group_syntax_used: List[bool],
+    model_group_ids: List[str],
+    component_name: Optional[str] = None,
+) -> None:
+    """Validate the LLM configuration of a component.
 
-    count = schema.count_nodes_of_a_given_type(
-        LLMBasedCommandGenerator, include_subtypes=True
-    )
+    Checks if the llm is defined using the new syntax or the old syntax.
+    If the new syntax is used, it checks that no other parameters are present.
 
-    if count > 1:
-        structlogger.error(
-            "validation.command_generator.multiple_command_generator_defined",
-            event_info=(
-                "Multiple LLM based command generators are defined in the config. "
-                "Please use only one LLM based command generator."
-            ),
-        )
-        sys.exit(1)
+    Args:
+        component_config: The config of the component
+        key: either 'llm' or 'embeddings'
+        model_group_syntax_used:
+            list of booleans indicating whether the new syntax is used
+        model_group_ids: list of model group ids
+        component_name: the name of the component
+    """
+    if key not in component_config:
+        # no llm configuration present
+        return
 
+    if MODEL_GROUP_CONFIG_KEY in component_config[key]:
+        model_group_syntax_used.append(True)
+        model_group_ids.append(component_config[key][MODEL_GROUP_CONFIG_KEY])
 
-def validate_command_generator_setup(
-    model_configuration: GraphModelConfiguration,
-) -> None:
-    schema = model_configuration.predict_schema
-    validate_command_generator_exclusivity(schema)
+        if len(component_config[key]) > 1:
+            structlogger.error(
+                "validation.validate_model_client_configuration_setup"
+                ".only_model_group_reference_key_is_allowed",
+                event_info=(
+                    f"You specified a '{MODEL_GROUP_CONFIG_KEY}' for the '{key}' "
+                    f"config key for the component "
+                    f"'{component_name or component_config['name']}'. "
+                    "No other parameters are allowed under the "
+                    f"'{key}' key in that case. Please update your config."
+                ),
+                component_name=component_name or component_config["name"],
+                component_client_config_key=key,
+            )
+            sys.exit(1)
+    else:
+        model_group_syntax_used.append(False)
+
+        # check that any of the sensitive data keys is not set in config
+        for secret_key in SENSITIVE_DATA:
+            if secret_key in component_config[key]:
+                structlogger.error(
+                    "validation.validate_model_client_configuration_setup"
+                    ".secret_key_not_allowed_in_the_config",
+                    event_info=(
+                        f"You specified '{secret_key}' in the config for "
+                        f"'{component_name or component_config['name']}', "
+                        f"which is not allowed. "
+                        "Set secret keys through environment variables."
+                    ),
+                    component_name=component_name or component_config["name"],
+                    component_client_config_key=key,
+                    secret_key=secret_key,
+                )
+                sys.exit(1)
 
 
-def validate_model_client_configuration_setup(config: Dict[str, Any]) -> None:
+def validate_model_client_configuration_setup_during_training_time(
+    config: Dict[str, Any],
+) -> None:
     """Validates the model client configuration setup.
 
+    Checks the model configuration of the components in the pipeline.
     Validation fails, if
-    - the LLM/embeddings provider is defined using 'api_type' key for providers other
-    than 'openai' or 'azure'
+    - the LLM/embeddings is/are defined using the old and the new syntax at
+      the same time (either at component level itself or across different components)
+    - the LLM/embeddings is/are defined using the new syntax, but no model
+      group is defined or the referenced model group does not exist
 
     Args:
         config: The config dictionary
     """
+
+    def is_uniform_bool_list(bool_list: List[bool]) -> bool:
+        # check if list contains only True or False
+        return all(bool_list) or not any(bool_list)
+
+    model_group_syntax_used: List[bool] = []
+    model_group_ids: List[str] = []
+
     for outer_key in ["pipeline", "policies"]:
         if outer_key not in config or config[outer_key] is None:
             continue
 
-        for component_config in config[outer_key]:
+        for component in config[outer_key]:
             for key in [LLM_CONFIG_KEY, EMBEDDINGS_CONFIG_KEY]:
-                validate_api_type_config_key_usage(component_config, key)
+                _validate_component_model_client_config(
+                    component, key, model_group_syntax_used, model_group_ids
+                )
 
             # as flow retrieval is not a component itself, we need to
             # check it separately
-            if (
-                FLOW_RETRIEVAL_KEY in component_config
-                and EMBEDDINGS_CONFIG_KEY in component_config[FLOW_RETRIEVAL_KEY]
-            ):
-                validate_api_type_config_key_usage(
-                    component_config[FLOW_RETRIEVAL_KEY],
-                    EMBEDDINGS_CONFIG_KEY,
-                    component_config["name"] + "." + FLOW_RETRIEVAL_KEY,
-                )
+            if FLOW_RETRIEVAL_KEY in component:
+                if EMBEDDINGS_CONFIG_KEY in component[FLOW_RETRIEVAL_KEY]:
+                    _validate_component_model_client_config(
+                        component[FLOW_RETRIEVAL_KEY],
+                        EMBEDDINGS_CONFIG_KEY,
+                        model_group_syntax_used,
+                        model_group_ids,
+                        component["name"] + "." + FLOW_RETRIEVAL_KEY,
+                    )
+
+    # also include the ContextualResponseRephraser component
+    endpoints = AvailableEndpoints.get_instance()
+    if endpoints.nlg is not None:
+        _validate_component_model_client_config(
+            endpoints.nlg.kwargs,
+            LLM_CONFIG_KEY,
+            model_group_syntax_used,
+            model_group_ids,
+            ContextualResponseRephraser.__name__,
+        )
 
+    if not is_uniform_bool_list(model_group_syntax_used):
+        structlogger.error(
+            "validation.validate_model_client_configuration_setup"
+            ".inconsistent_use_of_model_group_syntax",
+            event_info=(
+                "Some of your components refer to an LLM using the "
+                f"'{MODEL_GROUP_CONFIG_KEY}' parameter, other components directly"
+                f" define the LLM under the '{LLM_CONFIG_KEY}' or the "
+                f"'{EMBEDDINGS_CONFIG_KEY}' key. You cannot use"
+                " both types of definitions. Please chose one syntax "
+                "and update your config."
+            ),
+        )
+        sys.exit(1)
 
-def validate_api_type_config_key_usage(
-    component_config: Dict[str, Any],
-    key: Literal["llm", "embeddings"],
-    component_name: Optional[str] = None,
+    # Print a deprecation warning in case the old syntax is used.
+    if len(model_group_syntax_used) > 0 and model_group_syntax_used[0] is False:
+        structlogger.warning(
+            "validate_llm_configuration_setup",
+            event_info=(
+                "Defining the LLM configuration in the config.yml file itself is"
+                " deprecated and will be removed in Rasa 4.0.0. "
+                "Please use the new syntax and define your LLM configuration"
+                "in the endpoints.yml file."
+            ),
+        )
+
+    endpoints = AvailableEndpoints.get_instance()
+    if len(model_group_ids) > 0 and endpoints.model_groups is None:
+        structlogger.error(
+            "validation.validate_model_client_configuration_setup"
+            ".referencing_model_group_but_none_are_defined",
+            event_info=(
+                "You are referring to (a) model group(s) in your "
+                "config.yml file, but no model group was defined in "
+                "the endpoints.yml file. Please define the model "
+                "group(s)."
+            ),
+        )
+        sys.exit(1)
+
+    if endpoints.model_groups is None:
+        return
+
+    existing_model_group_ids = [
+        model_group[MODEL_GROUP_ID_CONFIG_KEY] for model_group in endpoints.model_groups
+    ]
+
+    for model_group_id in model_group_ids:
+        if model_group_id not in existing_model_group_ids:
+            structlogger.error(
+                "validation.validate_model_client_configuration_setup"
+                ".referencing_undefined_model_group",
+                event_info=(
+                    "One of your components is referring to the model group "
+                    f"'{model_group_id}', but this model group does not exist in the "
+                    f"endpoints.yml file. Please chose one of the existing "
+                    f"model groups ({existing_model_group_ids}) or define "
+                    f"the model group for '{model_group_id}'."
+                ),
+                referencing_model_group_id=model_group_id,
+                existing_model_group_ids=existing_model_group_ids,
+            )
+            sys.exit(1)
+
+
+def _validate_component_model_client_config_has_references_to_endpoints(
+    component_config: Dict[Text, Any],
+    key: str,
+    component_name: Optional[Text] = None,
 ) -> None:
-    """Validate the LLM/embeddings configuration of a component.
+    """Validates that the specified client configuration references a valid model group
+    defined in the `endpoints.yml` file.
 
-    Validation fails, if
-    - the LLM/embeddings provider is defined using 'api_type' key for providers other
-    than 'openai' or 'azure'
+    This function ensures that when the client configuration for a component uses the
+    `model_group` key, the referenced model group exists in the `endpoints.yml` file.
+    If the referenced model group is missing or invalid, an error is raised.
 
     Args:
-        component_config: The config of the component
-        key: either 'llm' or 'embeddings'
-        component_name: the name of the component
+        component_config: The configuration dictionary for the component being
+            validated.
+        key: 'llm' or 'embeddings'
+        component_name: Optional; the name of the component being validated, used for
+            error messages.
+
+    Raises:
+        SystemExit: If the referenced model group is missing or invalid.
     """
-    if component_config is None or key not in component_config:
+    if key not in component_config:
+        # no llm/embeddings configuration present
         return
 
-    if API_TYPE_CONFIG_KEY in component_config[key]:
-        api_type = component_config[key][API_TYPE_CONFIG_KEY]
-        if api_type not in VALID_PROVIDERS_FOR_API_TYPE_CONFIG_KEY:
+    endpoints = AvailableEndpoints.get_instance()
+
+    if MODEL_GROUP_CONFIG_KEY in component_config[key]:
+        referencing_model_group_id = component_config[key][MODEL_GROUP_CONFIG_KEY]
+
+        if endpoints.model_groups is None:
+            structlogger.error(
+                "validation.validate_model_client_config_correctly_references_endpoints"
+                ".no_model_groups_defined",
+                event_info=(
+                    f"Your {component_name or component_config.get('name') or ''} "
+                    f"component's '{key}' configuration of the trained model "
+                    f"references the model group '{referencing_model_group_id}', "
+                    f"but NO MODEL GROUPS ARE DEFINED in the endpoints.yml file. "
+                    f"Please add a definition for the required model group in the "
+                    f"endpoints.yml file."
+                ),
+                component_name=component_name or component_config.get("name"),
+                model_group_id=referencing_model_group_id,
+                component_client_config_key=key,
+            )
+            sys.exit(1)
+
+        existing_model_group_ids = [
+            model_group[MODEL_GROUP_ID_CONFIG_KEY]
+            for model_group in endpoints.model_groups
+        ]
+
+        if referencing_model_group_id not in existing_model_group_ids:
+            structlogger.error(
+                "validation.validate_model_client_config_correctly_references_endpoints"
+                ".referenced_model_group_does_not_exist",
+                event_info=(
+                    f"Your {component_name or component_config.get('name') or ''} "
+                    f"component's '{key}' configuration of the trained model "
+                    f"references the model group '{referencing_model_group_id}', "
+                    f"but this model group DOES NOT EXIST in the endpoints.yml file. "
+                    f"The endpoints.yml defines the following model groups: "
+                    f"{existing_model_group_ids}. "
+                    f"Please add a definition for the required model group in the "
+                    f"endpoints.yml file."
+                ),
+                model_group_id=referencing_model_group_id,
+                existing_model_group_ids=existing_model_group_ids,
+                component_client_config_key=key,
+            )
+            sys.exit(1)
+
+
+def validate_model_client_configuration_setup_during_inference_time(
+    model_metadata: ModelMetadata,
+) -> None:
+    for (
+        component_node_name,
+        component_node,
+    ) in model_metadata.predict_schema.nodes.items():
+        for client_config_key in [EMBEDDINGS_CONFIG_KEY, LLM_CONFIG_KEY]:
+            if client_config_key not in component_node.config:
+                continue
+
+            _validate_component_model_client_config_has_references_to_endpoints(
+                component_config=component_node.config,
+                key=client_config_key,
+                component_name=component_node_name,
+            )
+
+            # as flow retrieval is not a component itself, we need to
+            # check it separately
+            if FLOW_RETRIEVAL_KEY in component_node.config:
+                if EMBEDDINGS_CONFIG_KEY in component_node.config[FLOW_RETRIEVAL_KEY]:
+                    _validate_component_model_client_config_has_references_to_endpoints(
+                        component_config=component_node.config[FLOW_RETRIEVAL_KEY],
+                        key=EMBEDDINGS_CONFIG_KEY,
+                        component_name=component_node_name + "." + FLOW_RETRIEVAL_KEY,
+                    )
+
+    # also include the ContextualResponseRephraser component
+    endpoints = AvailableEndpoints.get_instance()
+    if endpoints.nlg is not None:
+        _validate_component_model_client_config_has_references_to_endpoints(
+            component_config=endpoints.nlg.kwargs,
+            key=LLM_CONFIG_KEY,
+            component_name=ContextualResponseRephraser.__name__,
+        )
+
+
+def _validate_unique_model_group_ids(model_groups: List[Dict[str, Any]]) -> None:
+    # Each model id must be unique within the model_groups
+    model_ids = [model_group[MODEL_GROUP_ID_CONFIG_KEY] for model_group in model_groups]
+    if len(model_ids) != len(set(model_ids)):
+        structlogger.error(
+            "validate_model_group_configuration_setup.non_unique_model_group_ids",
+            event_info=(
+                "Each model group id must be unique. Please make sure that "
+                "the model group ids are unique in your endpoints.yml file."
+            ),
+        )
+        sys.exit(1)
+
+
+def _validate_model_group_with_multiple_models(
+    model_groups: List[Dict[str, Any]],
+) -> None:
+    # You cannot define multiple models within a model group, when no router is defined.
+    for model_group in model_groups:
+        if (
+            len(model_group[MODELS_CONFIG_KEY]) > 1
+            and ROUTER_CONFIG_KEY not in model_group
+        ):
             structlogger.error(
-                "validation.component.api_type_config_key_invalid",
+                "validate_model_group_configuration_setup.router_not_present",
                 event_info=(
-                    f"You specified '{API_TYPE_CONFIG_KEY}: {api_type}' for "
-                    f"'{component_name or component_config['name']}', which is not "
-                    f"allowed. "
-                    f"The '{API_TYPE_CONFIG_KEY}' key can only be used for the "
-                    f"following providers: {VALID_PROVIDERS_FOR_API_TYPE_CONFIG_KEY}. "
-                    f"For other providers, please use the '{PROVIDER_CONFIG_KEY}' key."
+                    f"You defined multiple models for the model group "
+                    f"'{model_group[MODEL_GROUP_ID_CONFIG_KEY]}', but no router. "
+                    "If a model group contains multiple models, a router must be "
+                    "defined. Please define a router for the model group "
+                    f"'{model_group[MODEL_GROUP_ID_CONFIG_KEY]}'."
                 ),
+                model_group_id=model_group[MODEL_GROUP_ID_CONFIG_KEY],
             )
             sys.exit(1)
+
+
+def _validate_model_group_router_setting(
+    model_groups: List[Dict[str, Any]],
+) -> None:
+    # You cannot define multiple models within a model group, when no router is defined.
+    for model_group in model_groups:
+        if ROUTER_CONFIG_KEY not in model_group:
+            continue
+
+        for model_config in model_group.get(MODELS_CONFIG_KEY, []):
+            if USE_CHAT_COMPLETIONS_ENDPOINT_CONFIG_KEY in model_config:
+                structlogger.error(
+                    "validation.validate_model_group_configuration_setup"
+                    f".{USE_CHAT_COMPLETIONS_ENDPOINT_CONFIG_KEY}_set_incorrectly",
+                    event_info=(
+                        f"You defined the '{USE_CHAT_COMPLETIONS_ENDPOINT_CONFIG_KEY}' "
+                        f"in the model group "
+                        f"'{model_group[MODEL_GROUP_ID_CONFIG_KEY]}'. This key is not "
+                        f"allowed in the model configuration as the router is defined. "
+                        f"Please remove this key from your model configuration and "
+                        f"update it in the '{ROUTER_CONFIG_KEY} configuration, as it "
+                        f"is a router level setting."
+                    ),
+                    model_group_id=model_group[MODEL_GROUP_ID_CONFIG_KEY],
+                )
+                sys.exit(1)
+
+        router_config = model_group[ROUTER_CONFIG_KEY]
+        if ROUTING_STRATEGY_CONFIG_KEY in router_config:
+            routing_strategy = router_config.get(ROUTING_STRATEGY_CONFIG_KEY)
+            if routing_strategy and routing_strategy not in VALID_ROUTING_STRATEGIES:
+                structlogger.error(
+                    "validation.validate_model_group_configuration_setup"
+                    ".invalid_routing_strategy",
+                    event_info=(
+                        f"The routing strategy '{routing_strategy}' you defined for "
+                        f"the model group '{model_group[MODEL_GROUP_ID_CONFIG_KEY]}' "
+                        f"is not valid. Valid routing strategies are categorized as "
+                        f"follows:\n"
+                        f"- Strategies requiring Redis caching: "
+                        f"{', '.join(ROUTING_STRATEGIES_REQUIRING_REDIS_CACHE)}\n"
+                        f"- Strategies not requiring caching: "
+                        f"{', '.join(ROUTING_STRATEGIES_NOT_REQUIRING_CACHE)}"
+                    ),
+                    model_group_id=model_group[MODEL_GROUP_ID_CONFIG_KEY],
+                    invalid_routing_strategy=routing_strategy,
+                    supported_routing_strategies_requiring_redis_cache=(
+                        ROUTING_STRATEGIES_REQUIRING_REDIS_CACHE
+                    ),
+                    supported_routing_strategies_not_requiring_redis_cache=(
+                        ROUTING_STRATEGIES_NOT_REQUIRING_CACHE
+                    ),
+                )
+                sys.exit(1)
+            if (
+                routing_strategy in ROUTING_STRATEGIES_REQUIRING_REDIS_CACHE
+                and REDIS_HOST_CONFIG_KEY not in router_config
+            ):
+                structlogger.warning(
+                    "validation.routing_strategy.redis_host_not_defined",
+                    event_info=(
+                        f"The routing strategy '{routing_strategy}' requires a Redis "
+                        f"host to be defined. Without a Redis host, the system "
+                        f"defaults to 'in-memory' caching. Please add the "
+                        f"'{REDIS_HOST_CONFIG_KEY}' to the router configuration for "
+                        f"the model group '{model_group[MODEL_GROUP_ID_CONFIG_KEY]}'."
+                    ),
+                    model_group_id=model_group[MODEL_GROUP_ID_CONFIG_KEY],
+                )
+
+
+def _validate_usage_of_environment_variables_in_model_group_config(
+    model_groups: List[Dict[str, Any]],
+) -> None:
+    # Limit the use of ${env_var} in the model_groups config to the following variables:
+    # - deployment,
+    # - api_base, api_version and api_key,
+    # - aws_region_name, aws_access_key_id, aws_secret_access_key, and aws_session_token
+    allowed_env_vars = {
+        DEPLOYMENT_CONFIG_KEY,
+        API_BASE_CONFIG_KEY,
+        API_KEY,
+        API_VERSION_CONFIG_KEY,
+        AWS_REGION_NAME_CONFIG_KEY,
+        AWS_ACCESS_KEY_ID_CONFIG_KEY,
+        AWS_SECRET_ACCESS_KEY_CONFIG_KEY,
+        AWS_SESSION_TOKEN_CONFIG_KEY,
+    }
+
+    for model_group in model_groups:
+        for model_config in model_group[MODELS_CONFIG_KEY]:
+            for key, value in model_config.items():
+                if isinstance(value, str):
+                    if re.match(r"\${(\w+)}", value) and key not in allowed_env_vars:
+                        structlogger.error(
+                            "validation.validate_model_group_configuration_setup"
+                            ".invalid_use_of_environment_variables",
+                            event_info=(
+                                f"You defined '{key}' as environment variable in model "
+                                f"group '{model_group[MODEL_GROUP_ID_CONFIG_KEY]}', "
+                                f"which is not allowed. "
+                                f"You can only use environment variables for the "
+                                f"following keys: {', '.join(allowed_env_vars)}. "
+                                f"Please update your config."
+                            ),
+                            model_group_id=model_group[MODEL_GROUP_ID_CONFIG_KEY],
+                            key=key,
+                            allowed_keys_for_env_vars=allowed_env_vars,
+                        )
+                        sys.exit(1)
+
+
+def _validate_sensitive_keys_are_an_environment_variables_for_model_groups(
+    model_groups: List[Dict[str, Any]],
+) -> None:
+    # the api key can only be set as an environment variable
+    for model_group in model_groups:
+        for model_config in model_group[MODELS_CONFIG_KEY]:
+            for key, value in model_config.items():
+                if key in SENSITIVE_DATA:
+                    if isinstance(value, str):
+                        if not re.match(r"\${(\w+)}", value):
+                            structlogger.error(
+                                "validation.validate_model_group_configuration_setup"
+                                ".sensitive_key_string_value_must_be_set_as_env_var",
+                                event_info=(
+                                    f"You defined the '{key}' in model group "
+                                    f"'{model_group[MODEL_GROUP_ID_CONFIG_KEY]}' as a "
+                                    f"string. The '{key}' must be set as an "
+                                    f"environment variable. Please update your config."
+                                ),
+                                key=key,
+                                model_group_id=model_group[MODEL_GROUP_ID_CONFIG_KEY],
+                            )
+                            sys.exit(1)
+                    else:
+                        structlogger.error(
+                            "validation.validate_model_group_configuration_setup"
+                            ".sensitive_key_must_be_set_as_env_var",
+                            event_info=(
+                                f"You should define the '{key}' in model group "
+                                f"'{model_group[MODEL_GROUP_ID_CONFIG_KEY]}' using the "
+                                f"environment variable syntax - "
+                                f"${{ENV_VARIABLE_NAME}}. "
+                                f"Please update your config."
+                            ),
+                            key=key,
+                            model_group_id=model_group[MODEL_GROUP_ID_CONFIG_KEY],
+                        )
+                        sys.exit(1)
+
+
+def validate_model_group_configuration_setup() -> None:
+    """Validates the model group configuration setup in endpoints.yml."""
+    endpoints = AvailableEndpoints.get_instance()
+
+    if endpoints.model_groups is None:
+        return
+
+    _validate_unique_model_group_ids(endpoints.model_groups)
+    _validate_model_group_with_multiple_models(endpoints.model_groups)
+    _validate_usage_of_environment_variables_in_model_group_config(
+        endpoints.model_groups
+    )
+    _validate_sensitive_keys_are_an_environment_variables_for_model_groups(
+        endpoints.model_groups
+    )
+    _validate_model_group_router_setting(endpoints.model_groups)
+
+
+def validate_command_generator_exclusivity(schema: GraphSchema) -> None:
+    """Validate that multiple command generators are not defined at same time."""
+    from rasa.dialogue_understanding.generator import (
+        LLMBasedCommandGenerator,
+    )
+
+    count = schema.count_nodes_of_a_given_type(
+        LLMBasedCommandGenerator, include_subtypes=True
+    )
+
+    if count > 1:
+        structlogger.error(
+            "validation.command_generator.multiple_command_generator_defined",
+            event_info=(
+                "Multiple LLM based command generators are defined in the config. "
+                "Please use only one LLM based command generator."
+            ),
+        )
+        sys.exit(1)
+
+
+def validate_command_generator_setup(
+    model_configuration: GraphModelConfiguration,
+) -> None:
+    schema = model_configuration.predict_schema
+    validate_command_generator_exclusivity(schema)