rasa-pro 3.11.0rc2__py3-none-any.whl → 3.11.1__py3-none-any.whl

rasa/model_manager/warm_rasa_process.py

@@ -0,0 +1,187 @@
+import shlex
+import subprocess
+from rasa.__main__ import main
+import os
+from typing import List
+import structlog
+from dataclasses import dataclass
+import uuid
+
+from rasa.model_manager import config
+from rasa.model_manager.utils import ensure_base_directory_exists, logs_path
+
+structlogger = structlog.get_logger(__name__)
+
+warm_rasa_processes: List["WarmRasaProcess"] = []
+
+NUMBER_OF_INITIAL_PROCESSES = 3
+
+
+@dataclass
+class WarmRasaProcess:
+    """Data class to store a warm Rasa process.
+
+    A "warm" Rasa process is one where we've done the heavy lifting of
+    importing key modules ahead of time (e.g. litellm). This is to avoid
+    long import times when we actually want to run a command.
+
+    This is a started process waiting for a Rasa CLI command. It's
+    output is stored in a log file identified by `log_id`.
+    """
+
+    process: subprocess.Popen
+    log_id: str
+
+
+def _create_warm_rasa_process() -> WarmRasaProcess:
+    """Create a new warm Rasa process."""
+    command = [
+        config.RASA_PYTHON_PATH,
+        "-m",
+        "rasa.model_manager.warm_rasa_process",
+    ]
+
+    envs = os.environ.copy()
+    envs["RASA_TELEMETRY_ENABLED"] = "false"
+
+    log_id = uuid.uuid4().hex
+    log_path = logs_path(log_id)
+
+    ensure_base_directory_exists(log_path)
+
+    process = subprocess.Popen(
+        command,
+        stdout=open(log_path, "w"),
+        stderr=subprocess.STDOUT,
+        stdin=subprocess.PIPE,
+        env=envs,
+    )
+
+    structlogger.debug(
+        "model_trainer.created_warm_rasa_process",
+        pid=process.pid,
+        command=command,
+        log_path=log_path,
+    )
+
+    return WarmRasaProcess(process=process, log_id=log_id)
+
+
+def initialize_warm_rasa_process() -> None:
+    """Initialize the warm Rasa processes."""
+    global warm_rasa_processes
+    for _ in range(NUMBER_OF_INITIAL_PROCESSES):
+        warm_rasa_processes.append(_create_warm_rasa_process())
+
+
+def shutdown_warm_rasa_processes() -> None:
+    """Shutdown all warm Rasa processes."""
+    global warm_rasa_processes
+    for warm_rasa_process in warm_rasa_processes:
+        warm_rasa_process.process.terminate()
+    warm_rasa_processes = []
+
+
+def start_rasa_process(cwd: str, arguments: List[str]) -> WarmRasaProcess:
+    """Start a Rasa process.
+
+    This will start a Rasa process with the given current working directory
+    and arguments. The process will be a warm one, meaning that it has already
+    imported all necessary modules.
+    """
+    warm_rasa_process = _get_warm_rasa_process()
+    _pass_arguments_to_process(warm_rasa_process.process, cwd, arguments)
+    return warm_rasa_process
+
+
+def _get_warm_rasa_process() -> WarmRasaProcess:
+    """Get a warm Rasa process.
+
+    This will return a warm Rasa process from the pool and create a
+    new one to replace it.
+    """
+    global warm_rasa_processes
+
+    if not warm_rasa_processes:
+        warm_rasa_processes = [_create_warm_rasa_process()]
+
+    previous_warm_rasa_process = warm_rasa_processes.pop(0)
+
+    if previous_warm_rasa_process.process.poll() is not None:
+        # process has finished (for some reason...)
+        # back up plan is to create a new one on the spot.
+        # this should not happen, but let's be safe
+        structlogger.warning(
+            "model_trainer.warm_rasa_process_finished_unexpectedly",
+            pid=previous_warm_rasa_process.process.pid,
+        )
+        previous_warm_rasa_process = _create_warm_rasa_process()
+
+    warm_rasa_processes.append(_create_warm_rasa_process())
+    return previous_warm_rasa_process
+
+
+def _pass_arguments_to_process(
+    process: subprocess.Popen, cwd: str, arguments: List[str]
+) -> None:
+    """Pass arguments to a warm Rasa process.
+
+    The process is waiting for input on stdin. We pass the current working
+    directory and the arguments to run a Rasa CLI command.
+    """
+    arguments_string = " ".join(arguments)
+    # send arguments to stdin
+    process.stdin.write(cwd.encode())  # type: ignore[union-attr]
+    process.stdin.write("\n".encode())  # type: ignore[union-attr]
+    process.stdin.write(arguments_string.encode())  # type: ignore[union-attr]
+    process.stdin.write("\n".encode())  # type: ignore[union-attr]
+    process.stdin.flush()  # type: ignore[union-attr]
+
+
+def warmup() -> None:
+    """Import all necessary modules to warm up the process.
+
+    This should include all the modules that take a long time to import.
+    We import them now, so that the training / deployment can later
+    directly start.
+    """
+    try:
+        import presidio_analyzer  # noqa: F401
+        import litellm  # noqa: F401
+        import langchain  # noqa: F401
+        import tensorflow  # noqa: F401
+        import matplotlib  # noqa: F401
+        import pandas  # noqa: F401
+        import numpy  # noqa: F401
+        import spacy  # noqa: F401
+        import rasa.validator  # noqa: F401
+    except ImportError:
+        pass
+
+
+def warm_rasa_main() -> None:
+    """Entry point for processes waiting for their command to run.
+
+    The process will wait for the current working directory and the command
+    to run. These will be send on stdin by the parent process. After receiving
+    the input, we will kick things of starting or running a bot.
+
+    Uses the normal Rasa CLI entry point (e.g. `rasa train --data ...`).
+    """
+    warmup()
+
+    cwd = input()
+
+    # this should be `train --data ...` or similar
+    cli_arguments_str = input()
+    # splits the arguments string into a list of arguments as expected by `argparse`
+    arguments = shlex.split(cli_arguments_str)
+
+    # needed to make sure the passed arguments are relative to the working directory
+    os.chdir(cwd)
+
+    main(arguments)
+
+
+if __name__ == "__main__":
+    warm_rasa_main()

rasa/model_service.py

@@ -8,7 +8,7 @@ from rasa.core.persistor import RemoteStorageType, get_persistor
 from rasa.core.utils import list_routes
 from rasa.model_manager import model_api
 from rasa.model_manager import config
-from rasa.model_manager.config import SERVER_BASE_URL
+from rasa.model_manager.config import SERVER_BASE_URL, SERVER_PORT
 from rasa.utils.common import configure_logging_and_warnings
 import rasa.utils.licensing
 from urllib.parse import urlparse
@@ -18,8 +18,6 @@ from rasa.utils.sanic_error_handler import register_custom_sanic_error_handler
 
 structlogger = structlog.get_logger()
 
-MODEL_SERVICE_PORT = 8000
-
 
 def url_prefix_from_base_url() -> str:
     """Return the path prefix from the base URL."""
@@ -93,7 +91,7 @@ def main() -> None:
 
     validate_model_storage_type()
 
-    structlogger.debug("model_api.starting_server", port=MODEL_SERVICE_PORT)
+    structlogger.debug("model_api.starting_server", port=SERVER_PORT)
 
     url_prefix = url_prefix_from_base_url()
     # configure the sanic application
@@ -107,7 +105,7 @@ def main() -> None:
 
     register_custom_sanic_error_handler(app)
 
-    app.run(host="0.0.0.0", port=MODEL_SERVICE_PORT, legacy=True, motd=False)
+    app.run(host="0.0.0.0", port=SERVER_PORT, legacy=True, motd=False)
 
 
 if __name__ == "__main__":

rasa/model_training.py

@@ -322,8 +322,10 @@ async def _train_graph(
     rasa.engine.validation.validate_coexistance_routing_setup(
         domain, model_configuration, flows
     )
-    rasa.engine.validation.validate_model_client_configuration_setup(config)
     rasa.engine.validation.validate_model_group_configuration_setup()
+    rasa.engine.validation.validate_model_client_configuration_setup_during_training_time(
+        config
+    )
     rasa.engine.validation.validate_flow_component_dependencies(
         flows, model_configuration
     )

rasa/shared/constants.py

@@ -149,6 +149,10 @@ AZURE_AD_TOKEN_ENV_VAR = "AZURE_AD_TOKEN"
 AZURE_API_BASE_ENV_VAR = "AZURE_API_BASE"
 AZURE_API_VERSION_ENV_VAR = "AZURE_API_VERSION"
 AZURE_API_TYPE_ENV_VAR = "AZURE_API_TYPE"
+AZURE_SPEECH_API_KEY_ENV_VAR = "AZURE_SPEECH_API_KEY"
+
+DEEPGRAM_API_KEY_ENV_VAR = "DEEPGRAM_API_KEY"
+CARTESIA_API_KEY_ENV_VAR = "CARTESIA_API_KEY"
 
 OPENAI_API_KEY_ENV_VAR = "OPENAI_API_KEY"
 OPENAI_API_TYPE_ENV_VAR = "OPENAI_API_TYPE"
@@ -159,6 +163,9 @@ OPENAI_API_BASE_CONFIG_KEY = "openai_api_base"
 OPENAI_API_TYPE_CONFIG_KEY = "openai_api_type"
 OPENAI_API_VERSION_CONFIG_KEY = "openai_api_version"
 
+AWS_BEDROCK_PROVIDER = "bedrock"
+AWS_SAGEMAKER_PROVIDER = "sagemaker"
+
 API_BASE_CONFIG_KEY = "api_base"
 API_TYPE_CONFIG_KEY = "api_type"
 API_VERSION_CONFIG_KEY = "api_version"
@@ -184,19 +191,19 @@ N_REPHRASES_CONFIG_KEY = "n"
 USE_CHAT_COMPLETIONS_ENDPOINT_CONFIG_KEY = "use_chat_completions_endpoint"
 
 ROUTER_CONFIG_KEY = "router"
-ROUTER_STRATEGY_CONFIG_KEY = "router_strategy"
+ROUTING_STRATEGY_CONFIG_KEY = "routing_strategy"
 REDIS_HOST_CONFIG_KEY = "redis_host"
-ROUTER_STRATEGIES_REQUIRING_REDIS_CACHE = [
+ROUTING_STRATEGIES_REQUIRING_REDIS_CACHE = [
     "cost-based-routing",
     "usage-based-routing",
 ]
-ROUTER_STRATEGIES_NOT_REQUIRING_CACHE = [
+ROUTING_STRATEGIES_NOT_REQUIRING_CACHE = [
     "latency-based-routing",
     "least-busy",
     "simple-shuffle",
 ]
-VALID_ROUTER_STRATEGIES = (
-    ROUTER_STRATEGIES_REQUIRING_REDIS_CACHE + ROUTER_STRATEGIES_NOT_REQUIRING_CACHE
+VALID_ROUTING_STRATEGIES = (
+    ROUTING_STRATEGIES_REQUIRING_REDIS_CACHE + ROUTING_STRATEGIES_NOT_REQUIRING_CACHE
 )
 
 MODELS_CONFIG_KEY = "models"
@@ -219,6 +226,14 @@ AZURE_API_VERSION_ENV_VAR = "AZURE_API_VERSION"
 AZURE_API_TYPE_ENV_VAR = "AZURE_API_TYPE"
 
 AWS_REGION_NAME_CONFIG_KEY = "aws_region_name"
+AWS_ACCESS_KEY_ID_CONFIG_KEY = "aws_access_key_id"
+AWS_SECRET_ACCESS_KEY_CONFIG_KEY = "aws_secret_access_key"
+AWS_SESSION_TOKEN_CONFIG_KEY = "aws_session_token"
+
+AWS_ACCESS_KEY_ID_ENV_VAR = "AWS_ACCESS_KEY_ID"
+AWS_SECRET_ACCESS_KEY_ENV_VAR = "AWS_SECRET_ACCESS_KEY"
+AWS_REGION_NAME_ENV_VAR = "AWS_REGION_NAME"
+AWS_SESSION_TOKEN_ENV_VAR = "AWS_SESSION_TOKEN"
 
 HUGGINGFACE_MULTIPROCESS_CONFIG_KEY = "multi_process"
 HUGGINGFACE_CACHE_FOLDER_CONFIG_KEY = "cache_folder"
@@ -280,3 +295,10 @@ RASA_PATTERN_CANNOT_HANDLE_INVALID_INTENT = (
 )
 
 ROUTE_TO_CALM_SLOT = "route_session_to_calm"
+
+SENSITIVE_DATA = [
+    API_KEY,
+    AWS_ACCESS_KEY_ID_CONFIG_KEY,
+    AWS_SECRET_ACCESS_KEY_CONFIG_KEY,
+    AWS_SESSION_TOKEN_CONFIG_KEY,
+]

rasa/shared/core/constants.py

@@ -110,8 +110,8 @@ FLOW_SLOT_NAMES = [FLOW_HASHES_SLOT]
 
 # slots for audio timeout
 SLOT_SILENCE_TIMEOUT = "silence_timeout"
-SILENCE_TIMEOUT_DEFAULT_VALUE = 6.0
 SLOT_CONSECUTIVE_SILENCE_TIMEOUTS = "consecutive_silence_timeouts"
+SILENCE_TIMEOUT_DEFAULT_VALUE = 6.0
 SILENCE_SLOTS = [SLOT_SILENCE_TIMEOUT, SLOT_CONSECUTIVE_SILENCE_TIMEOUTS]
 # slots for knowledge base
 SLOT_LISTED_ITEMS = "knowledge_base_listed_objects"

rasa/shared/core/domain.py

@@ -3,7 +3,6 @@ from __future__ import annotations
 import collections
 import copy
 import json
-import math
 import os
 from dataclasses import dataclass
 from functools import cached_property
@@ -58,7 +57,6 @@ from rasa.shared.core.events import SlotSet, UserUttered
 from rasa.shared.core.slots import (
     AnySlot,
     CategoricalSlot,
-    FloatSlot,
     ListSlot,
     Slot,
     TextSlot,
@@ -198,6 +196,7 @@ class Domain:
     """
 
     validate_yaml: ClassVar[bool] = True
+    expand_env_vars: ClassVar[bool] = True
 
     @classmethod
     def empty(cls) -> Domain:
@@ -1084,7 +1083,6 @@ class Domain:
         self._add_knowledge_base_slots()
         self._add_categorical_slot_default_value()
         self._add_session_metadata_slot()
-        self._add_audio_slots()
 
     def _add_categorical_slot_default_value(self) -> None:
         """Add a default value to all categorical slots.
@@ -1139,29 +1137,6 @@ class Domain:
                 )
             )
 
-    def _add_audio_slots(self) -> None:
-        """Add slots relevant for audio channels."""
-        self.slots.append(
-            FloatSlot(
-                rasa.shared.core.constants.SLOT_SILENCE_TIMEOUT,
-                mappings=[],
-                influence_conversation=False,
-                is_builtin=True,
-                initial_value=rasa.shared.core.constants.SILENCE_TIMEOUT_DEFAULT_VALUE,
-                max_value=math.inf,
-            )
-        )
-        self.slots.append(
-            FloatSlot(
-                rasa.shared.core.constants.SLOT_CONSECUTIVE_SILENCE_TIMEOUTS,
-                mappings=[],
-                influence_conversation=False,
-                is_builtin=True,
-                initial_value=0.0,
-                max_value=math.inf,
-            )
-        )
-
     def _add_knowledge_base_slots(self) -> None:
         """Add slots for the knowledge base action to slots.
 
@@ -1981,8 +1956,8 @@ class Domain:
         """Check whether the domain is empty."""
         return self.as_dict() == Domain.empty().as_dict()
 
-    @staticmethod
-    def is_domain_file(filename: Union[Text, Path]) -> bool:
+    @classmethod
+    def is_domain_file(cls, filename: Union[Text, Path]) -> bool:
         """Checks whether the given file path is a Rasa domain file.
 
         Args:
@@ -2001,7 +1976,7 @@ class Domain:
             return False
 
         try:
-            content = read_yaml_file(filename)
+            content = read_yaml_file(filename, expand_env_vars=cls.expand_env_vars)
         except (RasaException, YamlSyntaxException):
             structlogger.warning(
                 "domain.cannot_load_domain_file",
@@ -2130,10 +2105,12 @@ class Domain:
                 "domain.from_yaml.validating",
             )
             validate_raw_yaml_using_schema_file_with_responses(
-                raw_yaml_content, DOMAIN_SCHEMA_FILE
+                raw_yaml_content,
+                DOMAIN_SCHEMA_FILE,
+                expand_env_vars=cls.expand_env_vars,
             )
 
-        return read_yaml(raw_yaml_content)
+        return read_yaml(raw_yaml_content, expand_env_vars=cls.expand_env_vars)
 
 
 def warn_about_duplicates_found_during_domain_merging(

rasa/shared/core/flows/yaml_flows_io.py

@@ -1,5 +1,5 @@
 from pathlib import Path
-from typing import Any, Dict, List, Optional, Text, Union
+from typing import Any, ClassVar, Dict, List, Optional, Text, Union
 
 import jsonschema
 import ruamel.yaml.nodes as yaml_nodes
@@ -25,6 +25,8 @@ KEY_FLOWS = "flows"
 class YAMLFlowsReader:
     """Class that reads flows information in YAML format."""
 
+    expand_env_vars: ClassVar[bool] = True
+
     @classmethod
     def read_from_file(
         cls, filename: Union[Text, Path], add_line_numbers: bool = True
@@ -217,14 +219,21 @@ class YAMLFlowsReader:
             `Flow`s read from `string`.
         """
         validate_yaml_with_jsonschema(
-            string, FLOWS_SCHEMA_FILE, humanize_error=cls.humanize_flow_error
+            string,
+            FLOWS_SCHEMA_FILE,
+            humanize_error=cls.humanize_flow_error,
+            expand_env_vars=cls.expand_env_vars,
         )
         if add_line_numbers:
-            yaml_content = read_yaml(string, custom_constructor=line_number_constructor)
+            yaml_content = read_yaml(
+                string,
+                custom_constructor=line_number_constructor,
+                expand_env_vars=cls.expand_env_vars,
+            )
             yaml_content = process_yaml_content(yaml_content)
 
         else:
-            yaml_content = read_yaml(string)
+            yaml_content = read_yaml(string, expand_env_vars=cls.expand_env_vars)
 
         return FlowsList.from_json(yaml_content.get(KEY_FLOWS, {}), file_path=file_path)
 

rasa/shared/importers/importer.py

@@ -1,7 +1,18 @@
 import logging
 from abc import ABC, abstractmethod
 from functools import reduce
-from typing import Any, Dict, List, Optional, Set, Text, Tuple, Type, Union, cast
+from typing import (
+    Any,
+    Dict,
+    List,
+    Optional,
+    Set,
+    Text,
+    Tuple,
+    Type,
+    Union,
+    cast,
+)
 
 import importlib_resources
 
@@ -167,6 +178,7 @@ class TrainingDataImporter(ABC):
         domain_path: Optional[Text] = None,
         training_data_paths: Optional[List[Text]] = None,
         args: Optional[Dict[Text, Any]] = None,
+        expand_env_vars: bool = True,
     ) -> "TrainingDataImporter":
         """Loads a `TrainingDataImporter` instance from a dictionary."""
         from rasa.shared.importers.rasa import RasaFileImporter
@@ -182,7 +194,12 @@ class TrainingDataImporter(ABC):
         importers = [importer for importer in importers if importer]
         if not importers:
             importers = [
-                RasaFileImporter(config_path, domain_path, training_data_paths)
+                RasaFileImporter(
+                    config_path,
+                    domain_path,
+                    training_data_paths,
+                    expand_env_vars=expand_env_vars,
+                )
             ]
 
         return E2EImporter(

rasa/shared/importers/rasa.py

@@ -29,7 +29,9 @@ class RasaFileImporter(TrainingDataImporter):
         config_file: Optional[Text] = None,
         domain_path: Optional[Text] = None,
         training_data_paths: Optional[Union[List[Text], Text]] = None,
+        expand_env_vars: bool = True,
     ):
+        self.expand_env_vars = expand_env_vars
         self._domain_path = domain_path
 
         self._nlu_files = rasa.shared.data.get_data_files(
@@ -54,7 +56,9 @@ class RasaFileImporter(TrainingDataImporter):
             logger.debug("No configuration file was provided to the RasaFileImporter.")
             return {}
 
-        config = read_model_configuration(self.config_file)
+        config = read_model_configuration(
+            self.config_file, expand_env_vars=self.expand_env_vars
+        )
         return config
 
     def get_config_file_for_auto_config(self) -> Optional[Text]:

rasa/shared/nlu/training_data/formats/rasa_yaml.py

@@ -1,7 +1,18 @@
 import logging
 from collections import OrderedDict
 from pathlib import Path
-from typing import Text, Any, List, Dict, Tuple, Union, Iterator, Optional, Callable
+from typing import (
+    ClassVar,
+    Text,
+    Any,
+    List,
+    Dict,
+    Tuple,
+    Union,
+    Iterator,
+    Optional,
+    Callable,
+)
 
 import rasa.shared.data
 from rasa.shared.core.domain import Domain
@@ -55,6 +66,8 @@ STRIP_SYMBOLS = "\n\r "
 class RasaYAMLReader(TrainingDataReader):
     """Reads YAML training data and creates a TrainingData object."""
 
+    expand_env_vars: ClassVar[bool] = True
+
     def __init__(self) -> None:
         super().__init__()
         self.training_examples: List[Message] = []
@@ -69,7 +82,9 @@ class RasaYAMLReader(TrainingDataReader):
         If the string is not in the right format, an exception will be raised.
         """
         try:
-            validate_raw_yaml_using_schema_file_with_responses(string, NLU_SCHEMA_FILE)
+            validate_raw_yaml_using_schema_file_with_responses(
+                string, NLU_SCHEMA_FILE, expand_env_vars=self.expand_env_vars
+            )
         except YamlException as e:
             e.filename = self.filename
             raise e
@@ -88,7 +103,7 @@ class RasaYAMLReader(TrainingDataReader):
         """
         self.validate(string)
 
-        yaml_content = read_yaml(string)
+        yaml_content = read_yaml(string, expand_env_vars=self.expand_env_vars)
 
         if not validate_training_data_format_version(yaml_content, self.filename):
             return TrainingData()

rasa/shared/providers/_configs/litellm_router_client_config.py

@@ -14,6 +14,7 @@ from rasa.shared.constants import (
     API_TYPE_CONFIG_KEY,
     MODEL_CONFIG_KEY,
     MODEL_LIST_KEY,
+    USE_CHAT_COMPLETIONS_ENDPOINT_CONFIG_KEY,
 )
 from rasa.shared.providers._configs.model_group_config import (
     ModelGroupConfig,
@@ -29,6 +30,7 @@ _LITELLM_UNSUPPORTED_KEYS = [
     PROVIDER_CONFIG_KEY,
     DEPLOYMENT_CONFIG_KEY,
     API_TYPE_CONFIG_KEY,
+    USE_CHAT_COMPLETIONS_ENDPOINT_CONFIG_KEY,
 ]
 
 
@@ -84,6 +86,7 @@ class LiteLLMRouterClientConfig:
 
     _model_group_config: ModelGroupConfig
     router: Dict[str, Any]
+    _use_chat_completions_endpoint: bool = True
     extra_parameters: dict = field(default_factory=dict)
 
     @property
@@ -98,6 +101,14 @@ class LiteLLMRouterClientConfig:
     def litellm_model_list(self) -> List[Dict[str, Any]]:
         return self._convert_models_to_litellm_model_list()
 
+    @property
+    def litellm_router_settings(self) -> Dict[str, Any]:
+        return self._convert_router_to_litellm_router_settings()
+
+    @property
+    def use_chat_completions_endpoint(self) -> bool:
+        return self._use_chat_completions_endpoint
+
     def __post_init__(self) -> None:
         if not self.router:
             message = "Router cannot be empty."
@@ -121,7 +132,6 @@ class LiteLLMRouterClientConfig:
         Returns:
             LiteLLMRouterClientConfig
         """
-
         model_group_config = ModelGroupConfig.from_dict(config)
 
         # Copy config to avoid mutating the original
@@ -130,13 +140,18 @@ class LiteLLMRouterClientConfig:
         config_copy.pop(MODEL_GROUP_ID_CONFIG_KEY, None)
         config_copy.pop(MODELS_CONFIG_KEY, None)
         # Get the router settings
-        router_settings = config_copy.pop(ROUTER_CONFIG_KEY, None)
+        router_settings = config_copy.pop(ROUTER_CONFIG_KEY, {})
+        # Get the use_chat_completions_endpoint setting
+        use_chat_completions_endpoint = router_settings.get(
+            USE_CHAT_COMPLETIONS_ENDPOINT_CONFIG_KEY, True
+        )
         # The rest is considered as extra parameters
         extra_parameters = config_copy
 
         this = LiteLLMRouterClientConfig(
             _model_group_config=model_group_config,
             router=router_settings,
+            _use_chat_completions_endpoint=use_chat_completions_endpoint,
             extra_parameters=extra_parameters,
         )
         return this
@@ -150,14 +165,17 @@ class LiteLLMRouterClientConfig:
         return d
 
     def to_litellm_dict(self) -> dict:
-        litellm_model_list = self._convert_models_to_litellm_model_list()
-        d = {
+        return {
             **self.extra_parameters,
             MODEL_GROUP_ID_CONFIG_KEY: self.model_group_id,
-            MODEL_LIST_KEY: litellm_model_list,
-            ROUTER_CONFIG_KEY: self.router,
+            MODEL_LIST_KEY: self._convert_models_to_litellm_model_list(),
+            ROUTER_CONFIG_KEY: self._convert_router_to_litellm_router_settings(),
         }
-        return d
+
+    def _convert_router_to_litellm_router_settings(self) -> Dict[str, Any]:
+        _router_settings_copy = copy.deepcopy(self.router)
+        _router_settings_copy.pop(USE_CHAT_COMPLETIONS_ENDPOINT_CONFIG_KEY, None)
+        return _router_settings_copy
 
     def _convert_models_to_litellm_model_list(self) -> List[Dict[str, Any]]:
         litellm_model_list = []
@@ -172,7 +190,7 @@ class LiteLLMRouterClientConfig:
             prefix = get_prefix_from_provider(provider)
 
             # Determine whether to use model or deployment key based on the provider.
-            litellm_model_name_without_prefix = (
+            litellm_model_name = (
                 litellm_model_config[DEPLOYMENT_CONFIG_KEY]
                 if provider in DEPLOYMENT_CENTRIC_PROVIDERS
                 else litellm_model_config[MODEL_CONFIG_KEY]
@@ -180,7 +198,9 @@ class LiteLLMRouterClientConfig:
 
             # Set 'model' to a provider prefixed model name e.g. openai/gpt-4
             litellm_model_config[MODEL_CONFIG_KEY] = (
-                f"{prefix}/{litellm_model_name_without_prefix}"
+                litellm_model_name
+                if f"{prefix}/" in litellm_model_name
+                else f"{prefix}/{litellm_model_name}"
             )
 
             # Remove parameters that are None and not supported by LiteLLM.