rasa-pro 3.12.12.dev1__py3-none-any.whl → 3.12.14__py3-none-any.whl

rasa/llm_fine_tuning/llm_data_preparation_module.py

@@ -1,13 +1,23 @@
 from dataclasses import dataclass
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Optional, cast
 
 import structlog
 from tqdm import tqdm
 
+from rasa.core.agent import Agent
+from rasa.core.channels import UserMessage
 from rasa.dialogue_understanding.commands.prompt_command import PromptCommand
+from rasa.dialogue_understanding.utils import set_record_commands_and_prompts
 from rasa.llm_fine_tuning.conversations import Conversation, ConversationStep
 from rasa.llm_fine_tuning.storage import StorageContext
-from rasa.llm_fine_tuning.utils import commands_as_string
+from rasa.llm_fine_tuning.utils import (
+    commands_as_string,
+    make_mock_invoke_llm,
+    patch_invoke_llm_in_generators,
+)
+from rasa.shared.core.trackers import DialogueStateTracker
+from rasa.shared.nlu.constants import KEY_USER_PROMPT, PROMPTS
+from rasa.shared.utils.llm import generate_sender_id
 
 LLM_DATA_PREPARATION_MODULE_STORAGE_LOCATION = "3_llm_finetune_data/llm_ft_data.jsonl"
 
@@ -47,40 +57,8 @@ def _create_data_point(
     )
 
 
-def _update_prompt(
-    prompt: str,
-    original_user_steps: List[ConversationStep],
-    rephrased_user_steps: List[str],
-) -> Optional[str]:
-    if len(original_user_steps) != len(rephrased_user_steps):
-        structlogger.debug(
-            "llm_fine_tuning.llm_data_preparation_module.failed_to_update_prompt",
-            original_user_steps=[
-                step.original_test_step.text for step in original_user_steps
-            ],
-            rephrased_user_steps=rephrased_user_steps,
-        )
-        return None
-
-    updated_prompt = prompt
-    for user_step, rephrased_message in zip(original_user_steps, rephrased_user_steps):
-        # replace all occurrences of the original user message with the rephrased user
-        # message in the conversation history mentioned in the prompt
-        updated_prompt = updated_prompt.replace(
-            f"USER: {user_step.original_test_step.text}", f"USER: {rephrased_message}"
-        )
-
-    # replace the latest user message mentioned in the prompt
-    updated_prompt = updated_prompt.replace(
-        f"'''{original_user_steps[-1].original_test_step.text}'''",
-        f"'''{rephrased_user_steps[-1]}'''",
-    )
-
-    return updated_prompt
-
-
-def _convert_conversation_into_llm_data(
-    conversation: Conversation,
+async def _convert_conversation_into_llm_data(
+    conversation: Conversation, agent: Agent
 ) -> List[LLMDataExample]:
     data = []
 
@@ -95,18 +73,52 @@ def _convert_conversation_into_llm_data(
         # create data point for the original e2e test case
         data.append(_create_data_point(step.llm_prompt, step, conversation))
 
-        # create data points using the rephrasings, e.g. 'new_conversations'
-        for rephrased_user_steps in new_conversations:
-            # +1 to include the current user turn
-            prompt = _update_prompt(
-                step.llm_prompt,
-                original_user_steps[: i + 1],
-                rephrased_user_steps[: i + 1],
+    test_case_name = conversation.name
+
+    # create data points using the rephrasings, e.g. 'new_conversations'
+    for rephrased_user_steps in new_conversations:
+        sender_id = generate_sender_id(test_case_name)
+        # create a new tracker to be able to simulate the conversation from start
+        await agent.tracker_store.save(DialogueStateTracker(sender_id, slots=[]))
+        # simulate the conversation to get the prompts
+        for i, step in enumerate(original_user_steps):
+            rephrased_user_message = rephrased_user_steps[i]
+            user_message = UserMessage(rephrased_user_message, sender_id=sender_id)
+
+            expected_commands = "\n".join(
+                [command.to_dsl() for command in step.llm_commands]
+            )
+            fake_invoke_function = make_mock_invoke_llm(expected_commands)
+
+            with (
+                set_record_commands_and_prompts(),
+                patch_invoke_llm_in_generators(fake_invoke_function),
+            ):
+                await agent.handle_message(user_message)
+
+            rephrased_tracker = await agent.tracker_store.retrieve(sender_id)
+            if rephrased_tracker is None:
+                # if tracker doesn't exist, we can't create a data point
+                continue
+
+            latest_message = rephrased_tracker.latest_message
+            if latest_message is None:
+                # if there is no latest message, we don't create a data point
+                continue
+
+            # tell the type checker what we expect to find under "prompts"
+            prompts = cast(
+                Optional[List[Dict[str, Any]]], latest_message.parse_data.get(PROMPTS)
             )
-            if prompt:
+
+            if prompts:
+                # as we only use single step or compact command generator,
+                # there is always exactly one prompt
+                prompt = prompts[0]
+                user_prompt: Optional[str] = prompt.get(KEY_USER_PROMPT)
                 data.append(
                     _create_data_point(
-                        prompt, step, conversation, rephrased_user_steps[i]
+                        user_prompt, step, conversation, rephrased_user_message
                     )
                 )
 
@@ -149,7 +161,7 @@ def _construct_new_conversations(conversation: Conversation) -> List[List[str]]:
                 current_conversation.append(step.original_test_step.text)
                 continue
 
-            # some user steps might have less rephrasings than others
+            # some user steps might have fewer rephrasings than others
             # loop over the rephrasings
             index = i % len(step.passed_rephrasings)
             current_conversation.append(step.passed_rephrasings[index])
@@ -165,13 +177,18 @@ def _construct_new_conversations(conversation: Conversation) -> List[List[str]]:
     return new_conversations
 
 
-def convert_to_fine_tuning_data(
-    conversations: List[Conversation], storage_context: StorageContext
+async def convert_to_fine_tuning_data(
+    conversations: List[Conversation],
+    storage_context: StorageContext,
+    agent: Agent,
 ) -> List[LLMDataExample]:
     llm_data = []
 
     for i in tqdm(range(len(conversations))):
-        llm_data.extend(_convert_conversation_into_llm_data(conversations[i]))
+        conversation_llm_data = await _convert_conversation_into_llm_data(
+            conversations[i], agent
+        )
+        llm_data.extend(conversation_llm_data)
 
     storage_context.write_llm_data(
         llm_data, LLM_DATA_PREPARATION_MODULE_STORAGE_LOCATION

rasa/llm_fine_tuning/paraphrasing/conversation_rephraser.py

@@ -11,10 +11,12 @@ from rasa.llm_fine_tuning.paraphrasing.rephrased_user_message import (
 )
 from rasa.shared.constants import (
     LLM_CONFIG_KEY,
+    MAX_COMPLETION_TOKENS_CONFIG_KEY,
     MODEL_CONFIG_KEY,
     MODEL_NAME_CONFIG_KEY,
     PROMPT_TEMPLATE_CONFIG_KEY,
     PROVIDER_CONFIG_KEY,
+    TEMPERATURE_CONFIG_KEY,
     TIMEOUT_CONFIG_KEY,
 )
 from rasa.shared.exceptions import ProviderClientAPIException
@@ -39,8 +41,8 @@ DEFAULT_LLM_CONFIG = {
     PROVIDER_CONFIG_KEY: OPENAI_PROVIDER,
     MODEL_CONFIG_KEY: "gpt-4o-mini",
     TIMEOUT_CONFIG_KEY: 7,
-    "temperature": 0.0,
-    "max_tokens": 4096,
+    TEMPERATURE_CONFIG_KEY: 0.0,
+    MAX_COMPLETION_TOKENS_CONFIG_KEY: 4096,
 }
 
 structlogger = structlog.get_logger()

rasa/llm_fine_tuning/paraphrasing/rephrase_validator.py

@@ -1,45 +1,45 @@
-from typing import Any, Dict, List, Type
+from typing import List, Optional
 
 import structlog
 
+from rasa.core.agent import Agent
+from rasa.core.channels import UserMessage
 from rasa.dialogue_understanding.commands import Command, SetSlotCommand
-from rasa.dialogue_understanding.generator.llm_based_command_generator import (
-    LLMBasedCommandGenerator,
-)
 from rasa.llm_fine_tuning.conversations import Conversation, ConversationStep
 from rasa.llm_fine_tuning.paraphrasing.rephrased_user_message import (
     RephrasedUserMessage,
 )
 from rasa.shared.core.flows import FlowsList
-from rasa.shared.exceptions import ProviderClientAPIException
-from rasa.shared.utils.llm import llm_factory
+from rasa.shared.core.trackers import DialogueStateTracker
+from rasa.shared.utils.llm import (
+    create_tracker_for_user_step,
+    generate_sender_id,
+)
 
 structlogger = structlog.get_logger()
 
 
 class RephraseValidator:
-    def __init__(self, llm_config: Dict[str, Any], flows: FlowsList) -> None:
-        self.llm_config = llm_config
+    def __init__(self, flows: FlowsList) -> None:
         self.flows = flows
 
     async def validate_rephrasings(
         self,
+        agent: Agent,
         rephrasings: List[RephrasedUserMessage],
         conversation: Conversation,
-        llm_command_generator: Type[LLMBasedCommandGenerator],
     ) -> List[RephrasedUserMessage]:
         """Split rephrased user messages into passing and failing.
 
-        Call an LLM using the same config of the former trained model with an updated
-        prompt from the original user message (replace all occurrences of the original
-        user message with the rephrased user message). Check if the
-        rephrased user message is producing the same commands as the original user
-        message. The rephase is passing if the commands match and failing otherwise.
+        Handle the rephrased messages using agent the same way the original
+        message was handled. Check if the rephrased user message is producing
+        the same commands as the original user message. The rephrase is passing
+        if the commands match and failing otherwise.
 
         Args:
+            agent: Rasa agent
             rephrasings: The rephrased user messages.
             conversation: The conversation.
-            llm_command_generator: A LLM based command generator class.
 
         Returns:
             A list of rephrased user messages including the passing and failing
@@ -52,7 +52,11 @@ class RephraseValidator:
 
             for rephrase in current_rephrasings.rephrasings:
                 if await self._validate_rephrase_is_passing(
-                    rephrase, step, llm_command_generator
+                    agent,
+                    rephrase,
+                    step,
+                    conversation.name,
+                    conversation.tracker,
                 ):
                     current_rephrasings.passed_rephrasings.append(rephrase)
                 else:
@@ -62,40 +66,29 @@ class RephraseValidator:
 
     async def _validate_rephrase_is_passing(
         self,
+        agent: Agent,
         rephrase: str,
         step: ConversationStep,
-        llm_command_generator: Type[LLMBasedCommandGenerator],
+        test_case_name: str,
+        tracker: DialogueStateTracker,
     ) -> bool:
-        prompt = self._update_prompt(
-            rephrase, step.original_test_step.text, step.llm_prompt
-        )
-
-        action_list = await self._invoke_llm(
-            prompt, llm_command_generator.get_default_llm_config()
+        rephrased_tracker = await self._send_rephrased_message_to_agent(
+            rephrase, step, test_case_name, agent, tracker
         )
+        if not (rephrased_tracker and rephrased_tracker.latest_message):
+            return False
 
         commands_from_original_utterance = step.llm_commands
-        commands_from_rephrased_utterance = llm_command_generator.parse_commands(  # type: ignore
-            action_list, None, self.flows
-        )
+
+        commands_from_rephrased_utterance = [
+            Command.command_from_json(command_json)
+            for command_json in rephrased_tracker.latest_message.commands
+        ]
+
         return self._check_commands_match(
             commands_from_original_utterance, commands_from_rephrased_utterance
         )
 
-    async def _invoke_llm(self, prompt: str, default_llm_config: Dict[str, Any]) -> str:
-        llm = llm_factory(self.llm_config, default_llm_config)
-
-        try:
-            llm_response = await llm.acompletion(prompt)
-            return llm_response.choices[0]
-        except Exception as e:
-            # unfortunately, langchain does not wrap LLM exceptions which means
-            # we have to catch all exceptions here
-            structlogger.error(
-                "rephrase_validator.validate_conversation.llm.error", error=e
-            )
-            raise ProviderClientAPIException(e, message="LLM call exception")
-
     @staticmethod
     def _check_commands_match(
         expected_commands: List[Command], actual_commands: List[Command]
@@ -120,7 +113,22 @@ class RephraseValidator:
         return True
 
     @staticmethod
-    def _update_prompt(
-        rephrased_user_message: str, original_user_message: str, prompt: str
-    ) -> str:
-        return prompt.replace(original_user_message, rephrased_user_message)
+    async def _send_rephrased_message_to_agent(
+        rephrased_user_message: str,
+        step: ConversationStep,
+        test_case_name: str,
+        agent: Agent,
+        tracker: DialogueStateTracker,
+    ) -> Optional[DialogueStateTracker]:
+        # create a rephrased UserMessage
+        sender_id = generate_sender_id(test_case_name)
+        user_message = UserMessage(rephrased_user_message, sender_id=sender_id)
+
+        await create_tracker_for_user_step(
+            sender_id, agent, tracker, step.tracker_event_index
+        )
+
+        await agent.handle_message(user_message)
+        rephrased_tracker = await agent.tracker_store.retrieve(sender_id)
+
+        return rephrased_tracker

rasa/llm_fine_tuning/paraphrasing_module.py

@@ -1,11 +1,9 @@
-from typing import Any, Dict, List, Tuple, Type
+from typing import Any, Dict, List, Tuple
 
 import structlog
 from tqdm import tqdm
 
-from rasa.dialogue_understanding.generator.llm_based_command_generator import (
-    LLMBasedCommandGenerator,
-)
+from rasa.core.agent import Agent
 from rasa.llm_fine_tuning.conversations import Conversation
 from rasa.llm_fine_tuning.paraphrasing.conversation_rephraser import (
     ConversationRephraser,
@@ -28,8 +26,7 @@ async def create_paraphrased_conversations(
     rephrase_config: Dict[str, Any],
     num_rephrases: int,
     flows: FlowsList,
-    llm_command_generator: Type[LLMBasedCommandGenerator],
-    llm_command_generator_config: Dict[str, Any],
+    agent: Agent,
     storage_context: StorageContext,
 ) -> Tuple[List[Conversation], Dict[str, Any]]:
     """Create paraphrased conversations.
@@ -42,7 +39,7 @@ async def create_paraphrased_conversations(
         rephrase_config: The path to the rephrase configuration file.
         num_rephrases: The number of rephrases to produce per user message.
         flows: All flows.
-        llm_command_generator_config: The configuration of the trained model.
+        agent: The Rasa agent.
         storage_context: The storage context.
 
     Returns:
@@ -50,7 +47,7 @@ async def create_paraphrased_conversations(
         rephrasing.
     """
     rephraser = ConversationRephraser(rephrase_config)
-    validator = RephraseValidator(llm_command_generator_config, flows)
+    validator = RephraseValidator(flows)
 
     if num_rephrases <= 0:
         structlogger.info(
@@ -64,18 +61,19 @@ async def create_paraphrased_conversations(
     rephrased_conversations: List[Conversation] = []
     for i in tqdm(range(len(conversations))):
         current_conversation = conversations[i]
-
         try:
             # rephrase all user messages even if rephrase=False is set
             # to not confuse the LLM and get valid output
             rephrasings = await rephraser.rephrase_conversation(
-                conversations[i], num_rephrases
+                current_conversation, num_rephrases
             )
             # filter out the rephrasings for user messages that have rephrase=False set
-            rephrasings = _filter_rephrasings(rephrasings, conversations[i])
+            rephrasings = _filter_rephrasings(rephrasings, current_conversation)
             # check if the rephrasings are still producing the same commands
             rephrasings = await validator.validate_rephrasings(
-                rephrasings, current_conversation, llm_command_generator
+                agent,
+                rephrasings,
+                current_conversation,
             )
         except ProviderClientAPIException as e:
             structlogger.error(

rasa/llm_fine_tuning/storage.py

@@ -96,9 +96,9 @@ class FileStorageStrategy(StorageStrategy):
         file_path = self._get_file_path(storage_location)
         self._create_output_dir(file_path)
 
-        with open(str(file_path), "w") as outfile:
+        with open(str(file_path), "w", encoding="utf-8") as outfile:
             for example in llm_data:
-                json.dump(example.as_dict(), outfile)
+                json.dump(example.as_dict(), outfile, ensure_ascii=False)
                 outfile.write("\n")
 
     def write_formatted_finetuning_data(
@@ -110,9 +110,9 @@ class FileStorageStrategy(StorageStrategy):
         file_path = self._get_file_path(module_storage_location, file_name)
         self._create_output_dir(file_path)
 
-        with open(str(file_path), "w") as file:
+        with open(str(file_path), "w", encoding="utf-8") as file:
             for example in formatted_data:
-                json.dump(example.as_dict(), file)
+                json.dump(example.as_dict(), file, ensure_ascii=False)
                 file.write("\n")
 
     def write_e2e_test_suite_to_yaml_file(

rasa/llm_fine_tuning/utils.py

@@ -1,7 +1,69 @@
-from typing import List
+from contextlib import contextmanager
+from datetime import datetime
+from typing import Callable, Generator, List, Union
+
+import structlog
 
 from rasa.dialogue_understanding.commands.prompt_command import PromptCommand
+from rasa.dialogue_understanding.generator import LLMBasedCommandGenerator
+from rasa.shared.providers.llm.llm_response import LLMResponse
+
+structlogger = structlog.get_logger()
 
 
 def commands_as_string(commands: List[PromptCommand], delimiter: str = "\n") -> str:
     return delimiter.join([command.to_dsl() for command in commands])
+
+
+def make_mock_invoke_llm(commands: str) -> Callable:
+    """Capture the `commands` in a closure so the resulting async function
+    can use it as its response.
+
+    Args:
+        commands: The commands to return from the mock LLM call.
+    """
+
+    async def _mock_invoke_llm(
+        self: LLMBasedCommandGenerator, prompt: Union[List[dict], List[str], str]
+    ) -> LLMResponse:
+        structlogger.debug(
+            f"LLM call intercepted, response mocked. "
+            f"Responding with the following commands: '{commands}' "
+            f"to the prompt: {prompt}"
+        )
+        fake_response_dict = {
+            "id": "",
+            "choices": [commands],
+            "created": int(datetime.now().timestamp()),
+            "model": "mocked-llm",
+        }
+        return LLMResponse.from_dict(fake_response_dict)
+
+    return _mock_invoke_llm
+
+
+@contextmanager
+def patch_invoke_llm_in_generators(mock_impl: Callable) -> Generator:
+    """Replace CommandGenerator.invoke_llm in the base class AND in all
+    current subclasses (recursively).  Everything is restored on exit.
+    """
+    originals = {}
+
+    def collect(cls: type[LLMBasedCommandGenerator]) -> None:
+        # store current attribute, then recurse
+        originals[cls] = cls.invoke_llm
+        for sub in cls.__subclasses__():
+            collect(sub)
+
+    # collect every existing subclass of CommandGenerator
+    collect(LLMBasedCommandGenerator)  # type: ignore[type-abstract]
+
+    try:
+        # apply the monkey-patch everywhere
+        for cls in originals:
+            cls.invoke_llm = mock_impl  # type: ignore[assignment]
+        yield
+    finally:
+        # restore originals (even if an exception happened)
+        for cls, orig in originals.items():
+            cls.invoke_llm = orig  # type: ignore[assignment]

rasa/server.py

@@ -522,12 +522,15 @@ def configure_cors(
     )
 
 
-def add_root_route(app: Sanic) -> None:
+def add_root_route(app: Sanic, is_inspector_enabled: bool = False) -> None:
     """Add '/' route to return hello."""
 
     @app.get("/")
     async def hello(request: Request) -> HTTPResponse:
         """Check if the server is running and responds with the version."""
+        if not is_inspector_enabled:
+            return response.text("Hello from Rasa: " + rasa.__version__)
+
         html_content = f"""
         <html>
             <body>
@@ -688,6 +691,7 @@ def create_app(
     jwt_private_key: Optional[Text] = None,
     jwt_method: Text = "HS256",
     endpoints: Optional[AvailableEndpoints] = None,
+    is_inspector_enabled: bool = False,
 ) -> Sanic:
     """Class representing a Rasa HTTP server."""
     app = Sanic("rasa_server")
@@ -733,7 +737,7 @@ def create_app(
     ) -> HTTPResponse:
         return response.json(exception.error_info, status=exception.status)
 
-    add_root_route(app)
+    add_root_route(app, is_inspector_enabled)
 
     @app.get("/version")
     async def version(request: Request) -> HTTPResponse:

rasa/shared/constants.py

@@ -197,7 +197,10 @@ PROVIDER_CONFIG_KEY = "provider"
 REQUEST_TIMEOUT_CONFIG_KEY = "request_timeout"  # deprecated
 TIMEOUT_CONFIG_KEY = "timeout"
 
+LOGIT_BIAS_CONFIG_KEY = "logit_bias"
+MAX_RETRIES_CONFIG_KEY = "max_retries"
 TEMPERATURE_CONFIG_KEY = "temperature"
+MAX_COMPLETION_TOKENS_CONFIG_KEY = "max_completion_tokens"
 MAX_TOKENS_CONFIG_KEY = "max_tokens"
 
 DEPLOYMENT_NAME_CONFIG_KEY = "deployment_name"

rasa/shared/exceptions.py

@@ -165,3 +165,7 @@ class ProviderClientAPIException(RasaException):
 
 class ProviderClientValidationError(RasaException):
     """Raised for errors that occur during validation of the API client."""
+
+
+class FinetuningDataPreparationException(RasaException):
+    """Raised when there is an error in data preparation for fine-tuning."""

rasa/shared/providers/_configs/azure_openai_client_config.py

@@ -23,6 +23,8 @@ from rasa.shared.constants import (
     DEPLOYMENT_NAME_CONFIG_KEY,
     ENGINE_CONFIG_KEY,
     LANGCHAIN_TYPE_CONFIG_KEY,
+    MAX_COMPLETION_TOKENS_CONFIG_KEY,
+    MAX_TOKENS_CONFIG_KEY,
     MODEL_CONFIG_KEY,
     MODEL_NAME_CONFIG_KEY,
     N_REPHRASES_CONFIG_KEY,
@@ -71,6 +73,8 @@ DEPRECATED_ALIASES_TO_STANDARD_KEY_MAPPING = {
     MODEL_NAME_CONFIG_KEY: MODEL_CONFIG_KEY,
     # Timeout aliases
     REQUEST_TIMEOUT_CONFIG_KEY: TIMEOUT_CONFIG_KEY,
+    # Max tokens aliases
+    MAX_TOKENS_CONFIG_KEY: MAX_COMPLETION_TOKENS_CONFIG_KEY,
 }
 
 REQUIRED_KEYS = [DEPLOYMENT_CONFIG_KEY]

rasa/shared/providers/_configs/openai_client_config.py

@@ -10,6 +10,8 @@ from rasa.shared.constants import (
     API_TYPE_CONFIG_KEY,
     API_VERSION_CONFIG_KEY,
     LANGCHAIN_TYPE_CONFIG_KEY,
+    MAX_COMPLETION_TOKENS_CONFIG_KEY,
+    MAX_TOKENS_CONFIG_KEY,
     MODEL_CONFIG_KEY,
     MODEL_NAME_CONFIG_KEY,
     N_REPHRASES_CONFIG_KEY,
@@ -48,6 +50,8 @@ DEPRECATED_ALIASES_TO_STANDARD_KEY_MAPPING = {
     OPENAI_API_VERSION_CONFIG_KEY: API_VERSION_CONFIG_KEY,
     # Timeout aliases
     REQUEST_TIMEOUT_CONFIG_KEY: TIMEOUT_CONFIG_KEY,
+    # Max tokens aliases
+    MAX_TOKENS_CONFIG_KEY: MAX_COMPLETION_TOKENS_CONFIG_KEY,
 }
 
 REQUIRED_KEYS = [MODEL_CONFIG_KEY]

rasa/shared/providers/embedding/_base_litellm_embedding_client.py

@@ -70,7 +70,10 @@ class _BaseLiteLLMEmbeddingClient:
     def _embedding_fn_args(self) -> Dict[str, Any]:
         """Returns the arguments to be passed to the embedding function."""
         return {
+            # Parameters set through config, can override drop_params
             **self._litellm_extra_parameters,
+            # Model name is constructed in the LiteLLM format from the provided config
+            # Non-overridable to ensure consistency
             "model": self._litellm_model_name,
         }
 

rasa/shared/providers/llm/_base_litellm_client.py

@@ -84,12 +84,15 @@ class _BaseLiteLLMClient:
     @property
     def _completion_fn_args(self) -> dict:
         return {
-            **self._litellm_extra_parameters,
-            "model": self._litellm_model_name,
             # Since all providers covered by LiteLLM use the OpenAI format, but
             # not all support every OpenAI parameter, raise an exception if
             # provider/model uses unsupported parameter
             "drop_params": False,
+            # All other parameters set through config, can override drop_params
+            **self._litellm_extra_parameters,
+            # Model name is constructed in the LiteLLM format from the provided config
+            # Non-overridable to ensure consistency
+            "model": self._litellm_model_name,
         }
 
     def validate_client_setup(self) -> None:

rasa/shared/utils/llm.py

@@ -2,6 +2,7 @@ import importlib.resources
 import json
 import logging
 from copy import deepcopy
+from datetime import datetime
 from functools import wraps
 from typing import (
     TYPE_CHECKING,
@@ -64,6 +65,7 @@ from rasa.shared.providers.mappings import (
 from rasa.shared.utils.constants import LOG_COMPONENT_SOURCE_METHOD_INIT
 
 if TYPE_CHECKING:
+    from rasa.core.agent import Agent
     from rasa.shared.core.trackers import DialogueStateTracker
 
 
@@ -886,3 +888,29 @@ def resolve_model_client_config(
         )
 
     return model_group[0]
+
+
+def generate_sender_id(test_case_name: str) -> str:
+    # add timestamp suffix to ensure sender_id is unique
+    return f"{test_case_name}_{datetime.now()}"
+
+
+async def create_tracker_for_user_step(
+    step_sender_id: str,
+    agent: "Agent",
+    test_case_tracker: "DialogueStateTracker",
+    index_user_uttered_event: int,
+) -> None:
+    """Creates a tracker for the user step."""
+    tracker = test_case_tracker.copy()
+    # modify the sender id so that the original tracker is not overwritten
+    tracker.sender_id = step_sender_id
+
+    if tracker.events:
+        # get the timestamp of the event just before the user uttered event
+        timestamp = tracker.events[index_user_uttered_event - 1].timestamp
+        # revert the tracker to the event just before the user uttered event
+        tracker = tracker.travel_back_in_time(timestamp)
+
+    # store the tracker with the unique sender id
+    await agent.tracker_store.save(tracker)