rasa-pro 3.12.0rc1__py3-none-any.whl → 3.12.0rc3__py3-none-any.whl

rasa/dialogue_understanding/generator/single_step/compact_llm_command_generator.py

@@ -1,3 +1,4 @@
+import copy
 from typing import Any, Dict, List, Optional, Text
 
 import structlog
@@ -19,10 +20,16 @@ from rasa.dialogue_understanding.generator.command_parser import (
 )
 from rasa.dialogue_understanding.generator.constants import (
     COMMAND_PROMPT_FILE_NAME,
-    DEFAULT_LLM_CONFIG,
+    DEFAULT_OPENAI_MAX_GENERATED_TOKENS,
     FLOW_RETRIEVAL_KEY,
     LLM_BASED_COMMAND_GENERATOR_CONFIG_FILE,
     LLM_CONFIG_KEY,
+    MODEL_CONFIG_KEY,
+    MODEL_NAME_CLAUDE_3_5_SONNET_20240620,
+    MODEL_NAME_GPT_4O_2024_11_20,
+    OPENAI_PROVIDER,
+    PROVIDER_CONFIG_KEY,
+    TIMEOUT_CONFIG_KEY,
     USER_INPUT_CONFIG_KEY,
 )
 from rasa.dialogue_understanding.generator.flow_retrieval import FlowRetrieval
@@ -36,9 +43,14 @@ from rasa.engine.recipes.default_recipe import DefaultV1Recipe
 from rasa.engine.storage.resource import Resource
 from rasa.engine.storage.storage import ModelStorage
 from rasa.shared.constants import (
+    ANTHROPIC_PROVIDER,
+    AWS_BEDROCK_PROVIDER,
+    AZURE_OPENAI_PROVIDER,
     EMBEDDINGS_CONFIG_KEY,
+    MAX_TOKENS_CONFIG_KEY,
     PROMPT_TEMPLATE_CONFIG_KEY,
     ROUTE_TO_CALM_SLOT,
+    TEMPERATURE_CONFIG_KEY,
 )
 from rasa.shared.core.flows import FlowsList
 from rasa.shared.core.trackers import DialogueStateTracker
@@ -61,19 +73,92 @@ from rasa.utils.log_utils import log_llm
 structlogger = structlog.get_logger()
 
 
+DEFAULT_LLM_CONFIG = {
+    PROVIDER_CONFIG_KEY: OPENAI_PROVIDER,
+    MODEL_CONFIG_KEY: MODEL_NAME_GPT_4O_2024_11_20,
+    TEMPERATURE_CONFIG_KEY: 0.0,
+    MAX_TOKENS_CONFIG_KEY: DEFAULT_OPENAI_MAX_GENERATED_TOKENS,
+    TIMEOUT_CONFIG_KEY: 7,
+}
+
 MODEL_PROMPT_MAPPER = {
-    "openai/gpt-4o-2024-11-20": "command_prompt_v2_gpt_4o_2024_11_20_template.jinja2",
-    "azure/gpt-4o-2024-11-20": "command_prompt_v2_gpt_4o_2024_11_20_template.jinja2",
-    "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0": (
+    f"{OPENAI_PROVIDER}/{MODEL_NAME_GPT_4O_2024_11_20}": (
+        "command_prompt_v2_gpt_4o_2024_11_20_template.jinja2"
+    ),
+    f"{AZURE_OPENAI_PROVIDER}/{MODEL_NAME_GPT_4O_2024_11_20}": (
+        "command_prompt_v2_gpt_4o_2024_11_20_template.jinja2"
+    ),
+    f"{AWS_BEDROCK_PROVIDER}/anthropic.{MODEL_NAME_CLAUDE_3_5_SONNET_20240620}-v1:0": (
         "command_prompt_v2_claude_3_5_sonnet_20240620_template.jinja2"
     ),
-    "anthropic/claude-3-5-sonnet-20240620": (
+    f"{ANTHROPIC_PROVIDER}/{MODEL_NAME_CLAUDE_3_5_SONNET_20240620}": (
         "command_prompt_v2_claude_3_5_sonnet_20240620_template.jinja2"
     ),
 }
 
+# When model is not configured, then we use the default prompt template
+DEFAULT_COMMAND_PROMPT_TEMPLATE_FILE_NAME = (
+    "command_prompt_v2_gpt_4o_2024_11_20_template.jinja2"
+)
+# When the configured model is not found in the model prompt mapper, then we use the
+# fallback prompt template
+FALLBACK_COMMAND_PROMPT_TEMPLATE_FILE_NAME = (
+    "command_prompt_v2_fallback_other_models_template.jinja2"
+)
+
+
+class CommandParserValidatorSingleton:
+    """Singleton class to validate the command parser.
+
+    This class is used to validate the command parser. It keeps track of the number of
+    consecutive turns where no commands are parsed by the command parser. If the
+    number of consecutive turns exceeds a certain threshold, a warning is logged.
+    The prompt can use a DSL syntax that can be incompatible with the command syntax
+    used by the command parser. This class helps to detect such incompatibilities.
+    """
+
+    MAX_CONSECUTIVE_TURNS_NO_COMMAND_PREDICTED = 5
+    _NO_COMMAND_PREDICTED_TURN_COUNTER = 0
+    _command_parser_validated = False
+
+    @classmethod
+    def get_no_command_predicted_turn_counter(cls) -> int:
+        return cls._NO_COMMAND_PREDICTED_TURN_COUNTER
+
+    @classmethod
+    def should_validate_command_parser(cls) -> bool:
+        return not cls._command_parser_validated
 
-DEFAULT_COMMAND_PROMPT_TEMPLATE_FILE_NAME = "command_prompt_v2_default.jinja2"
+    @classmethod
+    def reset_command_parser_validation(cls) -> None:
+        cls._NO_COMMAND_PREDICTED_TURN_COUNTER = 0
+        cls._command_parser_validated = False
+
+    @classmethod
+    def validate_if_commands_are_parsed_from_llm_response(
+        cls, commands: List[Command], llm_response: str
+    ) -> None:
+        if llm_response and not commands:
+            cls._NO_COMMAND_PREDICTED_TURN_COUNTER += 1
+        else:
+            # Reset the counter if commands are generated, and mark
+            # the command parser as validated.
+            cls._NO_COMMAND_PREDICTED_TURN_COUNTER = 0
+            cls._command_parser_validated = True
+            return
+
+        if (
+            cls._NO_COMMAND_PREDICTED_TURN_COUNTER
+            >= cls.MAX_CONSECUTIVE_TURNS_NO_COMMAND_PREDICTED
+        ):
+            structlogger.warning(
+                "llm_command_generator.predict_commands.command_parser_not_working",
+                event_info=(
+                    f"No commands were generated by the command parser for the last "
+                    f"{cls._NO_COMMAND_PREDICTED_TURN_COUNTER} times. Check if you "
+                    "are running incompatible prompt and LLM command generator."
+                ),
+            )
 
 
 @DefaultV1Recipe.register(
@@ -101,23 +186,19 @@ class CompactLLMCommandGenerator(LLMBasedCommandGenerator):
             **kwargs,
         )
 
-        # Get the default prompt template based on the model name
-        default_command_prompt_template = get_default_prompt_template_based_on_model(
-            config, MODEL_PROMPT_MAPPER, DEFAULT_COMMAND_PROMPT_TEMPLATE_FILE_NAME
+        # Get the prompt template from the config or the default prompt template.
+        self.prompt_template = self.resolve_component_prompt_template(
+            self.config, prompt_template
         )
 
-        # Set the prompt template either from the config or the default prompt template.
-        self.prompt_template = prompt_template or get_prompt_template(
-            config.get(PROMPT_TEMPLATE_CONFIG_KEY),
-            default_command_prompt_template,
+        # Set the command syntax version to v2
+        CommandSyntaxManager.set_syntax_version(
+            self.get_component_command_syntax_version()
         )
 
         self.trace_prompt_tokens = self.config.get("trace_prompt_tokens", False)
         self.repeat_command_enabled = self.is_repeat_command_enabled()
 
-        # Set the command syntax version to v2
-        CommandSyntaxManager.set_syntax_version(CommandSyntaxVersion.v2)
-
     ### Implementations of LLMBasedCommandGenerator parent
     @staticmethod
     def get_default_config() -> Dict[str, Any]:
@@ -164,7 +245,7 @@ class CompactLLMCommandGenerator(LLMBasedCommandGenerator):
         llm_config = resolve_model_client_config(config.get(LLM_CONFIG_KEY, {}))
         cls.perform_llm_health_check(
             llm_config,
-            DEFAULT_LLM_CONFIG,
+            cls.get_default_llm_config(),
             "compact_llm_command_generator.load",
             cls.__name__,
         )
@@ -230,7 +311,7 @@ class CompactLLMCommandGenerator(LLMBasedCommandGenerator):
             structlogger.warning(
                 "llm_command_generator.predict_commands",
                 message="No commands were predicted as the LLM response could "
-                "not be parsed or the LLM responded with an invalid command."
+                "not be parsed or the LLM responded with an invalid command. "
                 "Returning a CannotHandleCommand instead.",
             )
             commands = [CannotHandleCommand()]
@@ -306,6 +387,11 @@ class CompactLLMCommandGenerator(LLMBasedCommandGenerator):
 
         commands = self.parse_commands(action_list, tracker, flows)
 
+        if CommandParserValidatorSingleton.should_validate_command_parser():
+            CommandParserValidatorSingleton.validate_if_commands_are_parsed_from_llm_response(
+                commands, action_list
+            )
+
         self._update_message_parse_data_for_fine_tuning(message, commands, flow_prompt)
         add_commands_to_message_parse_data(message, self.__class__.__name__, commands)
         add_prompt_to_message_parse_data(
@@ -350,8 +436,8 @@ class CompactLLMCommandGenerator(LLMBasedCommandGenerator):
         """
         commands = parse_commands_using_command_parsers(actions, flows)
         if not commands:
-            structlogger.debug(
-                f"{cls.__class__.__name__}.parse_commands",
+            structlogger.warning(
+                f"{cls.__name__}.parse_commands",
                 message="No commands were parsed from the LLM actions.",
                 actions=actions,
             )
@@ -441,13 +527,6 @@ class CompactLLMCommandGenerator(LLMBasedCommandGenerator):
     def fingerprint_addon(cls: Any, config: Dict[str, Any]) -> Optional[str]:
         """Add a fingerprint for the graph."""
         # Get the default prompt template based on the model name
-        default_command_prompt_template = get_default_prompt_template_based_on_model(
-            config, MODEL_PROMPT_MAPPER, DEFAULT_COMMAND_PROMPT_TEMPLATE_FILE_NAME
-        )
-        prompt_template = get_prompt_template(
-            config.get(PROMPT_TEMPLATE_CONFIG_KEY),
-            default_command_prompt_template,
-        )
         llm_config = resolve_model_client_config(
             config.get(LLM_CONFIG_KEY), CompactLLMCommandGenerator.__name__
         )
@@ -455,6 +534,41 @@ class CompactLLMCommandGenerator(LLMBasedCommandGenerator):
             config.get(FLOW_RETRIEVAL_KEY, {}).get(EMBEDDINGS_CONFIG_KEY),
             FlowRetrieval.__name__,
         )
+
+        # Create a copy of the config to avoid modifying the original config
+        # and update the llm config with the resolved llm config.
+        _config_copy = copy.deepcopy(config)
+        _config_copy[LLM_CONFIG_KEY] = llm_config
+        prompt_template = cls.resolve_component_prompt_template(_config_copy)
+
         return deep_container_fingerprint(
             [prompt_template, llm_config, embedding_config]
         )
+
+    @staticmethod
+    def get_default_llm_config() -> Dict[str, Any]:
+        """Get the default LLM config for the command generator."""
+        return DEFAULT_LLM_CONFIG
+
+    @staticmethod
+    def get_component_command_syntax_version() -> CommandSyntaxVersion:
+        return CommandSyntaxVersion.v2
+
+    @staticmethod
+    def resolve_component_prompt_template(
+        config: Dict[str, Any], prompt_template: Optional[str] = None
+    ) -> Optional[str]:
+        """Get the prompt template from the config or the default prompt template."""
+        # Get the default prompt template based on the model name.
+        default_command_prompt_template = get_default_prompt_template_based_on_model(
+            config.get(LLM_CONFIG_KEY, {}) or {},
+            MODEL_PROMPT_MAPPER,
+            DEFAULT_COMMAND_PROMPT_TEMPLATE_FILE_NAME,
+            FALLBACK_COMMAND_PROMPT_TEMPLATE_FILE_NAME,
+        )
+
+        # Return the prompt template either from the config or the default prompt.
+        return prompt_template or get_prompt_template(
+            config.get(PROMPT_TEMPLATE_CONFIG_KEY),
+            default_command_prompt_template,
+        )

rasa/dialogue_understanding/generator/single_step/single_step_llm_command_generator.py

@@ -8,6 +8,7 @@ from rasa.dialogue_understanding.commands.command_syntax_manager import (
     CommandSyntaxVersion,
 )
 from rasa.dialogue_understanding.generator.constants import (
+    DEFAULT_LLM_CONFIG,
     FLOW_RETRIEVAL_KEY,
     LLM_CONFIG_KEY,
     USER_INPUT_CONFIG_KEY,
@@ -71,18 +72,14 @@ class SingleStepLLMCommandGenerator(CompactLLMCommandGenerator):
                     "Please use the config parameter 'prompt_template' instead. "
                 ),
             )
-        config_prompt = (
-            config.get(PROMPT_CONFIG_KEY)
-            or config.get(PROMPT_TEMPLATE_CONFIG_KEY)
-            or None
-        )
-        self.prompt_template = prompt_template or get_prompt_template(
-            config_prompt,
-            DEFAULT_COMMAND_PROMPT_TEMPLATE,
+        self.prompt_template = self.resolve_component_prompt_template(
+            config, prompt_template
         )
 
         # Set the command syntax version to v1
-        CommandSyntaxManager.set_syntax_version(CommandSyntaxVersion.v1)
+        CommandSyntaxManager.set_syntax_version(
+            self.get_component_command_syntax_version()
+        )
 
     @staticmethod
     def get_default_config() -> Dict[str, Any]:
@@ -98,15 +95,7 @@ class SingleStepLLMCommandGenerator(CompactLLMCommandGenerator):
     @classmethod
     def fingerprint_addon(cls: Any, config: Dict[str, Any]) -> Optional[str]:
         """Add a fingerprint for the graph."""
-        config_prompt = (
-            config.get(PROMPT_CONFIG_KEY)
-            or config.get(PROMPT_TEMPLATE_CONFIG_KEY)
-            or None
-        )
-        prompt_template = get_prompt_template(
-            config_prompt,
-            DEFAULT_COMMAND_PROMPT_TEMPLATE,
-        )
+        prompt_template = cls.resolve_component_prompt_template(config)
         llm_config = resolve_model_client_config(
             config.get(LLM_CONFIG_KEY), SingleStepLLMCommandGenerator.__name__
         )
@@ -117,3 +106,28 @@ class SingleStepLLMCommandGenerator(CompactLLMCommandGenerator):
         return deep_container_fingerprint(
             [prompt_template, llm_config, embedding_config]
         )
+
+    @staticmethod
+    def get_default_llm_config() -> Dict[str, Any]:
+        """Get the default LLM config for the command generator."""
+        return DEFAULT_LLM_CONFIG
+
+    @staticmethod
+    def get_component_command_syntax_version() -> CommandSyntaxVersion:
+        return CommandSyntaxVersion.v1
+
+    @staticmethod
+    def resolve_component_prompt_template(
+        config: Dict[str, Any], prompt_template: Optional[str] = None
+    ) -> Optional[str]:
+        """Get the prompt template from the config or the default prompt template."""
+        # Get the default prompt template based on the model name.
+        config_prompt = (
+            config.get(PROMPT_CONFIG_KEY)
+            or config.get(PROMPT_TEMPLATE_CONFIG_KEY)
+            or None
+        )
+        return prompt_template or get_prompt_template(
+            config_prompt,
+            DEFAULT_COMMAND_PROMPT_TEMPLATE,
+        )

rasa/dialogue_understanding/processor/command_processor.py

@@ -42,6 +42,7 @@ from rasa.dialogue_understanding.stack.frames import (
 from rasa.dialogue_understanding.stack.utils import (
     filled_slots_for_active_flow,
     top_flow_frame,
+    top_user_flow_frame,
 )
 from rasa.engine.graph import ExecutionContext
 from rasa.shared.constants import (
@@ -263,10 +264,10 @@ def execute_commands(
         events.extend(new_events)
         tracker.update_with_events(new_events)
 
-        new_events = push_stack_frames_to_follow_commands(
-            tracker, stack_frames_to_follow_commands
-        )
-        events.extend(new_events)
+    new_events = push_stack_frames_to_follow_commands(
+        tracker, stack_frames_to_follow_commands
+    )
+    events.extend(new_events)
 
     return remove_duplicated_set_slots(events)
 
@@ -430,28 +431,22 @@ def clean_up_commands(
                 command=command,
             )
 
-        elif isinstance(command, StartFlowCommand) and command.flow == active_flow:
-            # drop a start flow command if the starting flow is equal to the currently
-            # active flow
-            structlogger.debug(
-                "command_processor.clean_up_commands.skip_command_flow_already_active",
-                command=command,
+        elif isinstance(command, StartFlowCommand):
+            top_user_frame = top_user_flow_frame(
+                tracker.stack, ignore_call_and_link_frames=False
             )
+            top_flow_id = top_user_frame.flow_id if top_user_frame else ""
 
-        elif isinstance(command, StartFlowCommand) and active_flow is not None:
-            # push handle digressions command if we are at a collect step of
-            # a flow and a new flow is started
-            collect_info = get_current_collect_step(tracker.stack, all_flows)
-            current_flow = all_flows.flow_by_id(active_flow)
-            current_flow_condition = current_flow and (
-                current_flow.ask_confirm_digressions or current_flow.block_digressions
-            )
+            if top_flow_id == command.flow:
+                # drop a start flow command if the starting flow is equal
+                # to the currently active flow
+                structlogger.debug(
+                    "command_processor.clean_up_commands.skip_command_flow_already_active",
+                    command=command,
+                )
+                continue
 
-            if collect_info and (
-                collect_info.ask_confirm_digressions
-                or collect_info.block_digressions
-                or current_flow_condition
-            ):
+            if should_add_handle_digressions_command(tracker, all_flows, top_flow_id):
                 clean_commands.append(HandleDigressionsCommand(flow=command.flow))
                 structlogger.debug(
                     "command_processor.clean_up_commands.push_handle_digressions",
@@ -848,3 +843,28 @@ def filter_cannot_handle_command_for_skipped_slots(
             and CANNOT_HANDLE_REASON == command.reason
         )
     ]
+
+
+def should_add_handle_digressions_command(
+    tracker: DialogueStateTracker, all_flows: FlowsList, top_flow_id: str
+) -> bool:
+    """Check if a handle digressions command should be added to the commands.
+
+    The command should replace a StartFlow command only if we are at a collect step of
+    a flow and a new flow is predicted by the command generator to start.
+    """
+    current_flow = all_flows.flow_by_id(top_flow_id)
+    current_flow_condition = current_flow and (
+        current_flow.ask_confirm_digressions or current_flow.block_digressions
+    )
+
+    collect_info = get_current_collect_step(tracker.stack, all_flows)
+
+    if collect_info and (
+        collect_info.ask_confirm_digressions
+        or collect_info.block_digressions
+        or current_flow_condition
+    ):
+        return True
+
+    return False

rasa/dialogue_understanding/stack/utils.py

@@ -4,6 +4,9 @@ from typing import List, Optional, Set, Tuple
 from rasa.dialogue_understanding.patterns.collect_information import (
     CollectInformationPatternFlowStackFrame,
 )
+from rasa.dialogue_understanding.patterns.continue_interrupted import (
+    ContinueInterruptedPatternFlowStackFrame,
+)
 from rasa.dialogue_understanding.stack.dialogue_stack import DialogueStack
 from rasa.dialogue_understanding.stack.frames import (
     BaseFlowStackFrame,
@@ -60,7 +63,9 @@ def top_flow_frame(
     return None
 
 
-def top_user_flow_frame(dialogue_stack: DialogueStack) -> Optional[UserFlowStackFrame]:
+def top_user_flow_frame(
+    dialogue_stack: DialogueStack, ignore_call_and_link_frames: bool = True
+) -> Optional[UserFlowStackFrame]:
     """Returns the topmost user flow frame from the tracker.
 
     User flows are flows that are created by developers of an assistant and
@@ -72,16 +77,19 @@ def top_user_flow_frame(dialogue_stack: DialogueStack) -> Optional[UserFlowStack
 
     Args:
         dialogue_stack: The dialogue stack to use.
+        ignore_call_and_link_frames: Whether to ignore user frames of type `call`
+            and `link`. By default, these frames are ignored.
 
     Returns:
     The topmost user flow frame from the tracker.
     """
     for frame in reversed(dialogue_stack.frames):
-        if (
-            isinstance(frame, UserFlowStackFrame)
-            and frame.frame_type != FlowStackFrameType.CALL
-            and frame.frame_type != FlowStackFrameType.LINK
-        ):
+        if isinstance(frame, UserFlowStackFrame):
+            if ignore_call_and_link_frames and (
+                frame.frame_type == FlowStackFrameType.CALL
+                or frame.frame_type == FlowStackFrameType.LINK
+            ):
+                continue
             return frame
     return None
 
@@ -213,3 +221,38 @@ def get_collect_steps_excluding_ask_before_filling_for_active_flow(
         for step in active_flow.get_collect_steps()
         if not step.ask_before_filling
     )
+
+
+def remove_digression_from_stack(stack: DialogueStack, flow_id: str) -> DialogueStack:
+    """Remove a specific flow frame from the stack and other frames that reference it.
+
+    The main use-case is to prevent duplicate digressions from being added to the stack.
+
+    Args:
+        stack: The dialogue stack.
+        flow_id: The flow to remove.
+
+    Returns:
+        The updated dialogue stack.
+    """
+    updated_stack = stack.copy()
+    original_frames = updated_stack.frames[:]
+    found_digression_index = -1
+    for index, frame in enumerate(original_frames):
+        if isinstance(frame, BaseFlowStackFrame) and frame.flow_id == flow_id:
+            updated_stack.frames.pop(index)
+            found_digression_index = index
+
+        # we also need to remove the `ContinueInterruptedPatternFlowStackFrame`
+        elif (
+            isinstance(frame, ContinueInterruptedPatternFlowStackFrame)
+            and frame.previous_flow_name == flow_id
+            and found_digression_index + 1 == index
+        ):
+            # we know that this frame is always added after the digressing flow frame
+            # that was blocked previously by action_block_digressions,
+            # so this check would occur after the digressing flow was popped.
+            # Therefore, we need to update the index dynamically before popping.
+            updated_stack.frames.pop(index - 1)
+
+    return updated_stack

rasa/dialogue_understanding_test/du_test_case.py

@@ -85,15 +85,33 @@ class DialogueUnderstandingOutput(BaseModel):
             for command in predicted_commands
         ]
 
-    def get_component_names_that_predicted_commands(self) -> List[str]:
-        """Get all component names that have predicted commands."""
-        if self.commands is None:
-            return []
-        return [
-            component_name
-            for component_name, predicted_commands in self.commands.items()
-            if predicted_commands
-        ]
+    def get_component_names_that_predicted_commands_or_have_llm_response(
+        self,
+    ) -> List[str]:
+        """Get all component names that have predicted commands or recieved
+        non-empty response from LLM.
+        """
+        component_names_that_predicted_commands = (
+            [
+                component_name
+                for component_name, predicted_commands in self.commands.items()
+                if predicted_commands
+            ]
+            if self.commands
+            else []
+        )
+        components_with_prompts = (
+            [
+                str(prompt.get(KEY_COMPONENT_NAME, None))
+                for prompt in self.prompts
+                if prompt.get(KEY_LLM_RESPONSE_METADATA, None)
+            ]
+            if self.prompts
+            else []
+        )
+        return list(
+            set(component_names_that_predicted_commands + components_with_prompts)
+        )
 
     def get_component_name_to_prompt_info(self) -> Dict[str, List[Dict[str, Any]]]:
         """Return a dictionary of component names to prompt information.
@@ -120,7 +138,9 @@ class DialogueUnderstandingOutput(BaseModel):
             return {}
 
         data: Dict[str, List[Dict[str, Any]]] = {}
-        relevant_component_names = self.get_component_names_that_predicted_commands()
+        relevant_component_names = (
+            self.get_component_names_that_predicted_commands_or_have_llm_response()
+        )
 
         for prompt_data in self.prompts:
             component_name = prompt_data[KEY_COMPONENT_NAME]

rasa/dialogue_understanding_test/du_test_result.py

@@ -84,7 +84,7 @@ class FailedTestStep(BaseModel):
 
         if step.dialogue_understanding_output:
             predicted_commands = step.dialogue_understanding_output.commands
-            command_generators = step.dialogue_understanding_output.get_component_names_that_predicted_commands()  # noqa: E501
+            command_generators = step.dialogue_understanding_output.get_component_names_that_predicted_commands_or_have_llm_response()  # noqa: E501
             prompts = (
                 step.dialogue_understanding_output.get_component_name_to_prompt_info()
             )

rasa/e2e_test/assertions.py

@@ -972,7 +972,7 @@ class GenerativeResponseMixin(Assertion):
         data.pop("metric_adjective")
         return data
 
-    def _render_prompt(self, step_text: str, matching_event: BotUttered) -> str:
+    def _render_prompt(self, matching_event: BotUttered) -> str:
         raise NotImplementedError
 
     def _get_processed_output(self, parsed_llm_output: Dict[str, Any]) -> List[Any]:
@@ -999,7 +999,7 @@ class GenerativeResponseMixin(Assertion):
     ) -> Tuple[Optional[AssertionFailure], Optional[Event]]:
         """Run the LLM evaluation on the given event."""
         bot_message = matching_event.text
-        prompt = self._render_prompt(step_text, matching_event)
+        prompt = self._render_prompt(matching_event)
         llm_response = self._invoke_llm(llm_judge_config, prompt)
 
         try:
@@ -1160,9 +1160,9 @@ class GenerativeResponseIsRelevantAssertion(GenerativeResponseMixin):
     def type(cls) -> str:
         return AssertionType.GENERATIVE_RESPONSE_IS_RELEVANT.value
 
-    def _render_prompt(self, step_text: str, matching_event: BotUttered) -> str:
+    def _render_prompt(self, matching_event: BotUttered) -> str:
         """Render the prompt."""
-        inputs = _get_prompt_inputs(self.type(), step_text, matching_event)
+        inputs = _get_prompt_inputs(self.type(), matching_event)
         prompt_template = _get_default_prompt_template(
             DEFAULT_ANSWER_RELEVANCE_PROMPT_TEMPLATE_FILE_NAME
         )
@@ -1206,11 +1206,10 @@ class GenerativeResponseIsGroundedAssertion(GenerativeResponseMixin):
     def type(cls) -> str:
         return AssertionType.GENERATIVE_RESPONSE_IS_GROUNDED.value
 
-    def _render_prompt(self, step_text: str, matching_event: BotUttered) -> str:
+    def _render_prompt(self, matching_event: BotUttered) -> str:
         """Render the prompt."""
         inputs = _get_prompt_inputs(
             assertion_type=self.type(),
-            step_text=step_text,
             matching_event=matching_event,
             ground_truth=self.ground_truth,
         )
@@ -1336,12 +1335,11 @@ def _get_default_prompt_template(default_prompt_template_file_name: str) -> str:
 
 def _get_prompt_inputs(
     assertion_type: str,
-    step_text: str,
     matching_event: BotUttered,
     ground_truth: Optional[str] = None,
 ) -> Dict[str, Any]:
     if assertion_type == AssertionType.GENERATIVE_RESPONSE_IS_RELEVANT.value:
-        return {"num_variations": "3", "user_message": step_text}
+        return {"num_variations": "3", "bot_message": matching_event.text}
     elif assertion_type == AssertionType.GENERATIVE_RESPONSE_IS_GROUNDED.value:
         ground_truth_event_metadata = matching_event.metadata.get(
             SEARCH_RESULTS_METADATA_KEY, ""

rasa/e2e_test/llm_judge_prompts/answer_relevance_prompt_template.jinja2

@@ -84,6 +84,10 @@ These are a few examples of how to generate question variations and identify non
 
 ## Task Inputs
 
-- **Input Answer**: {{ user_message }}
+- **Input Answer**: {{ bot_message }}
+
+## Task Outputs
+
+Do not include any additional explanations in your output. Only provide the JSON object as described in the task steps.
 
 Your output:

rasa/e2e_test/llm_judge_prompts/groundedness_prompt_template.jinja2

@@ -162,4 +162,8 @@ These are a few examples of how to evaluate the correctness of the answer based
 - **Input Answer**: {{ bot_message }}
 - **Ground Truth**: {{ ground_truth }}
 
+## Task Outputs
+
+Do not include any additional explanations in your output. Only provide the JSON object as described in the task steps.
+
 Your output: