rasa-pro 3.10.15__py3-none-any.whl → 3.11.0__py3-none-any.whl

rasa/dialogue_understanding/generator/llm_based_command_generator.py

@@ -16,6 +16,7 @@ from rasa.dialogue_understanding.generator.constants import (
     LLM_CONFIG_KEY,
     FLOW_RETRIEVAL_KEY,
     FLOW_RETRIEVAL_ACTIVE_KEY,
+    FLOW_RETRIEVAL_FLOW_THRESHOLD,
 )
 from rasa.dialogue_understanding.generator.flow_retrieval import FlowRetrieval
 from rasa.engine.graph import GraphComponent, ExecutionContext
@@ -31,10 +32,11 @@ from rasa.shared.exceptions import ProviderClientAPIException
 from rasa.shared.nlu.constants import FLOWS_IN_PROMPT
 from rasa.shared.nlu.training_data.message import Message
 from rasa.shared.nlu.training_data.training_data import TrainingData
+from rasa.shared.utils.health_check.llm_health_check_mixin import LLMHealthCheckMixin
 from rasa.shared.utils.llm import (
     allowed_values_for_slot,
     llm_factory,
-    try_instantiate_llm_client,
+    resolve_model_client_config,
 )
 from rasa.utils.log_utils import log_llm
 
@@ -47,7 +49,9 @@ structlogger = structlog.get_logger()
     ],
     is_trainable=True,
 )
-class LLMBasedCommandGenerator(GraphComponent, CommandGenerator, ABC):
+class LLMBasedCommandGenerator(
+    LLMHealthCheckMixin, GraphComponent, CommandGenerator, ABC
+):
     """An abstract class defining interface and common functionality
     of an LLM-based command generators.
     """
@@ -61,6 +65,9 @@ class LLMBasedCommandGenerator(GraphComponent, CommandGenerator, ABC):
     ) -> None:
         super().__init__(config)
         self.config = {**self.get_default_config(), **config}
+        self.config[LLM_CONFIG_KEY] = resolve_model_client_config(
+            self.config.get(LLM_CONFIG_KEY), LLMBasedCommandGenerator.__name__
+        )
         self._model_storage = model_storage
         self._resource = resource
         self.flow_retrieval: Optional[FlowRetrieval]
@@ -70,17 +77,9 @@ class LLMBasedCommandGenerator(GraphComponent, CommandGenerator, ABC):
                 self.config[FLOW_RETRIEVAL_KEY], model_storage, resource
             )
             structlogger.info("llm_based_command_generator.flow_retrieval.enabled")
+            self.config[FLOW_RETRIEVAL_KEY] = self.flow_retrieval.config
         else:
             self.flow_retrieval = None
-            structlogger.warn(
-                "llm_based_command_generator.flow_retrieval.disabled",
-                event_info=(
-                    "Disabling flow retrieval can cause issues when there are a "
-                    "large number of flows to be included in the prompt. For more"
-                    "information see:\n"
-                    "https://rasa.com/docs/rasa-pro/concepts/dialogue-understanding#how-the-llmcommandgenerator-works"
-                ),
-            )
 
     ### Abstract methods
     @staticmethod
@@ -168,14 +167,34 @@ class LLMBasedCommandGenerator(GraphComponent, CommandGenerator, ABC):
         """Train the llm based command generator. Stores all flows into a vector
         store.
         """
-        # Validate llm configuration
-        try_instantiate_llm_client(
+        self.perform_llm_health_check(
             self.config.get(LLM_CONFIG_KEY),
             DEFAULT_LLM_CONFIG,
             "llm_based_command_generator.train",
-            "LLMBasedCommandGenerator",
+            LLMBasedCommandGenerator.__name__,
         )
 
+        if (
+            self.flow_retrieval is None
+            and len(flows.user_flows) > FLOW_RETRIEVAL_FLOW_THRESHOLD
+        ):
+            structlogger.warn(
+                "llm_based_command_generator.flow_retrieval.disabled",
+                event_info=(
+                    f"You have {len(flows.user_flows)} user flows but flow "
+                    f"retrieval is disabled. "
+                    f"It is recommended to enable flow retrieval if the "
+                    f"total number of user flows exceed "
+                    f"{FLOW_RETRIEVAL_FLOW_THRESHOLD}. "
+                    f"Keeping it disabled can result in deterioration of "
+                    f"command generator's functional "
+                    f"performance and higher costs because of increased "
+                    f"number of tokens in the prompt. For more"
+                    "information see:\n"
+                    "https://rasa.com/docs/rasa-pro/concepts/dialogue-understanding#how-the-llmcommandgenerator-works"
+                ),
+            )
+
         # flow retrieval is populated with only user-defined flows
         try:
             if self.flow_retrieval is not None and not flows.is_empty():
@@ -187,6 +206,7 @@ class LLMBasedCommandGenerator(GraphComponent, CommandGenerator, ABC):
                 error=e,
             )
             raise
+
         self.persist()
         return self._resource
 
@@ -224,7 +244,10 @@ class LLMBasedCommandGenerator(GraphComponent, CommandGenerator, ABC):
 
     @classmethod
     def load_flow_retrival(
-        cls, config: Dict[Text, Any], model_storage: ModelStorage, resource: Resource
+        cls,
+        config: Dict[str, Any],
+        model_storage: ModelStorage,
+        resource: Resource,
     ) -> Optional[FlowRetrieval]:
         """Load the FlowRetrieval component if it is enabled in the configuration."""
         enable_flow_retrieval = config.get(FLOW_RETRIEVAL_KEY, {}).get(

rasa/dialogue_understanding/generator/llm_command_generator.py

@@ -1,4 +1,4 @@
-from typing import Dict, Any, Optional, Text
+from typing import Any, Dict, Optional, Text
 
 import structlog
 from deprecated import deprecated  # type: ignore[import]

rasa/dialogue_understanding/generator/multi_step/multi_step_llm_command_generator.py

@@ -40,7 +40,10 @@ from rasa.engine.graph import ExecutionContext
 from rasa.engine.recipes.default_recipe import DefaultV1Recipe
 from rasa.engine.storage.resource import Resource
 from rasa.engine.storage.storage import ModelStorage
-from rasa.shared.constants import RASA_PATTERN_CANNOT_HANDLE_NOT_SUPPORTED
+from rasa.shared.constants import (
+    RASA_PATTERN_CANNOT_HANDLE_NOT_SUPPORTED,
+    EMBEDDINGS_CONFIG_KEY,
+)
 from rasa.shared.constants import ROUTE_TO_CALM_SLOT
 from rasa.shared.core.flows import FlowStep, Flow, FlowsList
 from rasa.shared.core.flows.steps.collect import CollectInformationFlowStep
@@ -54,7 +57,7 @@ from rasa.shared.utils.llm import (
     tracker_as_readable_transcript,
     sanitize_message_for_prompt,
     allowed_values_for_slot,
-    try_instantiate_llm_client,
+    resolve_model_client_config,
 )
 
 # multistep template keys
@@ -72,6 +75,7 @@ DEFAULT_HANDLE_FLOWS_TEMPLATE = importlib.resources.read_text(
 DEFAULT_FILL_SLOTS_TEMPLATE = importlib.resources.read_text(
     "rasa.dialogue_understanding.generator.multi_step", "fill_slots_prompt.jinja2"
 ).strip()
+MULTI_STEP_LLM_COMMAND_GENERATOR_CONFIG_FILE = "config.json"
 
 # dictionary of template names and associated file names and default values
 PROMPT_TEMPLATES = {
@@ -140,27 +144,32 @@ class MultiStepLLMCommandGenerator(LLMBasedCommandGenerator):
         **kwargs: Any,
     ) -> "MultiStepLLMCommandGenerator":
         """Loads trained component (see parent class for full docstring)."""
-        prompts = cls._load_prompt_templates(model_storage, resource)
-        # init base command generator
-        command_generator = cls(config, model_storage, resource, prompts)
-        try_instantiate_llm_client(
-            command_generator.config.get(LLM_CONFIG_KEY),
+
+        # Perform health check of the LLM client config
+        llm_config = resolve_model_client_config(config.get(LLM_CONFIG_KEY, {}))
+        cls.perform_llm_health_check(
+            llm_config,
             DEFAULT_LLM_CONFIG,
             "multi_step_llm_command_generator.load",
             MultiStepLLMCommandGenerator.__name__,
         )
+
+        prompts = cls._load_prompt_templates(model_storage, resource)
+
+        # init base command generator
+        command_generator = cls(config, model_storage, resource, prompts)
         # load flow retrieval if enabled
         if command_generator.enabled_flow_retrieval:
             command_generator.flow_retrieval = cls.load_flow_retrival(
                 command_generator.config, model_storage, resource
             )
+
         return command_generator
 
     def persist(self) -> None:
         """Persist this component to disk for future loading."""
-        # persist prompt template
         self._persist_prompt_templates()
-        # persist flow retrieval
+        self._persist_config()
         if self.flow_retrieval is not None:
             self.flow_retrieval.persist()
 
@@ -393,6 +402,13 @@ class MultiStepLLMCommandGenerator(LLMBasedCommandGenerator):
                 file_path = path / file_name
                 rasa.shared.utils.io.write_text_file(template, file_path)
 
+    def _persist_config(self) -> None:
+        """Persist config as a source of truth for resolved clients."""
+        with self._model_storage.write_to(self._resource) as path:
+            rasa.shared.utils.io.dump_obj_as_json_to_file(
+                path / MULTI_STEP_LLM_COMMAND_GENERATOR_CONFIG_FILE, self.config
+            )
+
     async def _predict_commands_with_multi_step(
         self,
         message: Message,
@@ -786,11 +802,17 @@ class MultiStepLLMCommandGenerator(LLMBasedCommandGenerator):
             .get(FILE_PATH_KEY),
             DEFAULT_FILL_SLOTS_TEMPLATE,
         )
+
+        llm_config = resolve_model_client_config(
+            config.get(LLM_CONFIG_KEY), MultiStepLLMCommandGenerator.__name__
+        )
+        embedding_config = resolve_model_client_config(
+            config.get(FLOW_RETRIEVAL_KEY, {}).get(EMBEDDINGS_CONFIG_KEY),
+            FlowRetrieval.__name__,
+        )
+
         return deep_container_fingerprint(
-            [
-                handle_flows_template,
-                fill_slots_template,
-            ]
+            [handle_flows_template, fill_slots_template, llm_config, embedding_config]
         )
 
     @staticmethod

rasa/dialogue_understanding/generator/single_step/command_prompt_template.jinja2

@@ -41,6 +41,9 @@ Based on this information generate a list of actions you want to take. Your job
 * Responding to knowledge-oriented user messages, described by "SearchAndReply()"
 * Responding to a casual, non-task-oriented user message, described by "ChitChat()".
 * Handing off to a human, in case the user seems frustrated or explicitly asks to speak to one, described by "HumanHandoff()".
+{% if is_repeat_command_enabled %}
+* Repeat the last bot messages, described by "RepeatLastBotMessages()". This is useful when the user asks to repeat the last bot messages.
+{% endif %}
 
 ===
 Write out the actions you want to take, one per line, in the order they should take place.

rasa/dialogue_understanding/generator/single_step/single_step_llm_command_generator.py

@@ -16,6 +16,7 @@ from rasa.dialogue_understanding.commands import (
     KnowledgeAnswerCommand,
     ClarifyCommand,
     CannotHandleCommand,
+    RepeatBotMessagesCommand,
 )
 from rasa.dialogue_understanding.generator.constants import (
     LLM_CONFIG_KEY,
@@ -38,6 +39,7 @@ from rasa.shared.constants import (
     ROUTE_TO_CALM_SLOT,
     PROMPT_CONFIG_KEY,
     PROMPT_TEMPLATE_CONFIG_KEY,
+    EMBEDDINGS_CONFIG_KEY,
 )
 from rasa.shared.core.flows import FlowsList
 from rasa.shared.core.trackers import DialogueStateTracker
@@ -49,8 +51,9 @@ from rasa.shared.utils.llm import (
     get_prompt_template,
     tracker_as_readable_transcript,
     sanitize_message_for_prompt,
-    try_instantiate_llm_client,
+    resolve_model_client_config,
 )
+from rasa.utils.beta import ensure_beta_feature_is_enabled, BetaNotEnabledException
 from rasa.utils.log_utils import log_llm
 
 COMMAND_PROMPT_FILE_NAME = "command_prompt.jinja2"
@@ -59,6 +62,7 @@ DEFAULT_COMMAND_PROMPT_TEMPLATE = importlib.resources.read_text(
     "rasa.dialogue_understanding.generator.single_step",
     "command_prompt_template.jinja2",
 )
+SINGLE_STEP_LLM_COMMAND_GENERATOR_CONFIG_FILE = "config.json"
 
 structlogger = structlog.get_logger()
 
@@ -109,6 +113,7 @@ class SingleStepLLMCommandGenerator(LLMBasedCommandGenerator):
         )
 
         self.trace_prompt_tokens = self.config.get("trace_prompt_tokens", False)
+        self.repeat_command_enabled = self.is_repeat_command_enabled()
 
     ### Implementations of LLMBasedCommandGenerator parent
     @staticmethod
@@ -132,35 +137,51 @@ class SingleStepLLMCommandGenerator(LLMBasedCommandGenerator):
         **kwargs: Any,
     ) -> "SingleStepLLMCommandGenerator":
         """Loads trained component (see parent class for full docstring)."""
+
+        # Perform health check of the LLM API endpoint
+        llm_config = resolve_model_client_config(config.get(LLM_CONFIG_KEY, {}))
+        cls.perform_llm_health_check(
+            llm_config,
+            DEFAULT_LLM_CONFIG,
+            "single_step_llm_command_generator.load",
+            SingleStepLLMCommandGenerator.__name__,
+        )
+
         # load prompt template from the model storage.
         prompt_template = cls.load_prompt_template_from_model_storage(
             model_storage, resource, COMMAND_PROMPT_FILE_NAME
         )
+
         # init base command generator
         command_generator = cls(config, model_storage, resource, prompt_template)
-        try_instantiate_llm_client(
-            command_generator.config.get(LLM_CONFIG_KEY),
-            DEFAULT_LLM_CONFIG,
-            "single_step_llm_command_generator.load",
-            SingleStepLLMCommandGenerator.__name__,
-        )
         # load flow retrieval if enabled
         if command_generator.enabled_flow_retrieval:
             command_generator.flow_retrieval = cls.load_flow_retrival(
                 command_generator.config, model_storage, resource
             )
+
         return command_generator
 
     def persist(self) -> None:
         """Persist this component to disk for future loading."""
-        # persist prompt template
+        self._persist_prompt_template()
+        self._persist_config()
+        if self.flow_retrieval is not None:
+            self.flow_retrieval.persist()
+
+    def _persist_prompt_template(self) -> None:
+        """Persist prompt template for future loading."""
         with self._model_storage.write_to(self._resource) as path:
             rasa.shared.utils.io.write_text_file(
                 self.prompt_template, path / COMMAND_PROMPT_FILE_NAME
             )
-        # persist flow retrieval
-        if self.flow_retrieval is not None:
-            self.flow_retrieval.persist()
+
+    def _persist_config(self) -> None:
+        """Persist config as a source of truth for resolved clients."""
+        with self._model_storage.write_to(self._resource) as path:
+            rasa.shared.utils.io.dump_obj_as_json_to_file(
+                path / SINGLE_STEP_LLM_COMMAND_GENERATOR_CONFIG_FILE, self.config
+            )
 
     async def predict_commands(
         self,
@@ -309,6 +330,7 @@ class SingleStepLLMCommandGenerator(LLMBasedCommandGenerator):
         knowledge_re = re.compile(r"SearchAndReply\(\)")
         humand_handoff_re = re.compile(r"HumanHandoff\(\)")
         clarify_re = re.compile(r"Clarify\(([\"\'a-zA-Z0-9_, ]+)\)")
+        repeat_re = re.compile(r"RepeatLastBotMessages\(\)")
 
         for action in actions.strip().splitlines():
             if match := slot_set_re.search(action):
@@ -335,6 +357,8 @@ class SingleStepLLMCommandGenerator(LLMBasedCommandGenerator):
                 commands.append(KnowledgeAnswerCommand())
             elif humand_handoff_re.search(action):
                 commands.append(HumanHandoffCommand())
+            elif repeat_re.search(action):
+                commands.append(RepeatBotMessagesCommand())
             elif match := clarify_re.search(action):
                 options = sorted([opt.strip() for opt in match.group(1).split(",")])
                 # Remove surrounding quotes if present
@@ -366,7 +390,7 @@ class SingleStepLLMCommandGenerator(LLMBasedCommandGenerator):
 
     @classmethod
     def fingerprint_addon(cls: Any, config: Dict[str, Any]) -> Optional[str]:
-        """Add a fingerprint of the knowledge base for the graph."""
+        """Add a fingerprint for the graph."""
         config_prompt = (
             config.get(PROMPT_CONFIG_KEY)
             or config.get(PROMPT_TEMPLATE_CONFIG_KEY)
@@ -376,7 +400,16 @@ class SingleStepLLMCommandGenerator(LLMBasedCommandGenerator):
             config_prompt,
             DEFAULT_COMMAND_PROMPT_TEMPLATE,
         )
-        return deep_container_fingerprint(prompt_template)
+        llm_config = resolve_model_client_config(
+            config.get(LLM_CONFIG_KEY), SingleStepLLMCommandGenerator.__name__
+        )
+        embedding_config = resolve_model_client_config(
+            config.get(FLOW_RETRIEVAL_KEY, {}).get(EMBEDDINGS_CONFIG_KEY),
+            FlowRetrieval.__name__,
+        )
+        return deep_container_fingerprint(
+            [prompt_template, llm_config, embedding_config]
+        )
 
     ### Helper methods
     def render_template(
@@ -426,6 +459,20 @@ class SingleStepLLMCommandGenerator(LLMBasedCommandGenerator):
             "current_slot": current_slot,
             "current_slot_description": current_slot_description,
             "user_message": latest_user_message,
+            "is_repeat_command_enabled": self.repeat_command_enabled,
         }
 
         return self.compile_template(self.prompt_template).render(**inputs)
+
+    def is_repeat_command_enabled(self) -> bool:
+        """Check for feature flag"""
+        RASA_PRO_BETA_REPEAT_COMMAND_ENV_VAR_NAME = "RASA_PRO_BETA_REPEAT_COMMAND"
+        try:
+            ensure_beta_feature_is_enabled(
+                "Repeat Command",
+                env_flag=RASA_PRO_BETA_REPEAT_COMMAND_ENV_VAR_NAME,
+            )
+        except BetaNotEnabledException:
+            return False
+
+        return True

rasa/dialogue_understanding/patterns/default_flows_for_patterns.yml

@@ -4,6 +4,11 @@ responses:
   utter_ask_rephrase:
     - text: I’m sorry I am unable to understand you, could you please rephrase?
 
+  utter_ask_still_there:
+    - text: "Hello, are you still there?"
+      metadata:
+        rephrase: True
+
   utter_boolean_slot_rejection:
     - text: "Sorry, the value you provided, `{{value}}`, is not valid. Please respond with a valid value."
       metadata:
@@ -75,6 +80,11 @@ responses:
       metadata:
         rephrase: True
 
+  utter_inform_hangup:
+    - text: It seems you are not there anymore. I will hang up shortly.
+      metadata:
+        rephrase: True
+
   utter_internal_error_rasa:
     - text: Sorry, I am having trouble with that. Please try again in a few minutes.
 
@@ -101,6 +111,15 @@ slots:
     type: bool
     mappings:
       - type: from_llm
+  silence_timeout:
+    type: float
+    initial_value: 6.0
+    max_value: 1000000
+  consecutive_silence_timeouts:
+    type: float
+    initial_value: 0.0
+    max_value: 1000000
+
 
 flows:
   pattern_cancel_flow:
@@ -217,6 +236,11 @@ flows:
               - action: utter_internal_error_rasa
                 next: END
 
+  pattern_repeat_bot_messages:
+    description: Voice conversation repair pattern to repeat bot messages
+    name: pattern repeat bot messages
+    steps:
+      - action: action_repeat_bot_messages
 
   pattern_restart:
     description: Flow for restarting the conversation
@@ -246,3 +270,32 @@ flows:
     name: pattern skip question
     steps:
       - action: utter_skip_question_answer
+
+  pattern_user_silence:
+    description: Reacting to user silence in voice bots
+    name: pattern react to silence
+    nlu_trigger:
+      - intent: silence_timeout
+    persisted_slots:
+      - consecutive_silence_timeouts
+    steps:
+      - noop: true
+        next:
+          - if: "slots.consecutive_silence_timeouts = 0.0"
+            then:
+              - set_slots:
+                  - consecutive_silence_timeouts: 1.0
+              - action: action_repeat_bot_messages
+                next: END
+          - if: "slots.consecutive_silence_timeouts = 1.0"
+            then:
+              - set_slots:
+                  - consecutive_silence_timeouts: 2.0
+              - action: utter_ask_still_there
+                next: END
+          - if: "slots.consecutive_silence_timeouts > 1.0"
+            then:
+              - action: utter_inform_hangup
+              - action: action_hangup
+                next: END
+          - else: END

rasa/dialogue_understanding/patterns/repeat.py

@@ -0,0 +1,37 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Dict
+
+from rasa.dialogue_understanding.stack.frames import PatternFlowStackFrame
+from rasa.shared.constants import RASA_DEFAULT_FLOW_PATTERN_PREFIX
+
+FLOW_PATTERN_REPEAT = RASA_DEFAULT_FLOW_PATTERN_PREFIX + "repeat_bot_messages"
+
+
+@dataclass
+class RepeatBotMessagesPatternFlowStackFrame(PatternFlowStackFrame):
+    """A flow stack frame that can get added when bot messages should be repeated"""
+
+    flow_id: str = FLOW_PATTERN_REPEAT
+    """The ID of the flow."""
+
+    @classmethod
+    def type(cls) -> str:
+        """Returns the type of the frame."""
+        return FLOW_PATTERN_REPEAT
+
+    @staticmethod
+    def from_dict(data: Dict[str, Any]) -> RepeatBotMessagesPatternFlowStackFrame:
+        """Creates a `DialogueStackFrame` from a dictionary.
+
+        Args:
+            data: The dictionary to create the `DialogueStackFrame` from.
+
+        Returns:
+            The created `DialogueStackFrame`.
+        """
+        return RepeatBotMessagesPatternFlowStackFrame(
+            frame_id=data["frame_id"],
+            step_id=data["step_id"],
+        )

rasa/dialogue_understanding/patterns/user_silence.py

@@ -0,0 +1,37 @@
+from __future__ import annotations
+from dataclasses import dataclass
+from typing import Any, Dict
+
+from rasa.dialogue_understanding.stack.frames import PatternFlowStackFrame
+from rasa.shared.constants import RASA_DEFAULT_FLOW_PATTERN_PREFIX
+
+
+FLOW_PATTERN_USER_SILENCE = RASA_DEFAULT_FLOW_PATTERN_PREFIX + "user_silence"
+
+
+@dataclass
+class UserSilencePatternFlowStackFrame(PatternFlowStackFrame):
+    """A flow stack frame that can get added when the silence timeout trips."""
+
+    flow_id: str = FLOW_PATTERN_USER_SILENCE
+    """The ID of the flow."""
+
+    @classmethod
+    def type(cls) -> str:
+        """Returns the type of the frame."""
+        return FLOW_PATTERN_USER_SILENCE
+
+    @staticmethod
+    def from_dict(data: Dict[str, Any]) -> UserSilencePatternFlowStackFrame:
+        """Creates a `DialogueStackFrame` from a dictionary.
+
+        Args:
+            data: The dictionary to create the `DialogueStackFrame` from.
+
+        Returns:
+            The created `DialogueStackFrame`.
+        """
+        return UserSilencePatternFlowStackFrame(
+            frame_id=data["frame_id"],
+            step_id=data["step_id"],
+        )

rasa/dialogue_understanding/processor/command_processor.py

@@ -8,6 +8,7 @@ from rasa.dialogue_understanding.commands import (
     Command,
     CorrectSlotsCommand,
     CorrectedSlot,
+    RepeatBotMessagesCommand,
     SetSlotCommand,
     StartFlowCommand,
     FreeFormAnswerCommand,
@@ -422,6 +423,9 @@ def clean_up_commands(
     elif not tracker.has_coexistence_routing_slot and len(clean_commands) > 1:
         clean_commands = filter_cannot_handle_command_for_skipped_slots(clean_commands)
 
+    clean_commands = ensure_max_number_of_command_type(
+        clean_commands, RepeatBotMessagesCommand, 1
+    )
     structlogger.debug(
         "command_processor.clean_up_commands.final_commands",
         command=clean_commands,
@@ -430,6 +434,22 @@ def clean_up_commands(
     return clean_commands
 
 
+def ensure_max_number_of_command_type(
+    commands: List[Command], command_type: Type[Command], n: int
+) -> List[Command]:
+    """Ensures that for a given command type only the first n stay in the list."""
+    filtered: List[Command] = []
+    count = 0
+    for c in commands:
+        if isinstance(c, command_type):
+            if count >= n:
+                continue
+            else:
+                count += 1
+        filtered.append(c)
+    return filtered
+
+
 def clean_up_clarify_command(
     commands_so_far: List[Command],
     all_commands: List[Command],
@@ -452,7 +472,7 @@ def clean_up_clarify_command(
         if not (isinstance(c, SetSlotCommand) and c.name == ROUTE_TO_CALM_SLOT)
     ]
 
-    # if there are multiple clarify commands, do add the first one
+    # if all commands are clarify commands, add the first one only, otherwise add none
     if all(
         isinstance(c, ClarifyCommand) for c in commands_without_route_to_calm_set_slot
     ):

rasa/e2e_test/aggregate_test_stats_calculator.py

@@ -35,7 +35,6 @@ class AggregateTestStatsCalculator:
         self.test_cases = test_cases
 
         self.failed_assertion_set: Set["Assertion"] = set()
-        self.failed_test_cases_without_assertion_failure: Set[str] = set()
         self.passed_count_mapping = {
             subclass_type: 0
             for subclass_type in _get_all_assertion_subclasses().keys()
@@ -90,14 +89,8 @@ class AggregateTestStatsCalculator:
         passed_test_case_names = [
             passed.test_case.name for passed in self.passed_results
         ]
-        # We filter out test cases that failed without an assertion failure
-        filtered_test_cases = [
-            test_case
-            for test_case in self.test_cases
-            if test_case.name not in self.failed_test_cases_without_assertion_failure
-        ]
 
-        for test_case in filtered_test_cases:
+        for test_case in self.test_cases:
             if test_case.name in passed_test_case_names:
                 for step in test_case.steps:
                     if step.assertions is None:
@@ -125,9 +118,6 @@ class AggregateTestStatsCalculator:
                     "no_assertion_failure_in_failed_result",
                     test_case=failed.test_case.name,
                 )
-                self.failed_test_cases_without_assertion_failure.add(
-                    failed.test_case.name
-                )
                 continue
 
             self.failed_assertion_set.add(failed.assertion_failure.assertion)