rasa-pro 3.13.0rc1__py3-none-any.whl → 3.13.0rc3__py3-none-any.whl

rasa/core/policies/enterprise_search_policy.py

@@ -1,8 +1,10 @@
 import dataclasses
+import glob
 import importlib.resources
 import json
+import os.path
 import re
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Text
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Text, Tuple
 
 import dotenv
 import structlog
@@ -137,6 +139,8 @@ DEFAULT_ENTERPRISE_SEARCH_PROMPT_WITH_RELEVANCY_CHECK_AND_CITATION_TEMPLATE = (
 
 _ENTERPRISE_SEARCH_ANSWER_NOT_RELEVANT_PATTERN = re.compile(r"\[NO_RAG_ANSWER\]")
 
+_ENTERPRISE_SEARCH_CITATION_PATTERN = re.compile(r"\[([^\]]+)\]")
+
 
 class VectorStoreConnectionError(RasaException):
     """Exception raised for errors in connecting to the vector store."""
@@ -351,9 +355,11 @@ class EnterpriseSearchPolicy(LLMHealthCheckMixin, EmbeddingsHealthCheckMixin, Po
 
         if self.vector_store_type == DEFAULT_VECTOR_STORE_TYPE:
             structlogger.info("enterprise_search_policy.train.faiss")
+            docs_folder = self.vector_store_config.get(SOURCE_PROPERTY)
+            self._validate_documents_folder(docs_folder)
             with self._model_storage.write_to(self._resource) as path:
                 self.vector_store = FAISS_Store(
-                    docs_folder=self.vector_store_config.get(SOURCE_PROPERTY),
+                    docs_folder=docs_folder,
                     embeddings=embeddings,
                     index_path=path,
                     create_index=True,
@@ -773,6 +779,33 @@ class EnterpriseSearchPolicy(LLMHealthCheckMixin, EmbeddingsHealthCheckMixin, Po
             result[domain.index_for_action(action_name)] = score  # type: ignore[assignment]
         return result
 
+    @classmethod
+    def _validate_documents_folder(cls, docs_folder: str) -> None:
+        if not os.path.exists(docs_folder) or not os.path.isdir(docs_folder):
+            error_message = (
+                f"Document source directory does not exist or is not a "
+                f"directory: '{docs_folder}'. "
+                "Please specify a valid path to the documents source directory in the "
+                "vector_store configuration."
+            )
+            structlogger.error(
+                "enterprise_search_policy.train.faiss.invalid_source_directory",
+                message=error_message,
+            )
+            print_error_and_exit(error_message)
+
+        docs = glob.glob(os.path.join(docs_folder, "*.txt"), recursive=True)
+        if not docs or len(docs) < 1:
+            error_message = (
+                f"Document source directory is empty: '{docs_folder}'. "
+                "Please add documents to this directory or specify a different one."
+            )
+            structlogger.error(
+                "enterprise_search_policy.train.faiss.source_directory_empty",
+                message=error_message,
+            )
+            print_error_and_exit(error_message)
+
     @classmethod
     def load(
         cls,
@@ -864,7 +897,12 @@ class EnterpriseSearchPolicy(LLMHealthCheckMixin, EmbeddingsHealthCheckMixin, Po
         e.g. FAISS, to ensure that the graph component is retrained when the knowledge
         base is updated.
         """
-        if store_type != DEFAULT_VECTOR_STORE_TYPE or not source:
+        if (
+            store_type != DEFAULT_VECTOR_STORE_TYPE
+            or not source
+            or not os.path.exists(source)
+            or not os.path.isdir(source)
+        ):
             return None
 
         docs = FAISS_Store.load_documents(source)
@@ -908,10 +946,18 @@ class EnterpriseSearchPolicy(LLMHealthCheckMixin, EmbeddingsHealthCheckMixin, Po
 
     @staticmethod
     def post_process_citations(llm_answer: str) -> str:
-        """Post-process the LLM answer.
-
-         Re-writes the bracketed numbers to start from 1 and
-         re-arranges the sources to follow the enumeration order.
+        """Post-processes the LLM answer to correctly number and sort citations and
+        sources.
+
+        - Handles both single `[1]` and grouped `[1, 3]` citations.
+        - Rewrites the numbers in square brackets in the answer text to start from 1
+        and be sorted within each group.
+        - Reorders the sources according to the order of their first appearance
+        in the text.
+        - Removes citations from the text that point to sources missing from
+        the source list.
+        - Keeps sources that are not cited in the text, placing them at the end
+        of the list.
 
         Args:
             llm_answer: The LLM answer.
@@ -925,77 +971,160 @@ class EnterpriseSearchPolicy(LLMHealthCheckMixin, EmbeddingsHealthCheckMixin, Po
 
         # Split llm_answer into answer and citations
         try:
-            answer, citations = llm_answer.rsplit("Sources:", 1)
+            answer_part, sources_part = llm_answer.rsplit("Sources:", 1)
         except ValueError:
-            # if there is no "Sources:" in the llm_answer
-            return llm_answer
-
-        # Find all source references in the answer
-        pattern = r"\[\s*(\d+(?:\s*,\s*\d+)*)\s*\]"
-        matches = re.findall(pattern, answer)
-        old_source_indices = [
-            int(num.strip()) for match in matches for num in match.split(",")
-        ]
+            # if there is no "Sources:" separator, return the original llm_answer
+            return llm_answer.strip()
 
-        # Map old source references to the correct enumeration
-        renumber_mapping = {num: idx + 1 for idx, num in enumerate(old_source_indices)}
-
-        # remove whitespace from original source citations in answer
-        for match in matches:
-            answer = answer.replace(f"[{match}]", f"[{match.replace(' ', '')}]")
-
-        new_answer = []
-        for word in answer.split():
-            matches = re.findall(pattern, word)
-            if matches:
-                for match in matches:
-                    if "," in match:
-                        old_indices = [
-                            int(num.strip()) for num in match.split(",") if num
-                        ]
-                        new_indices = [
-                            renumber_mapping[old_index]
-                            for old_index in old_indices
-                            if old_index in renumber_mapping
-                        ]
-                        if not new_indices:
-                            continue
-
-                        word = word.replace(
-                            match, f"{', '.join(map(str, new_indices))}"
-                        )
-                    else:
-                        old_index = int(match.strip("[].,:;?!"))
-                        new_index = renumber_mapping.get(old_index)
-                        if not new_index:
-                            continue
+        # Parse the sources block to extract valid sources and other lines
+        valid_sources, other_source_lines = EnterpriseSearchPolicy._parse_sources_block(
+            sources_part
+        )
 
-                        word = word.replace(str(old_index), str(new_index))
-            new_answer.append(word)
+        # Find all unique, valid citations in the answer text in their order
+        # of appearance
+        cited_order = EnterpriseSearchPolicy._get_cited_order(
+            answer_part, valid_sources
+        )
 
-        # join the words
-        joined_answer = " ".join(new_answer)
-        joined_answer += "\nSources:\n"
+        # Create a mapping from the old source numbers to the new, sequential numbers.
+        # For example, if the citation order in the text was [3, 1, 2], this map
+        # becomes {3: 1, 1: 2, 2: 3}. This allows for a quick lookup when rewriting
+        # the citations
+        renumbering_map = {
+            old_num: new_num + 1 for new_num, old_num in enumerate(cited_order)
+        }
 
-        new_sources: List[str] = []
+        # Rewrite the citations in the answer text based on the renumbering map
+        processed_answer = EnterpriseSearchPolicy._rewrite_answer_citations(
+            answer_part, renumbering_map
+        )
 
-        for line in citations.split("\n"):
-            pattern = r"(?<=\[)\d+"
-            match = re.search(pattern, line)
+        # Build the new list of sources
+        new_sources_list = EnterpriseSearchPolicy._build_final_sources_list(
+            cited_order,
+            renumbering_map,
+            valid_sources,
+            other_source_lines,
+        )
+
+        if len(new_sources_list) > 0:
+            processed_answer += "\nSources:\n" + "\n".join(new_sources_list)
+
+        return processed_answer
+
+    @staticmethod
+    def _parse_sources_block(sources_part: str) -> Tuple[Dict[int, str], List[str]]:
+        """Parses the sources block from the LLM response.
+        Returns a tuple containing:
+        - A dictionary of valid sources matching the "[1] ..." format,
+        where the key is the source number
+        - A list of other source lines that do not match the specified format
+        """
+        valid_sources: Dict[int, str] = {}
+        other_source_lines: List[str] = []
+        source_line_pattern = re.compile(r"^\s*\[(\d+)\](.*)")
+
+        source_lines = sources_part.strip().split("\n")
+
+        for line in source_lines:
+            line = line.strip()
+            if not line:
+                continue
+
+            match = source_line_pattern.match(line)
             if match:
-                old_index = int(match.group(0))
-                new_index = renumber_mapping[old_index]
-                # replace only the first occurrence of the old index
-                line = line.replace(f"[{old_index}]", f"[{new_index}]", 1)
+                num = int(match.group(1))
+                valid_sources[num] = line
+            else:
+                other_source_lines.append(line)
+
+        return valid_sources, other_source_lines
+
+    @staticmethod
+    def _get_cited_order(
+        answer_part: str, available_sources: Dict[int, str]
+    ) -> List[int]:
+        """Find all unique, valid citations in the answer text in their order
+        # of appearance
+        """
+        cited_order: List[int] = []
+        seen_indices = set()
+
+        for match in _ENTERPRISE_SEARCH_CITATION_PATTERN.finditer(answer_part):
+            content = match.group(1)
+            indices_str = [s.strip() for s in content.split(",")]
+            for index_str in indices_str:
+                if index_str.isdigit():
+                    index = int(index_str)
+                    if index in available_sources and index not in seen_indices:
+                        cited_order.append(index)
+                        seen_indices.add(index)
+
+        return cited_order
+
+    @staticmethod
+    def _rewrite_answer_citations(
+        answer_part: str, renumber_map: Dict[int, int]
+    ) -> str:
+        """Rewrites the citations in the answer text based on the renumbering map."""
+
+        def replacer(match: re.Match) -> str:
+            content = match.group(1)
+            old_indices_str = [s.strip() for s in content.split(",")]
+            new_indices = [
+                renumber_map[int(s)]
+                for s in old_indices_str
+                if s.isdigit() and int(s) in renumber_map
+            ]
+            if not new_indices:
+                return ""
+
+            return f"[{', '.join(map(str, sorted(list(set(new_indices)))))}]"
+
+        processed_answer = _ENTERPRISE_SEARCH_CITATION_PATTERN.sub(
+            replacer, answer_part
+        )
+
+        # Clean up formatting after replacements
+        processed_answer = re.sub(r"\s+([,.?])", r"\1", processed_answer)
+        processed_answer = processed_answer.replace("[]", " ")
+        processed_answer = re.sub(r"\s+", " ", processed_answer)
+        processed_answer = processed_answer.strip()
+
+        return processed_answer
+
+    @staticmethod
+    def _build_final_sources_list(
+        cited_order: List[int],
+        renumbering_map: Dict[int, int],
+        valid_sources: Dict[int, str],
+        other_source_lines: List[str],
+    ) -> List[str]:
+        """Builds the final list of sources based on the cited order and
+        renumbering map.
+        """
+        new_sources_list: List[str] = []
+
+        # First, add the sorted, used sources
+        for old_num in cited_order:
+            new_num = renumbering_map[old_num]
+            source_line = valid_sources[old_num]
+            new_sources_list.append(
+                source_line.replace(f"[{old_num}]", f"[{new_num}]", 1)
+            )
 
-                # insert the line into the new_index position
-                new_sources.insert(new_index - 1, line)
-            elif line.strip():
-                new_sources.append(line)
+        # Then, add the unused but validly numbered sources
+        used_source_nums = set(cited_order)
+        # Sort by number to ensure a consistent order for uncited sources
+        for num, line in sorted(valid_sources.items()):
+            if num not in used_source_nums:
+                new_sources_list.append(line)
 
-        joined_sources = "\n".join(new_sources)
+        # Finally, add any other source lines
+        new_sources_list.extend(other_source_lines)
 
-        return joined_answer + joined_sources
+        return new_sources_list
 
     @classmethod
     def _perform_health_checks(

rasa/core/policies/enterprise_search_prompt_with_relevancy_check_and_citation_template.jinja2

@@ -1,8 +1,11 @@
-Based on the provided documents and the recent conversation context, answer the following question.
+{% if check_relevancy %}Based on the provided documents and the recent conversation context, answer the following question.
 Before responding, ensure the answer is directly supported by the documents or context.
 Do not make assumptions or infer beyond the given information.
 Only answer if you are more than 80% confident that the response is fully supported.
 If the answer cannot be determined, respond with: [NO_RAG_ANSWER]
+{% else %}Given the following information, please provide an answer based on the provided documents and the context of the recent conversation.
+If the answer is not known or cannot be determined from the provided documents or context, please state that you do not know to the user.
+{% endif %}
 
 ### Relevant Documents
 Use the following documents to answer the question:

rasa/core/policies/flows/flow_executor.py

@@ -41,6 +41,7 @@ from rasa.dialogue_understanding.patterns.internal_error import (
     InternalErrorPatternFlowStackFrame,
 )
 from rasa.dialogue_understanding.patterns.search import SearchPatternFlowStackFrame
+from rasa.dialogue_understanding.patterns.user_silence import FLOW_PATTERN_USER_SILENCE
 from rasa.dialogue_understanding.stack.dialogue_stack import DialogueStack
 from rasa.dialogue_understanding.stack.frames import (
     BaseFlowStackFrame,
@@ -590,9 +591,9 @@ def run_step(
         initial_events.append(FlowStarted(flow.id, metadata=stack.current_context()))
 
     # FLow does not start with collect step or we are not in collect information pattern
-    if _first_step_is_not_collect(
-        step, previous_step_id
-    ) and not _in_collect_information_pattern(flow):
+    if _first_step_is_not_collect(step, previous_step_id) and not (
+        _in_collect_information_pattern(flow) or _in_pattern_user_silence(flow)
+    ):
         _append_global_silence_timeout_event(initial_events, tracker)
 
     if isinstance(step, CollectInformationFlowStep):
@@ -650,6 +651,11 @@ def _in_collect_information_pattern(flow: Flow) -> bool:
     return flow.id == FLOW_PATTERN_COLLECT_INFORMATION
 
 
+def _in_pattern_user_silence(flow: Flow) -> bool:
+    """Check if the current flow is a user silence pattern."""
+    return flow.id == FLOW_PATTERN_USER_SILENCE
+
+
 def _run_end_step(
     flow: Flow,
     flows: FlowsList,

rasa/core/processor.py

@@ -237,6 +237,12 @@ class MessageProcessor:
             )
             return None
 
+        if not self.privacy_manager.event_brokers:
+            structlogger.debug(
+                "processor.trigger_anonymization.skipping.no_event_brokers",
+            )
+            return None
+
         structlogger.info(
             "rasa.core.processor.trigger_anonymization",
             sender_id=tracker.sender_id,

rasa/core/tracker_stores/redis_tracker_store.py

@@ -88,17 +88,21 @@ class RedisTrackerStore(TrackerStore, SerializedTrackerAsText):
         if not timeout and self.record_exp:
             timeout = self.record_exp
 
-        stored = self.red.get(self.key_prefix + tracker.sender_id)
+        # if the sender_id starts with the key prefix, we remove it
+        # this is used to avoid storing the prefix twice
+        sender_id = tracker.sender_id
+        if sender_id.startswith(self.key_prefix):
+            sender_id = sender_id[len(self.key_prefix) :]
+
+        stored = self.red.get(self.key_prefix + sender_id)
 
         if stored is not None:
-            prior_tracker = self.deserialise_tracker(tracker.sender_id, stored)
+            prior_tracker = self.deserialise_tracker(sender_id, stored)
 
             tracker = self._merge_trackers(prior_tracker, tracker)
 
         serialised_tracker = self.serialise_tracker(tracker)
-        self.red.set(
-            self.key_prefix + tracker.sender_id, serialised_tracker, ex=timeout
-        )
+        self.red.set(self.key_prefix + sender_id, serialised_tracker, ex=timeout)
 
     async def delete(self, sender_id: Text) -> None:
         """Delete tracker for the given sender_id.
@@ -113,6 +117,9 @@ class RedisTrackerStore(TrackerStore, SerializedTrackerAsText):
             )
             return None
 
+        if sender_id.startswith(self.key_prefix):
+            sender_id = sender_id[len(self.key_prefix) :]
+
         self.red.delete(self.key_prefix + sender_id)
         structlogger.info(
             "redis_tracker_store.delete.deleted_tracker",
@@ -156,6 +163,9 @@ class RedisTrackerStore(TrackerStore, SerializedTrackerAsText):
             sender_id: Conversation ID to fetch the tracker for.
             fetch_all_sessions: Whether to fetch all sessions or only the last one.
         """
+        if sender_id.startswith(self.key_prefix):
+            sender_id = sender_id[len(self.key_prefix) :]
+
         stored = self.red.get(self.key_prefix + sender_id)
         if stored is None:
             structlogger.debug(

rasa/dialogue_understanding/coexistence/llm_based_router.py

@@ -21,6 +21,7 @@ from rasa.engine.recipes.default_recipe import DefaultV1Recipe
 from rasa.engine.storage.resource import Resource
 from rasa.engine.storage.storage import ModelStorage
 from rasa.shared.constants import (
+    LOGIT_BIAS_CONFIG_KEY,
     MAX_COMPLETION_TOKENS_CONFIG_KEY,
     MODEL_CONFIG_KEY,
     OPENAI_PROVIDER,
@@ -57,12 +58,22 @@ DEFAULT_COMMAND_PROMPT_TEMPLATE = importlib.resources.read_text(
 )
 LLM_BASED_ROUTER_CONFIG_FILE_NAME = "config.json"
 
+# Token ids for gpt-4o corresponding to space + capitalized Letter
+A_TO_C_TOKEN_IDS_CHATGPT = [
+    355,  # " A"
+    418,  # " B"
+    363,  # " C"
+]
+
 DEFAULT_LLM_CONFIG = {
     PROVIDER_CONFIG_KEY: OPENAI_PROVIDER,
     MODEL_CONFIG_KEY: DEFAULT_OPENAI_CHAT_MODEL_NAME,
     TIMEOUT_CONFIG_KEY: 7,
     TEMPERATURE_CONFIG_KEY: 0.0,
     MAX_COMPLETION_TOKENS_CONFIG_KEY: 1,
+    LOGIT_BIAS_CONFIG_KEY: {
+        str(token_id): 100 for token_id in A_TO_C_TOKEN_IDS_CHATGPT
+    },
 }
 
 structlogger = structlog.get_logger()

rasa/dialogue_understanding/generator/multi_step/multi_step_llm_command_generator.py

@@ -125,8 +125,9 @@ class MultiStepLLMCommandGenerator(LLMBasedCommandGenerator):
         raise_deprecation_warning(
             message=(
                 "Support for `MultiStepLLMCommandGenerator` will be removed in Rasa "
-                "`4.0.0`. Please modify your assistant's configuration to use other "
-                "LLM command generators like the `SingleStepLLMCommandGenerator`."
+                "`4.0.0`. Please modify your assistant's configuration to use the "
+                "`CompactLLMCommandGenerator` or `SearchReadyLLMCommandGenerator` "
+                "instead."
             )
         )
 

rasa/dialogue_understanding/generator/single_step/single_step_llm_command_generator.py

@@ -16,6 +16,7 @@ from rasa.shared.constants import (
     PROMPT_CONFIG_KEY,
     PROMPT_TEMPLATE_CONFIG_KEY,
 )
+from rasa.shared.utils.io import raise_deprecation_warning
 from rasa.shared.utils.llm import (
     check_prompt_config_keys_and_warn_if_deprecated,
     get_prompt_template,
@@ -47,6 +48,14 @@ class SingleStepLLMCommandGenerator(SingleStepBasedLLMCommandGenerator):
         prompt_template: Optional[Text] = None,
         **kwargs: Any,
     ) -> None:
+        raise_deprecation_warning(
+            message=(
+                "Support for `SingleStepLLMCommandGenerator` will be removed in Rasa "
+                "`4.0.0`. Please modify your assistant's configuration to use the "
+                "`CompactLLMCommandGenerator` or `SearchReadyLLMCommandGenerator` "
+                "instead."
+            )
+        )
         super().__init__(
             config,
             model_storage,

rasa/dialogue_understanding/patterns/default_flows_for_patterns.yml

@@ -139,7 +139,8 @@ flows:
     steps:
       - noop: true
         next:
-          # Chitchat fallback
+          # Fallback for ChitChat command when IntentlessPolicy isn't set, but
+          # pattern_chitchat invokes action_trigger_chitchat
           - if: context.reason is "cannot_handle_chitchat"
             then:
               - action: utter_cannot_handle
@@ -164,7 +165,9 @@ flows:
     description: Conversation repair flow for off-topic interactions that won't disrupt the main conversation
     name: pattern chitchat
     steps:
-      - action: action_trigger_chitchat
+      - action: utter_cannot_handle
+      # To enable free-form response use:
+      # - action: utter_free_chitchat_response
 
   pattern_clarification:
     description: Conversation repair flow for handling ambiguous requests that could match multiple flows

rasa/dialogue_understanding/processor/command_processor.py

@@ -64,12 +64,6 @@ from rasa.shared.nlu.constants import COMMANDS
 
 structlogger = structlog.get_logger()
 
-CANNOT_HANDLE_REASON = (
-    "A command generator attempted to set a slot "
-    "with a value extracted by an extractor "
-    "that is incompatible with the slot mapping type."
-)
-
 
 def contains_command(commands: List[Command], typ: Type[Command]) -> bool:
     """Check if a list of commands contains a command of a given type.
@@ -587,6 +581,11 @@ def clean_up_slot_command(
             "command_processor.clean_up_slot_command.skip_command_slot_not_in_domain",
             command=command,
         )
+        resulting_commands.append(
+            CannotHandleCommand(
+                reason="The slot predicted by the LLM is not defined in the domain."
+            )
+        )
         return resulting_commands
 
     if not should_slot_be_set(slot, command, resulting_commands):
@@ -605,7 +604,10 @@ def clean_up_slot_command(
             for command in resulting_commands
         )
 
-        cannot_handle = CannotHandleCommand(reason=CANNOT_HANDLE_REASON)
+        cannot_handle = CannotHandleCommand(
+            reason="A command generator attempted to set a slot with a value extracted "
+            "by an extractor that is incompatible with the slot mapping type."
+        )
         if not slot_command_exists_already and cannot_handle not in resulting_commands:
             resulting_commands.append(cannot_handle)
 
@@ -639,9 +641,9 @@ def clean_up_slot_command(
             resulting_commands.append(command)
             return resulting_commands
 
-        if (slot := tracker.slots.get(command.name)) is not None and slot.value == str(
-            command.value
-        ):
+        if (slot := tracker.slots.get(command.name)) is not None and str(
+            slot.value
+        ) == str(command.value):
             # the slot is already set, we don't need to set it again
             structlogger.debug(
                 "command_processor.clean_up_slot_command.skip_command_slot_already_set",

rasa/e2e_test/constants.py

@@ -40,7 +40,7 @@ DEFAULT_GROUNDEDNESS_PROMPT_TEMPLATE_FILE_NAME = "groundedness_prompt_template.j
 DEFAULT_ANSWER_RELEVANCE_PROMPT_TEMPLATE_FILE_NAME = (
     "answer_relevance_prompt_template.jinja2"
 )
-DEFAULT_E2E_TESTING_MODEL = "gpt-4o-mini"
+DEFAULT_E2E_TESTING_MODEL = "gpt-4.1-mini-2025-04-14"
 KEY_SCORE = "score"
 KEY_JUSTIFICATION = "justification"
 KEY_EXTRA_PARAMETERS = "extra_parameters"

rasa/llm_fine_tuning/annotation_module.py

@@ -9,8 +9,8 @@ from rasa.e2e_test.e2e_test_case import ActualStepOutput, TestCase, TestStep, Te
 from rasa.e2e_test.e2e_test_runner import TEST_TURNS_TYPE, E2ETestRunner
 from rasa.llm_fine_tuning.conversations import Conversation, ConversationStep
 from rasa.llm_fine_tuning.storage import StorageContext
-from rasa.shared.core.constants import USER
-from rasa.shared.core.events import UserUttered
+from rasa.shared.core.constants import BOT, USER
+from rasa.shared.core.events import BotUttered, UserUttered
 from rasa.shared.core.trackers import DialogueStateTracker
 from rasa.shared.exceptions import FinetuningDataPreparationException
 from rasa.shared.nlu.constants import LLM_COMMANDS, LLM_PROMPT
@@ -83,16 +83,18 @@ def generate_conversation(
         Conversation.
     """
     steps = []
-    tracker_event_indices = [
-        i for i, event in enumerate(tracker.events) if isinstance(event, UserUttered)
-    ]
-
-    if len(test_case.steps) != len(tracker_event_indices):
-        raise FinetuningDataPreparationException(
-            "Number of test case steps and tracker events do not match."
-        )
 
     if assertions_used:
+        tracker_event_indices = [
+            i
+            for i, event in enumerate(tracker.events)
+            if isinstance(event, UserUttered)
+        ]
+        if len(test_case.steps) != len(tracker_event_indices):
+            raise FinetuningDataPreparationException(
+                "Number of test case steps and tracker events do not match."
+            )
+
         # we only have user steps, extract the bot response from the bot uttered
         # events of the test turn
         for i, (original_step, tracker_event_index) in enumerate(
@@ -110,8 +112,30 @@ def generate_conversation(
             )
             steps.extend(_create_bot_test_steps(test_turns[i]))
     else:
+        tracker_event_indices = [
+            i
+            for i, event in enumerate(tracker.events)
+            if isinstance(event, UserUttered) or isinstance(event, BotUttered)
+        ]
+
+        # Generally, we expect one or more bot response(s) for each user utterance
+        # in the test case, so that we can evaluate the actual bot response.
+        # If the test case ends with one or more user utterance(s) instead,
+        # we should thus trim those from the test case steps.
+        # This only applies to test cases that have at least one bot utterance;
+        # otherwise, all test case steps would be removed.
+        has_bot_utterance = any(step.actor == BOT for step in test_case.steps)
+        i = len(test_case.steps)
+        if has_bot_utterance:
+            while i > 0 and test_case.steps[i - 1].actor == USER:
+                i -= 1
+        test_case_steps = test_case.steps[:i]
+
+        # If the number of test case steps and tracker events differ,
+        # using zip ensures we only process pairs that exist in both lists.
+        # Prevents index errors and ensures we don't process unmatched steps or events.
         for i, (original_step, tracker_event_index) in enumerate(
-            zip(test_case.steps, tracker_event_indices)
+            zip(test_case_steps, tracker_event_indices)
         ):
             if original_step.actor == USER:
                 previous_turn = _get_previous_actual_step_output(test_turns, i)
@@ -127,6 +151,14 @@ def generate_conversation(
             else:
                 steps.append(original_step)
 
+        # the tracker should only include events up to the last bot utterance
+        # so that the resulting transcript ends with the last bot utterance too
+        # only applies to test cases that have at least one bot utterance
+        if has_bot_utterance and test_case.steps and test_case.steps[-1].actor == USER:
+            event_to_go_to = tracker_event_indices[len(test_case_steps)] - 1
+            timestamp = tracker.events[event_to_go_to].timestamp
+            tracker = tracker.travel_back_in_time(timestamp)
+
     # Some messages in an e2e test case could be mapped to commands via
     # 'NLUCommandAdapter', e.g. the message will not be annotated with a prompt and
     # commands pair. Only convert steps that have a prompt and commands present into a