rasa-pro 3.12.12.dev1__py3-none-any.whl → 3.12.14__py3-none-any.whl

rasa/cli/llm_fine_tuning.py

@@ -208,10 +208,7 @@ def prepare_llm_fine_tuning_data(args: argparse.Namespace) -> None:
         sys.exit(0)
 
     flows = asyncio.run(e2e_test_runner.agent.processor.get_flows())
-    llm_command_generator_config = _get_llm_command_generator_config(e2e_test_runner)
-    llm_command_generator: Type[LLMBasedCommandGenerator] = _get_llm_command_generator(
-        e2e_test_runner
-    )
+    _validate_llm_command_generator_present(e2e_test_runner)
 
     # set up storage context
     storage_context = create_storage_context(StorageType.FILE, output_dir)
@@ -242,11 +239,11 @@ def prepare_llm_fine_tuning_data(args: argparse.Namespace) -> None:
             rephrase_config,
             args.num_rephrases,
             flows,
-            llm_command_generator,
-            llm_command_generator_config,
+            e2e_test_runner.agent,
             storage_context,
         )
     )
+
     statistics["num_passing_rephrased_user_messages"] = sum(
         [conversation.get_number_of_rephrases(True) for conversation in conversations]
     )
@@ -257,7 +254,11 @@ def prepare_llm_fine_tuning_data(args: argparse.Namespace) -> None:
 
     # 3. create fine-tuning dataset
     log_start_of_module("LLM Data Preparation")
-    llm_fine_tuning_data = convert_to_fine_tuning_data(conversations, storage_context)
+    llm_fine_tuning_data = asyncio.run(
+        convert_to_fine_tuning_data(
+            conversations, storage_context, e2e_test_runner.agent
+        )
+    )
     statistics["num_ft_data_points"] = len(llm_fine_tuning_data)
     log_end_of_module("LLM Data Preparation", statistics)
 
@@ -311,9 +312,9 @@ def _get_llm_command_generator_config(e2e_test_runner: E2ETestRunner) -> Dict[st
     sys.exit(1)
 
 
-def _get_llm_command_generator(
+def _validate_llm_command_generator_present(
     e2e_test_runner: E2ETestRunner,
-) -> Type[LLMBasedCommandGenerator]:
+) -> None:
     train_schema = e2e_test_runner.agent.processor.model_metadata.train_schema  # type: ignore
 
     for _, node in train_schema.nodes.items():
@@ -322,7 +323,7 @@ def _get_llm_command_generator(
         ) and not node.matches_type(
             MultiStepLLMCommandGenerator, include_subtypes=True
         ):
-            return cast(Type[LLMBasedCommandGenerator], node.uses)
+            return
 
     rasa.shared.utils.cli.print_error(
         "The provided model is not trained using 'SingleStepLLMCommandGenerator' or "

rasa/core/nlg/contextual_response_rephraser.py

@@ -8,12 +8,14 @@ from rasa.core.nlg.response import TemplatedNaturalLanguageGenerator
 from rasa.core.nlg.summarize import summarize_conversation
 from rasa.shared.constants import (
     LLM_CONFIG_KEY,
+    MAX_COMPLETION_TOKENS_CONFIG_KEY,
     MODEL_CONFIG_KEY,
     MODEL_GROUP_ID_CONFIG_KEY,
     MODEL_NAME_CONFIG_KEY,
     OPENAI_PROVIDER,
     PROMPT_CONFIG_KEY,
     PROVIDER_CONFIG_KEY,
+    TEMPERATURE_CONFIG_KEY,
     TIMEOUT_CONFIG_KEY,
 )
 from rasa.shared.core.domain import KEY_RESPONSES_TEXT, Domain
@@ -57,8 +59,8 @@ DEFAULT_MAX_HISTORICAL_TURNS = 5
 DEFAULT_LLM_CONFIG = {
     PROVIDER_CONFIG_KEY: OPENAI_PROVIDER,
     MODEL_CONFIG_KEY: DEFAULT_OPENAI_GENERATE_MODEL_NAME,
-    "temperature": 0.3,
-    "max_tokens": DEFAULT_OPENAI_MAX_GENERATED_TOKENS,
+    TEMPERATURE_CONFIG_KEY: 0.3,
+    MAX_COMPLETION_TOKENS_CONFIG_KEY: DEFAULT_OPENAI_MAX_GENERATED_TOKENS,
     TIMEOUT_CONFIG_KEY: 5,
 }
 

rasa/core/policies/enterprise_search_policy.py

@@ -46,12 +46,15 @@ from rasa.graph_components.providers.forms_provider import Forms
 from rasa.graph_components.providers.responses_provider import Responses
 from rasa.shared.constants import (
     EMBEDDINGS_CONFIG_KEY,
+    MAX_COMPLETION_TOKENS_CONFIG_KEY,
+    MAX_RETRIES_CONFIG_KEY,
     MODEL_CONFIG_KEY,
     MODEL_GROUP_ID_CONFIG_KEY,
     MODEL_NAME_CONFIG_KEY,
     OPENAI_PROVIDER,
     PROMPT_CONFIG_KEY,
     PROVIDER_CONFIG_KEY,
+    TEMPERATURE_CONFIG_KEY,
     TIMEOUT_CONFIG_KEY,
 )
 from rasa.shared.core.constants import (
@@ -135,14 +138,14 @@ DEFAULT_LLM_CONFIG = {
     PROVIDER_CONFIG_KEY: OPENAI_PROVIDER,
     MODEL_CONFIG_KEY: DEFAULT_OPENAI_CHAT_MODEL_NAME,
     TIMEOUT_CONFIG_KEY: 10,
-    "temperature": 0.0,
-    "max_tokens": 256,
-    "max_retries": 1,
+    TEMPERATURE_CONFIG_KEY: 0.0,
+    MAX_COMPLETION_TOKENS_CONFIG_KEY: 256,
+    MAX_RETRIES_CONFIG_KEY: 1,
 }
 
 DEFAULT_EMBEDDINGS_CONFIG = {
     PROVIDER_CONFIG_KEY: OPENAI_PROVIDER,
-    "model": DEFAULT_OPENAI_EMBEDDING_MODEL_NAME,
+    MODEL_CONFIG_KEY: DEFAULT_OPENAI_EMBEDDING_MODEL_NAME,
 }
 
 ENTERPRISE_SEARCH_PROMPT_FILE_NAME = "enterprise_search_policy_prompt.jinja2"

rasa/core/policies/intentless_policy.py

@@ -31,12 +31,14 @@ from rasa.graph_components.providers.responses_provider import Responses
 from rasa.shared.constants import (
     EMBEDDINGS_CONFIG_KEY,
     LLM_CONFIG_KEY,
+    MAX_COMPLETION_TOKENS_CONFIG_KEY,
     MODEL_CONFIG_KEY,
     MODEL_GROUP_ID_CONFIG_KEY,
     MODEL_NAME_CONFIG_KEY,
     OPENAI_PROVIDER,
     PROMPT_CONFIG_KEY,
     PROVIDER_CONFIG_KEY,
+    TEMPERATURE_CONFIG_KEY,
     TIMEOUT_CONFIG_KEY,
 )
 from rasa.shared.core.constants import ACTION_LISTEN_NAME
@@ -111,14 +113,14 @@ NLU_ABSTENTION_THRESHOLD = "nlu_abstention_threshold"
 DEFAULT_LLM_CONFIG = {
     PROVIDER_CONFIG_KEY: OPENAI_PROVIDER,
     MODEL_CONFIG_KEY: DEFAULT_OPENAI_CHAT_MODEL_NAME,
-    "temperature": 0.0,
-    "max_tokens": DEFAULT_OPENAI_MAX_GENERATED_TOKENS,
+    TEMPERATURE_CONFIG_KEY: 0.0,
+    MAX_COMPLETION_TOKENS_CONFIG_KEY: DEFAULT_OPENAI_MAX_GENERATED_TOKENS,
     TIMEOUT_CONFIG_KEY: 5,
 }
 
 DEFAULT_EMBEDDINGS_CONFIG = {
     PROVIDER_CONFIG_KEY: OPENAI_PROVIDER,
-    "model": DEFAULT_OPENAI_EMBEDDING_MODEL_NAME,
+    MODEL_CONFIG_KEY: DEFAULT_OPENAI_EMBEDDING_MODEL_NAME,
 }
 
 DEFAULT_INTENTLESS_PROMPT_TEMPLATE = importlib.resources.open_text(
@@ -344,8 +346,6 @@ class IntentlessPolicy(LLMHealthCheckMixin, EmbeddingsHealthCheckMixin, Policy):
             # ensures that the policy will not override a deterministic policy
             # which utilizes the nlu predictions confidence (e.g. Memoization).
             NLU_ABSTENTION_THRESHOLD: 0.9,
-            LLM_CONFIG_KEY: DEFAULT_LLM_CONFIG,
-            EMBEDDINGS_CONFIG_KEY: DEFAULT_EMBEDDINGS_CONFIG,
             PROMPT_CONFIG_KEY: DEFAULT_INTENTLESS_PROMPT_TEMPLATE,
         }
 
@@ -381,13 +381,19 @@ class IntentlessPolicy(LLMHealthCheckMixin, EmbeddingsHealthCheckMixin, Policy):
         super().__init__(config, model_storage, resource, execution_context, featurizer)
 
         # Resolve LLM config
-        self.config[LLM_CONFIG_KEY] = resolve_model_client_config(
-            self.config.get(LLM_CONFIG_KEY), IntentlessPolicy.__name__
+        self.config[LLM_CONFIG_KEY] = combine_custom_and_default_config(
+            resolve_model_client_config(
+                self.config.get(LLM_CONFIG_KEY), IntentlessPolicy.__name__
+            ),
+            DEFAULT_LLM_CONFIG,
         )
 
         # Resolve embeddings config
-        self.config[EMBEDDINGS_CONFIG_KEY] = resolve_model_client_config(
-            self.config.get(EMBEDDINGS_CONFIG_KEY), IntentlessPolicy.__name__
+        self.config[EMBEDDINGS_CONFIG_KEY] = combine_custom_and_default_config(
+            resolve_model_client_config(
+                self.config.get(EMBEDDINGS_CONFIG_KEY), IntentlessPolicy.__name__
+            ),
+            DEFAULT_EMBEDDINGS_CONFIG,
         )
 
         self.nlu_abstention_threshold: float = self.config[NLU_ABSTENTION_THRESHOLD]

rasa/core/run.py

@@ -86,13 +86,15 @@ def _create_single_channel(channel: Text, credentials: Dict[Text, Any]) -> Any:
             )
 
 
-def _create_app_without_api(cors: Optional[Union[Text, List[Text]]] = None) -> Sanic:
+def _create_app_without_api(
+    cors: Optional[Union[Text, List[Text]]] = None, is_inspector_enabled: bool = False
+) -> Sanic:
     app = Sanic("rasa_core_no_api", configure_logging=False)
 
     # Reset Sanic warnings filter that allows the triggering of Sanic warnings
     warnings.filterwarnings("ignore", category=DeprecationWarning, module=r"sanic.*")
 
-    server.add_root_route(app)
+    server.add_root_route(app, is_inspector_enabled)
     server.configure_cors(app, cors)
     return app
 
@@ -127,6 +129,7 @@ def configure_app(
     server_listeners: Optional[List[Tuple[Callable, Text]]] = None,
     use_uvloop: Optional[bool] = True,
     keep_alive_timeout: int = constants.DEFAULT_KEEP_ALIVE_TIMEOUT,
+    is_inspector_enabled: bool = False,
 ) -> Sanic:
     """Run the agent."""
     rasa.core.utils.configure_file_logging(
@@ -144,6 +147,7 @@ def configure_app(
                 jwt_private_key=jwt_private_key,
                 jwt_method=jwt_method,
                 endpoints=endpoints,
+                is_inspector_enabled=is_inspector_enabled,
             )
         )
     else:
@@ -259,6 +263,7 @@ def serve_application(
         syslog_protocol=syslog_protocol,
         request_timeout=request_timeout,
         server_listeners=server_listeners,
+        is_inspector_enabled=inspect,
     )
 
     ssl_context = server.create_ssl_context(

rasa/core/utils.py

@@ -244,6 +244,10 @@ class AvailableEndpoints:
             cls._instance = cls.read_endpoints(endpoint_file)
         return cls._instance
 
+    @classmethod
+    def reset_instance(cls) -> None:
+        cls._instance = None
+
 
 def read_endpoints_from_path(
     endpoints_path: Optional[Union[Path, Text]] = None,

rasa/dialogue_understanding/coexistence/llm_based_router.py

@@ -23,11 +23,14 @@ from rasa.engine.recipes.default_recipe import DefaultV1Recipe
 from rasa.engine.storage.resource import Resource
 from rasa.engine.storage.storage import ModelStorage
 from rasa.shared.constants import (
+    LOGIT_BIAS_CONFIG_KEY,
+    MAX_COMPLETION_TOKENS_CONFIG_KEY,
     MODEL_CONFIG_KEY,
     OPENAI_PROVIDER,
     PROMPT_CONFIG_KEY,
     PROVIDER_CONFIG_KEY,
     ROUTE_TO_CALM_SLOT,
+    TEMPERATURE_CONFIG_KEY,
     TIMEOUT_CONFIG_KEY,
 )
 from rasa.shared.core.trackers import DialogueStateTracker
@@ -66,9 +69,11 @@ DEFAULT_LLM_CONFIG = {
     PROVIDER_CONFIG_KEY: OPENAI_PROVIDER,
     MODEL_CONFIG_KEY: DEFAULT_OPENAI_CHAT_MODEL_NAME,
     TIMEOUT_CONFIG_KEY: 7,
-    "temperature": 0.0,
-    "max_tokens": 1,
-    "logit_bias": {str(token_id): 100 for token_id in A_TO_C_TOKEN_IDS_CHATGPT},
+    TEMPERATURE_CONFIG_KEY: 0.0,
+    MAX_COMPLETION_TOKENS_CONFIG_KEY: 1,
+    LOGIT_BIAS_CONFIG_KEY: {
+        str(token_id): 100 for token_id in A_TO_C_TOKEN_IDS_CHATGPT
+    },
 }
 
 structlogger = structlog.get_logger()

rasa/dialogue_understanding/commands/clarify_command.py

@@ -117,9 +117,9 @@ class ClarifyCommand(Command):
     @staticmethod
     def regex_pattern() -> str:
         mapper = {
-            CommandSyntaxVersion.v1: r"Clarify\(([\"\'a-zA-Z0-9_, ]*)\)",
+            CommandSyntaxVersion.v1: r"Clarify\(([\"\'a-zA-Z0-9_, -]*)\)",
             CommandSyntaxVersion.v2: (
-                r"""^[\s\W\d]*disambiguate flows (["'a-zA-Z0-9_, ]*)['"`]*$"""
+                r"""^[\s\W\d]*disambiguate flows (["'a-zA-Z0-9_, -]*)[\W\\n]*$"""
             ),
         }
         return mapper.get(

rasa/dialogue_understanding/commands/set_slot_command.py

@@ -190,7 +190,7 @@ class SetSlotCommand(Command):
                 r"""SetSlot\(['"]?([a-zA-Z_][a-zA-Z0-9_-]*)['"]?, ?['"]?(.*)['"]?\)"""
             ),
             CommandSyntaxVersion.v2: (
-                r"""^[\s\W\d]*set slot ['"`]?([a-zA-Z_][a-zA-Z0-9_-]*)['"`]? ['"`]?(.+?)['"`]*$"""  # noqa: E501
+                r"""^[\s\W\d]*set slot ['"`]?([a-zA-Z_][a-zA-Z0-9_-]*)['"`]? ['"`]?(.+?)[\W\\n]*$"""  # noqa: E501
             ),
         }
         return mapper.get(

rasa/dialogue_understanding/generator/constants.py

@@ -1,5 +1,5 @@
 from rasa.shared.constants import (
-    MAX_TOKENS_CONFIG_KEY,
+    MAX_COMPLETION_TOKENS_CONFIG_KEY,
     MODEL_CONFIG_KEY,
     OPENAI_PROVIDER,
     PROVIDER_CONFIG_KEY,
@@ -15,7 +15,7 @@ DEFAULT_LLM_CONFIG = {
     PROVIDER_CONFIG_KEY: OPENAI_PROVIDER,
     MODEL_CONFIG_KEY: DEFAULT_OPENAI_CHAT_MODEL_NAME_ADVANCED,
     TEMPERATURE_CONFIG_KEY: 0.0,
-    MAX_TOKENS_CONFIG_KEY: DEFAULT_OPENAI_MAX_GENERATED_TOKENS,
+    MAX_COMPLETION_TOKENS_CONFIG_KEY: DEFAULT_OPENAI_MAX_GENERATED_TOKENS,
     TIMEOUT_CONFIG_KEY: 7,
 }
 

rasa/dialogue_understanding/generator/llm_based_command_generator.py

@@ -528,7 +528,7 @@ class LLMBasedCommandGenerator(
         either contain a StartFlowCommand or a SetSlot command
         for the current collect step.
         """
-        return self.config.get(KEY_MINIMIZE_NUM_CALLS, False) and (
+        return self.config.get(KEY_MINIMIZE_NUM_CALLS, True) and (
             self._prior_commands_contain_start_flow(prior_commands)
             or self._prior_commands_contain_set_slot_for_active_collect_step(
                 prior_commands, flows, tracker

rasa/dialogue_understanding/generator/single_step/compact_llm_command_generator.py

@@ -47,7 +47,7 @@ from rasa.shared.constants import (
     AWS_BEDROCK_PROVIDER,
     AZURE_OPENAI_PROVIDER,
     EMBEDDINGS_CONFIG_KEY,
-    MAX_TOKENS_CONFIG_KEY,
+    MAX_COMPLETION_TOKENS_CONFIG_KEY,
     PROMPT_TEMPLATE_CONFIG_KEY,
     ROUTE_TO_CALM_SLOT,
     TEMPERATURE_CONFIG_KEY,
@@ -81,7 +81,7 @@ DEFAULT_LLM_CONFIG = {
     PROVIDER_CONFIG_KEY: OPENAI_PROVIDER,
     MODEL_CONFIG_KEY: MODEL_NAME_GPT_4O_2024_11_20,
     TEMPERATURE_CONFIG_KEY: 0.0,
-    MAX_TOKENS_CONFIG_KEY: DEFAULT_OPENAI_MAX_GENERATED_TOKENS,
+    MAX_COMPLETION_TOKENS_CONFIG_KEY: DEFAULT_OPENAI_MAX_GENERATED_TOKENS,
     TIMEOUT_CONFIG_KEY: 7,
 }
 

rasa/dialogue_understanding_test/du_test_runner.py

@@ -33,6 +33,7 @@ from rasa.e2e_test.e2e_test_runner import E2ETestRunner
 from rasa.shared.core.events import UserUttered
 from rasa.shared.core.trackers import DialogueStateTracker
 from rasa.shared.nlu.constants import PREDICTED_COMMANDS, PROMPTS
+from rasa.shared.utils.llm import create_tracker_for_user_step
 from rasa.utils.endpoints import EndpointConfig
 
 structlogger = structlog.get_logger()
@@ -178,8 +179,9 @@ class DialogueUnderstandingTestRunner:
             # create and save the tracker at the time just
             # before the user message was sent
             step_sender_id = f"{sender_id}_{user_step_index}"
-            await self._create_tracker_for_user_step(
+            await create_tracker_for_user_step(
                 step_sender_id,
+                self.agent,
                 test_case_tracker,
                 user_uttered_event_indices[user_step_index],
             )
@@ -280,26 +282,6 @@ class DialogueUnderstandingTestRunner:
 
         return user_uttered_event
 
-    async def _create_tracker_for_user_step(
-        self,
-        step_sender_id: str,
-        test_case_tracker: DialogueStateTracker,
-        index_user_uttered_event: int,
-    ) -> None:
-        """Creates a tracker for the user step."""
-        tracker = test_case_tracker.copy()
-        # modify the sender id so that the test case tracker is not overwritten
-        tracker.sender_id = step_sender_id
-
-        if tracker.events:
-            # get timestamp of the event just before the user uttered event
-            timestamp = tracker.events[index_user_uttered_event - 1].timestamp
-            # revert the tracker to the event just before the user uttered event
-            tracker = tracker.travel_back_in_time(timestamp)
-
-        # store the tracker with the unique sender id
-        await self.agent.tracker_store.save(tracker)
-
     async def _send_user_message(
         self,
         sender_id: str,

rasa/dialogue_understanding_test/test_case_simulation/test_case_tracker_simulator.py

@@ -1,4 +1,3 @@
-from datetime import datetime
 from typing import List, Optional
 
 import structlog
@@ -24,6 +23,7 @@ from rasa.shared.core.constants import SlotMappingType
 from rasa.shared.core.events import BotUttered, SlotSet, UserUttered
 from rasa.shared.core.trackers import DialogueStateTracker
 from rasa.shared.nlu.constants import COMMANDS, ENTITIES, INTENT
+from rasa.shared.utils.llm import generate_sender_id
 
 structlogger = structlog.get_logger()
 
@@ -52,7 +52,7 @@ class TestCaseTrackerSimulator:
         self.test_case = test_case
         self.output_channel = output_channel or CollectingOutputChannel()
 
-        self.sender_id = self._generate_sender_id()
+        self.sender_id = generate_sender_id(self.test_case.name)
 
     async def simulate_test_case(
         self,
@@ -150,10 +150,6 @@ class TestCaseTrackerSimulator:
             user_uttered_event_indices=user_uttered_event_indices,
         )
 
-    def _generate_sender_id(self) -> str:
-        # add timestamp suffix to ensure sender_id is unique
-        return f"{self.test_case.name}_{datetime.now()}"
-
     @staticmethod
     async def _get_latest_user_uttered_event_index(
         tracker: DialogueStateTracker, user_uttered_event_indices: List[int]

rasa/llm_fine_tuning/annotation_module.py

@@ -10,7 +10,9 @@ from rasa.e2e_test.e2e_test_runner import TEST_TURNS_TYPE, E2ETestRunner
 from rasa.llm_fine_tuning.conversations import Conversation, ConversationStep
 from rasa.llm_fine_tuning.storage import StorageContext
 from rasa.shared.core.constants import USER
+from rasa.shared.core.events import UserUttered
 from rasa.shared.core.trackers import DialogueStateTracker
+from rasa.shared.exceptions import FinetuningDataPreparationException
 from rasa.shared.nlu.constants import LLM_COMMANDS, LLM_PROMPT
 from rasa.shared.utils.llm import tracker_as_readable_transcript
 
@@ -37,7 +39,7 @@ def annotate_e2e_tests(
     storage_context: StorageContext,
 ) -> List[Conversation]:
     with set_preparing_fine_tuning_data():
-        converations = asyncio.run(
+        conversations = asyncio.run(
             e2e_test_runner.run_tests_for_fine_tuning(
                 test_suite.test_cases,
                 test_suite.fixtures,
@@ -46,10 +48,11 @@ def annotate_e2e_tests(
         )
 
     storage_context.write_conversations(
-        converations, ANNOTATION_MODULE_STORAGE_LOCATION
+        conversations,
+        ANNOTATION_MODULE_STORAGE_LOCATION,
     )
 
-    return converations
+    return conversations
 
 
 def _get_previous_actual_step_output(
@@ -80,25 +83,45 @@ def generate_conversation(
         Conversation.
     """
     steps = []
+    tracker_event_indices = [
+        i for i, event in enumerate(tracker.events) if isinstance(event, UserUttered)
+    ]
+
+    if len(test_case.steps) != len(tracker_event_indices):
+        raise FinetuningDataPreparationException(
+            "Number of test case steps and tracker events do not match."
+        )
 
     if assertions_used:
         # we only have user steps, extract the bot response from the bot uttered
         # events of the test turn
-        for i, original_step in enumerate(test_case.steps):
+        for i, (original_step, tracker_event_index) in enumerate(
+            zip(test_case.steps, tracker_event_indices)
+        ):
             previous_turn = _get_previous_actual_step_output(test_turns, i)
             steps.append(
                 _convert_to_conversation_step(
-                    original_step, test_turns[i], test_case.name, previous_turn
+                    original_step,
+                    test_turns[i],
+                    test_case.name,
+                    previous_turn,
+                    tracker_event_index,
                 )
             )
             steps.extend(_create_bot_test_steps(test_turns[i]))
     else:
-        for i, original_step in enumerate(test_case.steps):
+        for i, (original_step, tracker_event_index) in enumerate(
+            zip(test_case.steps, tracker_event_indices)
+        ):
             if original_step.actor == USER:
                 previous_turn = _get_previous_actual_step_output(test_turns, i)
                 steps.append(
                     _convert_to_conversation_step(
-                        original_step, test_turns[i], test_case.name, previous_turn
+                        original_step,
+                        test_turns[i],
+                        test_case.name,
+                        previous_turn,
+                        tracker_event_index,
                     )
                 )
             else:
@@ -120,7 +143,7 @@ def generate_conversation(
 
     transcript = tracker_as_readable_transcript(tracker, max_turns=None)
 
-    return Conversation(test_case.name, test_case, steps, transcript)
+    return Conversation(test_case.name, test_case, steps, transcript, tracker)
 
 
 def _create_bot_test_steps(current_turn: ActualStepOutput) -> List[TestStep]:
@@ -140,6 +163,7 @@ def _convert_to_conversation_step(
     current_turn: ActualStepOutput,
     test_case_name: str,
     previous_turn: Optional[ActualStepOutput],
+    tracker_event_index: Optional[int] = None,
 ) -> Union[TestStep, ConversationStep]:
     if not current_step.text == current_turn.text or not isinstance(
         current_turn, ActualStepOutput
@@ -169,7 +193,13 @@ def _convert_to_conversation_step(
     commands = [Command.command_from_json(data) for data in llm_commands]
     rephrase = _should_be_rephrased(current_turn.text, previous_turn, test_case_name)
 
-    return ConversationStep(current_step, commands, llm_prompt, rephrase=rephrase)
+    return ConversationStep(
+        current_step,
+        commands,
+        llm_prompt,
+        rephrase=rephrase,
+        tracker_event_index=tracker_event_index,
+    )
 
 
 def _should_be_rephrased(

rasa/llm_fine_tuning/conversations.py

@@ -4,6 +4,7 @@ from typing import Any, Dict, Iterator, List, Optional, Union
 from rasa.dialogue_understanding.commands.prompt_command import PromptCommand
 from rasa.e2e_test.e2e_test_case import TestCase, TestStep
 from rasa.shared.core.constants import USER
+from rasa.shared.core.trackers import DialogueStateTracker
 
 
 @dataclass
@@ -14,6 +15,7 @@ class ConversationStep:
     failed_rephrasings: List[str] = field(default_factory=list)
     passed_rephrasings: List[str] = field(default_factory=list)
     rephrase: bool = True
+    tracker_event_index: Optional[int] = None
 
     def as_dict(self) -> Dict[str, Any]:
         data = {
@@ -40,6 +42,7 @@ class Conversation:
     original_e2e_test_case: TestCase
     steps: List[Union[TestStep, ConversationStep]]
     transcript: str
+    tracker: Optional[DialogueStateTracker] = None
 
     def iterate_over_annotated_user_steps(
         self, rephrase: Optional[bool] = None