rasa-pro 3.13.0.dev7__py3-none-any.whl → 3.13.0.dev9__py3-none-any.whl

rasa/dialogue_understanding/utils.py

@@ -1,7 +1,9 @@
 from contextlib import contextmanager
 from typing import Any, Dict, Generator, List, Optional, Text
 
-from rasa.dialogue_understanding.commands import Command
+import structlog
+
+from rasa.dialogue_understanding.commands import Command, NoopCommand, SetSlotCommand
 from rasa.dialogue_understanding.constants import (
     RASA_RECORD_COMMANDS_AND_PROMPTS_ENV_VAR_NAME,
 )
@@ -16,7 +18,6 @@ from rasa.shared.nlu.constants import (
     KEY_USER_PROMPT,
     PREDICTED_COMMANDS,
     PROMPTS,
-    SET_SLOT_COMMAND,
 )
 from rasa.shared.nlu.training_data.message import Message
 from rasa.shared.providers.llm.llm_response import LLMResponse
@@ -26,6 +27,8 @@ record_commands_and_prompts = get_bool_env_variable(
     RASA_RECORD_COMMANDS_AND_PROMPTS_ENV_VAR_NAME, False
 )
 
+structlogger = structlog.get_logger()
+
 
 @contextmanager
 def set_record_commands_and_prompts() -> Generator:
@@ -144,21 +147,74 @@ def _handle_via_nlu_in_coexistence(
     if not tracker:
         return False
 
+    commands = message.get(COMMANDS, [])
+
+    # If coexistence routing slot is not active, this setup doesn't
+    # support dual routing -> default to CALM
     if not tracker.has_coexistence_routing_slot:
+        structlogger.debug(
+            "utils.handle_via_nlu_in_coexistence"
+            ".tracker_missing_route_session_to_calm_slot",
+            event_info=(
+                f"Tracker doesn't have the '{ROUTE_TO_CALM_SLOT}' slot."
+                f"Routing to CALM."
+            ),
+            route_session_to_calm=commands,
+        )
         return False
 
+    # Check if the routing decision is stored in the tracker slot
+    # If slot is true -> route to CALM
+    # If slot is false -> route to DM1
     value = tracker.get_slot(ROUTE_TO_CALM_SLOT)
     if value is not None:
+        structlogger.debug(
+            "utils.handle_via_nlu_in_coexistence"
+            ".tracker_route_session_to_calm_slot_value",
+            event_info=(
+                f"Tracker slot '{ROUTE_TO_CALM_SLOT}' set to '{value}'. "
+                f"Routing to "
+                f"{'CALM' if value else 'NLU system'}."
+            ),
+            route_session_to_calm_value_in_tracker=value,
+        )
         return not value
 
-    # routing slot has been reset so we need to check
-    # the command issued by the Router component
-    if message.get(COMMANDS):
-        for command in message.get(COMMANDS):
-            if (
-                command.get("command") == SET_SLOT_COMMAND
-                and command.get("name") == ROUTE_TO_CALM_SLOT
-            ):
-                return not command.get("value")
-
+    # Non-sticky routing to DM1 is only allowed if NoopCommand is the sole predicted
+    # command. In that case, route to DM1
+    if len(commands) == 1 and commands[0].get("command") == NoopCommand.command():
+        structlogger.debug(
+            "utils.handle_via_nlu_in_coexistence.noop_command_detected",
+            event_info="NoopCommand found. Routing to NLU system non-sticky.",
+            commands=commands,
+        )
+        return True
+
+    # If the slot was reset (e.g. new session), try to infer routing from
+    # attached commands. Look for a SetSlotCommand targeting the ROUTE_TO_CALM_SLOT
+    for command in message.get(COMMANDS, []):
+        # If slot is true -> route to CALM
+        # If slot is false -> route to DM1
+        if (
+            command.get("command") == SetSlotCommand.command()
+            and command.get("name") == ROUTE_TO_CALM_SLOT
+        ):
+            structlogger.debug(
+                "utils.handle_via_nlu_in_coexistence.set_slot_command_detected",
+                event_info=(
+                    f"SetSlotCommand setting the '{ROUTE_TO_CALM_SLOT}' to "
+                    f"'{command.get('value')}'. "
+                    f"Routing to "
+                    f"{'CALM' if command.get('value') else 'NLU system'}."
+                ),
+                commands=commands,
+            )
+            return not command.get("value")
+
+    # If no routing info is available -> default to CALM
+    structlogger.debug(
+        "utils.handle_via_nlu_in_coexistence.no_routing_info_available",
+        event_info="No routing info available. Routing to CALM.",
+        commands=commands,
+    )
     return False

rasa/dialogue_understanding_test/du_test_case.py

@@ -3,9 +3,9 @@ from typing import Any, Dict, Iterator, List, Optional, Tuple
 
 from pydantic import BaseModel, Field
 
-from rasa.core import IntentlessPolicy
 from rasa.core.nlg.contextual_response_rephraser import ContextualResponseRephraser
 from rasa.core.policies.enterprise_search_policy import EnterpriseSearchPolicy
+from rasa.core.policies.intentless_policy import IntentlessPolicy
 from rasa.dialogue_understanding.commands.prompt_command import PromptCommand
 from rasa.dialogue_understanding.generator.command_parser import parse_commands
 from rasa.dialogue_understanding_test.command_comparison import are_command_lists_equal

rasa/dialogue_understanding_test/du_test_runner.py

@@ -5,10 +5,10 @@ from typing import Any, Dict, List, Optional, Text
 import structlog
 from tqdm import tqdm
 
+from rasa.core.available_endpoints import AvailableEndpoints
 from rasa.core.channels import CollectingOutputChannel, UserMessage
 from rasa.core.exceptions import AgentNotReady
 from rasa.core.persistor import StorageType
-from rasa.core.utils import AvailableEndpoints
 from rasa.dialogue_understanding.commands import Command
 from rasa.dialogue_understanding.utils import set_record_commands_and_prompts
 from rasa.dialogue_understanding_test.du_test_case import (
@@ -34,6 +34,7 @@ from rasa.e2e_test.e2e_test_runner import E2ETestRunner
 from rasa.shared.core.events import UserUttered
 from rasa.shared.core.trackers import DialogueStateTracker
 from rasa.shared.nlu.constants import PREDICTED_COMMANDS, PROMPTS
+from rasa.shared.utils.llm import create_tracker_for_user_step
 from rasa.utils.endpoints import EndpointConfig
 
 structlogger = structlog.get_logger()
@@ -179,8 +180,9 @@ class DialogueUnderstandingTestRunner:
             # create and save the tracker at the time just
             # before the user message was sent
             step_sender_id = f"{sender_id}_{user_step_index}"
-            await self._create_tracker_for_user_step(
+            await create_tracker_for_user_step(
                 step_sender_id,
+                self.agent,
                 test_case_tracker,
                 user_uttered_event_indices[user_step_index],
             )
@@ -289,26 +291,6 @@ class DialogueUnderstandingTestRunner:
 
         return user_uttered_event
 
-    async def _create_tracker_for_user_step(
-        self,
-        step_sender_id: str,
-        test_case_tracker: DialogueStateTracker,
-        index_user_uttered_event: int,
-    ) -> None:
-        """Creates a tracker for the user step."""
-        tracker = test_case_tracker.copy()
-        # modify the sender id so that the test case tracker is not overwritten
-        tracker.sender_id = step_sender_id
-
-        if tracker.events:
-            # get timestamp of the event just before the user uttered event
-            timestamp = tracker.events[index_user_uttered_event - 1].timestamp
-            # revert the tracker to the event just before the user uttered event
-            tracker = tracker.travel_back_in_time(timestamp)
-
-        # store the tracker with the unique sender id
-        await self.agent.tracker_store.save(tracker)
-
     async def _send_user_message(
         self,
         sender_id: str,

rasa/dialogue_understanding_test/test_case_simulation/test_case_tracker_simulator.py

@@ -1,4 +1,3 @@
-from datetime import datetime
 from typing import List, Optional
 
 import structlog
@@ -24,6 +23,7 @@ from rasa.shared.core.constants import SlotMappingType
 from rasa.shared.core.events import BotUttered, SlotSet, UserUttered
 from rasa.shared.core.trackers import DialogueStateTracker
 from rasa.shared.nlu.constants import COMMANDS, ENTITIES, INTENT
+from rasa.shared.utils.llm import generate_sender_id
 
 structlogger = structlog.get_logger()
 
@@ -52,7 +52,7 @@ class TestCaseTrackerSimulator:
         self.test_case = test_case
         self.output_channel = output_channel or CollectingOutputChannel()
 
-        self.sender_id = self._generate_sender_id()
+        self.sender_id = generate_sender_id(self.test_case.name)
 
     async def simulate_test_case(
         self,
@@ -150,10 +150,6 @@ class TestCaseTrackerSimulator:
             user_uttered_event_indices=user_uttered_event_indices,
         )
 
-    def _generate_sender_id(self) -> str:
-        # add timestamp suffix to ensure sender_id is unique
-        return f"{self.test_case.name}_{datetime.now()}"
-
     @staticmethod
     async def _get_latest_user_uttered_event_index(
         tracker: DialogueStateTracker, user_uttered_event_indices: List[int]

rasa/e2e_test/e2e_test_runner.py

@@ -13,11 +13,11 @@ import structlog
 from tqdm import tqdm
 
 import rasa.shared.utils.io
+from rasa.core.available_endpoints import AvailableEndpoints
 from rasa.core.channels import CollectingOutputChannel, UserMessage
 from rasa.core.constants import ACTIVE_FLOW_METADATA_KEY, STEP_ID_METADATA_KEY
 from rasa.core.exceptions import AgentNotReady
 from rasa.core.persistor import StorageType
-from rasa.core.utils import AvailableEndpoints
 from rasa.dialogue_understanding_test.du_test_case import DialogueUnderstandingTestCase
 from rasa.e2e_test.constants import TEST_CASE_NAME, TEST_FILE_NAME
 from rasa.e2e_test.e2e_config import create_llm_judge_config

rasa/engine/constants.py

@@ -1,7 +1,7 @@
 from typing import List, Optional
 
+from rasa.core.available_endpoints import AvailableEndpoints
 from rasa.core.channels import UserMessage
-from rasa.core.utils import AvailableEndpoints
 from rasa.shared.core.trackers import DialogueStateTracker
 from rasa.shared.importers.importer import TrainingDataImporter
 

rasa/engine/graph.py

@@ -500,9 +500,9 @@ class GraphNode:
                 structlogger.warning(
                     "graph.node.input_not_resolved",
                     node_name=self._node_name,
-                    input_name=i,
+                    input_name=i,  # no PII
                     event_info=(
-                        "Node input was not resolved, there is no putput. "
+                        "Node input was not resolved, there is no output. "
                         "Another component should have provided this as an output."
                     ),
                 )

rasa/engine/recipes/default_recipe.py

@@ -396,7 +396,9 @@ class DefaultV1Recipe(Recipe):
         return preprocessors
 
     def _get_needs_from_args(
-        self, component: Type[GraphComponent], fn_name: str
+        self,
+        component: Type[GraphComponent],
+        fn_name: str,
     ) -> Dict[str, str]:
         """Get the needed arguments from the method on the component.
 
@@ -434,6 +436,7 @@ class DefaultV1Recipe(Recipe):
         parameters = {
             name
             for name, param in sig.parameters.items()
+            # only consider parameters which are positional or keyword
             if param.kind == param.POSITIONAL_OR_KEYWORD
         }
 
@@ -752,8 +755,28 @@ class DefaultV1Recipe(Recipe):
                 predict_config, predict_nodes, train_nodes, preprocessors
             )
 
+        # The `story_graph_provider` is only needed if the intentless policy is used.
+        # If it is not used, we can remove it from the nodes as it slows down the
+        # loading time if users have a large number of stories.
+        if not self._intentless_policy_used(predict_nodes):
+            # Removes the `story_graph_provider` from the nodes
+            predict_nodes.pop("story_graph_provider", None)
+            if "command_processor" in predict_nodes:
+                # Removes story_graph from the command processor inputs
+                predict_nodes["command_processor"].needs.pop("story_graph", None)
+
         return predict_nodes
 
+    @staticmethod
+    def _intentless_policy_used(nodes: Dict[Text, SchemaNode]) -> bool:
+        """Checks if the intentless policy is used in the nodes."""
+        from rasa.core.policies.intentless_policy import IntentlessPolicy
+
+        for schema_node in nodes.values():
+            if schema_node.matches_type(IntentlessPolicy):
+                return True
+        return False
+
     def _add_nlu_predict_nodes(
         self,
         last_run_node: Text,
@@ -924,7 +947,8 @@ class DefaultV1Recipe(Recipe):
         predict_nodes["command_processor"] = SchemaNode(
             **DEFAULT_PREDICT_KWARGS,
             needs=self._get_needs_from_args(
-                CommandProcessorComponent, "execute_commands"
+                CommandProcessorComponent,
+                "execute_commands",
             ),
             uses=CommandProcessorComponent,
             fn="execute_commands",

rasa/engine/validation.py

@@ -23,9 +23,10 @@ import structlog
 import typing_utils
 
 import rasa.utils.common
-from rasa.core import ContextualResponseRephraser, IntentlessPolicy
+from rasa.core.available_endpoints import AvailableEndpoints
+from rasa.core.nlg.contextual_response_rephraser import ContextualResponseRephraser
+from rasa.core.policies.intentless_policy import IntentlessPolicy
 from rasa.core.policies.policy import PolicyPrediction
-from rasa.core.utils import AvailableEndpoints
 from rasa.dialogue_understanding.coexistence.constants import (
     CALM_ENTRY,
     NLU_ENTRY,

rasa/hooks.py

@@ -9,7 +9,6 @@ import pluggy
 #  across the codebase.
 
 if TYPE_CHECKING:
-    from rasa.anonymization.anonymization_pipeline import AnonymizationPipeline
     from rasa.cli import SubParsersAction
     from rasa.core.brokers.broker import EventBroker
     from rasa.core.tracker_stores.tracker_store import TrackerStore
@@ -88,30 +87,3 @@ def create_tracker_store(
             endpoint_config=endpoint_config, domain=domain, event_broker=event_broker
         )
     return endpoint_config
-
-
-@hookimpl  # type: ignore[misc]
-def init_anonymization_pipeline(endpoints_file: Optional[Text]) -> None:
-    """Hook implementation for initializing the anonymization pipeline."""
-    from rasa.anonymization.anonymization_pipeline import load_anonymization_pipeline
-
-    load_anonymization_pipeline(endpoints_file)
-
-
-@hookimpl  # type: ignore[misc]
-def get_anonymization_pipeline() -> Optional["AnonymizationPipeline"]:
-    """Hook implementation for getting the anonymization pipeline."""
-    from rasa.anonymization.anonymization_pipeline import AnonymizationPipelineProvider
-
-    return AnonymizationPipelineProvider().get_anonymization_pipeline()
-
-
-@hookimpl  # type: ignore[misc]
-def after_server_stop() -> None:
-    """Hook implementation for stopping the anonymization pipeline."""
-    from rasa.anonymization.anonymization_pipeline import AnonymizationPipelineProvider
-
-    anon_pipeline = AnonymizationPipelineProvider().get_anonymization_pipeline()
-
-    if anon_pipeline is not None:
-        anon_pipeline.stop()

rasa/llm_fine_tuning/annotation_module.py

@@ -10,7 +10,9 @@ from rasa.e2e_test.e2e_test_runner import TEST_TURNS_TYPE, E2ETestRunner
 from rasa.llm_fine_tuning.conversations import Conversation, ConversationStep
 from rasa.llm_fine_tuning.storage import StorageContext
 from rasa.shared.core.constants import USER
+from rasa.shared.core.events import UserUttered
 from rasa.shared.core.trackers import DialogueStateTracker
+from rasa.shared.exceptions import FinetuningDataPreparationException
 from rasa.shared.nlu.constants import LLM_COMMANDS, LLM_PROMPT
 from rasa.shared.utils.llm import tracker_as_readable_transcript
 
@@ -37,7 +39,7 @@ def annotate_e2e_tests(
     storage_context: StorageContext,
 ) -> List[Conversation]:
     with set_preparing_fine_tuning_data():
-        converations = asyncio.run(
+        conversations = asyncio.run(
             e2e_test_runner.run_tests_for_fine_tuning(
                 test_suite.test_cases,
                 test_suite.fixtures,
@@ -46,10 +48,11 @@ def annotate_e2e_tests(
         )
 
     storage_context.write_conversations(
-        converations, ANNOTATION_MODULE_STORAGE_LOCATION
+        conversations,
+        ANNOTATION_MODULE_STORAGE_LOCATION,
     )
 
-    return converations
+    return conversations
 
 
 def _get_previous_actual_step_output(
@@ -80,25 +83,45 @@ def generate_conversation(
         Conversation.
     """
     steps = []
+    tracker_event_indices = [
+        i for i, event in enumerate(tracker.events) if isinstance(event, UserUttered)
+    ]
+
+    if len(test_case.steps) != len(tracker_event_indices):
+        raise FinetuningDataPreparationException(
+            "Number of test case steps and tracker events do not match."
+        )
 
     if assertions_used:
         # we only have user steps, extract the bot response from the bot uttered
         # events of the test turn
-        for i, original_step in enumerate(test_case.steps):
+        for i, (original_step, tracker_event_index) in enumerate(
+            zip(test_case.steps, tracker_event_indices)
+        ):
             previous_turn = _get_previous_actual_step_output(test_turns, i)
             steps.append(
                 _convert_to_conversation_step(
-                    original_step, test_turns[i], test_case.name, previous_turn
+                    original_step,
+                    test_turns[i],
+                    test_case.name,
+                    previous_turn,
+                    tracker_event_index,
                 )
             )
             steps.extend(_create_bot_test_steps(test_turns[i]))
     else:
-        for i, original_step in enumerate(test_case.steps):
+        for i, (original_step, tracker_event_index) in enumerate(
+            zip(test_case.steps, tracker_event_indices)
+        ):
             if original_step.actor == USER:
                 previous_turn = _get_previous_actual_step_output(test_turns, i)
                 steps.append(
                     _convert_to_conversation_step(
-                        original_step, test_turns[i], test_case.name, previous_turn
+                        original_step,
+                        test_turns[i],
+                        test_case.name,
+                        previous_turn,
+                        tracker_event_index,
                     )
                 )
             else:
@@ -120,7 +143,7 @@ def generate_conversation(
 
     transcript = tracker_as_readable_transcript(tracker, max_turns=None)
 
-    return Conversation(test_case.name, test_case, steps, transcript)
+    return Conversation(test_case.name, test_case, steps, transcript, tracker)
 
 
 def _create_bot_test_steps(current_turn: ActualStepOutput) -> List[TestStep]:
@@ -140,6 +163,7 @@ def _convert_to_conversation_step(
     current_turn: ActualStepOutput,
     test_case_name: str,
     previous_turn: Optional[ActualStepOutput],
+    tracker_event_index: Optional[int] = None,
 ) -> Union[TestStep, ConversationStep]:
     if not current_step.text == current_turn.text or not isinstance(
         current_turn, ActualStepOutput
@@ -169,7 +193,13 @@ def _convert_to_conversation_step(
     commands = [Command.command_from_json(data) for data in llm_commands]
     rephrase = _should_be_rephrased(current_turn.text, previous_turn, test_case_name)
 
-    return ConversationStep(current_step, commands, llm_prompt, rephrase=rephrase)
+    return ConversationStep(
+        current_step,
+        commands,
+        llm_prompt,
+        rephrase=rephrase,
+        tracker_event_index=tracker_event_index,
+    )
 
 
 def _should_be_rephrased(

rasa/llm_fine_tuning/conversations.py

@@ -4,6 +4,7 @@ from typing import Any, Dict, Iterator, List, Optional, Union
 from rasa.dialogue_understanding.commands.prompt_command import PromptCommand
 from rasa.e2e_test.e2e_test_case import TestCase, TestStep
 from rasa.shared.core.constants import USER
+from rasa.shared.core.trackers import DialogueStateTracker
 
 
 @dataclass
@@ -14,6 +15,7 @@ class ConversationStep:
     failed_rephrasings: List[str] = field(default_factory=list)
     passed_rephrasings: List[str] = field(default_factory=list)
     rephrase: bool = True
+    tracker_event_index: Optional[int] = None
 
     def as_dict(self) -> Dict[str, Any]:
         data = {
@@ -40,6 +42,7 @@ class Conversation:
     original_e2e_test_case: TestCase
     steps: List[Union[TestStep, ConversationStep]]
     transcript: str
+    tracker: Optional[DialogueStateTracker] = None
 
     def iterate_over_annotated_user_steps(
         self, rephrase: Optional[bool] = None

rasa/llm_fine_tuning/llm_data_preparation_module.py

@@ -1,13 +1,23 @@
 from dataclasses import dataclass
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Optional, cast
 
 import structlog
 from tqdm import tqdm
 
+from rasa.core.agent import Agent
+from rasa.core.channels import UserMessage
 from rasa.dialogue_understanding.commands.prompt_command import PromptCommand
+from rasa.dialogue_understanding.utils import set_record_commands_and_prompts
 from rasa.llm_fine_tuning.conversations import Conversation, ConversationStep
 from rasa.llm_fine_tuning.storage import StorageContext
-from rasa.llm_fine_tuning.utils import commands_as_string
+from rasa.llm_fine_tuning.utils import (
+    commands_as_string,
+    make_mock_invoke_llm,
+    patch_invoke_llm_in_generators,
+)
+from rasa.shared.core.trackers import DialogueStateTracker
+from rasa.shared.nlu.constants import KEY_USER_PROMPT, PROMPTS
+from rasa.shared.utils.llm import generate_sender_id
 
 LLM_DATA_PREPARATION_MODULE_STORAGE_LOCATION = "3_llm_finetune_data/llm_ft_data.jsonl"
 
@@ -47,40 +57,8 @@ def _create_data_point(
     )
 
 
-def _update_prompt(
-    prompt: str,
-    original_user_steps: List[ConversationStep],
-    rephrased_user_steps: List[str],
-) -> Optional[str]:
-    if len(original_user_steps) != len(rephrased_user_steps):
-        structlogger.debug(
-            "llm_fine_tuning.llm_data_preparation_module.failed_to_update_prompt",
-            original_user_steps=[
-                step.original_test_step.text for step in original_user_steps
-            ],
-            rephrased_user_steps=rephrased_user_steps,
-        )
-        return None
-
-    updated_prompt = prompt
-    for user_step, rephrased_message in zip(original_user_steps, rephrased_user_steps):
-        # replace all occurrences of the original user message with the rephrased user
-        # message in the conversation history mentioned in the prompt
-        updated_prompt = updated_prompt.replace(
-            f"USER: {user_step.original_test_step.text}", f"USER: {rephrased_message}"
-        )
-
-    # replace the latest user message mentioned in the prompt
-    updated_prompt = updated_prompt.replace(
-        f"'''{original_user_steps[-1].original_test_step.text}'''",
-        f"'''{rephrased_user_steps[-1]}'''",
-    )
-
-    return updated_prompt
-
-
-def _convert_conversation_into_llm_data(
-    conversation: Conversation,
+async def _convert_conversation_into_llm_data(
+    conversation: Conversation, agent: Agent
 ) -> List[LLMDataExample]:
     data = []
 
@@ -95,18 +73,52 @@ def _convert_conversation_into_llm_data(
         # create data point for the original e2e test case
         data.append(_create_data_point(step.llm_prompt, step, conversation))
 
-        # create data points using the rephrasings, e.g. 'new_conversations'
-        for rephrased_user_steps in new_conversations:
-            # +1 to include the current user turn
-            prompt = _update_prompt(
-                step.llm_prompt,
-                original_user_steps[: i + 1],
-                rephrased_user_steps[: i + 1],
+    test_case_name = conversation.name
+
+    # create data points using the rephrasings, e.g. 'new_conversations'
+    for rephrased_user_steps in new_conversations:
+        sender_id = generate_sender_id(test_case_name)
+        # create a new tracker to be able to simulate the conversation from start
+        await agent.tracker_store.save(DialogueStateTracker(sender_id, slots=[]))
+        # simulate the conversation to get the prompts
+        for i, step in enumerate(original_user_steps):
+            rephrased_user_message = rephrased_user_steps[i]
+            user_message = UserMessage(rephrased_user_message, sender_id=sender_id)
+
+            expected_commands = "\n".join(
+                [command.to_dsl() for command in step.llm_commands]
+            )
+            fake_invoke_function = make_mock_invoke_llm(expected_commands)
+
+            with (
+                set_record_commands_and_prompts(),
+                patch_invoke_llm_in_generators(fake_invoke_function),
+            ):
+                await agent.handle_message(user_message)
+
+            rephrased_tracker = await agent.tracker_store.retrieve(sender_id)
+            if rephrased_tracker is None:
+                # if tracker doesn't exist, we can't create a data point
+                continue
+
+            latest_message = rephrased_tracker.latest_message
+            if latest_message is None:
+                # if there is no latest message, we don't create a data point
+                continue
+
+            # tell the type checker what we expect to find under "prompts"
+            prompts = cast(
+                Optional[List[Dict[str, Any]]], latest_message.parse_data.get(PROMPTS)
             )
-            if prompt:
+
+            if prompts:
+                # as we only use single step or compact command generator,
+                # there is always exactly one prompt
+                prompt = prompts[0]
+                user_prompt: Optional[str] = prompt.get(KEY_USER_PROMPT)
                 data.append(
                     _create_data_point(
-                        prompt, step, conversation, rephrased_user_steps[i]
+                        user_prompt, step, conversation, rephrased_user_message
                     )
                 )
 
@@ -149,7 +161,7 @@ def _construct_new_conversations(conversation: Conversation) -> List[List[str]]:
                 current_conversation.append(step.original_test_step.text)
                 continue
 
-            # some user steps might have less rephrasings than others
+            # some user steps might have fewer rephrasings than others
             # loop over the rephrasings
             index = i % len(step.passed_rephrasings)
             current_conversation.append(step.passed_rephrasings[index])
@@ -165,13 +177,18 @@ def _construct_new_conversations(conversation: Conversation) -> List[List[str]]:
     return new_conversations
 
 
-def convert_to_fine_tuning_data(
-    conversations: List[Conversation], storage_context: StorageContext
+async def convert_to_fine_tuning_data(
+    conversations: List[Conversation],
+    storage_context: StorageContext,
+    agent: Agent,
 ) -> List[LLMDataExample]:
     llm_data = []
 
     for i in tqdm(range(len(conversations))):
-        llm_data.extend(_convert_conversation_into_llm_data(conversations[i]))
+        conversation_llm_data = await _convert_conversation_into_llm_data(
+            conversations[i], agent
+        )
+        llm_data.extend(conversation_llm_data)
 
     storage_context.write_llm_data(
         llm_data, LLM_DATA_PREPARATION_MODULE_STORAGE_LOCATION

rasa/llm_fine_tuning/paraphrasing/conversation_rephraser.py

@@ -22,11 +22,7 @@ from rasa.shared.constants import (
 from rasa.shared.exceptions import ProviderClientAPIException
 from rasa.shared.providers.mappings import OPENAI_PROVIDER
 from rasa.shared.utils.constants import LOG_COMPONENT_SOURCE_METHOD_INIT
-from rasa.shared.utils.llm import (
-    USER,
-    get_prompt_template,
-    llm_factory,
-)
+from rasa.shared.utils.llm import USER, get_prompt_template, llm_factory
 
 SEPARATOR = "\n\n"
 BACKUP_SEPARATOR = "\nUSER:"