rasa-pro 3.9.18__py3-none-any.whl → 3.10.16__py3-none-any.whl

rasa/llm_fine_tuning/conversations.py

@@ -0,0 +1,144 @@
+from dataclasses import dataclass, field
+from typing import List, Union, Iterator, Any, Dict, Optional
+
+from rasa.dialogue_understanding.commands import (
+    Command,
+    StartFlowCommand,
+    SetSlotCommand,
+    CancelFlowCommand,
+    ChitChatAnswerCommand,
+    SkipQuestionCommand,
+    HumanHandoffCommand,
+    KnowledgeAnswerCommand,
+    ClarifyCommand,
+)
+from rasa.e2e_test.e2e_test_case import TestCase, TestStep
+from rasa.shared.core.constants import USER
+
+
+@dataclass
+class ConversationStep:
+    original_test_step: TestStep
+    llm_commands: List[Command]
+    llm_prompt: str
+    failed_rephrasings: List[str] = field(default_factory=list)
+    passed_rephrasings: List[str] = field(default_factory=list)
+    rephrase: bool = True
+
+    def as_dict(self) -> Dict[str, Any]:
+        data = {
+            "user": self.original_test_step.text,
+            "llm_commands": self._commands_to_str(),
+        }
+        if self.passed_rephrasings:
+            data["passing_rephrasings"] = self.passed_rephrasings
+        if self.failed_rephrasings:
+            data["failing_rephrasings"] = self.failed_rephrasings
+
+        return data
+
+    def _commands_to_str(self) -> List[str]:
+        output = []
+        for command in self.llm_commands:
+            if isinstance(command, StartFlowCommand):
+                output.append(f"StartFlow({command.flow})")
+            elif isinstance(command, SetSlotCommand):
+                output.append(f"SetSlot({command.name}, {command.value})")
+            elif isinstance(command, ClarifyCommand):
+                output.append(f"Clarify({command.options})")
+            elif isinstance(command, CancelFlowCommand):
+                output.append("CancelFlow()")
+            elif isinstance(command, ChitChatAnswerCommand):
+                output.append("ChitChat()")
+            elif isinstance(command, SkipQuestionCommand):
+                output.append("SkipQuestion()")
+            elif isinstance(command, KnowledgeAnswerCommand):
+                output.append("SearchAndReply()")
+            elif isinstance(command, HumanHandoffCommand):
+                output.append("HumanHandoff()")
+        return output
+
+    def commands_as_string(self) -> str:
+        return "\n".join(self._commands_to_str())
+
+
+@dataclass
+class Conversation:
+    name: str
+    original_e2e_test_case: TestCase
+    steps: List[Union[TestStep, ConversationStep]]
+    transcript: str
+
+    def iterate_over_annotated_user_steps(
+        self, rephrase: Optional[bool] = None
+    ) -> Iterator[ConversationStep]:
+        """Iterate over conversation steps.
+
+        Yield each step based on the rephrase parameter.
+
+        Args:
+            rephrase: Determines whether to yield steps based on their `rephrase`
+                attribute. Can be:
+                - None: Yield all ConversationStep instances regardless of their
+                    rephrase attribute.
+                - True: Yield only those ConversationStep instances where the
+                    rephrase attribute is True.
+                - False: Yield only those ConversationStep instances where the
+                    rephrase attribute is False.
+
+        Yields:
+            ConversationStep: The next conversation step that matches the specified
+            rephrase condition.
+        """
+        for step in self.steps:
+            if isinstance(step, ConversationStep):
+                if rephrase is None:
+                    yield step
+                elif rephrase is not None and step.rephrase == rephrase:
+                    yield step
+
+    def get_user_messages(self) -> List[str]:
+        return [
+            step.original_test_step.text
+            for step in self.iterate_over_annotated_user_steps()
+            if step.original_test_step.text
+        ]
+
+    def get_user_messages_to_rephrase(self) -> List[str]:
+        return [
+            step.original_test_step.text
+            for step in self.iterate_over_annotated_user_steps(rephrase=True)
+            if step.original_test_step.text
+        ]
+
+    def as_dict(self) -> Dict[str, Any]:
+        steps_data = []
+        for step in self.steps:
+            if isinstance(step, ConversationStep):
+                steps_data.append(step.as_dict())
+            elif isinstance(step, TestStep):
+                if step.text and step.actor == USER:
+                    steps_data.append({"user": step.text})
+                elif step.template:
+                    steps_data.append({"utter": step.template})
+                elif step.text:
+                    steps_data.append({"bot": step.text})
+
+        return {
+            "conversations": [
+                {
+                    "original_test_case": f"{self.original_e2e_test_case.file}::"
+                    f"{self.original_e2e_test_case.name}",
+                    "steps": steps_data,
+                }
+            ]
+        }
+
+    def get_number_of_rephrases(self, passing: bool) -> int:
+        return sum(
+            len(step.passed_rephrasings) if passing else len(step.failed_rephrasings)
+            for step in self.iterate_over_annotated_user_steps()
+        )
+
+    def get_full_name(self) -> str:
+        return f"{self.original_e2e_test_case.file}::{self.name}"

rasa/llm_fine_tuning/llm_data_preparation_module.py

@@ -0,0 +1,178 @@
+from dataclasses import dataclass
+from typing import List, Dict, Any, Optional
+
+import structlog
+from tqdm import tqdm
+
+from rasa.llm_fine_tuning.conversations import Conversation, ConversationStep
+from rasa.llm_fine_tuning.storage import StorageContext
+
+LLM_DATA_PREPARATION_MODULE_STORAGE_LOCATION = "3_llm_finetune_data/llm_ft_data.jsonl"
+
+structlogger = structlog.get_logger()
+
+
+@dataclass
+class LLMDataExample:
+    prompt: str
+    output: str
+    original_test_name: str
+    original_user_utterance: str
+    rephrased_user_utterance: str
+
+    def as_dict(self) -> Dict[str, Any]:
+        return {
+            "prompt": self.prompt,
+            "output": self.output,
+            "original_test_name": self.original_test_name,
+            "original_user_utterance": self.original_user_utterance,
+            "rephrased_user_utterance": self.rephrased_user_utterance,
+        }
+
+
+def _create_data_point(
+    prompt: str,
+    step: ConversationStep,
+    conversation: Conversation,
+    rephrased_user_message: Optional[str] = None,
+) -> LLMDataExample:
+    return LLMDataExample(
+        prompt,
+        step.commands_as_string(),
+        conversation.get_full_name(),
+        step.original_test_step.text,
+        rephrased_user_message,
+    )
+
+
+def _update_prompt(
+    prompt: str,
+    original_user_steps: List[ConversationStep],
+    rephrased_user_steps: List[str],
+) -> Optional[str]:
+    if len(original_user_steps) != len(rephrased_user_steps):
+        structlogger.debug(
+            "llm_fine_tuning.llm_data_preparation_module.failed_to_update_prompt",
+            original_user_steps=[
+                step.original_test_step.text for step in original_user_steps
+            ],
+            rephrased_user_steps=rephrased_user_steps,
+        )
+        return None
+
+    updated_prompt = prompt
+    for user_step, rephrased_message in zip(original_user_steps, rephrased_user_steps):
+        # replace all occurrences of the original user message with the rephrased user
+        # message in the conversation history mentioned in the prompt
+        updated_prompt = updated_prompt.replace(
+            f"USER: {user_step.original_test_step.text}", f"USER: {rephrased_message}"
+        )
+
+    # replace the latest user message mentioned in the prompt
+    updated_prompt = updated_prompt.replace(
+        f"'''{original_user_steps[-1].original_test_step.text}'''",
+        f"'''{rephrased_user_steps[-1]}'''",
+    )
+
+    return updated_prompt
+
+
+def _convert_conversation_into_llm_data(
+    conversation: Conversation,
+) -> List[LLMDataExample]:
+    data = []
+
+    # construct new conversations from the rephrasings
+    new_conversations = _construct_new_conversations(conversation)
+
+    original_user_steps = [
+        step for step in conversation.iterate_over_annotated_user_steps()
+    ]
+
+    for i, step in enumerate(original_user_steps):
+        # create data point for the original e2e test case
+        data.append(_create_data_point(step.llm_prompt, step, conversation))
+
+        # create data points using the rephrasings, e.g. 'new_conversations'
+        for rephrased_user_steps in new_conversations:
+            # +1 to include the current user turn
+            prompt = _update_prompt(
+                step.llm_prompt,
+                original_user_steps[: i + 1],
+                rephrased_user_steps[: i + 1],
+            )
+            if prompt:
+                data.append(
+                    _create_data_point(
+                        prompt, step, conversation, rephrased_user_steps[i]
+                    )
+                )
+
+    return data
+
+
+def _construct_new_conversations(conversation: Conversation) -> List[List[str]]:
+    """Construct new conversations from the rephrasings.
+
+    In general, we will combine the passing rephrasings at the same index position to
+    construct a new conversation. If for one particular user turn no other passing
+    rephrasing exists, we reset the index and take the first passing rephrasing again.
+
+    Args:
+        conversation: The conversation.
+
+    Returns:
+        A list of new conversations (only rephrased user turns).
+    """
+    max_passed_rephrasings = max(
+        [
+            len(step.passed_rephrasings)
+            for step in conversation.iterate_over_annotated_user_steps()
+        ]
+    )
+
+    new_conversations = []
+    for i in range(0, max_passed_rephrasings):
+        current_conversation = []
+        for step in conversation.iterate_over_annotated_user_steps():
+            # take the orginial user message in case no passing rephrasings exist
+            if not step.passed_rephrasings and step.original_test_step.text:
+                structlogger.debug(
+                    "llm_fine_tuning.llm_data_preparation_module."
+                    "construct_new_conversations.no_passed_rephrasings",
+                    conversation=conversation.get_full_name(),
+                    step=step.original_test_step.text,
+                    message="Take original user message instead of rephrasing.",
+                )
+                current_conversation.append(step.original_test_step.text)
+                continue
+
+            # some user steps might have less rephrasings than others
+            # loop over the rephrasings
+            index = i % len(step.passed_rephrasings)
+            current_conversation.append(step.passed_rephrasings[index])
+        if current_conversation:
+            new_conversations.append(current_conversation)
+
+    structlogger.debug(
+        "llm_fine_tuning.llm_data_preparation_module.construct_new_conversations",
+        conversation=conversation.get_full_name(),
+        new_conversations=new_conversations,
+    )
+
+    return new_conversations
+
+
+def convert_to_fine_tuning_data(
+    conversations: List[Conversation], storage_context: StorageContext
+) -> List[LLMDataExample]:
+    llm_data = []
+
+    for i in tqdm(range(len(conversations))):
+        llm_data.extend(_convert_conversation_into_llm_data(conversations[i]))
+
+    storage_context.write_llm_data(
+        llm_data, LLM_DATA_PREPARATION_MODULE_STORAGE_LOCATION
+    )
+
+    return llm_data