rasa-pro 3.12.0.dev1__py3-none-any.whl

rasa/llm_fine_tuning/paraphrasing/conversation_rephraser.py

@@ -0,0 +1,281 @@
+import importlib.resources
+import re
+from typing import Dict, Any, List, Tuple, Optional
+
+import structlog
+from jinja2 import Template
+
+from rasa.llm_fine_tuning.conversations import Conversation
+from rasa.llm_fine_tuning.paraphrasing.rephrased_user_message import (
+    RephrasedUserMessage,
+)
+from rasa.shared.constants import (
+    MODEL_NAME_CONFIG_KEY,
+    MODEL_CONFIG_KEY,
+    LLM_CONFIG_KEY,
+    PROMPT_TEMPLATE_CONFIG_KEY,
+    PROVIDER_CONFIG_KEY,
+    TIMEOUT_CONFIG_KEY,
+)
+from rasa.shared.exceptions import ProviderClientAPIException
+from rasa.shared.providers.mappings import OPENAI_PROVIDER
+from rasa.shared.utils.llm import (
+    get_prompt_template,
+    llm_factory,
+    USER,
+)
+
+SEPARATOR = "\n\n"
+BACKUP_SEPARATOR = "\nUSER:"
+
+REPHRASING_PROMPT_FILE_NAME = "default_rephrase_prompt_template.jina2"
+DEFAULT_REPHRASING_PROMPT_TEMPLATE = importlib.resources.read_text(
+    "rasa.llm_fine_tuning.paraphrasing",
+    REPHRASING_PROMPT_FILE_NAME,
+)
+
+DEFAULT_LLM_CONFIG = {
+    PROVIDER_CONFIG_KEY: OPENAI_PROVIDER,
+    MODEL_CONFIG_KEY: "gpt-4o-mini",
+    TIMEOUT_CONFIG_KEY: 7,
+    "temperature": 0.0,
+    "max_tokens": 4096,
+}
+
+structlogger = structlog.get_logger()
+
+
+class ConversationRephraser:
+    def __init__(
+        self,
+        config: Dict[str, Any],
+    ) -> None:
+        self.config = {**self.get_default_config(), **config}
+        self.prompt_template = get_prompt_template(
+            self.config.get(PROMPT_TEMPLATE_CONFIG_KEY),
+            DEFAULT_REPHRASING_PROMPT_TEMPLATE,
+        )
+
+    @staticmethod
+    def validate_config(config: Dict[str, Any]) -> None:
+        """Validate the rephrase_config."""
+        if LLM_CONFIG_KEY in config:
+            llm_config = config.get(LLM_CONFIG_KEY)
+
+            # Check if LLM configuration is set to None, {}.
+            if llm_config is None:
+                error = "LLM config is empty. Please provide a valid LLM config."
+                structlogger.error("rephrase_config.empty_llm_config", error=error)
+                raise ValueError(error)
+
+            # Validate LLM model name or model in config.
+            if not llm_config.get(MODEL_CONFIG_KEY) and not llm_config.get(
+                MODEL_NAME_CONFIG_KEY
+            ):
+                error = (
+                    "LLM model name is empty. Please provide a valid LLM model name."
+                )
+                structlogger.error("rephrase_config.llm_model_is_not_set", error=error)
+                raise ValueError(error)
+
+        # Check if the config contains only the allowed keys.
+        allowed_keys = {PROMPT_TEMPLATE_CONFIG_KEY, LLM_CONFIG_KEY}
+        if len(set(config.keys()) - allowed_keys) > 0:
+            error = (
+                f"Invalid rephrase config. Only the following keys are allowed: "
+                f"{', '.join(allowed_keys)}."
+            )
+            structlogger.error("rephrase_config.invalid_keys", error=error)
+            raise ValueError(error)
+
+    @staticmethod
+    def get_default_config() -> Dict[str, Any]:
+        """The component's default config (see parent class for full docstring)."""
+        return {
+            PROMPT_TEMPLATE_CONFIG_KEY: None,
+            LLM_CONFIG_KEY: DEFAULT_LLM_CONFIG,
+        }
+
+    async def rephrase_conversation(
+        self, conversation: Conversation, number_of_rephrasings: int = 10
+    ) -> List[RephrasedUserMessage]:
+        """Create rephrasings for each user message in the conversation.
+
+        For each user message create <number_of_rephrasings> number of rephrasings.
+        The rephrasings are created with an LLM.
+
+        Args:
+            conversation: The conversation.
+            number_of_rephrasings: The number of rephrasings to produce per user
+            message.
+
+        Returns:
+            A list of rephrased user messages.
+        """
+        prompt = self._render_template(conversation, number_of_rephrasings)
+
+        result = await self._invoke_llm(prompt)
+
+        rephrased_user_messages = self._parse_output(
+            result, conversation.get_user_messages()
+        )
+
+        # Check if user message was successfully rephrased
+        self._check_rephrasings(
+            rephrased_user_messages, number_of_rephrasings, result, conversation.name
+        )
+
+        return rephrased_user_messages
+
+    async def _invoke_llm(self, prompt: str) -> str:
+        llm = llm_factory(self.config.get(LLM_CONFIG_KEY), DEFAULT_LLM_CONFIG)
+        try:
+            llm_response = await llm.acompletion(prompt)
+            return llm_response.choices[0]
+        except Exception as e:
+            # unfortunately, langchain does not wrap LLM exceptions which means
+            # we have to catch all exceptions here
+            structlogger.error(
+                "conversation_rephraser.rephrase_conversation.llm.error", error=e
+            )
+            raise ProviderClientAPIException(e, message="LLM call exception")
+
+    def _render_template(
+        self,
+        conversation: Conversation,
+        number_of_rephrasings: int,
+    ) -> str:
+        user_messages = conversation.get_user_messages()
+        number_of_user_messages = len(user_messages)
+
+        return Template(self.prompt_template).render(
+            test_case_name=conversation.name,
+            transcript=conversation.transcript,
+            number_of_user_messages=number_of_user_messages,
+            number_of_rephrasings=number_of_rephrasings,
+            user_prefix=USER,
+            user_messages=user_messages,
+        )
+
+    @staticmethod
+    def _extract_rephrasings(block: str) -> Tuple[Optional[str], Optional[List[str]]]:
+        """Extract the rephrasings for a specific user message.
+
+        Expected format looks like this:
+        USER: <user message>
+        1. <rephrased message 1>
+        2. <rephrased message 2>
+        3. <rephrased message 3>
+        ...
+
+        Args:
+            block: String that contains the user message and its rephrasings.
+
+        Returns:
+            The original user message and the list of rephrasings.
+        """
+        if not block.strip():
+            return None, None
+
+        # Split the block by new line character
+        lines = block.strip().split("\n")
+        # Filter out empty lines and lines that are equal to """
+        lines = [line.strip() for line in lines]
+        lines = [line for line in lines if line and line != '"""']
+
+        # We need at least the original user message and one rephrasing
+        if len(lines) < 2:
+            return None, None
+
+        # Extract the user message from the first line
+        # (ideally prefixed with 'USER: ')
+        if lines[0].startswith("USER:"):
+            original_user_message = lines[0][len(f"{USER}:") :].strip()
+        else:
+            original_user_message = lines[0]
+
+        # Extract rephrasings
+        rephrasings = []
+        for line in lines[1:]:
+            # Remove bullets or numbering and any extra whitespace
+            line = re.sub(r"^\s*[-\d\.]+", "", line).strip()
+            if line.startswith("USER:"):
+                line = line[len(f"{USER}:") :].strip()
+            if line:
+                rephrasings.append(line)
+
+        return original_user_message, rephrasings
+
+    def _parse_output(
+        self, output: str, user_messages: List[str]
+    ) -> List[RephrasedUserMessage]:
+        rephrased_messages = [
+            RephrasedUserMessage(message, []) for message in user_messages
+        ]
+
+        message_blocks = self._get_message_blocks(output, len(rephrased_messages))
+        for block in message_blocks:
+            original_user_message, rephrasings = self._extract_rephrasings(block)
+
+            if not original_user_message or not rephrasings:
+                continue
+
+            # Add the rephrasings to the correct user message
+            for rephrased_message in rephrased_messages:
+                if (
+                    rephrased_message.original_user_message.lower()
+                    == original_user_message.lower()
+                ):
+                    rephrased_message.rephrasings = rephrasings
+
+        return rephrased_messages
+
+    @staticmethod
+    def _get_message_blocks(output: str, expected_number_of_blocks: int) -> List[str]:
+        # Each user message block is (ideally) separated by new line ("\n\n")
+        # USER: <user message>
+        # 1. <rephrasing>
+        #
+        # USER: <user message>
+        # 1. <rephrasing>
+        message_blocks = output.split(SEPARATOR)
+        if len(message_blocks) == expected_number_of_blocks:
+            return message_blocks
+
+        # In case the message blocks are not separated by new line, try to split it
+        # by "\nUSER:", e.g.
+        # USER: <user message>
+        # 1. <rephrasing>
+        # USER: <user message>
+        # 1. <rephrasing>
+        message_blocks = output.split(BACKUP_SEPARATOR)
+        if len(message_blocks) == expected_number_of_blocks:
+            return message_blocks
+
+        return []
+
+    @staticmethod
+    def _check_rephrasings(
+        rephrased_messages: List[RephrasedUserMessage],
+        number_of_rephrasings: int,
+        llm_output: str,
+        conversation_name: str,
+    ) -> None:
+        incorrect_rephrasings_for_messages = []
+
+        for message in rephrased_messages:
+            if (
+                not message.rephrasings
+                or len(message.rephrasings) != number_of_rephrasings
+            ):
+                incorrect_rephrasings_for_messages.append(message.original_user_message)
+
+        if incorrect_rephrasings_for_messages:
+            structlogger.warning(
+                "conversation_rephraser.rephrase_conversation.parse_llm_output",
+                warning="Failed to parse llm output correctly. Not all user messages"
+                "were successfully rephrased.",
+                llm_output=llm_output,
+                conversation_name=conversation_name,
+                incorrect_rephrasings_for_messages=incorrect_rephrasings_for_messages,
+            )

rasa/llm_fine_tuning/paraphrasing/default_rephrase_prompt_template.jina2

@@ -0,0 +1,44 @@
+Objective:
+Create multiple rephrasings of user messages tailored to the "{{ test_case_name }}" conversation scenario.
+
+===
+Conversation overview:
+{{ transcript or "Not provided." }}
+
+===
+Task:
+Produce {{ number_of_rephrasings }} rephrasings for each user message that are diverse yet contextually appropriate.
+Preserve the intent and content, but vary the structure, formality, and detail.
+Only rephrase messages prefixed with "{{ user_prefix }}:".
+
+Guidelines:
+- Use a variety of expressions from brief and casual to elaborate and formal.
+- Vary sentence structures, vocabularies, and expressions creatively.
+- Keep the core message intact with concise and simple modifications.
+
+Format:
+- Each original user message should be prefixed with "USER: ".
+- Enumerate the rephrasing.
+- Separate each user message set with a line break.
+
+===
+Example output for 3 rephrasings of 2 user messages:
+"""
+USER: Show invoices
+1. I want to see my bills.
+2. I mean bills
+3. Yes, I want to see the invoices.
+
+USER: I'd like to book a car
+1. I need to reserve a car.
+2. Could I arrange for a car rental?
+3. I'm interested in hiring a car.
+"""
+
+===
+Expected output:
+{{ number_of_rephrasings }} rephrasings for the following {{ number_of_user_messages }} user messages in the expected
+format:
+{% for message in user_messages -%}
+- {{ message }}
+{% endfor %}

rasa/llm_fine_tuning/paraphrasing/rephrase_validator.py

@@ -0,0 +1,121 @@
+from typing import Dict, Any, List
+
+import structlog
+
+from rasa.dialogue_understanding.commands import Command, SetSlotCommand
+from rasa.dialogue_understanding.generator import SingleStepLLMCommandGenerator
+from rasa.llm_fine_tuning.conversations import Conversation, ConversationStep
+from rasa.llm_fine_tuning.paraphrasing.rephrased_user_message import (
+    RephrasedUserMessage,
+)
+from rasa.shared.core.flows import FlowsList
+from rasa.shared.exceptions import ProviderClientAPIException
+from rasa.shared.utils.llm import (
+    llm_factory,
+)
+
+structlogger = structlog.get_logger()
+
+
+class RephraseValidator:
+    def __init__(self, llm_config: Dict[str, Any], flows: FlowsList) -> None:
+        self.llm_config = llm_config
+        self.flows = flows
+
+    async def validate_rephrasings(
+        self,
+        rephrasings: List[RephrasedUserMessage],
+        conversation: Conversation,
+    ) -> List[RephrasedUserMessage]:
+        """Split rephrased user messages into passing and failing.
+
+        Call an LLM using the same config of the former trained model with an updated
+        prompt from the original user message (replace all occurrences of the original
+        user message with the rephrased user message). Check if the
+        rephrased user message is producing the same commands as the original user
+        message. The rephase is passing if the commands match and failing otherwise.
+
+        Args:
+            rephrasings: The rephrased user messages.
+            conversation: The conversation.
+
+        Returns:
+            A list of rephrased user messages including the passing and failing
+            rephrases.
+        """
+        for i, step in enumerate(
+            conversation.iterate_over_annotated_user_steps(rephrase=True)
+        ):
+            current_rephrasings = rephrasings[i]
+
+            for rephrase in current_rephrasings.rephrasings:
+                if await self._validate_rephrase_is_passing(rephrase, step):
+                    current_rephrasings.passed_rephrasings.append(rephrase)
+                else:
+                    current_rephrasings.failed_rephrasings.append(rephrase)
+
+        return rephrasings
+
+    async def _validate_rephrase_is_passing(
+        self,
+        rephrase: str,
+        step: ConversationStep,
+    ) -> bool:
+        prompt = self._update_prompt(
+            rephrase, step.original_test_step.text, step.llm_prompt
+        )
+
+        action_list = await self._invoke_llm(prompt)
+
+        commands_from_original_utterance = step.llm_commands
+        commands_from_rephrased_utterance = (
+            SingleStepLLMCommandGenerator.parse_commands(action_list, None, self.flows)
+        )
+        return self._check_commands_match(
+            commands_from_original_utterance, commands_from_rephrased_utterance
+        )
+
+    async def _invoke_llm(self, prompt: str) -> str:
+        from rasa.dialogue_understanding.generator.constants import DEFAULT_LLM_CONFIG
+
+        llm = llm_factory(self.llm_config, DEFAULT_LLM_CONFIG)
+
+        try:
+            llm_response = await llm.acompletion(prompt)
+            return llm_response.choices[0]
+        except Exception as e:
+            # unfortunately, langchain does not wrap LLM exceptions which means
+            # we have to catch all exceptions here
+            structlogger.error(
+                "rephrase_validator.validate_conversation.llm.error", error=e
+            )
+            raise ProviderClientAPIException(e, message="LLM call exception")
+
+    @staticmethod
+    def _check_commands_match(
+        expected_commands: List[Command], actual_commands: List[Command]
+    ) -> bool:
+        if len(expected_commands) != len(actual_commands):
+            return False
+
+        for expected_command in expected_commands:
+            if isinstance(expected_command, SetSlotCommand):
+                slot_name = expected_command.name
+                match_found = False
+                for c in actual_commands:
+                    if isinstance(c, SetSlotCommand) and c.name == slot_name:
+                        match_found = True
+                        break
+                if not match_found:
+                    return False
+
+            elif expected_command not in actual_commands:
+                return False
+
+        return True
+
+    @staticmethod
+    def _update_prompt(
+        rephrased_user_message: str, original_user_message: str, prompt: str
+    ) -> str:
+        return prompt.replace(original_user_message, rephrased_user_message)

rasa/llm_fine_tuning/paraphrasing/rephrased_user_message.py

@@ -0,0 +1,10 @@
+from dataclasses import dataclass, field
+from typing import List
+
+
+@dataclass
+class RephrasedUserMessage:
+    original_user_message: str
+    rephrasings: List[str]
+    failed_rephrasings: List[str] = field(default_factory=list)
+    passed_rephrasings: List[str] = field(default_factory=list)

rasa/llm_fine_tuning/paraphrasing_module.py

@@ -0,0 +1,128 @@
+from typing import List, Dict, Any, Tuple
+
+import structlog
+from tqdm import tqdm
+
+from rasa.llm_fine_tuning.conversations import Conversation
+from rasa.llm_fine_tuning.paraphrasing.conversation_rephraser import (
+    ConversationRephraser,
+)
+from rasa.llm_fine_tuning.paraphrasing.rephrase_validator import RephraseValidator
+from rasa.llm_fine_tuning.paraphrasing.rephrased_user_message import (
+    RephrasedUserMessage,
+)
+from rasa.llm_fine_tuning.storage import StorageContext
+from rasa.shared.core.flows import FlowsList
+from rasa.shared.exceptions import ProviderClientAPIException
+
+PARAPHRASING_MODULE_STORAGE_LOCATION = "2_rephrasings"
+
+structlogger = structlog.get_logger()
+
+
+async def create_paraphrased_conversations(
+    conversations: List[Conversation],
+    rephrase_config: Dict[str, Any],
+    num_rephrases: int,
+    flows: FlowsList,
+    llm_command_generator_config: Dict[str, Any],
+    storage_context: StorageContext,
+) -> Tuple[List[Conversation], Dict[str, Any]]:
+    """Create paraphrased conversations.
+
+    Rephrase all user messages of a conversation and divide them into passing
+    and failing rephrasings.
+
+    Args:
+        conversations: The conversations.
+        rephrase_config: The path to the rephrase configuration file.
+        num_rephrases: The number of rephrases to produce per user message.
+        flows: All flows.
+        llm_command_generator_config: The configuration of the trained model.
+        storage_context: The storage context.
+
+    Returns:
+        The conversations including rephrasings and the configuration used for
+        rephrasing.
+    """
+    rephraser = ConversationRephraser(rephrase_config)
+    validator = RephraseValidator(llm_command_generator_config, flows)
+
+    if num_rephrases <= 0:
+        structlogger.info(
+            "paraphrasing_module.skip",
+            num_rephrases=num_rephrases,
+            message="Skipping paraphrasing module as user messages should not be "
+            "rephrased.",
+        )
+        return conversations, rephraser.config
+
+    rephrased_conversations: List[Conversation] = []
+    for i in tqdm(range(len(conversations))):
+        current_conversation = conversations[i]
+
+        try:
+            # rephrase all user messages even if rephrase=False is set
+            # to not confuse the LLM and get valid output
+            rephrasings = await rephraser.rephrase_conversation(
+                conversations[i], num_rephrases
+            )
+            # filter out the rephrasings for user messages that have rephrase=False set
+            rephrasings = _filter_rephrasings(rephrasings, conversations[i])
+            # check if the rephrasings are still producing the same commands
+            rephrasings = await validator.validate_rephrasings(
+                rephrasings, current_conversation
+            )
+        except ProviderClientAPIException as e:
+            structlogger.error(
+                "paraphrasing_module.skip_conversation",
+                conversation=current_conversation.name,
+                exception=str(e),
+            )
+            continue
+
+        for j, step in enumerate(
+            current_conversation.iterate_over_annotated_user_steps(rephrase=True)
+        ):
+            step.passed_rephrasings = rephrasings[j].passed_rephrasings
+            step.failed_rephrasings = rephrasings[j].failed_rephrasings
+
+        rephrased_conversations.append(current_conversation)
+
+        storage_context.write_conversations(
+            conversations, PARAPHRASING_MODULE_STORAGE_LOCATION
+        )
+
+    return rephrased_conversations, rephraser.config
+
+
+def _filter_rephrasings(
+    rephrasings: List[RephrasedUserMessage], conversation: Conversation
+) -> List[RephrasedUserMessage]:
+    """Filter rephrasings.
+
+    Return only those rephrasings for user messages that have rephrase=True.
+
+    Args:
+        rephrasings: All rephrased user messages of the conversation.
+        conversation: The conversation.
+
+    Returns:
+        Rephrasings for those user messages that have rephrase=True.
+    """
+    filtered_rephrasings = []
+    index = 0
+    user_messages = conversation.get_user_messages_to_rephrase()
+
+    for rephrasing in rephrasings:
+        if index >= len(user_messages):
+            break
+
+        # the user messages and the rephrasings are in the same order
+        # rephrasings might contain more user messages as the user messages that
+        # should be rephrased
+        if rephrasing.original_user_message == user_messages[index]:
+            filtered_rephrasings.append(rephrasing)
+            index += 1
+
+    return filtered_rephrasings