rasa-pro 3.12.0.dev5__py3-none-any.whl → 3.12.0.dev7__py3-none-any.whl

rasa/core/policies/enterprise_search_policy.py

@@ -76,7 +76,7 @@ from rasa.shared.providers.embedding._langchain_embedding_client_adapter import
     _LangchainEmbeddingClientAdapter,
 )
 from rasa.shared.providers.llm.llm_client import LLMClient
-from rasa.shared.providers.llm.llm_response import LLMResponse
+from rasa.shared.providers.llm.llm_response import LLMResponse, measure_llm_latency
 from rasa.shared.utils.cli import print_error_and_exit
 from rasa.shared.utils.health_check.embeddings_health_check_mixin import (
     EmbeddingsHealthCheckMixin,
@@ -639,6 +639,7 @@ class EnterpriseSearchPolicy(LLMHealthCheckMixin, EmbeddingsHealthCheckMixin, Po
         )
         return prompt
 
+    @measure_llm_latency
     async def _generate_llm_answer(
         self, llm: LLMClient, prompt: Text
     ) -> Optional[LLMResponse]:

rasa/dialogue_understanding/commands/can_not_handle_command.py

@@ -74,7 +74,7 @@ class CannotHandleCommand(Command):
 
     def to_dsl(self) -> str:
         """Converts the command to a DSL string."""
-        return "cannot handle"
+        return "CannotHandle()"
 
     @classmethod
     def from_dsl(cls, match: re.Match, **kwargs: Any) -> CannotHandleCommand:
@@ -86,4 +86,4 @@ class CannotHandleCommand(Command):
 
     @staticmethod
     def regex_pattern() -> str:
-        return r"^cannot handle$"
+        return r"CannotHandle\(\)"

rasa/dialogue_understanding/commands/cancel_flow_command.py

@@ -124,7 +124,7 @@ class CancelFlowCommand(Command):
 
     def to_dsl(self) -> str:
         """Converts the command to a DSL string."""
-        return "cancel"
+        return "CancelFlow()"
 
     @classmethod
     def from_dsl(cls, match: re.Match, **kwargs: Any) -> CancelFlowCommand:
@@ -133,4 +133,4 @@ class CancelFlowCommand(Command):
 
     @staticmethod
     def regex_pattern() -> str:
-        return r"^cancel$"
+        return r"CancelFlow\(\)"

rasa/dialogue_understanding/commands/change_flow_command.py

@@ -48,7 +48,7 @@ class ChangeFlowCommand(Command):
 
     def to_dsl(self) -> str:
         """Converts the command to a DSL string."""
-        return "change"
+        return "ChangeFlow()"
 
     @staticmethod
     def from_dsl(match: re.Match, **kwargs: Any) -> ChangeFlowCommand:
@@ -57,4 +57,4 @@ class ChangeFlowCommand(Command):
 
     @staticmethod
     def regex_pattern() -> str:
-        return r"^change"
+        return r"ChangeFlow\(\)"

rasa/dialogue_understanding/commands/chit_chat_answer_command.py

@@ -59,7 +59,7 @@ class ChitChatAnswerCommand(FreeFormAnswerCommand):
 
     def to_dsl(self) -> str:
         """Converts the command to a DSL string."""
-        return "chat"
+        return "ChitChat()"
 
     @classmethod
     def from_dsl(cls, match: re.Match, **kwargs: Any) -> ChitChatAnswerCommand:
@@ -68,4 +68,4 @@ class ChitChatAnswerCommand(FreeFormAnswerCommand):
 
     @staticmethod
     def regex_pattern() -> str:
-        return r"^chat$"
+        return r"ChitChat\(\)"

rasa/dialogue_understanding/commands/clarify_command.py

@@ -89,7 +89,7 @@ class ClarifyCommand(Command):
 
     def to_dsl(self) -> str:
         """Converts the command to a DSL string."""
-        return f"clarify {' '.join(self.options)}"
+        return f"Clarify({', '.join(self.options)})"
 
     @classmethod
     def from_dsl(cls, match: re.Match, **kwargs: Any) -> Optional[ClarifyCommand]:
@@ -99,4 +99,4 @@ class ClarifyCommand(Command):
 
     @staticmethod
     def regex_pattern() -> str:
-        return r"^clarify([\"\'a-zA-Z0-9_, ]*)$"
+        return r"Clarify\(([\"\'a-zA-Z0-9_, ]*)\)"

rasa/dialogue_understanding/commands/human_handoff_command.py

@@ -66,7 +66,7 @@ class HumanHandoffCommand(Command):
 
     def to_dsl(self) -> str:
         """Converts the command to a DSL string."""
-        return "hand over"
+        return "HumanHandoff()"
 
     @classmethod
     def from_dsl(cls, match: re.Match, **kwargs: Any) -> HumanHandoffCommand:
@@ -75,4 +75,4 @@ class HumanHandoffCommand(Command):
 
     @staticmethod
     def regex_pattern() -> str:
-        return r"^hand over$"
+        return r"HumanHandoff\(\)"

rasa/dialogue_understanding/commands/knowledge_answer_command.py

@@ -59,7 +59,7 @@ class KnowledgeAnswerCommand(FreeFormAnswerCommand):
 
     def to_dsl(self) -> str:
         """Converts the command to a DSL string."""
-        return "search"
+        return "SearchAndReply()"
 
     @classmethod
     def from_dsl(cls, match: re.Match, **kwargs: Any) -> KnowledgeAnswerCommand:
@@ -68,4 +68,4 @@ class KnowledgeAnswerCommand(FreeFormAnswerCommand):
 
     @staticmethod
     def regex_pattern() -> str:
-        return r"^search$"
+        return r"SearchAndReply\(\)"

rasa/dialogue_understanding/commands/repeat_bot_messages_command.py

@@ -60,7 +60,7 @@ class RepeatBotMessagesCommand(Command):
 
     def to_dsl(self) -> str:
         """Converts the command to a DSL string."""
-        return "repeat message"
+        return "RepeatLastBotMessages()"
 
     @classmethod
     def from_dsl(cls, match: re.Match, **kwargs: Any) -> RepeatBotMessagesCommand:
@@ -69,4 +69,4 @@ class RepeatBotMessagesCommand(Command):
 
     @staticmethod
     def regex_pattern() -> str:
-        return r"^repeat message$"
+        return r"RepeatLastBotMessages\(\)"

rasa/dialogue_understanding/commands/set_slot_command.py

@@ -170,7 +170,7 @@ class SetSlotCommand(Command):
 
     def to_dsl(self) -> str:
         """Converts the command to a DSL string."""
-        return f"set {self.name} {self.value}"
+        return f"SetSlot({self.name}, {self.value})"
 
     @classmethod
     def from_dsl(cls, match: re.Match, **kwargs: Any) -> SetSlotCommand:
@@ -181,4 +181,4 @@ class SetSlotCommand(Command):
 
     @staticmethod
     def regex_pattern() -> str:
-        return r"""^set ['"]?([a-zA-Z_][a-zA-Z0-9_-]*)['"]? ['"]?(.+?)['"]?$"""
+        return r"""SetSlot\(['"]?([a-zA-Z_][a-zA-Z0-9_-]*)['"]?, ?['"]?(.*)['"]?\)"""

rasa/dialogue_understanding/commands/skip_question_command.py

@@ -75,7 +75,7 @@ class SkipQuestionCommand(Command):
 
     def to_dsl(self) -> str:
         """Converts the command to a DSL string."""
-        return "skip"
+        return "SkipQuestion()"
 
     @classmethod
     def from_dsl(cls, match: re.Match, **kwargs: Any) -> SkipQuestionCommand:
@@ -84,4 +84,4 @@ class SkipQuestionCommand(Command):
 
     @staticmethod
     def regex_pattern() -> str:
-        return r"^skip$"
+        return r"SkipQuestion\(\)"

rasa/dialogue_understanding/commands/start_flow_command.py

@@ -110,7 +110,7 @@ class StartFlowCommand(Command):
 
     def to_dsl(self) -> str:
         """Converts the command to a DSL string."""
-        return f"start {self.flow}"
+        return f"StartFlow({self.flow})"
 
     @classmethod
     def from_dsl(cls, match: re.Match, **kwargs: Any) -> Optional[StartFlowCommand]:
@@ -119,4 +119,4 @@ class StartFlowCommand(Command):
 
     @staticmethod
     def regex_pattern() -> str:
-        return r"^start ['\"]?([a-zA-Z0-9_-]+)['\"]?$"
+        return r"StartFlow\(['\"]?([a-zA-Z0-9_-]+)['\"]?\)"

rasa/dialogue_understanding/commands/utils.py

@@ -27,7 +27,7 @@ def extract_cleaned_options(options_str: str) -> List[str]:
     """Extract and clean options from a string."""
     return sorted(
         opt.strip().strip('"').strip("'")
-        for opt in options_str.split(" ")
+        for opt in options_str.split(",")
         if opt.strip()
     )
 

rasa/dialogue_understanding/generator/command_parser.py

@@ -125,7 +125,7 @@ def _parse_standard_commands(
     commands: List[Command] = []
     for command_clz in standard_commands:
         pattern = _get_compiled_pattern(command_clz.regex_pattern())
-        if match := pattern.search(action.strip()):
+        if match := pattern.search(action):
             parsed_command = command_clz.from_dsl(match, **kwargs)
             if _additional_parsing_fn := _get_additional_parsing_logic(command_clz):
                 parsed_command = _additional_parsing_fn(parsed_command, flows, **kwargs)

rasa/dialogue_understanding/generator/llm_based_command_generator.py

@@ -30,7 +30,7 @@ from rasa.shared.exceptions import FileIOException, ProviderClientAPIException
 from rasa.shared.nlu.constants import FLOWS_IN_PROMPT
 from rasa.shared.nlu.training_data.message import Message
 from rasa.shared.nlu.training_data.training_data import TrainingData
-from rasa.shared.providers.llm.llm_response import LLMResponse
+from rasa.shared.providers.llm.llm_response import LLMResponse, measure_llm_latency
 from rasa.shared.utils.health_check.llm_health_check_mixin import LLMHealthCheckMixin
 from rasa.shared.utils.llm import (
     allowed_values_for_slot,
@@ -303,6 +303,7 @@ class LLMBasedCommandGenerator(
         )
         return filtered_flows
 
+    @measure_llm_latency
     async def invoke_llm(self, prompt: Text) -> Optional[LLMResponse]:
         """Use LLM to generate a response.
 
@@ -356,8 +357,7 @@ class LLMBasedCommandGenerator(
                     "slots": slots_with_info,
                 }
             )
-
-        return sorted(result, key=lambda x: x["name"])
+        return result
 
     @staticmethod
     def is_extractable(

rasa/dialogue_understanding/generator/single_step/command_prompt_template.jinja2

@@ -1,58 +1,60 @@
 Your task is to analyze the current conversation context and generate a list of actions to start new business processes that we call flows, to extract slots, or respond to small talk and knowledge requests.
 
-
-## Available Actions:
-* Starting a flow, described by "start flow_name". For example, "start transfer_money" or "start list_contacts"
-* Slot setting, described by "set slot_name slot_value". For example, "set transfer_money_recipient Freddy". Can be used to correct and change previously set values.
-* Cancelling the current flow, described by "cancel"
-* Clarifying which flow should be started in ambiguous cases. For example, "clarify list_contacts add_contact remove_contact" if the user just wrote "contacts" and there are multiple potential candidates.
-* Skipping the current question when the user explicitly asks for it, described by "skip".
-* Responding to knowledge-oriented user messages, described by "search"
-* Responding to a casual, non-task-oriented user message, described by "chat".
-* Handing over to a human, in case the user seems frustrated or explicitly asks to speak to one, described by "hand over".
-
-
-## General Tips
-* Do not fill slots with abstract values or placeholders.
-* Only use information provided by the user.
-* Use clarification in ambiguous cases.
-* Multiple flows can be started. If a user wants to digress into a second flow, you do not need to cancel the current flow.
-* Strictly adhere to the provided action format.
-* For categorical slots try to match the user message with potential slot values. Use "other" if you cannot match it
-* Focus on the last message and take it one step at a time.
-* Use the previous conversation steps only to aid understanding.
-
-
-## Available Flows:
+These are the flows that can be started, with their description and slots:
 {% for flow in available_flows %}
-* {{ flow.name }}: {{ flow.description }}
+{{ flow.name }}: {{ flow.description }}
     {% for slot in flow.slots -%}
-    * {{ slot.name }}{% if slot.description %} ({{ slot.description }}){% endif %}{% if slot.allowed_values %}, allowed values: {{ slot.allowed_values }}{% endif %}
+    slot: {{ slot.name }}{% if slot.description %} ({{ slot.description }}){% endif %}{% if slot.allowed_values %}, allowed values: {{ slot.allowed_values }}{% endif %}
     {% endfor %}
 {%- endfor %}
 
+===
+Here is what happened previously in the conversation:
+{{ current_conversation }}
 
-## Current State
+===
 {% if current_flow != None %}
 You are currently in the flow "{{ current_flow }}".
 You have just asked the user for the slot "{{ current_slot }}"{% if current_slot_description %} ({{ current_slot_description }}){% endif %}.
 
 {% if flow_slots|length > 0 %}
-Here are the slots of the flow "{{ current_flow }}":
+Here are the slots of the currently active flow:
 {% for slot in flow_slots -%}
-* name: {{ slot.name }}, value: {{ slot.value }}, type: {{ slot.type }}, description: {{ slot.description}}{% if slot.allowed_values %}, allowed values: {{ slot.allowed_values }}{% endif %}
+- name: {{ slot.name }}, value: {{ slot.value }}, type: {{ slot.type }}, description: {{ slot.description}}{% if slot.allowed_values %}, allowed values: {{ slot.allowed_values }}{% endif %}
 {% endfor %}
 {% endif %}
 {% else %}
-You are currently not inside any flow.
+You are currently not in any flow and so there are no active slots.
+This means you can only set a slot if you first start a flow that requires that slot.
+{% endif %}
+If you start a flow, first start the flow and then optionally fill that flow's slots with information the user provided in their message.
+
+The user just said """{{ user_message }}""".
+
+===
+Based on this information generate a list of actions you want to take. Your job is to start flows and to fill slots where appropriate. Any logic of what happens afterwards is handled by the flow engine. These are your available actions:
+* Slot setting, described by "SetSlot(slot_name, slot_value)". An example would be "SetSlot(recipient, Freddy)"
+* Starting another flow, described by "StartFlow(flow_name)". An example would be "StartFlow(transfer_money)"
+* Cancelling the current flow, described by "CancelFlow()"
+* Clarifying which flow should be started. An example would be Clarify(list_contacts, add_contact, remove_contact) if the user just wrote "contacts" and there are multiple potential candidates. It also works with a single flow name to confirm you understood correctly, as in Clarify(transfer_money).
+* Intercepting and handle user messages with the intent to bypass the current step in the flow, described by "SkipQuestion()". Examples of user skip phrases are: "Go to the next question", "Ask me something else".
+* Responding to knowledge-oriented user messages, described by "SearchAndReply()"
+* Responding to a casual, non-task-oriented user message, described by "ChitChat()".
+* Handing off to a human, in case the user seems frustrated or explicitly asks to speak to one, described by "HumanHandoff()".
+{% if is_repeat_command_enabled %}
+* Repeat the last bot messages, described by "RepeatLastBotMessages()". This is useful when the user asks to repeat the last bot messages.
 {% endif %}
 
-
-## Conversation History
-{{ current_conversation }}
-
-
-## Task
-Create an action list with one action per line in response to the users last message: """{{ user_message }}""".
-
-Your action list:
+===
+Write out the actions you want to take, one per line, in the order they should take place.
+Do not fill slots with abstract values or placeholders.
+Only use information provided by the user.
+Only start a flow if it's completely clear what the user wants. Imagine you were a person reading this message. If it's not 100% clear, clarify the next step.
+Don't be overly confident. Take a conservative approach and clarify before proceeding.
+If the user asks for two things which seem contradictory, clarify before starting a flow.
+If it's not clear whether the user wants to skip the step or to cancel the flow, cancel the flow.
+Strictly adhere to the provided action types listed above.
+Focus on the last message and take it one step at a time.
+Use the previous conversation steps only to aid understanding.
+
+Your action list:

rasa/dialogue_understanding_test/command_metric_calculation.py

@@ -79,12 +79,16 @@ def calculate_command_metrics(
     return metrics
 
 
+def _get_command_name(command: Command) -> str:
+    return command.command().replace(" ", "_")
+
+
 def _increase_total_count(
     commands: List[Command],
     metrics: Dict[str, CommandMetrics],
 ) -> None:
     for command in commands:
-        metrics[command.command()].total_count += 1
+        metrics[_get_command_name(command)].total_count += 1
 
 
 def _increase_tp(
@@ -92,7 +96,7 @@ def _increase_tp(
     metrics: Dict[str, CommandMetrics],
 ) -> None:
     for command in commands:
-        metrics[command.command()].tp += 1
+        metrics[_get_command_name(command)].tp += 1
 
 
 def _update_metrics_true_positive_and_false_negative(
@@ -101,7 +105,7 @@ def _update_metrics_true_positive_and_false_negative(
     metrics: Dict[str, CommandMetrics],
 ) -> None:
     for expected_command in expected_commands:
-        command_name = expected_command.command()
+        command_name = _get_command_name(expected_command)
         if is_command_present_in_list(expected_command, predicted_commands):
             metrics[command_name].tp += 1
         else:
@@ -115,4 +119,4 @@ def _update_metrics_false_positive(
 ) -> None:
     for predicted_command in predicted_commands:
         if not is_command_present_in_list(predicted_command, expected_commands):
-            metrics[predicted_command.command()].fp += 1
+            metrics[_get_command_name(predicted_command)].fp += 1

rasa/dialogue_understanding_test/du_test_result.py

@@ -27,6 +27,7 @@ OUTPUT_NUMBER_OF_FAILED_USER_UTTERANCES = "number_of_failed_user_utterances"
 OUTPUT_COMMAND_METRICS = "command_metrics"
 OUTPUT_NAMES_OF_FAILED_TESTS = "names_of_failed_tests"
 OUTPUT_NAMES_OF_PASSED_TESTS = "names_of_passed_tests"
+OUTPUT_LLM_COMMAND_GENERATOR_CONFIG = "llm_command_generator_config"
 
 
 class DialogueUnderstandingTestResult(BaseModel):
@@ -151,6 +152,7 @@ class DialogueUnderstandingTestSuiteResult:
         self.names_of_failed_tests: List[str] = []
         self.names_of_passed_tests: List[str] = []
         self.failed_test_steps: List[FailedTestStep] = []
+        self.llm_config: Optional[Dict[str, Any]] = None
 
     @classmethod
     def from_results(
@@ -158,6 +160,7 @@ class DialogueUnderstandingTestSuiteResult:
         failing_test_results: List[DialogueUnderstandingTestResult],
         passing_test_results: List[DialogueUnderstandingTestResult],
         command_metrics: Dict[str, "CommandMetrics"],
+        llm_config: Optional[Dict[str, Any]],
     ) -> "DialogueUnderstandingTestSuiteResult":
         """Create a DialogueUnderstandingTestSuiteResult object from the test results.
 
@@ -171,6 +174,7 @@ class DialogueUnderstandingTestSuiteResult:
                 representing the test cases that passed.
             command_metrics: A dictionary of command-specific performance metrics, keyed
                 by command name.
+            llm_config: A dictionary containing the command generator configuration.
 
         Returns:
             A DialogueUnderstandingTestSuiteResult object containing aggregated test
@@ -202,6 +206,8 @@ class DialogueUnderstandingTestSuiteResult:
             failing_test_results
         )
 
+        instance.llm_config = llm_config
+
         return instance
 
     def _set_user_utterance_metrics(
@@ -297,4 +303,7 @@ class DialogueUnderstandingTestSuiteResult:
 
         result_dict["failed_test_steps"] = failed_steps_list
 
+        if self.llm_config:
+            result_dict[OUTPUT_LLM_COMMAND_GENERATOR_CONFIG] = self.llm_config
+
         return result_dict