rasa-pro 3.12.0.dev7__py3-none-any.whl → 3.12.0.dev8__py3-none-any.whl

rasa/dialogue_understanding/commands/can_not_handle_command.py

@@ -74,7 +74,7 @@ class CannotHandleCommand(Command):
 
     def to_dsl(self) -> str:
         """Converts the command to a DSL string."""
-        return "CannotHandle()"
+        return "cannot handle"
 
     @classmethod
     def from_dsl(cls, match: re.Match, **kwargs: Any) -> CannotHandleCommand:
@@ -86,4 +86,4 @@ class CannotHandleCommand(Command):
 
     @staticmethod
     def regex_pattern() -> str:
-        return r"CannotHandle\(\)"
+        return r"^cannot handle$"

rasa/dialogue_understanding/commands/cancel_flow_command.py

@@ -124,7 +124,7 @@ class CancelFlowCommand(Command):
 
     def to_dsl(self) -> str:
         """Converts the command to a DSL string."""
-        return "CancelFlow()"
+        return "cancel"
 
     @classmethod
     def from_dsl(cls, match: re.Match, **kwargs: Any) -> CancelFlowCommand:
@@ -133,4 +133,4 @@ class CancelFlowCommand(Command):
 
     @staticmethod
     def regex_pattern() -> str:
-        return r"CancelFlow\(\)"
+        return r"^cancel$"

rasa/dialogue_understanding/commands/change_flow_command.py

@@ -48,7 +48,7 @@ class ChangeFlowCommand(Command):
 
     def to_dsl(self) -> str:
         """Converts the command to a DSL string."""
-        return "ChangeFlow()"
+        return "change"
 
     @staticmethod
     def from_dsl(match: re.Match, **kwargs: Any) -> ChangeFlowCommand:
@@ -57,4 +57,4 @@ class ChangeFlowCommand(Command):
 
     @staticmethod
     def regex_pattern() -> str:
-        return r"ChangeFlow\(\)"
+        return r"^change"

rasa/dialogue_understanding/commands/chit_chat_answer_command.py

@@ -59,7 +59,7 @@ class ChitChatAnswerCommand(FreeFormAnswerCommand):
 
     def to_dsl(self) -> str:
         """Converts the command to a DSL string."""
-        return "ChitChat()"
+        return "chat"
 
     @classmethod
     def from_dsl(cls, match: re.Match, **kwargs: Any) -> ChitChatAnswerCommand:
@@ -68,4 +68,4 @@ class ChitChatAnswerCommand(FreeFormAnswerCommand):
 
     @staticmethod
     def regex_pattern() -> str:
-        return r"ChitChat\(\)"
+        return r"^chat$"

rasa/dialogue_understanding/commands/clarify_command.py

@@ -89,7 +89,7 @@ class ClarifyCommand(Command):
 
     def to_dsl(self) -> str:
         """Converts the command to a DSL string."""
-        return f"Clarify({', '.join(self.options)})"
+        return f"clarify {' '.join(self.options)}"
 
     @classmethod
     def from_dsl(cls, match: re.Match, **kwargs: Any) -> Optional[ClarifyCommand]:
@@ -99,4 +99,4 @@ class ClarifyCommand(Command):
 
     @staticmethod
     def regex_pattern() -> str:
-        return r"Clarify\(([\"\'a-zA-Z0-9_, ]*)\)"
+        return r"^clarify([\"\'a-zA-Z0-9_, ]*)$"

rasa/dialogue_understanding/commands/human_handoff_command.py

@@ -66,7 +66,7 @@ class HumanHandoffCommand(Command):
 
     def to_dsl(self) -> str:
         """Converts the command to a DSL string."""
-        return "HumanHandoff()"
+        return "hand over"
 
     @classmethod
     def from_dsl(cls, match: re.Match, **kwargs: Any) -> HumanHandoffCommand:
@@ -75,4 +75,4 @@ class HumanHandoffCommand(Command):
 
     @staticmethod
     def regex_pattern() -> str:
-        return r"HumanHandoff\(\)"
+        return r"^hand over$"

rasa/dialogue_understanding/commands/knowledge_answer_command.py

@@ -59,7 +59,7 @@ class KnowledgeAnswerCommand(FreeFormAnswerCommand):
 
     def to_dsl(self) -> str:
         """Converts the command to a DSL string."""
-        return "SearchAndReply()"
+        return "search"
 
     @classmethod
     def from_dsl(cls, match: re.Match, **kwargs: Any) -> KnowledgeAnswerCommand:
@@ -68,4 +68,4 @@ class KnowledgeAnswerCommand(FreeFormAnswerCommand):
 
     @staticmethod
     def regex_pattern() -> str:
-        return r"SearchAndReply\(\)"
+        return r"^search$"

rasa/dialogue_understanding/commands/repeat_bot_messages_command.py

@@ -60,7 +60,7 @@ class RepeatBotMessagesCommand(Command):
 
     def to_dsl(self) -> str:
         """Converts the command to a DSL string."""
-        return "RepeatLastBotMessages()"
+        return "repeat message"
 
     @classmethod
     def from_dsl(cls, match: re.Match, **kwargs: Any) -> RepeatBotMessagesCommand:
@@ -69,4 +69,4 @@ class RepeatBotMessagesCommand(Command):
 
     @staticmethod
     def regex_pattern() -> str:
-        return r"RepeatLastBotMessages\(\)"
+        return r"^repeat message$"

rasa/dialogue_understanding/commands/set_slot_command.py

@@ -170,7 +170,7 @@ class SetSlotCommand(Command):
 
     def to_dsl(self) -> str:
         """Converts the command to a DSL string."""
-        return f"SetSlot({self.name}, {self.value})"
+        return f"set {self.name} {self.value}"
 
     @classmethod
     def from_dsl(cls, match: re.Match, **kwargs: Any) -> SetSlotCommand:
@@ -181,4 +181,4 @@ class SetSlotCommand(Command):
 
     @staticmethod
     def regex_pattern() -> str:
-        return r"""SetSlot\(['"]?([a-zA-Z_][a-zA-Z0-9_-]*)['"]?, ?['"]?(.*)['"]?\)"""
+        return r"""^set ['"]?([a-zA-Z_][a-zA-Z0-9_-]*)['"]? ['"]?(.+?)['"]?$"""

rasa/dialogue_understanding/commands/skip_question_command.py

@@ -75,7 +75,7 @@ class SkipQuestionCommand(Command):
 
     def to_dsl(self) -> str:
         """Converts the command to a DSL string."""
-        return "SkipQuestion()"
+        return "skip"
 
     @classmethod
     def from_dsl(cls, match: re.Match, **kwargs: Any) -> SkipQuestionCommand:
@@ -84,4 +84,4 @@ class SkipQuestionCommand(Command):
 
     @staticmethod
     def regex_pattern() -> str:
-        return r"SkipQuestion\(\)"
+        return r"^skip$"

rasa/dialogue_understanding/commands/start_flow_command.py

@@ -110,7 +110,7 @@ class StartFlowCommand(Command):
 
     def to_dsl(self) -> str:
         """Converts the command to a DSL string."""
-        return f"StartFlow({self.flow})"
+        return f"start {self.flow}"
 
     @classmethod
     def from_dsl(cls, match: re.Match, **kwargs: Any) -> Optional[StartFlowCommand]:
@@ -119,4 +119,4 @@ class StartFlowCommand(Command):
 
     @staticmethod
     def regex_pattern() -> str:
-        return r"StartFlow\(['\"]?([a-zA-Z0-9_-]+)['\"]?\)"
+        return r"^start ['\"]?([a-zA-Z0-9_-]+)['\"]?$"

rasa/dialogue_understanding/commands/utils.py

@@ -27,7 +27,7 @@ def extract_cleaned_options(options_str: str) -> List[str]:
     """Extract and clean options from a string."""
     return sorted(
         opt.strip().strip('"').strip("'")
-        for opt in options_str.split(",")
+        for opt in options_str.split(" ")
         if opt.strip()
     )
 

rasa/dialogue_understanding/generator/command_parser.py

@@ -125,7 +125,7 @@ def _parse_standard_commands(
     commands: List[Command] = []
     for command_clz in standard_commands:
         pattern = _get_compiled_pattern(command_clz.regex_pattern())
-        if match := pattern.search(action):
+        if match := pattern.search(action.strip()):
             parsed_command = command_clz.from_dsl(match, **kwargs)
             if _additional_parsing_fn := _get_additional_parsing_logic(command_clz):
                 parsed_command = _additional_parsing_fn(parsed_command, flows, **kwargs)

rasa/dialogue_understanding/generator/llm_based_command_generator.py

@@ -357,7 +357,8 @@ class LLMBasedCommandGenerator(
                     "slots": slots_with_info,
                 }
             )
-        return result
+
+        return sorted(result, key=lambda x: x["name"])
 
     @staticmethod
     def is_extractable(

rasa/dialogue_understanding/generator/single_step/command_prompt_template.jinja2

@@ -1,60 +1,58 @@
 Your task is to analyze the current conversation context and generate a list of actions to start new business processes that we call flows, to extract slots, or respond to small talk and knowledge requests.
 
-These are the flows that can be started, with their description and slots:
+
+## Available Actions:
+* Starting a flow, described by "start flow_name". For example, "start transfer_money" or "start list_contacts"
+* Slot setting, described by "set slot_name slot_value". For example, "set transfer_money_recipient Freddy". Can be used to correct and change previously set values.
+* Cancelling the current flow, described by "cancel"
+* Clarifying which flow should be started in ambiguous cases. For example, "clarify list_contacts add_contact remove_contact" if the user just wrote "contacts" and there are multiple potential candidates.
+* Skipping the current question when the user explicitly asks for it, described by "skip".
+* Responding to knowledge-oriented user messages, described by "search"
+* Responding to a casual, non-task-oriented user message, described by "chat".
+* Handing over to a human, in case the user seems frustrated or explicitly asks to speak to one, described by "hand over".
+
+
+## General Tips
+* Do not fill slots with abstract values or placeholders.
+* Only use information provided by the user.
+* Use clarification in ambiguous cases.
+* Multiple flows can be started. If a user wants to digress into a second flow, you do not need to cancel the current flow.
+* Strictly adhere to the provided action format.
+* For categorical slots try to match the user message with potential slot values. Use "other" if you cannot match it
+* Focus on the last message and take it one step at a time.
+* Use the previous conversation steps only to aid understanding.
+
+
+## Available Flows:
 {% for flow in available_flows %}
-{{ flow.name }}: {{ flow.description }}
+* {{ flow.name }}: {{ flow.description }}
     {% for slot in flow.slots -%}
-    slot: {{ slot.name }}{% if slot.description %} ({{ slot.description }}){% endif %}{% if slot.allowed_values %}, allowed values: {{ slot.allowed_values }}{% endif %}
+    * {{ slot.name }}{% if slot.description %} ({{ slot.description }}){% endif %}{% if slot.allowed_values %}, allowed values: {{ slot.allowed_values }}{% endif %}
     {% endfor %}
 {%- endfor %}
 
-===
-Here is what happened previously in the conversation:
-{{ current_conversation }}
 
-===
+## Current State
 {% if current_flow != None %}
 You are currently in the flow "{{ current_flow }}".
 You have just asked the user for the slot "{{ current_slot }}"{% if current_slot_description %} ({{ current_slot_description }}){% endif %}.
 
 {% if flow_slots|length > 0 %}
-Here are the slots of the currently active flow:
+Here are the slots of the flow "{{ current_flow }}":
 {% for slot in flow_slots -%}
-- name: {{ slot.name }}, value: {{ slot.value }}, type: {{ slot.type }}, description: {{ slot.description}}{% if slot.allowed_values %}, allowed values: {{ slot.allowed_values }}{% endif %}
+* name: {{ slot.name }}, value: {{ slot.value }}, type: {{ slot.type }}, description: {{ slot.description}}{% if slot.allowed_values %}, allowed values: {{ slot.allowed_values }}{% endif %}
 {% endfor %}
 {% endif %}
 {% else %}
-You are currently not in any flow and so there are no active slots.
-This means you can only set a slot if you first start a flow that requires that slot.
-{% endif %}
-If you start a flow, first start the flow and then optionally fill that flow's slots with information the user provided in their message.
-
-The user just said """{{ user_message }}""".
-
-===
-Based on this information generate a list of actions you want to take. Your job is to start flows and to fill slots where appropriate. Any logic of what happens afterwards is handled by the flow engine. These are your available actions:
-* Slot setting, described by "SetSlot(slot_name, slot_value)". An example would be "SetSlot(recipient, Freddy)"
-* Starting another flow, described by "StartFlow(flow_name)". An example would be "StartFlow(transfer_money)"
-* Cancelling the current flow, described by "CancelFlow()"
-* Clarifying which flow should be started. An example would be Clarify(list_contacts, add_contact, remove_contact) if the user just wrote "contacts" and there are multiple potential candidates. It also works with a single flow name to confirm you understood correctly, as in Clarify(transfer_money).
-* Intercepting and handle user messages with the intent to bypass the current step in the flow, described by "SkipQuestion()". Examples of user skip phrases are: "Go to the next question", "Ask me something else".
-* Responding to knowledge-oriented user messages, described by "SearchAndReply()"
-* Responding to a casual, non-task-oriented user message, described by "ChitChat()".
-* Handing off to a human, in case the user seems frustrated or explicitly asks to speak to one, described by "HumanHandoff()".
-{% if is_repeat_command_enabled %}
-* Repeat the last bot messages, described by "RepeatLastBotMessages()". This is useful when the user asks to repeat the last bot messages.
+You are currently not inside any flow.
 {% endif %}
 
-===
-Write out the actions you want to take, one per line, in the order they should take place.
-Do not fill slots with abstract values or placeholders.
-Only use information provided by the user.
-Only start a flow if it's completely clear what the user wants. Imagine you were a person reading this message. If it's not 100% clear, clarify the next step.
-Don't be overly confident. Take a conservative approach and clarify before proceeding.
-If the user asks for two things which seem contradictory, clarify before starting a flow.
-If it's not clear whether the user wants to skip the step or to cancel the flow, cancel the flow.
-Strictly adhere to the provided action types listed above.
-Focus on the last message and take it one step at a time.
-Use the previous conversation steps only to aid understanding.
-
-Your action list:
+
+## Conversation History
+{{ current_conversation }}
+
+
+## Task
+Create an action list with one action per line in response to the users last message: """{{ user_message }}""".
+
+Your action list:

rasa/dialogue_understanding_test/du_test_case.py

@@ -1,6 +1,6 @@
 from typing import Any, Dict, Iterator, List, Optional, Tuple
 
-from pydantic import BaseModel, ConfigDict, Field
+from pydantic import BaseModel, Field
 
 from rasa.dialogue_understanding.commands import Command
 from rasa.dialogue_understanding.generator.command_parser import parse_commands
@@ -20,10 +20,17 @@ from rasa.dialogue_understanding_test.constants import (
 from rasa.shared.core.flows import FlowsList
 from rasa.shared.nlu.constants import (
     KEY_COMPONENT_NAME,
+    KEY_LATENCY,
+    KEY_LLM_RESPONSE_METADATA,
     KEY_PROMPT_NAME,
+    KEY_SYSTEM_PROMPT,
     KEY_USER_PROMPT,
 )
 
+KEY_USAGE = "usage"
+KEY_PROMPT_TOKENS = "prompt_tokens"
+KEY_COMPLETION_TOKENS = "completion_tokens"
+
 
 class DialogueUnderstandingOutput(BaseModel):
     """Output containing prompts and generated commands by component.
@@ -62,8 +69,6 @@ class DialogueUnderstandingOutput(BaseModel):
     # List of prompts
     prompts: Optional[List[Dict[str, Any]]] = None
 
-    model_config = ConfigDict(frozen=True)
-
     def get_predicted_commands(self) -> List[Command]:
         """Get all commands from the output."""
         return [
@@ -82,15 +87,31 @@ class DialogueUnderstandingOutput(BaseModel):
             if predicted_commands
         ]
 
-    def get_component_name_to_user_prompts(self) -> Dict[str, List[Tuple[str, str]]]:
-        """Return a dictionary of component names to a list of prompts.
+    def get_component_name_to_prompt_info(self) -> Dict[str, List[Dict[str, Any]]]:
+        """Return a dictionary of component names to prompt information.
 
-        The prompts are represented as tuples of (prompt_name, user_prompt).
+        The prompt information includes the prompt name, user prompt, system prompt,
+        latency, and usage information.
+        The return dict is of the form:
+        {
+            "component_name": [
+                {
+                    "prompt_name": "...",
+                    "user_prompt": "...",
+                    "system_prompt": "...",
+                    "latency": 0.1,
+                    "prompt_tokens": 10,
+                    "completion_tokens": 20
+                },
+                ...
+            ],
+            ...
+        }
         """
         if self.prompts is None:
             return {}
 
-        data: Dict[str, List[Tuple[str, str]]] = {}
+        data: Dict[str, List[Dict[str, Any]]] = {}
         relevant_component_names = self.get_component_names_that_predicted_commands()
 
         for prompt_data in self.prompts:
@@ -99,12 +120,31 @@ class DialogueUnderstandingOutput(BaseModel):
             if component_name not in relevant_component_names:
                 continue
 
-            prompt_name = prompt_data[KEY_PROMPT_NAME]
-            user_prompt = prompt_data[KEY_USER_PROMPT]
-
             if component_name not in data:
                 data[component_name] = []
-            data[component_name].append((prompt_name, user_prompt))
+
+            prompt_info = {
+                KEY_PROMPT_NAME: prompt_data[KEY_PROMPT_NAME],
+                KEY_USER_PROMPT: prompt_data[KEY_USER_PROMPT],
+            }
+
+            latency = prompt_data.get(KEY_LLM_RESPONSE_METADATA, {}).get(KEY_LATENCY)
+            if latency:
+                prompt_info[KEY_LATENCY] = latency
+
+            if prompt_data.get(KEY_SYSTEM_PROMPT):
+                prompt_info[KEY_SYSTEM_PROMPT] = prompt_data[KEY_SYSTEM_PROMPT]
+
+            usage_object = prompt_data.get(KEY_LLM_RESPONSE_METADATA, {}).get(KEY_USAGE)
+            if usage_object:
+                if usage_object.get(KEY_PROMPT_TOKENS):
+                    prompt_info[KEY_PROMPT_TOKENS] = usage_object.get(KEY_PROMPT_TOKENS)
+                if usage_object.get(KEY_COMPLETION_TOKENS):
+                    prompt_info[KEY_COMPLETION_TOKENS] = usage_object.get(
+                        KEY_COMPLETION_TOKENS
+                    )
+
+            data[component_name].append(prompt_info)
 
         return data
 
@@ -149,7 +189,7 @@ class DialogueUnderstandingTestStep(BaseModel):
         Args:
             step: Dictionary containing the step.
             flows: List of flows.
-            custom_commands: Custom commands to use in the test case.
+            custom_command_classes: Custom commands to use in the test case.
             remove_default_commands: Default commands to remove from the test case.
 
         Returns:
@@ -210,6 +250,42 @@ class DialogueUnderstandingTestStep(BaseModel):
 
         return ""
 
+    def get_latencies(self) -> List[float]:
+        if self.dialogue_understanding_output is None:
+            return []
+
+        prompts = self.dialogue_understanding_output.get_component_name_to_prompt_info()
+
+        return [
+            prompt_data.get(KEY_LATENCY, 0.0)
+            for prompt in prompts.values()
+            for prompt_data in prompt
+        ]
+
+    def get_completion_tokens(self) -> List[int]:
+        if self.dialogue_understanding_output is None:
+            return []
+
+        prompts = self.dialogue_understanding_output.get_component_name_to_prompt_info()
+
+        return [
+            prompt_data.get(KEY_COMPLETION_TOKENS, 0)
+            for prompt in prompts.values()
+            for prompt_data in prompt
+        ]
+
+    def get_prompt_tokens(self) -> List[int]:
+        if self.dialogue_understanding_output is None:
+            return []
+
+        prompts = self.dialogue_understanding_output.get_component_name_to_prompt_info()
+
+        return [
+            prompt_data.get(KEY_PROMPT_TOKENS, 0)
+            for prompt in prompts.values()
+            for prompt_data in prompt
+        ]
+
 
 class DialogueUnderstandingTestCase(BaseModel):
     name: str

rasa/dialogue_understanding_test/du_test_result.py

@@ -1,6 +1,8 @@
+import copy
 import typing
 from typing import Any, Dict, List, Optional, Text
 
+import numpy as np
 from pydantic import BaseModel
 
 from rasa.dialogue_understanding.commands import Command
@@ -9,6 +11,10 @@ from rasa.dialogue_understanding_test.du_test_case import (
     DialogueUnderstandingTestStep,
 )
 from rasa.dialogue_understanding_test.utils import get_command_comparison
+from rasa.shared.nlu.constants import (
+    KEY_SYSTEM_PROMPT,
+    KEY_USER_PROMPT,
+)
 
 if typing.TYPE_CHECKING:
     from rasa.dialogue_understanding_test.command_metric_calculation import (
@@ -25,6 +31,9 @@ OUTPUT_USER_UTTERANCES_ACCURACY = "user_utterances_accuracy"
 OUTPUT_NUMBER_OF_PASSED_USER_UTTERANCES = "number_of_passed_user_utterances"
 OUTPUT_NUMBER_OF_FAILED_USER_UTTERANCES = "number_of_failed_user_utterances"
 OUTPUT_COMMAND_METRICS = "command_metrics"
+OUTPUT_LATENCY_METRICS = "latency"
+OUTPUT_COMPLETION_TOKEN_METRICS = "completion_token"
+OUTPUT_PROMPT_TOKEN_METRICS = "prompt_token"
 OUTPUT_NAMES_OF_FAILED_TESTS = "names_of_failed_tests"
 OUTPUT_NAMES_OF_PASSED_TESTS = "names_of_passed_tests"
 OUTPUT_LLM_COMMAND_GENERATOR_CONFIG = "llm_command_generator_config"
@@ -50,7 +59,7 @@ class FailedTestStep(BaseModel):
     error_line: int
     pass_status: bool
     command_generators: List[str]
-    prompt: Optional[Dict[str, Any]] = None
+    prompts: Optional[Dict[str, List[Dict[str, Any]]]] = None
     expected_commands: List[Command]
     predicted_commands: Dict[str, List[Command]]
     conversation_with_diff: List[str]
@@ -66,14 +75,14 @@ class FailedTestStep(BaseModel):
         line_number = step.line or -1
 
         predicted_commands: Dict[str, List[Command]] = {}
-        prompts: Optional[Dict[str, Any]] = None
+        prompts: Optional[Dict[str, List[Dict[str, Any]]]] = None
         command_generators: List[str] = []
 
         if step.dialogue_understanding_output:
             predicted_commands = step.dialogue_understanding_output.commands
             command_generators = step.dialogue_understanding_output.get_component_names_that_predicted_commands()  # noqa: E501
             prompts = (
-                step.dialogue_understanding_output.get_component_name_to_user_prompts()
+                step.dialogue_understanding_output.get_component_name_to_prompt_info()
             )
 
         step_index = test_case.steps.index(step)
@@ -89,13 +98,13 @@ class FailedTestStep(BaseModel):
             error_line=line_number,
             pass_status=False,
             command_generators=command_generators,
-            prompt=prompts,
+            prompts=prompts,
             expected_commands=step.commands or [],
             predicted_commands=predicted_commands,
             conversation_with_diff=conversation_with_diff,
         )
 
-    def to_dict(self, output_prompt: bool) -> Dict[Text, Any]:
+    def to_dict(self, output_prompt: bool) -> Dict[str, Any]:
         step_info = {
             "file": self.file,
             "test_case": self.test_case_name,
@@ -115,19 +124,17 @@ class FailedTestStep(BaseModel):
             ],
         }
 
-        if output_prompt and self.prompt:
-            step_info["prompts"] = [
-                {
-                    component: [
-                        {
-                            "prompt_name": prompt_name,
-                            "prompt_content": prompt_content,
-                        }
-                        for prompt_name, prompt_content in prompts
-                    ],
-                }
-                for component, prompts in self.prompt.items()
-            ]
+        if output_prompt and self.prompts:
+            step_info["prompts"] = copy.deepcopy(self.prompts)
+        elif self.prompts:
+            prompts = copy.deepcopy(self.prompts)
+            # remove user and system prompts
+            for prompt_data in prompts.values():
+                for prompt_info in prompt_data:
+                    prompt_info.pop(KEY_USER_PROMPT, None)
+                    prompt_info.pop(KEY_SYSTEM_PROMPT, None)
+
+                step_info["prompts"] = prompts
 
         return step_info
 
@@ -153,6 +160,9 @@ class DialogueUnderstandingTestSuiteResult:
         self.names_of_passed_tests: List[str] = []
         self.failed_test_steps: List[FailedTestStep] = []
         self.llm_config: Optional[Dict[str, Any]] = None
+        self.latency_metrics: Dict[str, float] = {}
+        self.prompt_token_metrics: Dict[str, float] = {}
+        self.completion_token_metrics: Dict[str, float] = {}
 
     @classmethod
     def from_results(
@@ -206,6 +216,16 @@ class DialogueUnderstandingTestSuiteResult:
             failing_test_results
         )
 
+        instance.latency_metrics = cls.get_latency_metrics(
+            failing_test_results, passing_test_results
+        )
+        instance.prompt_token_metrics = cls.get_prompt_token_metrics(
+            failing_test_results, passing_test_results
+        )
+        instance.completion_token_metrics = cls.get_completion_token_metrics(
+            failing_test_results, passing_test_results
+        )
+
         instance.llm_config = llm_config
 
         return instance
@@ -264,7 +284,60 @@ class DialogueUnderstandingTestSuiteResult:
 
         return failed_test_steps
 
-    def to_dict(self, output_prompt: bool = False) -> Dict[Text, Any]:
+    @staticmethod
+    def _calculate_percentiles(values: List[float]) -> Dict[str, float]:
+        return {
+            "p50": float(np.percentile(values, 50)) if values else 0.0,
+            "p90": float(np.percentile(values, 90)) if values else 0.0,
+            "p99": float(np.percentile(values, 99)) if values else 0.0,
+        }
+
+    @classmethod
+    def get_latency_metrics(
+        cls,
+        failing_test_results: List["DialogueUnderstandingTestResult"],
+        passing_test_results: List["DialogueUnderstandingTestResult"],
+    ) -> Dict[str, float]:
+        latencies = [
+            latency
+            for result in failing_test_results + passing_test_results
+            for step in result.test_case.steps
+            for latency in step.get_latencies()
+        ]
+
+        return cls._calculate_percentiles(latencies)
+
+    @classmethod
+    def get_prompt_token_metrics(
+        cls,
+        failing_test_results: List["DialogueUnderstandingTestResult"],
+        passing_test_results: List["DialogueUnderstandingTestResult"],
+    ) -> Dict[str, float]:
+        tokens = [
+            token_count
+            for result in failing_test_results + passing_test_results
+            for step in result.test_case.steps
+            for token_count in step.get_prompt_tokens()
+        ]
+
+        return cls._calculate_percentiles(tokens)
+
+    @classmethod
+    def get_completion_token_metrics(
+        cls,
+        failing_test_results: List["DialogueUnderstandingTestResult"],
+        passing_test_results: List["DialogueUnderstandingTestResult"],
+    ) -> Dict[str, float]:
+        tokens = [
+            token_count
+            for result in failing_test_results + passing_test_results
+            for step in result.test_case.steps
+            for token_count in step.get_completion_tokens()
+        ]
+
+        return cls._calculate_percentiles(tokens)
+
+    def to_dict(self, output_prompt: bool = False) -> Dict[str, Any]:
         """Builds a dictionary for writing test results to a YML file.
 
         Args:
@@ -292,6 +365,10 @@ class DialogueUnderstandingTestSuiteResult:
 
         result_dict[OUTPUT_COMMAND_METRICS] = cmd_metrics_output
 
+        result_dict[OUTPUT_LATENCY_METRICS] = self.latency_metrics
+        result_dict[OUTPUT_PROMPT_TOKEN_METRICS] = self.prompt_token_metrics
+        result_dict[OUTPUT_COMPLETION_TOKEN_METRICS] = self.completion_token_metrics
+
         result_dict[OUTPUT_NAMES_OF_PASSED_TESTS] = self.names_of_passed_tests
         result_dict[OUTPUT_NAMES_OF_FAILED_TESTS] = self.names_of_failed_tests
 

rasa/dialogue_understanding_test/io.py

@@ -7,6 +7,10 @@ import rich
 import rasa.shared.data
 from rasa.dialogue_understanding_test.command_metric_calculation import CommandMetrics
 from rasa.dialogue_understanding_test.constants import SCHEMA_FILE_PATH
+from rasa.dialogue_understanding_test.du_test_case import (
+    KEY_COMPLETION_TOKENS,
+    KEY_PROMPT_TOKENS,
+)
 from rasa.dialogue_understanding_test.du_test_result import (
     DialogueUnderstandingTestSuiteResult,
     FailedTestStep,
@@ -31,6 +35,12 @@ from rasa.e2e_test.utils.io import (
     validate_test_case,
 )
 from rasa.shared.core.flows import FlowsList
+from rasa.shared.nlu.constants import (
+    KEY_LATENCY,
+    KEY_PROMPT_NAME,
+    KEY_SYSTEM_PROMPT,
+    KEY_USER_PROMPT,
+)
 from rasa.shared.utils.yaml import (
     read_schema_file,
     validate_yaml_content_using_schema,
@@ -188,13 +198,11 @@ def print_test_results(
     """Print the result of the test run.
 
     Example output (truncated for brevity):
-        =================================================================== FAILURES
-        ===================================================================
+        ====== FAILURES ======
 
-        ---------------- test_case:
-        /Users/milos/Projects/rasa-calm-demo/dialogue_understanding_tests/
-        immediate_cancellation_and_start_of_new_flow.yml::
-        user immediately cancels and starts new flow -----------------
+        ---------------- test_case: rasa-calm-demo/dialogue_understanding_tests/
+        immediate_cancellation_and_start_of_new_flow.yml::user immediately cancels
+        and starts new flow -----------------
 
         == failure starting at user message 'I want to send money'.
 
@@ -208,7 +216,7 @@ def print_test_results(
         SetSlot(transfer_money_amount_of_money, 878) |
 
         ...
-
+        ====== COMMAND METRICS ======
         set slot (2 commands in total):
           tp: 0 fp: 1 fn: 2
           precision: 0.0000
@@ -221,9 +229,22 @@ def print_test_results(
           recall   : 1.0000
           f1       : 1.0000
 
-        =================================== 1 failed test cases, 0 passed test cases
-        ===================================
-        ======== 2 failed user steps, 1 passed user steps (accuracy: 0.3333) ========
+        ...
+        ====== LATENCY METRICS ======
+        p50: 0.00065571
+        p90: 0.00074687
+        p99: 0.00077837
+        ====== PROMPT TOKEN METRICS ======
+        p50: 1336.00
+        p90: 1389.50
+        p99: 1401.65
+        ====== COMPLETION TOKEN METRICS ======
+        p50: 12.00
+        p90: 15.80
+        p99: 16.88
+
+        ====== 1 failed test cases, 0 passed test cases ======
+        ====== 2 failed user steps, 1 passed user steps (accuracy: 0.3333) ======
 
     Args:
         test_suite_result: Test results suite containing the test results.
@@ -253,6 +274,7 @@ def print_test_results(
         print_failed_cases(test_suite_result, output_prompt=output_prompt)
 
     print_command_summary(test_suite_result.command_metrics)
+    print_latency_and_token_metrics(test_suite_result)
     print_final_line(test_suite_result)
 
 
@@ -290,16 +312,33 @@ def print_failed_cases(
 
 
 def print_prompt(step: FailedTestStep) -> None:
-    if step.prompt is None:
+    if step.prompts is None:
         return
-    prompt_data = step.prompt
+    prompts = step.prompts
 
     rich.print("\n[red3]-- PROMPT(s) --[/red3]")
-    for component, prompts in prompt_data.items():
+    for component, component_prompts in prompts.items():
         rich.print(f"[bold]{component}[/bold]")
-        for subcomponent, prompt in prompts:
-            rich.print(f"    [bold]{subcomponent}[/bold]")
-            rich.print(f"        {prompt}")
+        for prompt_data in component_prompts:
+            rich.print(
+                f"[bold]  prompt name      [/bold]: {prompt_data[KEY_PROMPT_NAME]}"
+            )
+            rich.print(
+                f"[bold]  prompt tokens    [/bold]: {prompt_data[KEY_PROMPT_TOKENS]}"
+            )
+            rich.print(
+                f"[bold]  completion tokens[/bold]: "
+                f"{prompt_data[KEY_COMPLETION_TOKENS]}"
+            )
+            rich.print(f"[bold]  latency          [/bold]: {prompt_data[KEY_LATENCY]}")
+            if KEY_SYSTEM_PROMPT in prompt_data:
+                rich.print(
+                    f"[bold]  system prompt    [/bold]: "
+                    f"{prompt_data[KEY_SYSTEM_PROMPT]}"
+                )
+            rich.print(
+                f"[bold]  user prompt      [/bold]: {prompt_data[KEY_USER_PROMPT]}"
+            )
 
 
 def print_command_summary(metrics: Dict[str, CommandMetrics]) -> None:
@@ -331,6 +370,24 @@ def print_command_summary(metrics: Dict[str, CommandMetrics]) -> None:
         )
 
 
+def print_latency_and_token_metrics(
+    result: DialogueUnderstandingTestSuiteResult,
+) -> None:
+    """Print the latency and token metrics."""
+    print()
+    rasa.shared.utils.cli.print_info(rasa.shared.utils.cli.pad("LATENCY METRICS"))
+    for key, value in result.latency_metrics.items():
+        rasa.shared.utils.cli.print_info(f"{key}: {value:.8f}")
+    rasa.shared.utils.cli.print_info(rasa.shared.utils.cli.pad("PROMPT TOKEN METRICS"))
+    for key, value in result.prompt_token_metrics.items():
+        rasa.shared.utils.cli.print_info(f"{key}: {value:.2f}")
+    rasa.shared.utils.cli.print_info(
+        rasa.shared.utils.cli.pad("COMPLETION TOKEN METRICS")
+    )
+    for key, value in result.completion_token_metrics.items():
+        rasa.shared.utils.cli.print_info(f"{key}: {value:.2f}")
+
+
 def print_final_line(test_suite_result: DialogueUnderstandingTestSuiteResult) -> None:
     """Print the final line of the test output.
 

rasa/shared/providers/llm/_base_litellm_client.py

@@ -181,6 +181,46 @@ class _BaseLiteLLMClient:
                 )
             raise ProviderClientAPIException(e, message)
 
+
+    @suppress_logs(log_level=logging.WARNING)
+    async def acompletion_with_system(self, formatted_messages: Union[List[str], str]) -> LLMResponse:
+        """Asynchronously generate completions for given list of messages.
+
+        Args:
+            messages: List of messages or a single message to generate the
+                completion for.
+
+        Returns:
+            List of message completions.
+
+        Raises:
+            ProviderClientAPIException: If the API request fails.
+        """
+        try:
+            # formatted_messages = self._format_messages(messages)
+            arguments = resolve_environment_variables(self._completion_fn_args)
+            response = await acompletion(messages=formatted_messages, **arguments)
+            return self._format_response(response)
+        except Exception as e:
+            message = ""
+            from rasa.shared.providers.llm.self_hosted_llm_client import (
+                SelfHostedLLMClient,
+            )
+
+            if isinstance(self, SelfHostedLLMClient):
+                message = (
+                    "If you are using 'provider=self-hosted' to call a hosted vllm "
+                    "server make sure your config is correctly setup. You should have "
+                    "the following mandatory keys in your config: "
+                    "provider=self-hosted; "
+                    "model='<your-vllm-model-name>'; "
+                    "api_base='your-hosted-vllm-serv'."
+                    "In case you are getting OpenAI connection errors, such as missing "
+                    "API key, your configuration is incorrect."
+                )
+            raise ProviderClientAPIException(e, message)
+
+
     def _format_messages(self, messages: Union[List[str], str]) -> List[Dict[str, str]]:
         """Formats messages (or a single message) to OpenAI format."""
         if isinstance(messages, str):

rasa/shared/utils/llm.py

@@ -12,6 +12,7 @@ from typing import (
     TypeVar,
     Union,
     cast,
+    List,
 )
 
 import structlog
@@ -237,6 +238,84 @@ def tracker_as_readable_transcript(
     return "\n".join(transcript)
 
 
+def sanitize_command_for_prompt(cmd_dict):
+    command = ""
+    if cmd_dict["command"] == "start flow":
+        command = f"start {cmd_dict['flow']}"
+    elif cmd_dict["command"] == "set slot":
+        command = f"set {cmd_dict['name']} {cmd_dict['value']}"
+    elif cmd_dict["command"] == "skip question":
+        command = "skip"
+    elif cmd_dict["command"] == "clarify":
+        command = f"clarify {' '.join(cmd_dict['options'])}"
+    elif cmd_dict["command"] == "knowledge":
+        command = "search"
+    elif cmd_dict["command"] == "chitchat":
+        command = "chat"
+    elif cmd_dict["command"] == "cancel flow":
+        command = "cancel"
+    else:
+        command = cmd_dict["command"]
+
+    return command
+
+def tracker_as_message_list(
+    tracker: "DialogueStateTracker",
+    max_turns: Optional[int] = 20,
+) -> List[str]:
+    """Creates a readable dialogue from a tracker.
+
+    Args:
+        tracker: the tracker to convert
+        max_turns: the maximum number of turns to include in the transcript
+
+    Example:
+        >>> tracker = Tracker(
+        ...     sender_id="test",
+        ...     slots=[],
+        ...     events=[
+        ...         UserUttered("hello"),
+        ...         BotUttered("hi"),
+        ...     ],
+        ... )
+        >>> tracker_as_readable_transcript(tracker)
+        USER: hello
+        AI: hi
+
+    Returns:
+    A string representing the transcript of the tracker
+    """
+    messages = []
+
+    # using `applied_events` rather than `events` means that only events after the
+    # most recent `Restart` or `SessionStarted` are included in the transcript
+    last_commands = None
+    for event in tracker.applied_events():
+        if isinstance(event, UserUttered):
+            if event.has_triggered_error:
+                first_error = event.error_commands[0]
+                error_type = first_error.get("error_type")
+                message = ERROR_PLACEHOLDER.get(
+                    error_type, ERROR_PLACEHOLDER["default"]
+                )
+            else:
+                message = sanitize_message_for_prompt(event.text)
+                last_commands = event.commands
+            messages.append({"role": "user", "content": message})
+            # messages.append({"role": "system", "content": ' \n '.join([sanitize_command_for_prompt(cmd) for cmd in last_commands])})
+            # transcript.append(f"{human_prefix}: {message}")
+
+        elif isinstance(event, BotUttered):
+            messages.append({"role": "assistant", "content": f"{sanitize_message_for_prompt(event.text)}"})
+            # transcript.append(f"{ai_prefix}: {sanitize_message_for_prompt(event.text)}")
+
+    if max_turns:
+        messages = messages[-max_turns:]
+        # transcript = transcript[-max_turns:]
+
+    return messages
+
+
 def sanitize_message_for_prompt(text: Optional[str]) -> str:
     """Removes new lines from a string.
 
@@ -680,7 +759,7 @@ def allowed_values_for_slot(slot: Slot) -> Union[str, None]:
     if isinstance(slot, BooleanSlot):
         return str([True, False])
     if isinstance(slot, CategoricalSlot):
-        return str([v for v in slot.values if v != "__other__"])
+        return str([v for v in slot.values if v != "__other__"] + ["other"])
     else:
         return None
 

rasa/tracing/instrumentation/attribute_extractors.py

@@ -24,12 +24,15 @@ from rasa.dialogue_understanding.stack.dialogue_stack import DialogueStack
 from rasa.dialogue_understanding_test.du_test_result import (
     KEY_TEST_CASES_ACCURACY,
     KEY_USER_UTTERANCES_ACCURACY,
+    OUTPUT_COMPLETION_TOKEN_METRICS,
+    OUTPUT_LATENCY_METRICS,
     OUTPUT_NAMES_OF_FAILED_TESTS,
     OUTPUT_NAMES_OF_PASSED_TESTS,
     OUTPUT_NUMBER_OF_FAILED_TESTS,
     OUTPUT_NUMBER_OF_FAILED_USER_UTTERANCES,
     OUTPUT_NUMBER_OF_PASSED_TESTS,
     OUTPUT_NUMBER_OF_PASSED_USER_UTTERANCES,
+    OUTPUT_PROMPT_TOKEN_METRICS,
     OUTPUT_TEST_CASES_ACCURACY,
     OUTPUT_USER_UTTERANCES_ACCURACY,
     DialogueUnderstandingTestSuiteResult,
@@ -636,6 +639,13 @@ def extract_attrs_for_du_print_test_results(
             for key, value in test_suite_result.llm_config.items():
                 attributes_dict[f"llm_config_0_{key}"] = value
 
+    for key, value in test_suite_result.latency_metrics.items():
+        attributes_dict[f"{OUTPUT_LATENCY_METRICS}_{key}"] = value
+    for key, value in test_suite_result.prompt_token_metrics.items():
+        attributes_dict[f"{OUTPUT_PROMPT_TOKEN_METRICS}_{key}"] = value
+    for key, value in test_suite_result.completion_token_metrics.items():
+        attributes_dict[f"{OUTPUT_COMPLETION_TOKEN_METRICS}_{key}"] = value
+
     return attributes_dict
 
 

rasa/version.py

@@ -1,3 +1,3 @@
 # this file will automatically be changed,
 # do not add anything but the version number here!
-__version__ = "3.12.0.dev7"
+__version__ = "3.12.0.dev8"

{rasa_pro-3.12.0.dev7.dist-info → rasa_pro-3.12.0.dev8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: rasa-pro
-Version: 3.12.0.dev7
+Version: 3.12.0.dev8
 Summary: State-of-the-art open-core Conversational AI framework for Enterprises that natively leverages generative AI for effortless assistant development.
 Home-page: https://rasa.com
 Keywords: nlp,machine-learning,machine-learning-library,bot,bots,botkit,rasa conversational-agents,conversational-ai,chatbot,chatbot-framework,bot-framework

{rasa_pro-3.12.0.dev7.dist-info → rasa_pro-3.12.0.dev8.dist-info}/RECORD

@@ -364,36 +364,36 @@ rasa/dialogue_understanding/coexistence/intent_based_router.py,sha256=JlYBZdScnh
 rasa/dialogue_understanding/coexistence/llm_based_router.py,sha256=UTLMZpynSRt8qnQSSV-dTgFQbBHaMEgrLIUWCMyH1BE,11354
 rasa/dialogue_understanding/coexistence/router_template.jinja2,sha256=CHWFreN0sv1EbPh-hf5AlCt3zxy2_llX1Pdn9Q11Y18,357
 rasa/dialogue_understanding/commands/__init__.py,sha256=F-pLETYRUjhIkjjDfXGUuPsK_ac1HcLmJkrUUP0RhME,2259
-rasa/dialogue_understanding/commands/can_not_handle_command.py,sha256=2ferTQRUR_Ibr2Qa-Dma71DzVT6Tfy3-6iUhFrFFeZ4,2831
-rasa/dialogue_understanding/commands/cancel_flow_command.py,sha256=Um2ouQkfR_Za66uRiKjjZKI5fuYG76uvk3hFW3w4XV8,4625
-rasa/dialogue_understanding/commands/change_flow_command.py,sha256=8tjULfNQXsH3ZWTsMlrzbVgHLU26catdHTWzcYgSBWI,1735
-rasa/dialogue_understanding/commands/chit_chat_answer_command.py,sha256=GtYU3FtpwoAQti2_zW_T3x7iJABu2Yyb0eqUd7takhg,2152
-rasa/dialogue_understanding/commands/clarify_command.py,sha256=y5iEESwalNmQ7MZDbS5ut5KZwqEp_4GF6kBrvwUlRBE,3432
+rasa/dialogue_understanding/commands/can_not_handle_command.py,sha256=z7aGn7aBMdXTsn7lzN1RvXlYi8WN7gtFm21qcJF8Hko,2829
+rasa/dialogue_understanding/commands/cancel_flow_command.py,sha256=Tc5xXQw_P8XlIApL6CblFf1nnfnWGUqsMSIkVV_K_bE,4613
+rasa/dialogue_understanding/commands/change_flow_command.py,sha256=gQyL1UqRNPdYXKVQi3hlAHOdpcCWLkhL34M3A_PzSR0,1722
+rasa/dialogue_understanding/commands/chit_chat_answer_command.py,sha256=JnOMAaWhyynlTTzLpi663BOSs8hYNZyxrjUnSq0Qle8,2140
+rasa/dialogue_understanding/commands/clarify_command.py,sha256=7ZlgA_gmOCj0xcP3sCNn7frk75SqiuWpS0O0HWuxjNE,3428
 rasa/dialogue_understanding/commands/command.py,sha256=riYYO0Lvm7T2vydDOfFFi7IquJWdyX_jH8d7qX0PB2c,2993
 rasa/dialogue_understanding/commands/correct_slots_command.py,sha256=lnSfYXNdTwQflOeMRXh1rV4qn6Bgx85S1yaXHtiKKAs,10461
 rasa/dialogue_understanding/commands/error_command.py,sha256=LTEsxkGGGZR6wEEGuTtQ4K4EK_u2UFhNK4eAKyPfyME,2436
 rasa/dialogue_understanding/commands/free_form_answer_command.py,sha256=XlQrHXrcOemzu1LHZiDhBAluiSlnUQ2V7ET5Z-aG7gc,224
 rasa/dialogue_understanding/commands/handle_code_change_command.py,sha256=Cp2e1iD0zacXmljJ8vDXHJu9Fp6BwB7cGx8NF748akw,2192
-rasa/dialogue_understanding/commands/human_handoff_command.py,sha256=WlGswi5Wlc3jsXp6hsU3ANMdZMqGk33rxeX0kAyiwgw,2285
-rasa/dialogue_understanding/commands/knowledge_answer_command.py,sha256=S2AP2_H-KFnSRUKqG3kz083dR_JTXKMWkjhzi9mEDPQ,2166
+rasa/dialogue_understanding/commands/human_handoff_command.py,sha256=bxSTNbLz05nN_gL14TsVural4S9-Du1teVXy1JDASP8,2275
+rasa/dialogue_understanding/commands/knowledge_answer_command.py,sha256=IvFcV8rhnDchzvWfJncJ34OFkXHiQ3yWytkGFWNuSho,2146
 rasa/dialogue_understanding/commands/noop_command.py,sha256=aIaLBjSV84qy9X4aGlJfMIYhF57maH5CiKNWL_-giD4,1485
-rasa/dialogue_understanding/commands/repeat_bot_messages_command.py,sha256=9SaRXsqdgC5p-2JIa8T3fsc6sVcsLf1FL90HtjW5Uek,2243
+rasa/dialogue_understanding/commands/repeat_bot_messages_command.py,sha256=6oitCpsYtLjOUBGe4cIx2OOZnw9ygi4nKwWOg66R2aA,2225
 rasa/dialogue_understanding/commands/restart_command.py,sha256=vvmucwlVtfh6VMgdOn5hZfsP9U5HhfbDeBSG2IndX0Y,1639
 rasa/dialogue_understanding/commands/session_end_command.py,sha256=ZecUpYZDTX_68_kV1Hv4i317bbeBeVHHyhW_A7r5yzs,1770
 rasa/dialogue_understanding/commands/session_start_command.py,sha256=FA4yocMnFt5bn2dmXj48S4Pq_yTlEnOBxgK_mq-qAxg,1704
-rasa/dialogue_understanding/commands/set_slot_command.py,sha256=NgrN3IgzjCCIWNjrySdSUyQBQSlomwdqYUFOE1HZSRw,5994
-rasa/dialogue_understanding/commands/skip_question_command.py,sha256=45NvtkqMaNX05T2KmTInwGSTWAeRokgdF7CXWpHnL48,2608
-rasa/dialogue_understanding/commands/start_flow_command.py,sha256=yQJNiAey5ETInxwYSNARabCL2f2QDVKuBQrIyvtPt1A,3793
+rasa/dialogue_understanding/commands/set_slot_command.py,sha256=ENpMuAbrlNA0hX3ABbIxQPNAL4a7sk7HZvNTZGhoiRE,5982
+rasa/dialogue_understanding/commands/skip_question_command.py,sha256=ZPstoD-A2jnW8AszKsZ10WtDEeAtoCh6swESwk7fpDA,2588
+rasa/dialogue_understanding/commands/start_flow_command.py,sha256=WYgynvL1sqlOr_I5VXI5OHqWaSRUexvFUvEsgI9WvPw,3783
 rasa/dialogue_understanding/commands/user_silence_command.py,sha256=DQjRfZk09sV1o2emnLkmX7cZpsJwBHNeJGBDQVkejjY,1686
-rasa/dialogue_understanding/commands/utils.py,sha256=vz3fNF31cCIUoWQ6HP6_nI4E30MBFIfacCZb6QKx634,1748
+rasa/dialogue_understanding/commands/utils.py,sha256=OV6hJjlb2WWOHCaIxtRXbbKHKSNGX2het_KZNoomrBI,1748
 rasa/dialogue_understanding/constants.py,sha256=YcELaIss69Hnroclvn90Dl4Suk3S6e3t0UoIbUaXG2A,83
 rasa/dialogue_understanding/generator/__init__.py,sha256=Ykeb2wQ1DuiUWAWO0hLIPSTK1_Ktiq9DZXF6D3ugN78,764
 rasa/dialogue_understanding/generator/command_generator.py,sha256=OTyE9_CcorpjTOgdI-u0J9ryBwlDdd2foJLA3jQJ9_Y,11959
-rasa/dialogue_understanding/generator/command_parser.py,sha256=HWLKJoXBOQVByk__rVbReJJ8u7au9Wit0MXvuHEAZHg,6422
+rasa/dialogue_understanding/generator/command_parser.py,sha256=E-P1giFoSzPwgg7FqPEx0-4XWUAKidn8HT2Bn70DzmM,6430
 rasa/dialogue_understanding/generator/constants.py,sha256=ntP5xmTlS3b_6uVT7LKvWzxbmf5_9tWZ5eFY1RLBtqU,716
 rasa/dialogue_understanding/generator/flow_document_template.jinja2,sha256=f4H6vVd-_nX_RtutMh1xD3ZQE_J2OyuPHAtiltfiAPY,253
 rasa/dialogue_understanding/generator/flow_retrieval.py,sha256=wlGnMj17-X1-siQmdSvOd7K61sRzBf82MQEL2pqDQMI,17891
-rasa/dialogue_understanding/generator/llm_based_command_generator.py,sha256=o8Dt-uIrgKqsRGvlMy0TpqLhf2DHSE4O5BDuHpKR5EQ,16161
+rasa/dialogue_understanding/generator/llm_based_command_generator.py,sha256=uQKQFNoayY5klvhPERje7jQmsM4oULlYQNlULx7fsko,16195
 rasa/dialogue_understanding/generator/llm_command_generator.py,sha256=QpNXhjB9ugtPV8XAHmKjbJtOiI1yE9rC2osbsI_A4ZY,2529
 rasa/dialogue_understanding/generator/multi_step/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 rasa/dialogue_understanding/generator/multi_step/fill_slots_prompt.jinja2,sha256=Y0m673tAML3cFPaLM-urMXDsBYUUcXIw9YUpkAhGUuA,2933
@@ -401,7 +401,7 @@ rasa/dialogue_understanding/generator/multi_step/handle_flows_prompt.jinja2,sha2
 rasa/dialogue_understanding/generator/multi_step/multi_step_llm_command_generator.py,sha256=U8Sj6KxQZw5v-RdkuH-5fJYEvKDfIG2SnPXfsEZuG2Y,31034
 rasa/dialogue_understanding/generator/nlu_command_adapter.py,sha256=-DFGkeRKQwU0-0ZJpVjxvgAUGq8pK9NXavAUONT0Wo8,9383
 rasa/dialogue_understanding/generator/single_step/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-rasa/dialogue_understanding/generator/single_step/command_prompt_template.jinja2,sha256=nMayu-heJYH1QmcL1cFmXb8SeiJzfdDR_9Oy5IRUXsM,3937
+rasa/dialogue_understanding/generator/single_step/command_prompt_template.jinja2,sha256=k6s9Gj4ULCwfuqRbX41yW0aJwQN_T_ShfWlGeZDT308,2863
 rasa/dialogue_understanding/generator/single_step/single_step_llm_command_generator.py,sha256=KznnZzf3RtvRK8sSfMw4tCaWzXMJf0Mw66S73lA22uk,15724
 rasa/dialogue_understanding/generator/utils.py,sha256=fRACEJ2bEgEgSq90TUrQFLTrLqtJqtbb64mgAXklMQ4,1911
 rasa/dialogue_understanding/patterns/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -441,11 +441,11 @@ rasa/dialogue_understanding_test/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRk
 rasa/dialogue_understanding_test/command_comparison.py,sha256=LvCZGgZVFpKjWqaZE5OqPClM5xDNdFZQ4FslvNerB7s,1812
 rasa/dialogue_understanding_test/command_metric_calculation.py,sha256=ys1BobRxqEhsfKk5Op9OB_IXUIsGGSiK6ox8246W-9E,3889
 rasa/dialogue_understanding_test/constants.py,sha256=G63FEzswDUOonTxoXQicEJwI6ICkSx3YP1ILkGH1ijw,790
-rasa/dialogue_understanding_test/du_test_case.py,sha256=P82xu4wkyIrhPdQWHp2wMYG6YVkUgaDFNSSDDN6Znv8,11277
-rasa/dialogue_understanding_test/du_test_result.py,sha256=ITnY5rtPAW-DTPXQA9yp3gyFfKRUjIZ_IVBTc3AjReY,11861
+rasa/dialogue_understanding_test/du_test_case.py,sha256=dRb3htpUpSKzArQZOgW8a3J2Wi85oVTLjk2gq4OELQU,13801
+rasa/dialogue_understanding_test/du_test_result.py,sha256=TlpmimfnwtNfu5UMFLogjSZrkfpBYrHgCeMOQdXQjhw,14822
 rasa/dialogue_understanding_test/du_test_runner.py,sha256=ZG-TNfu-Ak9l_gg9NNadzKzARgICJ9wlsYooCBi1WKU,11943
 rasa/dialogue_understanding_test/du_test_schema.yml,sha256=zgIhb6PE8LnoigVmv4NbU3cjSsr2SkGoO-5Xh4Et9KA,4767
-rasa/dialogue_understanding_test/io.py,sha256=ah26adacAU57YX7rhL3fMLNKeVddVURzUaWddXl8N0k,12707
+rasa/dialogue_understanding_test/io.py,sha256=nXmauJp14yMPvKTCHbSnIgvd1u8B_piSCXwxYpfSvWQ,14656
 rasa/dialogue_understanding_test/test_case_simulation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 rasa/dialogue_understanding_test/test_case_simulation/exception.py,sha256=RJV8CfoGKmfpC3d28y7IBKfmcAZSm2Vs6p0GkiCHlcc,1034
 rasa/dialogue_understanding_test/test_case_simulation/test_case_tracker_simulator.py,sha256=hL7NisfRWe_QuiETH1hkqN6U7IRi8fZ2ggQ4JleHnzk,12994
@@ -715,7 +715,7 @@ rasa/shared/providers/embedding/huggingface_local_embedding_client.py,sha256=Zo3
 rasa/shared/providers/embedding/litellm_router_embedding_client.py,sha256=Y5XNwWl1kxoOLs8ZMY3nhPn__XSPOeg8ED-dQQuegF4,4517
 rasa/shared/providers/embedding/openai_embedding_client.py,sha256=XNRGE7apo2v3kWRrtgxE-Gq4rvNko3IiXtvgC4krDYE,5429
 rasa/shared/providers/llm/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-rasa/shared/providers/llm/_base_litellm_client.py,sha256=Kwa8s96g5AijiKiOO9-UKaLWYJ4iHEuyepsc5ZkTEP4,10004
+rasa/shared/providers/llm/_base_litellm_client.py,sha256=f-aK_FHm1DT3BiloW2Z_mHEO_zCQTrmYaSimdrW8Cz0,11691
 rasa/shared/providers/llm/azure_openai_llm_client.py,sha256=HBm5f_bPwOXP2N-zEMfnhAdMQxDHGHp8JKC62gnj4mg,12976
 rasa/shared/providers/llm/default_litellm_llm_client.py,sha256=Gu9l-RNaKXAUsEmG8DIIGSPHtW9vMEyVNWvOVADiJcU,3940
 rasa/shared/providers/llm/litellm_router_llm_client.py,sha256=hPYGMZUUjpnRCFNvHSSuaKK_OxHV2spr9eItkwYYnBI,6816
@@ -737,7 +737,7 @@ rasa/shared/utils/health_check/embeddings_health_check_mixin.py,sha256=ASOzDtI3i
 rasa/shared/utils/health_check/health_check.py,sha256=izixrbc9BxFSsjzwoIw9U0w0VKSX5gMwhey8bcwe1wc,9709
 rasa/shared/utils/health_check/llm_health_check_mixin.py,sha256=ANP5Q68TRX8p4wWkRCAISsWBV1iYYeGnqWILnR1NawE,957
 rasa/shared/utils/io.py,sha256=AhuECoXGO367NvWRCBu99utEtTQnyxWVJyKOOpLePpg,15917
-rasa/shared/utils/llm.py,sha256=6mfcRdKZVvB7JLVN_2oj6tnIgPfiPqQ2hYnTyK8mwP4,25545
+rasa/shared/utils/llm.py,sha256=s3Y54gWAnzkK6BezpJliMZurZ8dNbU_zNpxja1UMqXw,28333
 rasa/shared/utils/pykwalify_extensions.py,sha256=36vfuD9gSreZvMIZ_qIPpykXhYGuZu2BpoEMVX50Is4,883
 rasa/shared/utils/schemas/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 rasa/shared/utils/schemas/config.yml,sha256=czxSADw9hOIZdhvFP8pVUQo810hs9_C8ZGfCPx17taM,27
@@ -760,7 +760,7 @@ rasa/tracing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 rasa/tracing/config.py,sha256=j5N6s-GX3idNH9FO-0z10KduVg2ovzsE-u5ve87249U,12860
 rasa/tracing/constants.py,sha256=N_MJLStE3IkmPKQCQv42epd3jdBMJ4Ith1dVO65N5ho,2425
 rasa/tracing/instrumentation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-rasa/tracing/instrumentation/attribute_extractors.py,sha256=yzs6mMCvJgPMxnuESJBn_QfRZq8lyUfhicVa4Y1-ONw,28955
+rasa/tracing/instrumentation/attribute_extractors.py,sha256=-LjX91Y94Jalccm5tMpbYvXF25abg0NVFr0MKJJOB-Q,29478
 rasa/tracing/instrumentation/instrumentation.py,sha256=K4h_u3VY-eKPKUP8UXKEHsCy5UpXw0QGQ2MdlQFBeWE,51861
 rasa/tracing/instrumentation/intentless_policy_instrumentation.py,sha256=RgixI0FVIzBz19E3onidUpSEwjkAh8paA5_w07PMzFo,4821
 rasa/tracing/instrumentation/metrics.py,sha256=N4Zxo8P84p4VH6vGai1oRurIUifXPtMrZh1BlpPB7kg,10534
@@ -801,9 +801,9 @@ rasa/utils/train_utils.py,sha256=ClJx-6x3-h3Vt6mskacgkcCUJTMXjFPe3zAcy_DfmaU,212
 rasa/utils/url_tools.py,sha256=dZ1HGkVdWTJB7zYEdwoDIrEuyX9HE5WsxKKFVsXBLE0,1218
 rasa/utils/yaml.py,sha256=KjbZq5C94ZP7Jdsw8bYYF7HASI6K4-C_kdHfrnPLpSI,2000
 rasa/validator.py,sha256=Uh5R1JDmIRl0aNprh9FfHND8UKiNJTNYBrMdBDVxEFM,67516
-rasa/version.py,sha256=ZidWAktMLMvwbHEMlVU7E2f6M0gBaoHOhrxbP9X0u-g,122
-rasa_pro-3.12.0.dev7.dist-info/METADATA,sha256=YNfDp8Z3qUiOSQ4He_M5ddASCrUenlBybBk7LTv3XEE,10730
-rasa_pro-3.12.0.dev7.dist-info/NOTICE,sha256=7HlBoMHJY9CL2GlYSfTQ-PZsVmLmVkYmMiPlTjhuCqA,218
-rasa_pro-3.12.0.dev7.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-rasa_pro-3.12.0.dev7.dist-info/entry_points.txt,sha256=ckJ2SfEyTPgBqj_I6vm_tqY9dZF_LAPJZA335Xp0Q9U,43
-rasa_pro-3.12.0.dev7.dist-info/RECORD,,
+rasa/version.py,sha256=X1Y_R4GCpb62i5wQ8SK2UG1DoYm2-pjjkdhp_xW_qhA,122
+rasa_pro-3.12.0.dev8.dist-info/METADATA,sha256=98LVdTnmeOxjsjJ56onShrhTUJ3fJAZX-8-IX2dLcUI,10730
+rasa_pro-3.12.0.dev8.dist-info/NOTICE,sha256=7HlBoMHJY9CL2GlYSfTQ-PZsVmLmVkYmMiPlTjhuCqA,218
+rasa_pro-3.12.0.dev8.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+rasa_pro-3.12.0.dev8.dist-info/entry_points.txt,sha256=ckJ2SfEyTPgBqj_I6vm_tqY9dZF_LAPJZA335Xp0Q9U,43
+rasa_pro-3.12.0.dev8.dist-info/RECORD,,