rasa-pro 3.12.0.dev7__tar.gz → 3.12.0.dev9__tar.gz

{rasa_pro-3.12.0.dev7 → rasa_pro-3.12.0.dev9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: rasa-pro
-Version: 3.12.0.dev7
+Version: 3.12.0.dev9
 Summary: State-of-the-art open-core Conversational AI framework for Enterprises that natively leverages generative AI for effortless assistant development.
 Home-page: https://rasa.com
 Keywords: nlp,machine-learning,machine-learning-library,bot,bots,botkit,rasa conversational-agents,conversational-ai,chatbot,chatbot-framework,bot-framework

{rasa_pro-3.12.0.dev7 → rasa_pro-3.12.0.dev9}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "rasa-pro"
-version = "3.12.0.dev7"
+version = "3.12.0.dev9"
 description = "State-of-the-art open-core Conversational AI framework for Enterprises that natively leverages generative AI for effortless assistant development."
 authors = [ "Rasa Technologies GmbH <hi@rasa.com>",]
 maintainers = [ "Tom Bocklisch <tom@rasa.com>",]

{rasa_pro-3.12.0.dev7 → rasa_pro-3.12.0.dev9}/rasa/dialogue_understanding/commands/can_not_handle_command.py

@@ -74,7 +74,7 @@ class CannotHandleCommand(Command):
 
     def to_dsl(self) -> str:
         """Converts the command to a DSL string."""
-        return "CannotHandle()"
+        return "cannot handle"
 
     @classmethod
     def from_dsl(cls, match: re.Match, **kwargs: Any) -> CannotHandleCommand:
@@ -86,4 +86,4 @@ class CannotHandleCommand(Command):
 
     @staticmethod
     def regex_pattern() -> str:
-        return r"CannotHandle\(\)"
+        return r"^cannot handle$"

{rasa_pro-3.12.0.dev7 → rasa_pro-3.12.0.dev9}/rasa/dialogue_understanding/commands/cancel_flow_command.py

@@ -124,7 +124,7 @@ class CancelFlowCommand(Command):
 
     def to_dsl(self) -> str:
         """Converts the command to a DSL string."""
-        return "CancelFlow()"
+        return "cancel"
 
     @classmethod
     def from_dsl(cls, match: re.Match, **kwargs: Any) -> CancelFlowCommand:
@@ -133,4 +133,4 @@ class CancelFlowCommand(Command):
 
     @staticmethod
     def regex_pattern() -> str:
-        return r"CancelFlow\(\)"
+        return r"^cancel$"

{rasa_pro-3.12.0.dev7 → rasa_pro-3.12.0.dev9}/rasa/dialogue_understanding/commands/change_flow_command.py

@@ -48,7 +48,7 @@ class ChangeFlowCommand(Command):
 
     def to_dsl(self) -> str:
         """Converts the command to a DSL string."""
-        return "ChangeFlow()"
+        return "change"
 
     @staticmethod
     def from_dsl(match: re.Match, **kwargs: Any) -> ChangeFlowCommand:
@@ -57,4 +57,4 @@ class ChangeFlowCommand(Command):
 
     @staticmethod
     def regex_pattern() -> str:
-        return r"ChangeFlow\(\)"
+        return r"^change"

{rasa_pro-3.12.0.dev7 → rasa_pro-3.12.0.dev9}/rasa/dialogue_understanding/commands/chit_chat_answer_command.py

@@ -59,7 +59,7 @@ class ChitChatAnswerCommand(FreeFormAnswerCommand):
 
     def to_dsl(self) -> str:
         """Converts the command to a DSL string."""
-        return "ChitChat()"
+        return "chat"
 
     @classmethod
     def from_dsl(cls, match: re.Match, **kwargs: Any) -> ChitChatAnswerCommand:
@@ -68,4 +68,4 @@ class ChitChatAnswerCommand(FreeFormAnswerCommand):
 
     @staticmethod
     def regex_pattern() -> str:
-        return r"ChitChat\(\)"
+        return r"^chat$"

{rasa_pro-3.12.0.dev7 → rasa_pro-3.12.0.dev9}/rasa/dialogue_understanding/commands/clarify_command.py

@@ -89,7 +89,7 @@ class ClarifyCommand(Command):
 
     def to_dsl(self) -> str:
         """Converts the command to a DSL string."""
-        return f"Clarify({', '.join(self.options)})"
+        return f"clarify {' '.join(self.options)}"
 
     @classmethod
     def from_dsl(cls, match: re.Match, **kwargs: Any) -> Optional[ClarifyCommand]:
@@ -99,4 +99,4 @@ class ClarifyCommand(Command):
 
     @staticmethod
     def regex_pattern() -> str:
-        return r"Clarify\(([\"\'a-zA-Z0-9_, ]*)\)"
+        return r"^clarify([\"\'a-zA-Z0-9_, ]*)$"

{rasa_pro-3.12.0.dev7 → rasa_pro-3.12.0.dev9}/rasa/dialogue_understanding/commands/human_handoff_command.py

@@ -66,7 +66,7 @@ class HumanHandoffCommand(Command):
 
     def to_dsl(self) -> str:
         """Converts the command to a DSL string."""
-        return "HumanHandoff()"
+        return "hand over"
 
     @classmethod
     def from_dsl(cls, match: re.Match, **kwargs: Any) -> HumanHandoffCommand:
@@ -75,4 +75,4 @@ class HumanHandoffCommand(Command):
 
     @staticmethod
     def regex_pattern() -> str:
-        return r"HumanHandoff\(\)"
+        return r"^hand over$"

{rasa_pro-3.12.0.dev7 → rasa_pro-3.12.0.dev9}/rasa/dialogue_understanding/commands/knowledge_answer_command.py

@@ -59,7 +59,7 @@ class KnowledgeAnswerCommand(FreeFormAnswerCommand):
 
     def to_dsl(self) -> str:
         """Converts the command to a DSL string."""
-        return "SearchAndReply()"
+        return "search"
 
     @classmethod
     def from_dsl(cls, match: re.Match, **kwargs: Any) -> KnowledgeAnswerCommand:
@@ -68,4 +68,4 @@ class KnowledgeAnswerCommand(FreeFormAnswerCommand):
 
     @staticmethod
     def regex_pattern() -> str:
-        return r"SearchAndReply\(\)"
+        return r"^search$"

{rasa_pro-3.12.0.dev7 → rasa_pro-3.12.0.dev9}/rasa/dialogue_understanding/commands/repeat_bot_messages_command.py

@@ -60,7 +60,7 @@ class RepeatBotMessagesCommand(Command):
 
     def to_dsl(self) -> str:
         """Converts the command to a DSL string."""
-        return "RepeatLastBotMessages()"
+        return "repeat message"
 
     @classmethod
     def from_dsl(cls, match: re.Match, **kwargs: Any) -> RepeatBotMessagesCommand:
@@ -69,4 +69,4 @@ class RepeatBotMessagesCommand(Command):
 
     @staticmethod
     def regex_pattern() -> str:
-        return r"RepeatLastBotMessages\(\)"
+        return r"^repeat message$"

{rasa_pro-3.12.0.dev7 → rasa_pro-3.12.0.dev9}/rasa/dialogue_understanding/commands/set_slot_command.py

@@ -170,7 +170,7 @@ class SetSlotCommand(Command):
 
     def to_dsl(self) -> str:
         """Converts the command to a DSL string."""
-        return f"SetSlot({self.name}, {self.value})"
+        return f"set {self.name} {self.value}"
 
     @classmethod
     def from_dsl(cls, match: re.Match, **kwargs: Any) -> SetSlotCommand:
@@ -181,4 +181,4 @@ class SetSlotCommand(Command):
 
     @staticmethod
     def regex_pattern() -> str:
-        return r"""SetSlot\(['"]?([a-zA-Z_][a-zA-Z0-9_-]*)['"]?, ?['"]?(.*)['"]?\)"""
+        return r"""^set ['"]?([a-zA-Z_][a-zA-Z0-9_-]*)['"]? ['"]?(.+?)['"]?$"""

{rasa_pro-3.12.0.dev7 → rasa_pro-3.12.0.dev9}/rasa/dialogue_understanding/commands/skip_question_command.py

@@ -75,7 +75,7 @@ class SkipQuestionCommand(Command):
 
     def to_dsl(self) -> str:
         """Converts the command to a DSL string."""
-        return "SkipQuestion()"
+        return "skip"
 
     @classmethod
     def from_dsl(cls, match: re.Match, **kwargs: Any) -> SkipQuestionCommand:
@@ -84,4 +84,4 @@ class SkipQuestionCommand(Command):
 
     @staticmethod
     def regex_pattern() -> str:
-        return r"SkipQuestion\(\)"
+        return r"^skip$"

{rasa_pro-3.12.0.dev7 → rasa_pro-3.12.0.dev9}/rasa/dialogue_understanding/commands/start_flow_command.py

@@ -110,7 +110,7 @@ class StartFlowCommand(Command):
 
     def to_dsl(self) -> str:
         """Converts the command to a DSL string."""
-        return f"StartFlow({self.flow})"
+        return f"start {self.flow}"
 
     @classmethod
     def from_dsl(cls, match: re.Match, **kwargs: Any) -> Optional[StartFlowCommand]:
@@ -119,4 +119,4 @@ class StartFlowCommand(Command):
 
     @staticmethod
     def regex_pattern() -> str:
-        return r"StartFlow\(['\"]?([a-zA-Z0-9_-]+)['\"]?\)"
+        return r"^start ['\"]?([a-zA-Z0-9_-]+)['\"]?$"

{rasa_pro-3.12.0.dev7 → rasa_pro-3.12.0.dev9}/rasa/dialogue_understanding/commands/utils.py

@@ -27,7 +27,7 @@ def extract_cleaned_options(options_str: str) -> List[str]:
     """Extract and clean options from a string."""
     return sorted(
         opt.strip().strip('"').strip("'")
-        for opt in options_str.split(",")
+        for opt in options_str.split(" ")
         if opt.strip()
     )
 

{rasa_pro-3.12.0.dev7 → rasa_pro-3.12.0.dev9}/rasa/dialogue_understanding/generator/command_parser.py

@@ -125,7 +125,7 @@ def _parse_standard_commands(
     commands: List[Command] = []
     for command_clz in standard_commands:
         pattern = _get_compiled_pattern(command_clz.regex_pattern())
-        if match := pattern.search(action):
+        if match := pattern.search(action.strip()):
             parsed_command = command_clz.from_dsl(match, **kwargs)
             if _additional_parsing_fn := _get_additional_parsing_logic(command_clz):
                 parsed_command = _additional_parsing_fn(parsed_command, flows, **kwargs)

{rasa_pro-3.12.0.dev7 → rasa_pro-3.12.0.dev9}/rasa/dialogue_understanding/generator/llm_based_command_generator.py

@@ -357,7 +357,8 @@ class LLMBasedCommandGenerator(
                     "slots": slots_with_info,
                 }
             )
-        return result
+
+        return sorted(result, key=lambda x: x["name"])
 
     @staticmethod
     def is_extractable(

rasa_pro-3.12.0.dev9/rasa/dialogue_understanding/generator/single_step/command_prompt_template.jinja2

@@ -0,0 +1,58 @@
+Your task is to analyze the current conversation context and generate a list of actions to start new business processes that we call flows, to extract slots, or respond to small talk and knowledge requests.
+
+
+## Available Actions:
+* Starting a flow, described by "start flow_name". For example, "start transfer_money" or "start list_contacts"
+* Slot setting, described by "set slot_name slot_value". For example, "set transfer_money_recipient Freddy". Can be used to correct and change previously set values.
+* Cancelling the current flow, described by "cancel"
+* Clarifying which flow should be started in ambiguous cases. For example, "clarify list_contacts add_contact remove_contact" if the user just wrote "contacts" and there are multiple potential candidates.
+* Skipping the current question when the user explicitly asks for it, described by "skip".
+* Responding to knowledge-oriented user messages, described by "search"
+* Responding to a casual, non-task-oriented user message, described by "chat".
+* Handing over to a human, in case the user seems frustrated or explicitly asks to speak to one, described by "hand over".
+
+
+## General Tips
+* Do not fill slots with abstract values or placeholders.
+* Only use information provided by the user.
+* Use clarification in ambiguous cases.
+* Multiple flows can be started. If a user wants to digress into a second flow, you do not need to cancel the current flow.
+* Strictly adhere to the provided action format.
+* For categorical slots try to match the user message with potential slot values. Use "other" if you cannot match it
+* Focus on the last message and take it one step at a time.
+* Use the previous conversation steps only to aid understanding.
+
+
+## Available Flows:
+{% for flow in available_flows %}
+* {{ flow.name }}: {{ flow.description }}
+    {% for slot in flow.slots -%}
+    * {{ slot.name }}{% if slot.description %} ({{ slot.description }}){% endif %}{% if slot.allowed_values %}, allowed values: {{ slot.allowed_values }}{% endif %}
+    {% endfor %}
+{%- endfor %}
+
+
+## Current State
+{% if current_flow != None %}
+You are currently in the flow "{{ current_flow }}".
+You have just asked the user for the slot "{{ current_slot }}"{% if current_slot_description %} ({{ current_slot_description }}){% endif %}.
+
+{% if flow_slots|length > 0 %}
+Here are the slots of the flow "{{ current_flow }}":
+{% for slot in flow_slots -%}
+* name: {{ slot.name }}, value: {{ slot.value }}, type: {{ slot.type }}, description: {{ slot.description}}{% if slot.allowed_values %}, allowed values: {{ slot.allowed_values }}{% endif %}
+{% endfor %}
+{% endif %}
+{% else %}
+You are currently not inside any flow.
+{% endif %}
+
+
+## Conversation History
+{{ current_conversation }}
+
+
+## Task
+Create an action list with one action per line in response to the users last message: """{{ user_message }}""".
+
+Your action list:

{rasa_pro-3.12.0.dev7 → rasa_pro-3.12.0.dev9}/rasa/dialogue_understanding_test/du_test_case.py

@@ -1,6 +1,6 @@
 from typing import Any, Dict, Iterator, List, Optional, Tuple
 
-from pydantic import BaseModel, ConfigDict, Field
+from pydantic import BaseModel, Field
 
 from rasa.dialogue_understanding.commands import Command
 from rasa.dialogue_understanding.generator.command_parser import parse_commands
@@ -20,10 +20,17 @@ from rasa.dialogue_understanding_test.constants import (
 from rasa.shared.core.flows import FlowsList
 from rasa.shared.nlu.constants import (
     KEY_COMPONENT_NAME,
+    KEY_LATENCY,
+    KEY_LLM_RESPONSE_METADATA,
     KEY_PROMPT_NAME,
+    KEY_SYSTEM_PROMPT,
     KEY_USER_PROMPT,
 )
 
+KEY_USAGE = "usage"
+KEY_PROMPT_TOKENS = "prompt_tokens"
+KEY_COMPLETION_TOKENS = "completion_tokens"
+
 
 class DialogueUnderstandingOutput(BaseModel):
     """Output containing prompts and generated commands by component.
@@ -62,8 +69,6 @@ class DialogueUnderstandingOutput(BaseModel):
     # List of prompts
     prompts: Optional[List[Dict[str, Any]]] = None
 
-    model_config = ConfigDict(frozen=True)
-
     def get_predicted_commands(self) -> List[Command]:
         """Get all commands from the output."""
         return [
@@ -82,15 +87,31 @@ class DialogueUnderstandingOutput(BaseModel):
             if predicted_commands
         ]
 
-    def get_component_name_to_user_prompts(self) -> Dict[str, List[Tuple[str, str]]]:
-        """Return a dictionary of component names to a list of prompts.
+    def get_component_name_to_prompt_info(self) -> Dict[str, List[Dict[str, Any]]]:
+        """Return a dictionary of component names to prompt information.
 
-        The prompts are represented as tuples of (prompt_name, user_prompt).
+        The prompt information includes the prompt name, user prompt, system prompt,
+        latency, and usage information.
+        The return dict is of the form:
+        {
+            "component_name": [
+                {
+                    "prompt_name": "...",
+                    "user_prompt": "...",
+                    "system_prompt": "...",
+                    "latency": 0.1,
+                    "prompt_tokens": 10,
+                    "completion_tokens": 20
+                },
+                ...
+            ],
+            ...
+        }
         """
         if self.prompts is None:
             return {}
 
-        data: Dict[str, List[Tuple[str, str]]] = {}
+        data: Dict[str, List[Dict[str, Any]]] = {}
         relevant_component_names = self.get_component_names_that_predicted_commands()
 
         for prompt_data in self.prompts:
@@ -99,12 +120,31 @@ class DialogueUnderstandingOutput(BaseModel):
             if component_name not in relevant_component_names:
                 continue
 
-            prompt_name = prompt_data[KEY_PROMPT_NAME]
-            user_prompt = prompt_data[KEY_USER_PROMPT]
-
             if component_name not in data:
                 data[component_name] = []
-            data[component_name].append((prompt_name, user_prompt))
+
+            prompt_info = {
+                KEY_PROMPT_NAME: prompt_data[KEY_PROMPT_NAME],
+                KEY_USER_PROMPT: prompt_data[KEY_USER_PROMPT],
+            }
+
+            latency = prompt_data.get(KEY_LLM_RESPONSE_METADATA, {}).get(KEY_LATENCY)
+            if latency:
+                prompt_info[KEY_LATENCY] = latency
+
+            if prompt_data.get(KEY_SYSTEM_PROMPT):
+                prompt_info[KEY_SYSTEM_PROMPT] = prompt_data[KEY_SYSTEM_PROMPT]
+
+            usage_object = prompt_data.get(KEY_LLM_RESPONSE_METADATA, {}).get(KEY_USAGE)
+            if usage_object:
+                if usage_object.get(KEY_PROMPT_TOKENS):
+                    prompt_info[KEY_PROMPT_TOKENS] = usage_object.get(KEY_PROMPT_TOKENS)
+                if usage_object.get(KEY_COMPLETION_TOKENS):
+                    prompt_info[KEY_COMPLETION_TOKENS] = usage_object.get(
+                        KEY_COMPLETION_TOKENS
+                    )
+
+            data[component_name].append(prompt_info)
 
         return data
 
@@ -149,7 +189,7 @@ class DialogueUnderstandingTestStep(BaseModel):
         Args:
             step: Dictionary containing the step.
             flows: List of flows.
-            custom_commands: Custom commands to use in the test case.
+            custom_command_classes: Custom commands to use in the test case.
             remove_default_commands: Default commands to remove from the test case.
 
         Returns:
@@ -210,6 +250,42 @@ class DialogueUnderstandingTestStep(BaseModel):
 
         return ""
 
+    def get_latencies(self) -> List[float]:
+        if self.dialogue_understanding_output is None:
+            return []
+
+        prompts = self.dialogue_understanding_output.get_component_name_to_prompt_info()
+
+        return [
+            prompt_data.get(KEY_LATENCY, 0.0)
+            for prompt in prompts.values()
+            for prompt_data in prompt
+        ]
+
+    def get_completion_tokens(self) -> List[int]:
+        if self.dialogue_understanding_output is None:
+            return []
+
+        prompts = self.dialogue_understanding_output.get_component_name_to_prompt_info()
+
+        return [
+            prompt_data.get(KEY_COMPLETION_TOKENS, 0)
+            for prompt in prompts.values()
+            for prompt_data in prompt
+        ]
+
+    def get_prompt_tokens(self) -> List[int]:
+        if self.dialogue_understanding_output is None:
+            return []
+
+        prompts = self.dialogue_understanding_output.get_component_name_to_prompt_info()
+
+        return [
+            prompt_data.get(KEY_PROMPT_TOKENS, 0)
+            for prompt in prompts.values()
+            for prompt_data in prompt
+        ]
+
 
 class DialogueUnderstandingTestCase(BaseModel):
     name: str

{rasa_pro-3.12.0.dev7 → rasa_pro-3.12.0.dev9}/rasa/dialogue_understanding_test/du_test_result.py

@@ -1,6 +1,8 @@
+import copy
 import typing
 from typing import Any, Dict, List, Optional, Text
 
+import numpy as np
 from pydantic import BaseModel
 
 from rasa.dialogue_understanding.commands import Command
@@ -9,6 +11,10 @@ from rasa.dialogue_understanding_test.du_test_case import (
     DialogueUnderstandingTestStep,
 )
 from rasa.dialogue_understanding_test.utils import get_command_comparison
+from rasa.shared.nlu.constants import (
+    KEY_SYSTEM_PROMPT,
+    KEY_USER_PROMPT,
+)
 
 if typing.TYPE_CHECKING:
     from rasa.dialogue_understanding_test.command_metric_calculation import (
@@ -25,6 +31,9 @@ OUTPUT_USER_UTTERANCES_ACCURACY = "user_utterances_accuracy"
 OUTPUT_NUMBER_OF_PASSED_USER_UTTERANCES = "number_of_passed_user_utterances"
 OUTPUT_NUMBER_OF_FAILED_USER_UTTERANCES = "number_of_failed_user_utterances"
 OUTPUT_COMMAND_METRICS = "command_metrics"
+OUTPUT_LATENCY_METRICS = "latency"
+OUTPUT_COMPLETION_TOKEN_METRICS = "completion_token"
+OUTPUT_PROMPT_TOKEN_METRICS = "prompt_token"
 OUTPUT_NAMES_OF_FAILED_TESTS = "names_of_failed_tests"
 OUTPUT_NAMES_OF_PASSED_TESTS = "names_of_passed_tests"
 OUTPUT_LLM_COMMAND_GENERATOR_CONFIG = "llm_command_generator_config"
@@ -50,7 +59,7 @@ class FailedTestStep(BaseModel):
     error_line: int
     pass_status: bool
     command_generators: List[str]
-    prompt: Optional[Dict[str, Any]] = None
+    prompts: Optional[Dict[str, List[Dict[str, Any]]]] = None
     expected_commands: List[Command]
     predicted_commands: Dict[str, List[Command]]
     conversation_with_diff: List[str]
@@ -66,14 +75,14 @@ class FailedTestStep(BaseModel):
         line_number = step.line or -1
 
         predicted_commands: Dict[str, List[Command]] = {}
-        prompts: Optional[Dict[str, Any]] = None
+        prompts: Optional[Dict[str, List[Dict[str, Any]]]] = None
         command_generators: List[str] = []
 
         if step.dialogue_understanding_output:
             predicted_commands = step.dialogue_understanding_output.commands
             command_generators = step.dialogue_understanding_output.get_component_names_that_predicted_commands()  # noqa: E501
             prompts = (
-                step.dialogue_understanding_output.get_component_name_to_user_prompts()
+                step.dialogue_understanding_output.get_component_name_to_prompt_info()
             )
 
         step_index = test_case.steps.index(step)
@@ -89,13 +98,13 @@ class FailedTestStep(BaseModel):
             error_line=line_number,
             pass_status=False,
             command_generators=command_generators,
-            prompt=prompts,
+            prompts=prompts,
             expected_commands=step.commands or [],
             predicted_commands=predicted_commands,
             conversation_with_diff=conversation_with_diff,
         )
 
-    def to_dict(self, output_prompt: bool) -> Dict[Text, Any]:
+    def to_dict(self, output_prompt: bool) -> Dict[str, Any]:
         step_info = {
             "file": self.file,
             "test_case": self.test_case_name,
@@ -115,19 +124,17 @@ class FailedTestStep(BaseModel):
             ],
         }
 
-        if output_prompt and self.prompt:
-            step_info["prompts"] = [
-                {
-                    component: [
-                        {
-                            "prompt_name": prompt_name,
-                            "prompt_content": prompt_content,
-                        }
-                        for prompt_name, prompt_content in prompts
-                    ],
-                }
-                for component, prompts in self.prompt.items()
-            ]
+        if output_prompt and self.prompts:
+            step_info["prompts"] = copy.deepcopy(self.prompts)
+        elif self.prompts:
+            prompts = copy.deepcopy(self.prompts)
+            # remove user and system prompts
+            for prompt_data in prompts.values():
+                for prompt_info in prompt_data:
+                    prompt_info.pop(KEY_USER_PROMPT, None)
+                    prompt_info.pop(KEY_SYSTEM_PROMPT, None)
+
+                step_info["prompts"] = prompts
 
         return step_info
 
@@ -153,6 +160,9 @@ class DialogueUnderstandingTestSuiteResult:
         self.names_of_passed_tests: List[str] = []
         self.failed_test_steps: List[FailedTestStep] = []
         self.llm_config: Optional[Dict[str, Any]] = None
+        self.latency_metrics: Dict[str, float] = {}
+        self.prompt_token_metrics: Dict[str, float] = {}
+        self.completion_token_metrics: Dict[str, float] = {}
 
     @classmethod
     def from_results(
@@ -206,6 +216,16 @@ class DialogueUnderstandingTestSuiteResult:
             failing_test_results
         )
 
+        instance.latency_metrics = cls.get_latency_metrics(
+            failing_test_results, passing_test_results
+        )
+        instance.prompt_token_metrics = cls.get_prompt_token_metrics(
+            failing_test_results, passing_test_results
+        )
+        instance.completion_token_metrics = cls.get_completion_token_metrics(
+            failing_test_results, passing_test_results
+        )
+
         instance.llm_config = llm_config
 
         return instance
@@ -264,7 +284,60 @@ class DialogueUnderstandingTestSuiteResult:
 
         return failed_test_steps
 
-    def to_dict(self, output_prompt: bool = False) -> Dict[Text, Any]:
+    @staticmethod
+    def _calculate_percentiles(values: List[float]) -> Dict[str, float]:
+        return {
+            "p50": float(np.percentile(values, 50)) if values else 0.0,
+            "p90": float(np.percentile(values, 90)) if values else 0.0,
+            "p99": float(np.percentile(values, 99)) if values else 0.0,
+        }
+
+    @classmethod
+    def get_latency_metrics(
+        cls,
+        failing_test_results: List["DialogueUnderstandingTestResult"],
+        passing_test_results: List["DialogueUnderstandingTestResult"],
+    ) -> Dict[str, float]:
+        latencies = [
+            latency
+            for result in failing_test_results + passing_test_results
+            for step in result.test_case.steps
+            for latency in step.get_latencies()
+        ]
+
+        return cls._calculate_percentiles(latencies)
+
+    @classmethod
+    def get_prompt_token_metrics(
+        cls,
+        failing_test_results: List["DialogueUnderstandingTestResult"],
+        passing_test_results: List["DialogueUnderstandingTestResult"],
+    ) -> Dict[str, float]:
+        tokens = [
+            token_count
+            for result in failing_test_results + passing_test_results
+            for step in result.test_case.steps
+            for token_count in step.get_prompt_tokens()
+        ]
+
+        return cls._calculate_percentiles(tokens)
+
+    @classmethod
+    def get_completion_token_metrics(
+        cls,
+        failing_test_results: List["DialogueUnderstandingTestResult"],
+        passing_test_results: List["DialogueUnderstandingTestResult"],
+    ) -> Dict[str, float]:
+        tokens = [
+            token_count
+            for result in failing_test_results + passing_test_results
+            for step in result.test_case.steps
+            for token_count in step.get_completion_tokens()
+        ]
+
+        return cls._calculate_percentiles(tokens)
+
+    def to_dict(self, output_prompt: bool = False) -> Dict[str, Any]:
         """Builds a dictionary for writing test results to a YML file.
 
         Args:
@@ -292,6 +365,10 @@ class DialogueUnderstandingTestSuiteResult:
 
         result_dict[OUTPUT_COMMAND_METRICS] = cmd_metrics_output
 
+        result_dict[OUTPUT_LATENCY_METRICS] = self.latency_metrics
+        result_dict[OUTPUT_PROMPT_TOKEN_METRICS] = self.prompt_token_metrics
+        result_dict[OUTPUT_COMPLETION_TOKEN_METRICS] = self.completion_token_metrics
+
         result_dict[OUTPUT_NAMES_OF_PASSED_TESTS] = self.names_of_passed_tests
         result_dict[OUTPUT_NAMES_OF_FAILED_TESTS] = self.names_of_failed_tests