rasa-pro 3.12.22__py3-none-any.whl → 3.13.0__py3-none-any.whl

rasa/dialogue_understanding_test/du_test_case.py

@@ -1,7 +1,11 @@
+from collections import defaultdict
 from typing import Any, Dict, Iterator, List, Optional, Tuple
 
 from pydantic import BaseModel, Field
 
+from rasa.core.nlg.contextual_response_rephraser import ContextualResponseRephraser
+from rasa.core.policies.enterprise_search_policy import EnterpriseSearchPolicy
+from rasa.core.policies.intentless_policy import IntentlessPolicy
 from rasa.dialogue_understanding.commands.prompt_command import PromptCommand
 from rasa.dialogue_understanding.generator.command_parser import parse_commands
 from rasa.dialogue_understanding_test.command_comparison import are_command_lists_equal
@@ -69,6 +73,8 @@ class DialogueUnderstandingOutput(BaseModel):
     commands: Dict[str, List[PromptCommand]]
     # List of prompts
     prompts: Optional[List[Dict[str, Any]]] = None
+    # Latency of the full message roundtrip
+    latency: Optional[float] = None
 
     class Config:
         """Skip validation for PromptCommand protocol as pydantic does not know how to
@@ -88,27 +94,41 @@ class DialogueUnderstandingOutput(BaseModel):
     def get_component_names_that_predicted_commands_or_have_llm_response(
         self,
     ) -> List[str]:
-        """Get all component names that have predicted commands or recieved
+        """Get all relevant component names.
+
+        Components are relevant if they have predicted commands or received a
         non-empty response from LLM.
         """
+        # Exclude components that are not related to Dialogue Understanding
+        component_names_to_exclude = [
+            EnterpriseSearchPolicy.__name__,
+            IntentlessPolicy.__name__,
+            ContextualResponseRephraser.__name__,
+        ]
+
         component_names_that_predicted_commands = (
             [
                 component_name
                 for component_name, predicted_commands in self.commands.items()
                 if predicted_commands
+                and component_name not in component_names_to_exclude
             ]
             if self.commands
             else []
         )
+
         components_with_prompts = (
             [
                 str(prompt.get(KEY_COMPONENT_NAME, None))
                 for prompt in self.prompts
                 if prompt.get(KEY_LLM_RESPONSE_METADATA, None)
+                and prompt.get(KEY_COMPONENT_NAME, None)
+                not in component_names_to_exclude
             ]
             if self.prompts
             else []
         )
+
         return list(
             set(component_names_that_predicted_commands + components_with_prompts)
         )
@@ -298,41 +318,54 @@ class DialogueUnderstandingTestStep(BaseModel):
 
         return ""
 
-    def get_latencies(self) -> List[float]:
+    def get_latencies(self) -> Dict[str, List[float]]:
         if self.dialogue_understanding_output is None:
-            return []
+            return {}
 
-        prompts = self.dialogue_understanding_output.get_component_name_to_prompt_info()
+        component_name_to_prompt_info = (
+            self.dialogue_understanding_output.get_component_name_to_prompt_info()
+        )
 
-        return [
-            prompt_data.get(KEY_LATENCY, 0.0)
-            for prompt in prompts.values()
-            for prompt_data in prompt
-        ]
+        latencies = defaultdict(list)
+        for component_name, prompt_info_list in component_name_to_prompt_info.items():
+            for prompt_info in prompt_info_list:
+                latencies[component_name].append(prompt_info.get(KEY_LATENCY, 0.0))
 
-    def get_completion_tokens(self) -> List[int]:
+        return latencies
+
+    def get_completion_tokens(self) -> Dict[str, List[float]]:
         if self.dialogue_understanding_output is None:
-            return []
+            return {}
 
-        prompts = self.dialogue_understanding_output.get_component_name_to_prompt_info()
+        component_name_to_prompt_info = (
+            self.dialogue_understanding_output.get_component_name_to_prompt_info()
+        )
 
-        return [
-            prompt_data.get(KEY_COMPLETION_TOKENS, 0)
-            for prompt in prompts.values()
-            for prompt_data in prompt
-        ]
+        completion_tokens = defaultdict(list)
+        for component_name, prompt_info_list in component_name_to_prompt_info.items():
+            for prompt_info in prompt_info_list:
+                completion_tokens[component_name].append(
+                    prompt_info.get(KEY_COMPLETION_TOKENS, 0.0)
+                )
+
+        return completion_tokens
 
-    def get_prompt_tokens(self) -> List[int]:
+    def get_prompt_tokens(self) -> Dict[str, List[float]]:
         if self.dialogue_understanding_output is None:
-            return []
+            return {}
 
-        prompts = self.dialogue_understanding_output.get_component_name_to_prompt_info()
+        component_name_to_prompt_info = (
+            self.dialogue_understanding_output.get_component_name_to_prompt_info()
+        )
 
-        return [
-            prompt_data.get(KEY_PROMPT_TOKENS, 0)
-            for prompt in prompts.values()
-            for prompt_data in prompt
-        ]
+        prompt_tokens = defaultdict(list)
+        for component_name, prompt_info_list in component_name_to_prompt_info.items():
+            for prompt_info in prompt_info_list:
+                prompt_tokens[component_name].append(
+                    prompt_info.get(KEY_PROMPT_TOKENS, 0.0)
+                )
+
+        return prompt_tokens
 
 
 class DialogueUnderstandingTestCase(BaseModel):

rasa/dialogue_understanding_test/du_test_result.py

@@ -1,11 +1,14 @@
 import copy
-import typing
+from collections import defaultdict
 from typing import Any, Dict, List, Optional, Text
 
 import numpy as np
 from pydantic import BaseModel
 
 from rasa.dialogue_understanding.commands.prompt_command import PromptCommand
+from rasa.dialogue_understanding_test.command_metrics import (
+    CommandMetrics,
+)
 from rasa.dialogue_understanding_test.du_test_case import (
     DialogueUnderstandingTestCase,
     DialogueUnderstandingTestStep,
@@ -13,26 +16,40 @@ from rasa.dialogue_understanding_test.du_test_case import (
 from rasa.dialogue_understanding_test.utils import get_command_comparison
 from rasa.shared.nlu.constants import KEY_SYSTEM_PROMPT, KEY_USER_PROMPT
 
-if typing.TYPE_CHECKING:
-    from rasa.dialogue_understanding_test.command_metric_calculation import (
-        CommandMetrics,
-    )
-
 KEY_TEST_CASES_ACCURACY = "test_cases"
 KEY_USER_UTTERANCES_ACCURACY = "user_utterances"
 
+KEY_COMMANDS_F1_MACRO = "macro"
+KEY_COMMANDS_F1_MICRO = "micro"
+KEY_COMMANDS_F1_WEIGHTED = "weighted_average"
+
+OUTPUT_DUT_ACCURACY = "accuracy"
+OUTPUT_DUT_ACCURACY_TEST_CASES = "test_cases"
+OUTPUT_DUT_ACCURACY_USER_UTTERANCES = "user_utterances"
+
+OUTPUT_COMMANDS_F1 = "f1_score"
+OUTPUT_COMMANDS_F1_MACRO = "macro"
+OUTPUT_COMMANDS_F1_MICRO = "micro"
+OUTPUT_COMMANDS_F1_WEIGHTED = "weighted_average"
+
 OUTPUT_NUMBER_OF_FAILED_TESTS = "number_of_failed_tests"
 OUTPUT_NUMBER_OF_PASSED_TESTS = "number_of_passed_tests"
-OUTPUT_TEST_CASES_ACCURACY = "test_cases_accuracy"
-OUTPUT_USER_UTTERANCES_ACCURACY = "user_utterances_accuracy"
 OUTPUT_NUMBER_OF_PASSED_USER_UTTERANCES = "number_of_passed_user_utterances"
 OUTPUT_NUMBER_OF_FAILED_USER_UTTERANCES = "number_of_failed_user_utterances"
+OUTPUT_NAMES_OF_FAILED_TESTS = "names_of_failed_tests"
+OUTPUT_NAMES_OF_PASSED_TESTS = "names_of_passed_tests"
+OUTPUT_FAILED_TEST_STEPS = "failed_test_steps"
+OUTPUT_TEST_CASES_ACCURACY = "test_cases_accuracy"
+OUTPUT_USER_UTTERANCES_ACCURACY = "user_utterances_accuracy"
 OUTPUT_COMMAND_METRICS = "command_metrics"
+OUTPUT_COMMANDS_F1_MACRO_INSTRUMENTATION_ATTR = "commands_f1_macro"
+OUTPUT_COMMANDS_F1_MICRO_INSTRUMENTATION_ATTR = "commands_f1_micro"
+OUTPUT_COMMANDS_F1_WEIGHTED_INSTRUMENTATION_ATTR = "commands_f1_weighted_average"
+
 OUTPUT_LATENCY_METRICS = "latency"
 OUTPUT_COMPLETION_TOKEN_METRICS = "completion_token"
 OUTPUT_PROMPT_TOKEN_METRICS = "prompt_token"
-OUTPUT_NAMES_OF_FAILED_TESTS = "names_of_failed_tests"
-OUTPUT_NAMES_OF_PASSED_TESTS = "names_of_passed_tests"
+
 OUTPUT_LLM_COMMAND_GENERATOR_CONFIG = "llm_command_generator_config"
 
 
@@ -60,6 +77,7 @@ class FailedTestStep(BaseModel):
     expected_commands: List[PromptCommand]
     predicted_commands: Dict[str, List[PromptCommand]]
     conversation_with_diff: List[str]
+    conversation_until_failed_user_utterance: List[str]
 
     class Config:
         """Skip validation for PromptCommand protocol as pydantic does not know how to
@@ -90,10 +108,12 @@ class FailedTestStep(BaseModel):
             )
 
         step_index = test_case.steps.index(step)
-
-        conversation_with_diff = test_case.to_readable_conversation(
+        conversation_until_failed_user_utterance = test_case.to_readable_conversation(
             until_step=step_index + 1
-        ) + get_command_comparison(step)
+        )
+        conversation_with_diff = (
+            conversation_until_failed_user_utterance + get_command_comparison(step)
+        )
 
         return cls(
             file=file_path,
@@ -106,12 +126,14 @@ class FailedTestStep(BaseModel):
             expected_commands=step.commands or [],
             predicted_commands=predicted_commands,
             conversation_with_diff=conversation_with_diff,
+            conversation_until_failed_user_utterance=conversation_until_failed_user_utterance,
         )
 
     def to_dict(self, output_prompt: bool) -> Dict[str, Any]:
         step_info = {
             "file": self.file,
             "test_case": self.test_case_name,
+            "conversation": self.conversation_until_failed_user_utterance,
             "failed_user_utterance": self.failed_user_utterance,
             "error_line": self.error_line,
             "pass_status": self.pass_status,
@@ -155,25 +177,32 @@ class DialogueUnderstandingTestSuiteResult:
             KEY_TEST_CASES_ACCURACY: 0.0,
             KEY_USER_UTTERANCES_ACCURACY: 0.0,
         }
+        self.f1_score = {
+            KEY_COMMANDS_F1_MACRO: 0.0,
+            KEY_COMMANDS_F1_MICRO: 0.0,
+            KEY_COMMANDS_F1_WEIGHTED: 0.0,
+        }
         self.number_of_passed_tests = 0
         self.number_of_failed_tests = 0
         self.number_of_passed_user_utterances = 0
         self.number_of_failed_user_utterances = 0
-        self.command_metrics: Optional[Dict[str, "CommandMetrics"]] = None
+        self.command_metrics: Optional[Dict[str, CommandMetrics]] = None
         self.names_of_failed_tests: List[str] = []
         self.names_of_passed_tests: List[str] = []
         self.failed_test_steps: List[FailedTestStep] = []
         self.llm_config: Optional[Dict[str, Any]] = None
-        self.latency_metrics: Dict[str, float] = {}
-        self.prompt_token_metrics: Dict[str, float] = {}
-        self.completion_token_metrics: Dict[str, float] = {}
+        # The performance metrics distribution per component
+        # For example: {"command_generator": {"p50": x, ...}, ...}
+        self.latency_metrics: Dict[str, Dict[str, float]] = {}
+        self.prompt_token_metrics: Dict[str, Dict[str, float]] = {}
+        self.completion_token_metrics: Dict[str, Dict[str, float]] = {}
 
     @classmethod
     def from_results(
         cls,
         failing_test_results: List[DialogueUnderstandingTestResult],
         passing_test_results: List[DialogueUnderstandingTestResult],
-        command_metrics: Dict[str, "CommandMetrics"],
+        command_metrics: Dict[str, CommandMetrics],
         llm_config: Optional[Dict[str, Any]],
     ) -> "DialogueUnderstandingTestSuiteResult":
         """Create a DialogueUnderstandingTestSuiteResult object from the test results.
@@ -207,6 +236,16 @@ class DialogueUnderstandingTestSuiteResult:
 
         instance.command_metrics = command_metrics
 
+        instance.f1_score[KEY_COMMANDS_F1_MACRO] = cls.calculate_f1_macro(
+            command_metrics
+        )
+        instance.f1_score[KEY_COMMANDS_F1_MICRO] = cls.calculate_f1_micro(
+            command_metrics
+        )
+        instance.f1_score[KEY_COMMANDS_F1_WEIGHTED] = cls.calculate_f1_weighted(
+            command_metrics
+        )
+
         instance.names_of_passed_tests = [
             passing_test_result.test_case.full_name()
             for passing_test_result in passing_test_results
@@ -234,131 +273,34 @@ class DialogueUnderstandingTestSuiteResult:
 
         return instance
 
-    def _set_user_utterance_metrics(
-        self,
-        failing_test_results: List[DialogueUnderstandingTestResult],
-        passing_test_results: List[DialogueUnderstandingTestResult],
-    ) -> None:
-        # Create list of booleans indicating whether each user utterance
-        # passed or failed
-        user_utterances_status = [
-            step.has_passed()
-            for test in failing_test_results + passing_test_results
-            for step in test.test_case.iterate_over_user_steps()
-        ]
-        # Calculate number of passed and failed user utterances
-        self.number_of_passed_user_utterances = sum(user_utterances_status)
-        self.number_of_failed_user_utterances = (
-            len(user_utterances_status) - self.number_of_passed_user_utterances
-        )
-        # Calculate user utterance accuracy
-        self.accuracy[KEY_USER_UTTERANCES_ACCURACY] = (
-            self.number_of_passed_user_utterances
-            / (
-                self.number_of_failed_user_utterances
-                + self.number_of_passed_user_utterances
-            )
-        )
-
-    @staticmethod
-    def _create_failed_steps_from_results(
-        failing_test_results: List["DialogueUnderstandingTestResult"],
-    ) -> List[FailedTestStep]:
-        """Create list of FailedTestStep objects from failing test results.
-
-        Given a list of failing DialogueUnderstandingTestResult objects,
-        create and return a list of FailedTestStep objects for each failing user step.
-
-        Args:
-            failing_test_results: Results of failing Dialogue Understanding tests.
-
-        Returns:
-            List of aggregated FailedTestStep objects for logging to console and file.
-        """
-        failed_test_steps: List[FailedTestStep] = []
-
-        for result in failing_test_results:
-            test_case = result.test_case
-            for step in test_case.failed_user_steps():
-                failed_test_steps.append(
-                    FailedTestStep.from_dialogue_understanding_test_step(
-                        step, test_case
-                    )
-                )
-
-        return failed_test_steps
-
-    @staticmethod
-    def _calculate_percentiles(values: List[float]) -> Dict[str, float]:
-        return {
-            "p50": float(np.percentile(values, 50)) if values else 0.0,
-            "p90": float(np.percentile(values, 90)) if values else 0.0,
-            "p99": float(np.percentile(values, 99)) if values else 0.0,
-        }
-
-    @classmethod
-    def get_latency_metrics(
-        cls,
-        failing_test_results: List["DialogueUnderstandingTestResult"],
-        passing_test_results: List["DialogueUnderstandingTestResult"],
-    ) -> Dict[str, float]:
-        latencies = [
-            latency
-            for result in failing_test_results + passing_test_results
-            for step in result.test_case.steps
-            for latency in step.get_latencies()
-        ]
-
-        return cls._calculate_percentiles(latencies)
-
-    @classmethod
-    def get_prompt_token_metrics(
-        cls,
-        failing_test_results: List["DialogueUnderstandingTestResult"],
-        passing_test_results: List["DialogueUnderstandingTestResult"],
-    ) -> Dict[str, float]:
-        tokens = [
-            token_count
-            for result in failing_test_results + passing_test_results
-            for step in result.test_case.steps
-            for token_count in step.get_prompt_tokens()
-        ]
-
-        return cls._calculate_percentiles(tokens)
-
-    @classmethod
-    def get_completion_token_metrics(
-        cls,
-        failing_test_results: List["DialogueUnderstandingTestResult"],
-        passing_test_results: List["DialogueUnderstandingTestResult"],
-    ) -> Dict[str, float]:
-        tokens = [
-            token_count
-            for result in failing_test_results + passing_test_results
-            for step in result.test_case.steps
-            for token_count in step.get_completion_tokens()
-        ]
-
-        return cls._calculate_percentiles(tokens)
-
     def to_dict(self, output_prompt: bool = False) -> Dict[str, Any]:
         """Builds a dictionary for writing test results to a YML file.
 
         Args:
             output_prompt: Whether to log the prompt or not.
         """
-        # 1. Accuracy block
         result_dict: Dict[Text, Any] = {
-            "accuracy": {
-                "test_cases": self.accuracy[KEY_TEST_CASES_ACCURACY],
-                "user_utterances": self.accuracy[KEY_USER_UTTERANCES_ACCURACY],
+            # Accuracy block
+            OUTPUT_DUT_ACCURACY: {
+                OUTPUT_DUT_ACCURACY_TEST_CASES: self.accuracy[KEY_TEST_CASES_ACCURACY],
+                OUTPUT_DUT_ACCURACY_USER_UTTERANCES: self.accuracy[
+                    KEY_USER_UTTERANCES_ACCURACY
+                ],
+            },
+            # F1 block
+            OUTPUT_COMMANDS_F1: {
+                OUTPUT_COMMANDS_F1_MACRO: self.f1_score[KEY_COMMANDS_F1_MACRO],
+                OUTPUT_COMMANDS_F1_MICRO: self.f1_score[KEY_COMMANDS_F1_MICRO],
+                OUTPUT_COMMANDS_F1_WEIGHTED: self.f1_score[KEY_COMMANDS_F1_WEIGHTED],
             },
+            # Other metrics block
             OUTPUT_NUMBER_OF_PASSED_TESTS: self.number_of_passed_tests,
             OUTPUT_NUMBER_OF_FAILED_TESTS: self.number_of_failed_tests,
             OUTPUT_NUMBER_OF_PASSED_USER_UTTERANCES: self.number_of_passed_user_utterances,  # noqa: E501
             OUTPUT_NUMBER_OF_FAILED_USER_UTTERANCES: self.number_of_failed_user_utterances,  # noqa: E501
         }
 
+        # Command metrics block
         cmd_metrics_output = {}
         if self.command_metrics:
             if isinstance(self.command_metrics, dict):
@@ -366,25 +308,179 @@ class DialogueUnderstandingTestSuiteResult:
                     cmd_metrics_output[cmd_name] = metrics_obj.as_dict()
             else:
                 pass
-
         result_dict[OUTPUT_COMMAND_METRICS] = cmd_metrics_output
 
+        # Latency and tokens metrics block
         result_dict[OUTPUT_LATENCY_METRICS] = self.latency_metrics
         result_dict[OUTPUT_PROMPT_TOKEN_METRICS] = self.prompt_token_metrics
         result_dict[OUTPUT_COMPLETION_TOKEN_METRICS] = self.completion_token_metrics
 
+        # Passed and failed test names block
         result_dict[OUTPUT_NAMES_OF_PASSED_TESTS] = self.names_of_passed_tests
         result_dict[OUTPUT_NAMES_OF_FAILED_TESTS] = self.names_of_failed_tests
 
+        # Failed test steps block
         failed_steps_list = []
         for failed_test_step in self.failed_test_steps:
             failed_steps_list.append(
                 failed_test_step.to_dict(output_prompt=output_prompt)
             )
+        result_dict[OUTPUT_FAILED_TEST_STEPS] = failed_steps_list
 
-        result_dict["failed_test_steps"] = failed_steps_list
-
+        # LLM config block
         if self.llm_config:
             result_dict[OUTPUT_LLM_COMMAND_GENERATOR_CONFIG] = self.llm_config
 
         return result_dict
+
+    @staticmethod
+    def calculate_f1_macro(command_metrics: Dict[str, CommandMetrics]) -> float:
+        f1_scores = [metrics.get_f1_score() for metrics in command_metrics.values()]
+        return sum(f1_scores) / len(f1_scores)
+
+    @staticmethod
+    def calculate_f1_micro(command_metrics: Dict[str, CommandMetrics]) -> float:
+        combined_metrics = CommandMetrics(
+            tp=sum([metrics.tp for metrics in command_metrics.values()]),
+            fp=sum([metrics.fp for metrics in command_metrics.values()]),
+            fn=sum([metrics.fn for metrics in command_metrics.values()]),
+            total_count=sum(m.total_count for m in command_metrics.values()),
+        )
+        return combined_metrics.get_f1_score()
+
+    @staticmethod
+    def calculate_f1_weighted(command_metrics: Dict[str, CommandMetrics]) -> float:
+        class_counts = []
+        f1_scores = []
+        for metrics in command_metrics.values():
+            class_counts.append(metrics.total_count)
+            f1_scores.append(metrics.get_f1_score())
+
+        total_count = sum(class_counts)
+        weighted_f1 = sum(
+            (count / total_count) * f1 for f1, count in zip(f1_scores, class_counts)
+        )
+        return weighted_f1
+
+    @classmethod
+    def get_latency_metrics(
+        cls,
+        failing_test_results: List["DialogueUnderstandingTestResult"],
+        passing_test_results: List["DialogueUnderstandingTestResult"],
+    ) -> Dict[str, Dict[str, float]]:
+        latencies = defaultdict(list)
+
+        for result in failing_test_results + passing_test_results:
+            for step in result.test_case.steps:
+                if (
+                    step.dialogue_understanding_output
+                    and step.dialogue_understanding_output.latency
+                ):
+                    latencies["total"].append(
+                        step.dialogue_understanding_output.latency
+                    )
+                for component_name, latency in step.get_latencies().items():
+                    latencies[component_name].extend(latency)
+
+        return {
+            component_name: cls._calculate_percentiles(latency_list)
+            for component_name, latency_list in latencies.items()
+        }
+
+    @classmethod
+    def get_prompt_token_metrics(
+        cls,
+        failing_test_results: List["DialogueUnderstandingTestResult"],
+        passing_test_results: List["DialogueUnderstandingTestResult"],
+    ) -> Dict[str, Dict[str, float]]:
+        tokens = defaultdict(list)
+
+        for result in failing_test_results + passing_test_results:
+            for step in result.test_case.steps:
+                for component_name, token_count in step.get_prompt_tokens().items():
+                    tokens[component_name].extend(token_count)
+
+        return {
+            component_name: cls._calculate_percentiles(latency_list)
+            for component_name, latency_list in tokens.items()
+        }
+
+    @classmethod
+    def get_completion_token_metrics(
+        cls,
+        failing_test_results: List["DialogueUnderstandingTestResult"],
+        passing_test_results: List["DialogueUnderstandingTestResult"],
+    ) -> Dict[str, Dict[str, float]]:
+        tokens = defaultdict(list)
+
+        for result in failing_test_results + passing_test_results:
+            for step in result.test_case.steps:
+                for component_name, token_count in step.get_completion_tokens().items():
+                    tokens[component_name].extend(token_count)
+
+        return {
+            component_name: cls._calculate_percentiles(latency_list)
+            for component_name, latency_list in tokens.items()
+        }
+
+    @staticmethod
+    def _calculate_percentiles(values: List[float]) -> Dict[str, float]:
+        return {
+            "p50": float(np.percentile(values, 50)) if values else 0.0,
+            "p90": float(np.percentile(values, 90)) if values else 0.0,
+            "p99": float(np.percentile(values, 99)) if values else 0.0,
+        }
+
+    @staticmethod
+    def _create_failed_steps_from_results(
+        failing_test_results: List["DialogueUnderstandingTestResult"],
+    ) -> List[FailedTestStep]:
+        """Create list of FailedTestStep objects from failing test results.
+
+        Given a list of failing DialogueUnderstandingTestResult objects,
+        create and return a list of FailedTestStep objects for each failing user step.
+
+        Args:
+            failing_test_results: Results of failing Dialogue Understanding tests.
+
+        Returns:
+            List of aggregated FailedTestStep objects for logging to console and file.
+        """
+        failed_test_steps: List[FailedTestStep] = []
+
+        for result in failing_test_results:
+            test_case = result.test_case
+            for step in test_case.failed_user_steps():
+                failed_test_steps.append(
+                    FailedTestStep.from_dialogue_understanding_test_step(
+                        step, test_case
+                    )
+                )
+
+        return failed_test_steps
+
+    def _set_user_utterance_metrics(
+        self,
+        failing_test_results: List[DialogueUnderstandingTestResult],
+        passing_test_results: List[DialogueUnderstandingTestResult],
+    ) -> None:
+        # Create list of booleans indicating whether each user utterance
+        # passed or failed
+        user_utterances_status = [
+            step.has_passed()
+            for test in failing_test_results + passing_test_results
+            for step in test.test_case.iterate_over_user_steps()
+        ]
+        # Calculate number of passed and failed user utterances
+        self.number_of_passed_user_utterances = sum(user_utterances_status)
+        self.number_of_failed_user_utterances = (
+            len(user_utterances_status) - self.number_of_passed_user_utterances
+        )
+        # Calculate user utterance accuracy
+        self.accuracy[KEY_USER_UTTERANCES_ACCURACY] = (
+            self.number_of_passed_user_utterances
+            / (
+                self.number_of_failed_user_utterances
+                + self.number_of_passed_user_utterances
+            )
+        )

rasa/dialogue_understanding_test/du_test_runner.py

@@ -1,13 +1,14 @@
 import asyncio
+import time
 from typing import Any, Dict, List, Optional, Text
 
 import structlog
 from tqdm import tqdm
 
+from rasa.core.available_endpoints import AvailableEndpoints
 from rasa.core.channels import CollectingOutputChannel, UserMessage
 from rasa.core.exceptions import AgentNotReady
 from rasa.core.persistor import StorageType
-from rasa.core.utils import AvailableEndpoints
 from rasa.dialogue_understanding.commands import Command
 from rasa.dialogue_understanding.utils import set_record_commands_and_prompts
 from rasa.dialogue_understanding_test.du_test_case import (
@@ -186,8 +187,12 @@ class DialogueUnderstandingTestRunner:
                 user_uttered_event_indices[user_step_index],
             )
 
+            # Total latency of a message roundtrip
+            latency = None
+
             # send the user message
             try:
+                start = time.time()
                 await self._send_user_message(
                     step_sender_id,
                     test_case,
@@ -195,6 +200,8 @@ class DialogueUnderstandingTestRunner:
                     metadata,
                     output_channel=output_channel,
                 )
+                end = time.time()
+                latency = end - start
             except Exception as e:
                 structlogger.error(
                     "dialogue_understanding_test_runner.send_user_message.failed",
@@ -212,7 +219,7 @@ class DialogueUnderstandingTestRunner:
             # get the dialogue understanding output
             tracker = await self.agent.tracker_store.retrieve(step_sender_id)
             dialogue_understanding_output = self.get_dialogue_understanding_output(
-                tracker, user_uttered_event_indices[user_step_index]
+                tracker, user_uttered_event_indices[user_step_index], latency
             )
             user_step.dialogue_understanding_output = dialogue_understanding_output
 
@@ -226,6 +233,7 @@ class DialogueUnderstandingTestRunner:
         self,
         tracker: DialogueStateTracker,
         index_user_uttered_event: int,
+        latency: Optional[float] = None,
     ) -> Optional[DialogueUnderstandingOutput]:
         """Returns the dialogue understanding output.
 
@@ -261,6 +269,7 @@ class DialogueUnderstandingTestRunner:
         return DialogueUnderstandingOutput(
             commands=commands,
             prompts=user_uttered_event.parse_data.get(PROMPTS, []),
+            latency=latency,
         )
 
     @staticmethod