rasa-pro 3.12.6.dev2__py3-none-any.whl → 3.13.0.dev2__py3-none-any.whl

rasa/dialogue_understanding_test/du_test_result.py

@@ -1,11 +1,14 @@
 import copy
-import typing
+from collections import defaultdict
 from typing import Any, Dict, List, Optional, Text
 
 import numpy as np
 from pydantic import BaseModel
 
 from rasa.dialogue_understanding.commands.prompt_command import PromptCommand
+from rasa.dialogue_understanding_test.command_metrics import (
+    CommandMetrics,
+)
 from rasa.dialogue_understanding_test.du_test_case import (
     DialogueUnderstandingTestCase,
     DialogueUnderstandingTestStep,
@@ -13,26 +16,40 @@ from rasa.dialogue_understanding_test.du_test_case import (
 from rasa.dialogue_understanding_test.utils import get_command_comparison
 from rasa.shared.nlu.constants import KEY_SYSTEM_PROMPT, KEY_USER_PROMPT
 
-if typing.TYPE_CHECKING:
-    from rasa.dialogue_understanding_test.command_metric_calculation import (
-        CommandMetrics,
-    )
-
 KEY_TEST_CASES_ACCURACY = "test_cases"
 KEY_USER_UTTERANCES_ACCURACY = "user_utterances"
 
+KEY_COMMANDS_F1_MACRO = "macro"
+KEY_COMMANDS_F1_MICRO = "micro"
+KEY_COMMANDS_F1_WEIGHTED = "weighted_average"
+
+OUTPUT_DUT_ACCURACY = "accuracy"
+OUTPUT_DUT_ACCURACY_TEST_CASES = "test_cases"
+OUTPUT_DUT_ACCURACY_USER_UTTERANCES = "user_utterances"
+
+OUTPUT_COMMANDS_F1 = "f1_score"
+OUTPUT_COMMANDS_F1_MACRO = "macro"
+OUTPUT_COMMANDS_F1_MICRO = "micro"
+OUTPUT_COMMANDS_F1_WEIGHTED = "weighted_average"
+
 OUTPUT_NUMBER_OF_FAILED_TESTS = "number_of_failed_tests"
 OUTPUT_NUMBER_OF_PASSED_TESTS = "number_of_passed_tests"
-OUTPUT_TEST_CASES_ACCURACY = "test_cases_accuracy"
-OUTPUT_USER_UTTERANCES_ACCURACY = "user_utterances_accuracy"
 OUTPUT_NUMBER_OF_PASSED_USER_UTTERANCES = "number_of_passed_user_utterances"
 OUTPUT_NUMBER_OF_FAILED_USER_UTTERANCES = "number_of_failed_user_utterances"
+OUTPUT_NAMES_OF_FAILED_TESTS = "names_of_failed_tests"
+OUTPUT_NAMES_OF_PASSED_TESTS = "names_of_passed_tests"
+OUTPUT_FAILED_TEST_STEPS = "failed_test_steps"
+OUTPUT_TEST_CASES_ACCURACY = "test_cases_accuracy"
+OUTPUT_USER_UTTERANCES_ACCURACY = "user_utterances_accuracy"
 OUTPUT_COMMAND_METRICS = "command_metrics"
+OUTPUT_COMMANDS_F1_MACRO_INSTRUMENTATION_ATTR = "commands_f1_macro"
+OUTPUT_COMMANDS_F1_MICRO_INSTRUMENTATION_ATTR = "commands_f1_micro"
+OUTPUT_COMMANDS_F1_WEIGHTED_INSTRUMENTATION_ATTR = "commands_f1_weighted_average"
+
 OUTPUT_LATENCY_METRICS = "latency"
 OUTPUT_COMPLETION_TOKEN_METRICS = "completion_token"
 OUTPUT_PROMPT_TOKEN_METRICS = "prompt_token"
-OUTPUT_NAMES_OF_FAILED_TESTS = "names_of_failed_tests"
-OUTPUT_NAMES_OF_PASSED_TESTS = "names_of_passed_tests"
+
 OUTPUT_LLM_COMMAND_GENERATOR_CONFIG = "llm_command_generator_config"
 
 
@@ -60,6 +77,7 @@ class FailedTestStep(BaseModel):
     expected_commands: List[PromptCommand]
     predicted_commands: Dict[str, List[PromptCommand]]
     conversation_with_diff: List[str]
+    conversation_until_failed_user_utterance: List[str]
 
     class Config:
         """Skip validation for PromptCommand protocol as pydantic does not know how to
@@ -90,10 +108,12 @@ class FailedTestStep(BaseModel):
             )
 
         step_index = test_case.steps.index(step)
-
-        conversation_with_diff = test_case.to_readable_conversation(
+        conversation_until_failed_user_utterance = test_case.to_readable_conversation(
             until_step=step_index + 1
-        ) + get_command_comparison(step)
+        )
+        conversation_with_diff = (
+            conversation_until_failed_user_utterance + get_command_comparison(step)
+        )
 
         return cls(
             file=file_path,
@@ -106,12 +126,14 @@ class FailedTestStep(BaseModel):
             expected_commands=step.commands or [],
             predicted_commands=predicted_commands,
             conversation_with_diff=conversation_with_diff,
+            conversation_until_failed_user_utterance=conversation_until_failed_user_utterance,
         )
 
     def to_dict(self, output_prompt: bool) -> Dict[str, Any]:
         step_info = {
             "file": self.file,
             "test_case": self.test_case_name,
+            "conversation": self.conversation_until_failed_user_utterance,
             "failed_user_utterance": self.failed_user_utterance,
             "error_line": self.error_line,
             "pass_status": self.pass_status,
@@ -155,25 +177,32 @@ class DialogueUnderstandingTestSuiteResult:
             KEY_TEST_CASES_ACCURACY: 0.0,
             KEY_USER_UTTERANCES_ACCURACY: 0.0,
         }
+        self.f1_score = {
+            KEY_COMMANDS_F1_MACRO: 0.0,
+            KEY_COMMANDS_F1_MICRO: 0.0,
+            KEY_COMMANDS_F1_WEIGHTED: 0.0,
+        }
         self.number_of_passed_tests = 0
         self.number_of_failed_tests = 0
         self.number_of_passed_user_utterances = 0
         self.number_of_failed_user_utterances = 0
-        self.command_metrics: Optional[Dict[str, "CommandMetrics"]] = None
+        self.command_metrics: Optional[Dict[str, CommandMetrics]] = None
         self.names_of_failed_tests: List[str] = []
         self.names_of_passed_tests: List[str] = []
         self.failed_test_steps: List[FailedTestStep] = []
         self.llm_config: Optional[Dict[str, Any]] = None
-        self.latency_metrics: Dict[str, float] = {}
-        self.prompt_token_metrics: Dict[str, float] = {}
-        self.completion_token_metrics: Dict[str, float] = {}
+        # The performance metrics distribution per component
+        # For example: {"command_generator": {"p50": x, ...}, ...}
+        self.latency_metrics: Dict[str, Dict[str, float]] = {}
+        self.prompt_token_metrics: Dict[str, Dict[str, float]] = {}
+        self.completion_token_metrics: Dict[str, Dict[str, float]] = {}
 
     @classmethod
     def from_results(
         cls,
         failing_test_results: List[DialogueUnderstandingTestResult],
         passing_test_results: List[DialogueUnderstandingTestResult],
-        command_metrics: Dict[str, "CommandMetrics"],
+        command_metrics: Dict[str, CommandMetrics],
         llm_config: Optional[Dict[str, Any]],
     ) -> "DialogueUnderstandingTestSuiteResult":
         """Create a DialogueUnderstandingTestSuiteResult object from the test results.
@@ -207,6 +236,16 @@ class DialogueUnderstandingTestSuiteResult:
 
         instance.command_metrics = command_metrics
 
+        instance.f1_score[KEY_COMMANDS_F1_MACRO] = cls.calculate_f1_macro(
+            command_metrics
+        )
+        instance.f1_score[KEY_COMMANDS_F1_MICRO] = cls.calculate_f1_micro(
+            command_metrics
+        )
+        instance.f1_score[KEY_COMMANDS_F1_WEIGHTED] = cls.calculate_f1_weighted(
+            command_metrics
+        )
+
         instance.names_of_passed_tests = [
             passing_test_result.test_case.full_name()
             for passing_test_result in passing_test_results
@@ -234,131 +273,34 @@ class DialogueUnderstandingTestSuiteResult:
 
         return instance
 
-    def _set_user_utterance_metrics(
-        self,
-        failing_test_results: List[DialogueUnderstandingTestResult],
-        passing_test_results: List[DialogueUnderstandingTestResult],
-    ) -> None:
-        # Create list of booleans indicating whether each user utterance
-        # passed or failed
-        user_utterances_status = [
-            step.has_passed()
-            for test in failing_test_results + passing_test_results
-            for step in test.test_case.iterate_over_user_steps()
-        ]
-        # Calculate number of passed and failed user utterances
-        self.number_of_passed_user_utterances = sum(user_utterances_status)
-        self.number_of_failed_user_utterances = (
-            len(user_utterances_status) - self.number_of_passed_user_utterances
-        )
-        # Calculate user utterance accuracy
-        self.accuracy[KEY_USER_UTTERANCES_ACCURACY] = (
-            self.number_of_passed_user_utterances
-            / (
-                self.number_of_failed_user_utterances
-                + self.number_of_passed_user_utterances
-            )
-        )
-
-    @staticmethod
-    def _create_failed_steps_from_results(
-        failing_test_results: List["DialogueUnderstandingTestResult"],
-    ) -> List[FailedTestStep]:
-        """Create list of FailedTestStep objects from failing test results.
-
-        Given a list of failing DialogueUnderstandingTestResult objects,
-        create and return a list of FailedTestStep objects for each failing user step.
-
-        Args:
-            failing_test_results: Results of failing Dialogue Understanding tests.
-
-        Returns:
-            List of aggregated FailedTestStep objects for logging to console and file.
-        """
-        failed_test_steps: List[FailedTestStep] = []
-
-        for result in failing_test_results:
-            test_case = result.test_case
-            for step in test_case.failed_user_steps():
-                failed_test_steps.append(
-                    FailedTestStep.from_dialogue_understanding_test_step(
-                        step, test_case
-                    )
-                )
-
-        return failed_test_steps
-
-    @staticmethod
-    def _calculate_percentiles(values: List[float]) -> Dict[str, float]:
-        return {
-            "p50": float(np.percentile(values, 50)) if values else 0.0,
-            "p90": float(np.percentile(values, 90)) if values else 0.0,
-            "p99": float(np.percentile(values, 99)) if values else 0.0,
-        }
-
-    @classmethod
-    def get_latency_metrics(
-        cls,
-        failing_test_results: List["DialogueUnderstandingTestResult"],
-        passing_test_results: List["DialogueUnderstandingTestResult"],
-    ) -> Dict[str, float]:
-        latencies = [
-            latency
-            for result in failing_test_results + passing_test_results
-            for step in result.test_case.steps
-            for latency in step.get_latencies()
-        ]
-
-        return cls._calculate_percentiles(latencies)
-
-    @classmethod
-    def get_prompt_token_metrics(
-        cls,
-        failing_test_results: List["DialogueUnderstandingTestResult"],
-        passing_test_results: List["DialogueUnderstandingTestResult"],
-    ) -> Dict[str, float]:
-        tokens = [
-            token_count
-            for result in failing_test_results + passing_test_results
-            for step in result.test_case.steps
-            for token_count in step.get_prompt_tokens()
-        ]
-
-        return cls._calculate_percentiles(tokens)
-
-    @classmethod
-    def get_completion_token_metrics(
-        cls,
-        failing_test_results: List["DialogueUnderstandingTestResult"],
-        passing_test_results: List["DialogueUnderstandingTestResult"],
-    ) -> Dict[str, float]:
-        tokens = [
-            token_count
-            for result in failing_test_results + passing_test_results
-            for step in result.test_case.steps
-            for token_count in step.get_completion_tokens()
-        ]
-
-        return cls._calculate_percentiles(tokens)
-
     def to_dict(self, output_prompt: bool = False) -> Dict[str, Any]:
         """Builds a dictionary for writing test results to a YML file.
 
         Args:
             output_prompt: Whether to log the prompt or not.
         """
-        # 1. Accuracy block
         result_dict: Dict[Text, Any] = {
-            "accuracy": {
-                "test_cases": self.accuracy[KEY_TEST_CASES_ACCURACY],
-                "user_utterances": self.accuracy[KEY_USER_UTTERANCES_ACCURACY],
+            # Accuracy block
+            OUTPUT_DUT_ACCURACY: {
+                OUTPUT_DUT_ACCURACY_TEST_CASES: self.accuracy[KEY_TEST_CASES_ACCURACY],
+                OUTPUT_DUT_ACCURACY_USER_UTTERANCES: self.accuracy[
+                    KEY_USER_UTTERANCES_ACCURACY
+                ],
+            },
+            # F1 block
+            OUTPUT_COMMANDS_F1: {
+                OUTPUT_COMMANDS_F1_MACRO: self.f1_score[KEY_COMMANDS_F1_MACRO],
+                OUTPUT_COMMANDS_F1_MICRO: self.f1_score[KEY_COMMANDS_F1_MICRO],
+                OUTPUT_COMMANDS_F1_WEIGHTED: self.f1_score[KEY_COMMANDS_F1_WEIGHTED],
             },
+            # Other metrics block
             OUTPUT_NUMBER_OF_PASSED_TESTS: self.number_of_passed_tests,
             OUTPUT_NUMBER_OF_FAILED_TESTS: self.number_of_failed_tests,
             OUTPUT_NUMBER_OF_PASSED_USER_UTTERANCES: self.number_of_passed_user_utterances,  # noqa: E501
             OUTPUT_NUMBER_OF_FAILED_USER_UTTERANCES: self.number_of_failed_user_utterances,  # noqa: E501
         }
 
+        # Command metrics block
         cmd_metrics_output = {}
         if self.command_metrics:
             if isinstance(self.command_metrics, dict):
@@ -366,25 +308,179 @@ class DialogueUnderstandingTestSuiteResult:
                     cmd_metrics_output[cmd_name] = metrics_obj.as_dict()
             else:
                 pass
-
         result_dict[OUTPUT_COMMAND_METRICS] = cmd_metrics_output
 
+        # Latency and tokens metrics block
         result_dict[OUTPUT_LATENCY_METRICS] = self.latency_metrics
         result_dict[OUTPUT_PROMPT_TOKEN_METRICS] = self.prompt_token_metrics
         result_dict[OUTPUT_COMPLETION_TOKEN_METRICS] = self.completion_token_metrics
 
+        # Passed and failed test names block
         result_dict[OUTPUT_NAMES_OF_PASSED_TESTS] = self.names_of_passed_tests
         result_dict[OUTPUT_NAMES_OF_FAILED_TESTS] = self.names_of_failed_tests
 
+        # Failed test steps block
         failed_steps_list = []
         for failed_test_step in self.failed_test_steps:
             failed_steps_list.append(
                 failed_test_step.to_dict(output_prompt=output_prompt)
             )
+        result_dict[OUTPUT_FAILED_TEST_STEPS] = failed_steps_list
 
-        result_dict["failed_test_steps"] = failed_steps_list
-
+        # LLM config block
         if self.llm_config:
             result_dict[OUTPUT_LLM_COMMAND_GENERATOR_CONFIG] = self.llm_config
 
         return result_dict
+
+    @staticmethod
+    def calculate_f1_macro(command_metrics: Dict[str, CommandMetrics]) -> float:
+        f1_scores = [metrics.get_f1_score() for metrics in command_metrics.values()]
+        return sum(f1_scores) / len(f1_scores)
+
+    @staticmethod
+    def calculate_f1_micro(command_metrics: Dict[str, CommandMetrics]) -> float:
+        combined_metrics = CommandMetrics(
+            tp=sum([metrics.tp for metrics in command_metrics.values()]),
+            fp=sum([metrics.fp for metrics in command_metrics.values()]),
+            fn=sum([metrics.fn for metrics in command_metrics.values()]),
+            total_count=sum(m.total_count for m in command_metrics.values()),
+        )
+        return combined_metrics.get_f1_score()
+
+    @staticmethod
+    def calculate_f1_weighted(command_metrics: Dict[str, CommandMetrics]) -> float:
+        class_counts = []
+        f1_scores = []
+        for metrics in command_metrics.values():
+            class_counts.append(metrics.total_count)
+            f1_scores.append(metrics.get_f1_score())
+
+        total_count = sum(class_counts)
+        weighted_f1 = sum(
+            (count / total_count) * f1 for f1, count in zip(f1_scores, class_counts)
+        )
+        return weighted_f1
+
+    @classmethod
+    def get_latency_metrics(
+        cls,
+        failing_test_results: List["DialogueUnderstandingTestResult"],
+        passing_test_results: List["DialogueUnderstandingTestResult"],
+    ) -> Dict[str, Dict[str, float]]:
+        latencies = defaultdict(list)
+
+        for result in failing_test_results + passing_test_results:
+            for step in result.test_case.steps:
+                if (
+                    step.dialogue_understanding_output
+                    and step.dialogue_understanding_output.latency
+                ):
+                    latencies["total"].append(
+                        step.dialogue_understanding_output.latency
+                    )
+                for component_name, latency in step.get_latencies().items():
+                    latencies[component_name].extend(latency)
+
+        return {
+            component_name: cls._calculate_percentiles(latency_list)
+            for component_name, latency_list in latencies.items()
+        }
+
+    @classmethod
+    def get_prompt_token_metrics(
+        cls,
+        failing_test_results: List["DialogueUnderstandingTestResult"],
+        passing_test_results: List["DialogueUnderstandingTestResult"],
+    ) -> Dict[str, Dict[str, float]]:
+        tokens = defaultdict(list)
+
+        for result in failing_test_results + passing_test_results:
+            for step in result.test_case.steps:
+                for component_name, token_count in step.get_prompt_tokens().items():
+                    tokens[component_name].extend(token_count)
+
+        return {
+            component_name: cls._calculate_percentiles(latency_list)
+            for component_name, latency_list in tokens.items()
+        }
+
+    @classmethod
+    def get_completion_token_metrics(
+        cls,
+        failing_test_results: List["DialogueUnderstandingTestResult"],
+        passing_test_results: List["DialogueUnderstandingTestResult"],
+    ) -> Dict[str, Dict[str, float]]:
+        tokens = defaultdict(list)
+
+        for result in failing_test_results + passing_test_results:
+            for step in result.test_case.steps:
+                for component_name, token_count in step.get_completion_tokens().items():
+                    tokens[component_name].extend(token_count)
+
+        return {
+            component_name: cls._calculate_percentiles(latency_list)
+            for component_name, latency_list in tokens.items()
+        }
+
+    @staticmethod
+    def _calculate_percentiles(values: List[float]) -> Dict[str, float]:
+        return {
+            "p50": float(np.percentile(values, 50)) if values else 0.0,
+            "p90": float(np.percentile(values, 90)) if values else 0.0,
+            "p99": float(np.percentile(values, 99)) if values else 0.0,
+        }
+
+    @staticmethod
+    def _create_failed_steps_from_results(
+        failing_test_results: List["DialogueUnderstandingTestResult"],
+    ) -> List[FailedTestStep]:
+        """Create list of FailedTestStep objects from failing test results.
+
+        Given a list of failing DialogueUnderstandingTestResult objects,
+        create and return a list of FailedTestStep objects for each failing user step.
+
+        Args:
+            failing_test_results: Results of failing Dialogue Understanding tests.
+
+        Returns:
+            List of aggregated FailedTestStep objects for logging to console and file.
+        """
+        failed_test_steps: List[FailedTestStep] = []
+
+        for result in failing_test_results:
+            test_case = result.test_case
+            for step in test_case.failed_user_steps():
+                failed_test_steps.append(
+                    FailedTestStep.from_dialogue_understanding_test_step(
+                        step, test_case
+                    )
+                )
+
+        return failed_test_steps
+
+    def _set_user_utterance_metrics(
+        self,
+        failing_test_results: List[DialogueUnderstandingTestResult],
+        passing_test_results: List[DialogueUnderstandingTestResult],
+    ) -> None:
+        # Create list of booleans indicating whether each user utterance
+        # passed or failed
+        user_utterances_status = [
+            step.has_passed()
+            for test in failing_test_results + passing_test_results
+            for step in test.test_case.iterate_over_user_steps()
+        ]
+        # Calculate number of passed and failed user utterances
+        self.number_of_passed_user_utterances = sum(user_utterances_status)
+        self.number_of_failed_user_utterances = (
+            len(user_utterances_status) - self.number_of_passed_user_utterances
+        )
+        # Calculate user utterance accuracy
+        self.accuracy[KEY_USER_UTTERANCES_ACCURACY] = (
+            self.number_of_passed_user_utterances
+            / (
+                self.number_of_failed_user_utterances
+                + self.number_of_passed_user_utterances
+            )
+        )

rasa/dialogue_understanding_test/du_test_runner.py

@@ -1,4 +1,5 @@
 import asyncio
+import time
 from typing import Any, Dict, List, Optional, Text
 
 import structlog
@@ -184,8 +185,12 @@ class DialogueUnderstandingTestRunner:
                 user_uttered_event_indices[user_step_index],
             )
 
+            # Total latency of a message roundtrip
+            latency = None
+
             # send the user message
             try:
+                start = time.time()
                 await self._send_user_message(
                     step_sender_id,
                     test_case,
@@ -193,6 +198,8 @@ class DialogueUnderstandingTestRunner:
                     metadata,
                     output_channel=output_channel,
                 )
+                end = time.time()
+                latency = end - start
             except Exception as e:
                 structlogger.error(
                     "dialogue_understanding_test_runner.send_user_message.failed",
@@ -210,7 +217,7 @@ class DialogueUnderstandingTestRunner:
             # get the dialogue understanding output
             tracker = await self.agent.tracker_store.retrieve(step_sender_id)
             dialogue_understanding_output = self.get_dialogue_understanding_output(
-                tracker, user_uttered_event_indices[user_step_index]
+                tracker, user_uttered_event_indices[user_step_index], latency
             )
             user_step.dialogue_understanding_output = dialogue_understanding_output
 
@@ -224,6 +231,7 @@ class DialogueUnderstandingTestRunner:
         self,
         tracker: DialogueStateTracker,
         index_user_uttered_event: int,
+        latency: Optional[float] = None,
     ) -> Optional[DialogueUnderstandingOutput]:
         """Returns the dialogue understanding output.
 
@@ -259,6 +267,7 @@ class DialogueUnderstandingTestRunner:
         return DialogueUnderstandingOutput(
             commands=commands,
             prompts=user_uttered_event.parse_data.get(PROMPTS, []),
+            latency=latency,
         )
 
     @staticmethod

rasa/dialogue_understanding_test/io.py

@@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Union
 import rich
 
 import rasa.shared.data
-from rasa.dialogue_understanding_test.command_metric_calculation import CommandMetrics
+from rasa.dialogue_understanding_test.command_metrics import CommandMetrics
 from rasa.dialogue_understanding_test.constants import SCHEMA_FILE_PATH
 from rasa.dialogue_understanding_test.du_test_case import (
     KEY_CHOICES,
@@ -13,6 +13,9 @@ from rasa.dialogue_understanding_test.du_test_case import (
     KEY_PROMPT_TOKENS,
 )
 from rasa.dialogue_understanding_test.du_test_result import (
+    KEY_COMMANDS_F1_MACRO,
+    KEY_COMMANDS_F1_MICRO,
+    KEY_COMMANDS_F1_WEIGHTED,
     DialogueUnderstandingTestSuiteResult,
     FailedTestStep,
 )
@@ -274,6 +277,7 @@ def print_test_results(
         # print failed test steps
         print_failed_cases(test_suite_result, output_prompt=output_prompt)
 
+    print_f1_summary(test_suite_result)
     print_command_summary(test_suite_result.command_metrics)
     print_latency_and_token_metrics(test_suite_result)
     print_final_line(test_suite_result)
@@ -325,14 +329,19 @@ def print_prompt(step: FailedTestStep) -> None:
             rich.print(
                 f"[bold]  prompt name      [/bold]: {prompt_data[KEY_PROMPT_NAME]}"
             )
-            rich.print(
-                f"[bold]  prompt tokens    [/bold]: {prompt_data[KEY_PROMPT_TOKENS]}"
-            )
-            rich.print(
-                f"[bold]  completion tokens[/bold]: "
-                f"{prompt_data[KEY_COMPLETION_TOKENS]}"
-            )
-            rich.print(f"[bold]  latency          [/bold]: {prompt_data[KEY_LATENCY]}")
+            if KEY_PROMPT_TOKENS in prompt_data:
+                rich.print(
+                    f"[bold]  prompt tokens    [/bold]: {prompt_data[KEY_PROMPT_TOKENS]}"  # noqa: E501
+                )
+            if KEY_COMPLETION_TOKENS in prompt_data:
+                rich.print(
+                    f"[bold]  completion tokens[/bold]: "
+                    f"{prompt_data[KEY_COMPLETION_TOKENS]}"
+                )
+            if KEY_LATENCY in prompt_data:
+                rich.print(
+                    f"[bold]  latency          [/bold]: {prompt_data[KEY_LATENCY]}"
+                )
             if KEY_SYSTEM_PROMPT in prompt_data:
                 rich.print(
                     f"[bold]  system prompt    [/bold]: "
@@ -350,11 +359,26 @@ def print_llm_output(step: FailedTestStep) -> None:
     for component, component_prompts in step.prompts.items():
         for prompt_data in component_prompts:
             if KEY_CHOICES in prompt_data:
-                rich.print("\n[red3]-- CHOICES --[/red3]")
+                rich.print(f"\n[red3]-- LLM ouptut for {component} --[/red3]")
                 rich.print(prompt_data.get(KEY_CHOICES))
                 rich.print("[red3]-------------[/red3]")
 
 
+def print_f1_summary(result: DialogueUnderstandingTestSuiteResult) -> None:
+    """Print the f1 summary."""
+    print()
+    rasa.shared.utils.cli.print_info(rasa.shared.utils.cli.pad("COMMANDS F1"))
+    rasa.shared.utils.cli.print_info(
+        f"macro           : {result.f1_score[KEY_COMMANDS_F1_MACRO]:.8f}"
+    )
+    rasa.shared.utils.cli.print_info(
+        f"micro           : {result.f1_score[KEY_COMMANDS_F1_MICRO]:.8f}"
+    )
+    rasa.shared.utils.cli.print_info(
+        f"weighted average: {result.f1_score[KEY_COMMANDS_F1_WEIGHTED]:.8f}"
+    )
+
+
 def print_command_summary(metrics: Dict[str, CommandMetrics]) -> None:
     """Print the command summary.
 
@@ -390,16 +414,24 @@ def print_latency_and_token_metrics(
     """Print the latency and token metrics."""
     print()
     rasa.shared.utils.cli.print_info(rasa.shared.utils.cli.pad("LATENCY METRICS"))
-    for key, value in result.latency_metrics.items():
-        rasa.shared.utils.cli.print_info(f"{key}: {value:.8f}")
+    for component, latency_metric in result.latency_metrics.items():
+        rasa.shared.utils.cli.print_info(f"--- {component} ---")
+        for key, value in latency_metric.items():
+            rasa.shared.utils.cli.print_info(f"{key}: {value:.8f}")
+
     rasa.shared.utils.cli.print_info(rasa.shared.utils.cli.pad("PROMPT TOKEN METRICS"))
-    for key, value in result.prompt_token_metrics.items():
-        rasa.shared.utils.cli.print_info(f"{key}: {value:.2f}")
+    for component, prompt_token_metric in result.prompt_token_metrics.items():
+        rasa.shared.utils.cli.print_info(f"--- {component} ---")
+        for key, value in prompt_token_metric.items():
+            rasa.shared.utils.cli.print_info(f"{key}: {value:.2f}")
+
     rasa.shared.utils.cli.print_info(
         rasa.shared.utils.cli.pad("COMPLETION TOKEN METRICS")
     )
-    for key, value in result.completion_token_metrics.items():
-        rasa.shared.utils.cli.print_info(f"{key}: {value:.2f}")
+    for component, completion_token_metric in result.completion_token_metrics.items():
+        rasa.shared.utils.cli.print_info(f"--- {component} ---")
+        for key, value in completion_token_metric.items():
+            rasa.shared.utils.cli.print_info(f"{key}: {value:.2f}")
 
 
 def print_final_line(test_suite_result: DialogueUnderstandingTestSuiteResult) -> None:

rasa/document_retrieval/constants.py

@@ -0,0 +1,32 @@
+# keys for storing information in the message object
+from rasa.shared.constants import OPENAI_PROVIDER, PROVIDER_CONFIG_KEY
+from rasa.shared.utils.llm import DEFAULT_OPENAI_EMBEDDING_MODEL_NAME
+
+SEARCH_QUERY_KEY = "search_query"
+RETRIEVED_DOCUMENTS_KEY = "retrieved_documents"
+POST_PROCESSED_DOCUMENTS_KEY = "post_processed_documents"
+
+# config keys
+THRESHOLD_CONFIG_KEY = "threshold"
+K_CONFIG_KEY = "k"
+VECTOR_STORE_TYPE_CONFIG_KEY = "type"
+VECTOR_STORE_CONFIG_KEY = "vector_store"
+CONNECTOR_CONFIG_KEY = "connector"
+SOURCE_PROPERTY = "source"
+POST_PROCESSING_CONFIG_KEY = "post_processing"
+QUERY_REWRITING_CONFIG_KEY = "query_rewriting"
+USE_LLM_PROPERTY = "use_generative_llm"
+
+# default values
+DEFAULT_THRESHOLD = 0.0
+DEFAULT_K = 3
+DEFAULT_VECTOR_STORE_TYPE = "faiss"
+DEFAULT_EMBEDDINGS_CONFIG = {
+    PROVIDER_CONFIG_KEY: OPENAI_PROVIDER,
+    "model": DEFAULT_OPENAI_EMBEDDING_MODEL_NAME,
+}
+DEFAULT_VECTOR_STORE = {
+    VECTOR_STORE_TYPE_CONFIG_KEY: DEFAULT_VECTOR_STORE_TYPE,
+    SOURCE_PROPERTY: "./docs",
+    THRESHOLD_CONFIG_KEY: DEFAULT_THRESHOLD,
+}