rasa-pro 3.12.0.dev2__py3-none-any.whl → 3.12.0.dev4__py3-none-any.whl

rasa/dialogue_understanding_test/du_test_result.py

@@ -1,11 +1,300 @@
-from typing import Optional
+import typing
+from typing import Any, Dict, List, Optional, Text
 
 from pydantic import BaseModel
 
-from rasa.dialogue_understanding_test.du_test_case import DialogueUnderstandingTestCase
+from rasa.dialogue_understanding.commands import Command
+from rasa.dialogue_understanding_test.du_test_case import (
+    DialogueUnderstandingTestCase,
+    DialogueUnderstandingTestStep,
+)
+from rasa.dialogue_understanding_test.utils import get_command_comparison
+
+if typing.TYPE_CHECKING:
+    from rasa.dialogue_understanding_test.command_metric_calculation import (
+        CommandMetrics,
+    )
+
+KEY_TEST_CASES_ACCURACY = "test_cases"
+KEY_USER_UTTERANCES_ACCURACY = "user_utterances"
+
+OUTPUT_NUMBER_OF_FAILED_TESTS = "number_of_failed_tests"
+OUTPUT_NUMBER_OF_PASSED_TESTS = "number_of_passed_tests"
+OUTPUT_TEST_CASES_ACCURACY = "test_cases_accuracy"
+OUTPUT_USER_UTTERANCES_ACCURACY = "user_utterances_accuracy"
+OUTPUT_NUMBER_OF_PASSED_USER_UTTERANCES = "number_of_passed_user_utterances"
+OUTPUT_NUMBER_OF_FAILED_USER_UTTERANCES = "number_of_failed_user_utterances"
+OUTPUT_COMMAND_METRICS = "command_metrics"
+OUTPUT_NAMES_OF_FAILED_TESTS = "names_of_failed_tests"
+OUTPUT_NAMES_OF_PASSED_TESTS = "names_of_passed_tests"
 
 
 class DialogueUnderstandingTestResult(BaseModel):
+    """Result of a single dialogue understanding test case."""
+
     test_case: DialogueUnderstandingTestCase
     passed: bool
     error_line: Optional[int] = None
+
+    def get_expected_commands(self) -> List[Command]:
+        return self.test_case.get_expected_commands()
+
+
+class FailedTestStep(BaseModel):
+    """Failed test step information."""
+
+    file: str
+    test_case_name: str
+    failed_user_utterance: str
+    error_line: int
+    pass_status: bool
+    command_generators: List[str]
+    prompt: Optional[Dict[str, Any]] = None
+    expected_commands: List[Command]
+    predicted_commands: Dict[str, List[Command]]
+    conversation_with_diff: List[str]
+
+    @classmethod
+    def from_dialogue_understanding_test_step(
+        cls,
+        step: DialogueUnderstandingTestStep,
+        test_case: DialogueUnderstandingTestCase,
+    ) -> "FailedTestStep":
+        file_path = test_case.file or ""
+        user_utterance = step.text or ""
+        line_number = step.line or -1
+
+        predicted_commands: Dict[str, List[Command]] = {}
+        prompts: Optional[Dict[str, Any]] = None
+        command_generators: List[str] = []
+
+        if step.dialogue_understanding_output:
+            predicted_commands = step.dialogue_understanding_output.commands
+            command_generators = step.dialogue_understanding_output.get_component_names_that_predicted_commands()  # noqa: E501
+            prompts = (
+                step.dialogue_understanding_output.get_component_name_to_user_prompts()
+            )
+
+        step_index = test_case.steps.index(step)
+
+        conversation_with_diff = test_case.to_readable_conversation(
+            until_step=step_index + 1
+        ) + get_command_comparison(step)
+
+        return cls(
+            file=file_path,
+            test_case_name=test_case.name,
+            failed_user_utterance=user_utterance,
+            error_line=line_number,
+            pass_status=False,
+            command_generators=command_generators,
+            prompt=prompts,
+            expected_commands=step.commands or [],
+            predicted_commands=predicted_commands,
+            conversation_with_diff=conversation_with_diff,
+        )
+
+    def to_dict(self, output_prompt: bool) -> Dict[Text, Any]:
+        step_info = {
+            "file": self.file,
+            "test_case": self.test_case_name,
+            "failed_user_utterance": self.failed_user_utterance,
+            "error_line": self.error_line,
+            "pass_status": self.pass_status,
+            "expected_commands": [
+                command.to_dsl() for command in self.expected_commands
+            ],
+            "predicted_commands": [
+                {
+                    "component": component,
+                    "commands": [command.to_dsl() for command in commands],
+                }
+                for component, commands in self.predicted_commands.items()
+                if commands
+            ],
+        }
+
+        if output_prompt and self.prompt:
+            step_info["prompts"] = [
+                {
+                    component: [
+                        {
+                            "prompt_name": prompt_name,
+                            "prompt_content": prompt_content,
+                        }
+                        for prompt_name, prompt_content in prompts
+                    ],
+                }
+                for component, prompts in self.prompt.items()
+            ]
+
+        return step_info
+
+
+class DialogueUnderstandingTestSuiteResult:
+    """Result of a dialogue understanding test suite.
+
+    Aggregates test results and provides metrics for the entire test suite
+    used to log the results to the console and write them to a file.
+    """
+
+    def __init__(self) -> None:
+        self.accuracy = {
+            KEY_TEST_CASES_ACCURACY: 0.0,
+            KEY_USER_UTTERANCES_ACCURACY: 0.0,
+        }
+        self.number_of_passed_tests = 0
+        self.number_of_failed_tests = 0
+        self.number_of_passed_user_utterances = 0
+        self.number_of_failed_user_utterances = 0
+        self.command_metrics: Optional[Dict[str, "CommandMetrics"]] = None
+        self.names_of_failed_tests: List[str] = []
+        self.names_of_passed_tests: List[str] = []
+        self.failed_test_steps: List[FailedTestStep] = []
+
+    @classmethod
+    def from_results(
+        cls,
+        failing_test_results: List[DialogueUnderstandingTestResult],
+        passing_test_results: List[DialogueUnderstandingTestResult],
+        command_metrics: Dict[str, "CommandMetrics"],
+    ) -> "DialogueUnderstandingTestSuiteResult":
+        """Create a DialogueUnderstandingTestSuiteResult object from the test results.
+
+        Create a new instance of DialogueUnderstandingTestSuiteResult by aggregating
+        metrics from passing and failing test results, along with command metrics.
+
+        Args:
+            failing_test_results: A list of DialogueUnderstandingTestResult objects
+                representing the test cases that did not pass.
+            passing_test_results: A list of DialogueUnderstandingTestResult objects
+                representing the test cases that passed.
+            command_metrics: A dictionary of command-specific performance metrics, keyed
+                by command name.
+
+        Returns:
+            A DialogueUnderstandingTestSuiteResult object containing aggregated test
+            suite metrics, including accuracy, counts of passed and failed test cases,
+            user utterance statistics, and command metrics.
+        """
+        instance = cls()
+
+        instance.number_of_passed_tests = len(passing_test_results)
+        instance.number_of_failed_tests = len(failing_test_results)
+        instance.accuracy[KEY_TEST_CASES_ACCURACY] = instance.number_of_passed_tests / (
+            instance.number_of_passed_tests + instance.number_of_failed_tests
+        )
+
+        instance._set_user_utterance_metrics(failing_test_results, passing_test_results)
+
+        instance.command_metrics = command_metrics
+
+        instance.names_of_passed_tests = [
+            passing_test_result.test_case.full_name()
+            for passing_test_result in passing_test_results
+        ]
+        instance.names_of_failed_tests = [
+            failing_test_result.test_case.full_name()
+            for failing_test_result in failing_test_results
+        ]
+
+        instance.failed_test_steps = cls._create_failed_steps_from_results(
+            failing_test_results
+        )
+
+        return instance
+
+    def _set_user_utterance_metrics(
+        self,
+        failing_test_results: List[DialogueUnderstandingTestResult],
+        passing_test_results: List[DialogueUnderstandingTestResult],
+    ) -> None:
+        # Create list of booleans indicating whether each user utterance
+        # passed or failed
+        user_utterances_status = [
+            step.has_passed()
+            for test in failing_test_results + passing_test_results
+            for step in test.test_case.iterate_over_user_steps()
+        ]
+        # Calculate number of passed and failed user utterances
+        self.number_of_passed_user_utterances = sum(user_utterances_status)
+        self.number_of_failed_user_utterances = (
+            len(user_utterances_status) - self.number_of_passed_user_utterances
+        )
+        # Calculate user utterance accuracy
+        self.accuracy[KEY_USER_UTTERANCES_ACCURACY] = (
+            self.number_of_passed_user_utterances
+            / (
+                self.number_of_failed_user_utterances
+                + self.number_of_passed_user_utterances
+            )
+        )
+
+    @staticmethod
+    def _create_failed_steps_from_results(
+        failing_test_results: List["DialogueUnderstandingTestResult"],
+    ) -> List[FailedTestStep]:
+        """Create list of FailedTestStep objects from failing test results.
+
+        Given a list of failing DialogueUnderstandingTestResult objects,
+        create and return a list of FailedTestStep objects for each failing user step.
+
+        Args:
+            failing_test_results: Results of failing Dialogue Understanding tests.
+
+        Returns:
+            List of aggregated FailedTestStep objects for logging to console and file.
+        """
+        failed_test_steps: List[FailedTestStep] = []
+
+        for result in failing_test_results:
+            test_case = result.test_case
+            for step in test_case.failed_user_steps():
+                failed_test_steps.append(
+                    FailedTestStep.from_dialogue_understanding_test_step(
+                        step, test_case
+                    )
+                )
+
+        return failed_test_steps
+
+    def to_dict(self, output_prompt: bool = False) -> Dict[Text, Any]:
+        """Builds a dictionary for writing test results to a YML file.
+
+        Args:
+            output_prompt: Whether to log the prompt or not.
+        """
+        # 1. Accuracy block
+        result_dict: Dict[Text, Any] = {
+            "accuracy": {
+                "test_cases": self.accuracy[KEY_TEST_CASES_ACCURACY],
+                "user_utterances": self.accuracy[KEY_USER_UTTERANCES_ACCURACY],
+            },
+            OUTPUT_NUMBER_OF_PASSED_TESTS: self.number_of_passed_tests,
+            OUTPUT_NUMBER_OF_FAILED_TESTS: self.number_of_failed_tests,
+            OUTPUT_NUMBER_OF_PASSED_USER_UTTERANCES: self.number_of_passed_user_utterances,  # noqa: E501
+            OUTPUT_NUMBER_OF_FAILED_USER_UTTERANCES: self.number_of_failed_user_utterances,  # noqa: E501
+        }
+
+        cmd_metrics_output = {}
+        if self.command_metrics:
+            if isinstance(self.command_metrics, dict):
+                for cmd_name, metrics_obj in self.command_metrics.items():
+                    cmd_metrics_output[cmd_name] = metrics_obj.as_dict()
+            else:
+                pass
+
+        result_dict[OUTPUT_COMMAND_METRICS] = cmd_metrics_output
+
+        result_dict[OUTPUT_NAMES_OF_PASSED_TESTS] = self.names_of_passed_tests
+        result_dict[OUTPUT_NAMES_OF_FAILED_TESTS] = self.names_of_failed_tests
+
+        failed_steps_list = []
+        for failed_test_step in self.failed_test_steps:
+            failed_steps_list.append(
+                failed_test_step.to_dict(output_prompt=output_prompt)
+            )
+
+        result_dict["failed_test_steps"] = failed_steps_list
+
+        return result_dict

rasa/dialogue_understanding_test/du_test_runner.py

@@ -1,32 +1,52 @@
 import asyncio
-from typing import Dict, Optional, Text, Union, List
+from typing import Any, Dict, List, Optional, Text
 
 import structlog
+from tqdm import tqdm
 
+from rasa.core.channels import CollectingOutputChannel, UserMessage
 from rasa.core.exceptions import AgentNotReady
 from rasa.core.persistor import StorageType
 from rasa.core.utils import AvailableEndpoints
-from rasa.dialogue_understanding_test.du_test_case import DialogueUnderstandingTestCase
+from rasa.dialogue_understanding.commands import Command
+from rasa.dialogue_understanding.utils import set_record_commands_and_prompts
+from rasa.dialogue_understanding_test.du_test_case import (
+    DialogueUnderstandingOutput,
+    DialogueUnderstandingTestCase,
+    DialogueUnderstandingTestStep,
+)
 from rasa.dialogue_understanding_test.du_test_result import (
     DialogueUnderstandingTestResult,
 )
+from rasa.dialogue_understanding_test.test_case_simulation.test_case_tracker_simulator import (  # noqa: E501
+    TestCaseTrackerSimulator,
+    TestCaseTrackerSimulatorException,
+    TestCaseTrackerSimulatorResult,
+)
+from rasa.dialogue_understanding_test.utils import filter_metadata
 from rasa.e2e_test.e2e_test_case import (
     KEY_STUB_CUSTOM_ACTIONS,
-    ActualStepOutput,
-    TestStep,
     Fixture,
     Metadata,
 )
 from rasa.e2e_test.e2e_test_runner import E2ETestRunner
+from rasa.shared.core.events import UserUttered
+from rasa.shared.core.trackers import DialogueStateTracker
+from rasa.shared.nlu.constants import PREDICTED_COMMANDS, PROMPTS
 from rasa.utils.endpoints import EndpointConfig
 
 structlogger = structlog.get_logger()
 
-TEST_TURNS_TYPE = Dict[int, Union[TestStep, ActualStepOutput]]
-
 
 class DialogueUnderstandingTestRunner:
-    """Dialogue Understanding test suite runner."""
+    """Dialogue Understanding test suite runner.
+
+    Responsible for executing dialogue understanding test cases by simulating
+    conversations and validating expected outputs against actual responses.
+
+    Attributes:
+        agent: The Rasa agent instance used for processing messages
+    """
 
     def __init__(
         self,
@@ -62,7 +82,8 @@ class DialogueUnderstandingTestRunner:
                 "load the trained model."
             )
 
-    def _check_action_server(self, endpoints: AvailableEndpoints) -> None:
+    @staticmethod
+    def _check_action_server(endpoints: AvailableEndpoints) -> None:
         """Check if the action server is reachable."""
         are_custom_actions_stubbed = (
             endpoints
@@ -74,7 +95,7 @@ class DialogueUnderstandingTestRunner:
                 endpoints, "dialogue_understanding_test_runner"
             )
 
-    async def run_tests(
+    async def run_test_cases(
         self,
         test_cases: List[DialogueUnderstandingTestCase],
         fixtures: List[Fixture],
@@ -90,4 +111,212 @@ class DialogueUnderstandingTestRunner:
         Returns:
             List[DialogueUnderstandingTestResult]: List of test results.
         """
-        return []
+        results = []
+        output_channel = CollectingOutputChannel()
+
+        structlogger.info("Starting dialogue understanding tests.")
+
+        for i in tqdm(range(len(test_cases))):
+            test_case = test_cases[i]
+
+            # set up the tracker by simulating the conversation
+            tracker_simulator = TestCaseTrackerSimulator(self.agent, test_case)
+            await tracker_simulator.initialize_tracker(fixtures)
+            try:
+                simulation_result = await tracker_simulator.simulate_test_case(metadata)
+            except TestCaseTrackerSimulatorException as e:
+                structlogger.error(
+                    "Failed to simulate test case. Skipping test case.",
+                    error=str(e),
+                )
+                continue
+
+            # run the actual test case
+            test_result = await self.run_test_case(
+                test_case, metadata, simulation_result, output_channel
+            )
+            results.append(test_result)
+
+        structlogger.info("Finished dialogue understanding tests.")
+
+        return results
+
+    async def run_test_case(
+        self,
+        test_case: DialogueUnderstandingTestCase,
+        metadata: List[Metadata],
+        simulation_result: TestCaseTrackerSimulatorResult,
+        output_channel: CollectingOutputChannel,
+    ) -> DialogueUnderstandingTestResult:
+        """Runs dialogue understanding test case.
+
+        Args:
+            test_case: The test case to run.
+            metadata: List of metadata.
+            simulation_result: The test case tracker.
+            output_channel: The output channel.
+
+        Returns:
+            A dialogue understanding test result.
+        """
+        test_passed = True
+
+        sender_id = simulation_result.sender_id
+        user_uttered_event_indices = simulation_result.user_uttered_event_indices
+        test_case_tracker = await self.agent.tracker_store.retrieve(sender_id)
+
+        for user_step_index, user_step in enumerate(
+            test_case.iterate_over_user_steps()
+        ):
+            if user_step_index >= len(user_uttered_event_indices):
+                # the conversation could not be simulated completely and was
+                # stopped early, no further user messages can be evaluated
+                return DialogueUnderstandingTestResult(
+                    test_case=test_case, passed=test_passed
+                )
+
+            # create and save the tracker at the time just
+            # before the user message was sent
+            step_sender_id = f"{sender_id}_{user_step_index}"
+            await self._create_tracker_for_user_step(
+                step_sender_id,
+                test_case_tracker,
+                user_uttered_event_indices[user_step_index],
+            )
+
+            # send the user message
+            try:
+                await self._send_user_message(
+                    step_sender_id,
+                    test_case,
+                    user_step,
+                    metadata,
+                    output_channel=output_channel,
+                )
+            except Exception as e:
+                structlogger.error(
+                    "dialogue_understanding_test_runner.send_user_message.failed",
+                    test_case=test_case.full_name(),
+                    user_message=user_step.text,
+                    error=str(e),
+                )
+                # as sending the user message failed, we cannot continue with the test
+                # as subsequent user message do not match the actual conversation
+                # return a test result up until this point
+                return DialogueUnderstandingTestResult(
+                    test_case=test_case, passed=test_passed
+                )
+
+            # get the dialogue understanding output
+            tracker = await self.agent.tracker_store.retrieve(step_sender_id)
+            dialogue_understanding_output = self.get_dialogue_understanding_output(
+                tracker, user_uttered_event_indices[user_step_index]
+            )
+            user_step.dialogue_understanding_output = dialogue_understanding_output
+
+            # check if we have a command match
+            if not user_step.has_passed():
+                test_passed = False
+
+        return DialogueUnderstandingTestResult(test_case=test_case, passed=test_passed)
+
+    def get_dialogue_understanding_output(
+        self,
+        tracker: DialogueStateTracker,
+        index_user_uttered_event: int,
+    ) -> Optional[DialogueUnderstandingOutput]:
+        """Returns the dialogue understanding output.
+
+        Creates the dialogue understanding output from the commands and prompts
+        added to the user uttered event.
+
+        Args:
+            tracker: The tracker for the current test step.
+            index_user_uttered_event: The index of the user uttered event.
+
+        Returns:
+        The dialogue understanding output with commands and optionally prompts.
+        """
+        user_uttered_event = self._get_user_uttered_event_from_tracker(
+            tracker, index_user_uttered_event
+        )
+        if user_uttered_event is None:
+            return None
+
+        predicted_commands: Dict[str, Any] = user_uttered_event.parse_data.get(  # type:ignore[assignment]
+            PREDICTED_COMMANDS, {}
+        )
+        # convert the predicted commands to Command objects
+        commands = {}
+        for component, list_of_commands in predicted_commands.items():
+            # remove any duplicate commands
+            commands[component] = list(
+                set(
+                    [Command.command_from_json(command) for command in list_of_commands]
+                )
+            )
+
+        return DialogueUnderstandingOutput(
+            commands=commands,
+            prompts=user_uttered_event.parse_data.get(PROMPTS, []),
+        )
+
+    @staticmethod
+    def _get_user_uttered_event_from_tracker(
+        tracker: DialogueStateTracker,
+        index_user_uttered_event: int,
+    ) -> Optional[UserUttered]:
+        """Returns the user uttered event from the tracker at the given index."""
+        if (
+            tracker.events is None
+            or len(list(tracker.events)) <= index_user_uttered_event
+        ):
+            return None
+
+        user_uttered_event = list(tracker.events)[index_user_uttered_event]
+
+        if not user_uttered_event or not isinstance(user_uttered_event, UserUttered):
+            return None
+
+        return user_uttered_event
+
+    async def _create_tracker_for_user_step(
+        self,
+        step_sender_id: str,
+        test_case_tracker: DialogueStateTracker,
+        index_user_uttered_event: int,
+    ) -> None:
+        """Creates a tracker for the user step."""
+        tracker = test_case_tracker.copy()
+        # modify the sender id so that the test case tracker is not overwritten
+        tracker.sender_id = step_sender_id
+
+        if tracker.events:
+            # get timestamp of the event just before the user uttered event
+            timestamp = tracker.events[index_user_uttered_event - 1].timestamp
+            # revert the tracker to the event just before the user uttered event
+            tracker = tracker.travel_back_in_time(timestamp)
+
+        # store the tracker with the unique sender id
+        await self.agent.tracker_store.save(tracker)
+
+    async def _send_user_message(
+        self,
+        sender_id: str,
+        test_case: DialogueUnderstandingTestCase,
+        user_step: DialogueUnderstandingTestStep,
+        metadata: List[Metadata],
+        output_channel: CollectingOutputChannel = CollectingOutputChannel(),
+    ) -> None:
+        """Sends a user message to the agent."""
+        metadata_for_step = filter_metadata(test_case, user_step, metadata, sender_id)
+
+        user_message = UserMessage(
+            user_step.text,
+            output_channel,
+            sender_id,
+            metadata=metadata_for_step,
+        )
+
+        with set_record_commands_and_prompts():
+            await self.agent.handle_message(user_message)