rasa-pro 3.12.0.dev1__py3-none-any.whl → 3.12.0.dev3__py3-none-any.whl

rasa/dialogue_understanding_test/du_test_schema.yml

@@ -0,0 +1,161 @@
+mapping:
+  fixtures:
+    allowempty: True
+    type: "seq"
+    sequence:
+      - type: map
+        mapping:
+          regex;(^[a-zA-Z_]+[a-zA-Z0-9_]*$):
+            type: "seq"
+            sequence:
+              - type: map
+                mapping:
+                  regex;(^[a-zA-Z_]+[a-zA-Z0-9_]*$):
+                    type: any
+
+  metadata:
+    allowempty: True
+    type: "seq"
+    sequence:
+      - type: map
+        mapping:
+          regex;(^[a-zA-Z_]+[a-zA-Z0-9_]*$):
+            type: map
+            mapping:
+              regex;(^[a-zA-Z_]+[a-zA-Z0-9_]*$):
+                type: any
+
+  stub_custom_actions:
+    type: map
+    mapping:
+      regex;(^[a-zA-Z_]+([a-zA-Z0-9_]*::?)*[a-zA-Z0-9_]*$):
+        type: map
+        mapping:
+          events:
+            type: seq
+            sequence:
+              - type: map
+                matching: any
+                mapping:
+                  event:
+                    type: "str"
+                    enum: ["user", "bot", "slot", "session_started", "action", "reset_slots", "restart", "reminder", "cancel_reminder", "pause", "resume", "followup", "export", "undo", "rewind", "entities", "active_loop", "agent", "loop_interrupted", "action_execution_rejected"]
+                  timestamp:
+                    type: float
+                  metadata:
+                    type: map
+                    mapping:
+                      regex;(^[a-zA-Z_]+[a-zA-Z0-9_]*$):
+                        type: any
+                  text:
+                    type: "str"
+                  parse_data:
+                    type: map
+                    mapping:
+                        regex;(^[a-zA-Z_]+[a-zA-Z0-9_]*$):
+                            type: any
+                  input_channel:
+                    type: "str"
+                  data:
+                    type: map
+                    mapping:
+                      regex;(^[a-zA-Z_]+[a-zA-Z0-9_]*$):
+                        type: any
+                  name:
+                    type: "str"
+                  value:
+                    type: any
+                  entities:
+                    type: any
+                  intent:
+                    type: "str"
+                  kill_on_user_message:
+                    type: bool
+                  date_time:
+                    type: "str"
+                  policy:
+                    type: "str"
+                  confidence:
+                    type: float
+                  is_interrupted:
+                    type: bool
+
+          responses:
+            type: seq
+            sequence:
+              - type: map
+                matching: any
+                mapping:
+                  text:
+                    type: "str"
+                  buttons:
+                    type: "seq"
+                    sequence:
+                      - type: map
+                        mapping:
+                          payload:
+                            type: "str"
+                          title:
+                            type: "str"
+                  response:
+                    type: "str"
+                  image:
+                    type: "str"
+                  attachment:
+                    type: "str"
+                  elements:
+                    type: "seq"
+                    sequence:
+                      - type: map
+                        mapping:
+                          regex;(^[a-zA-Z_]+[a-zA-Z0-9_]*$):
+                            type: any
+                  custom:
+                    type: map
+                    mapping:
+                      regex;(^[a-zA-Z_]+[a-zA-Z0-9_]*$):
+                        type: any
+
+  test_cases:
+    type: "seq"
+    sequence:
+      - type: map
+        mapping:
+          test_case:
+            type: "str"
+            required: True
+          fixtures:
+            allowempty: True
+            type: "seq"
+            sequence:
+              - type: "str"
+                pattern: ^[a-zA-Z_]+[a-zA-Z0-9_]*$
+          metadata:
+            type: "str"
+            pattern: ^[a-zA-Z_]+[a-zA-Z0-9_]*$
+          steps:
+            type: "seq"
+            matching: "any"
+            required: True
+            sequence:
+              - type: map
+                mapping:
+                  user:
+                    type: str
+                    required: True
+                  commands:
+                    type: "seq"
+                    required: True
+                    sequence:
+                      - type: "str"
+                  metadata:
+                    type: "str"
+                    pattern: ^[a-zA-Z_]+[a-zA-Z0-9_]*$
+              - type: map
+                mapping:
+                  utter:
+                    type: "str"
+              - type: map
+                mapping:
+                  bot:
+                    type: "str"

rasa/dialogue_understanding_test/io.py

@@ -1,54 +1,372 @@
-from typing import List, Dict
+from collections import defaultdict
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Dict, List, Union
 
+import rich
+
+import rasa.shared.data
+from rasa.dialogue_understanding_test.command_metric_calculation import CommandMetrics
+from rasa.dialogue_understanding_test.constants import SCHEMA_FILE_PATH
 from rasa.dialogue_understanding_test.du_test_result import (
-    DialogueUnderstandingTestResult,
+    DialogueUnderstandingTestSuiteResult,
+    FailedTestStep,
+)
+from rasa.e2e_test.constants import KEY_TEST_CASE, KEY_TEST_CASES
+from rasa.e2e_test.e2e_test_case import (
+    DialogueUnderstandingTestCase,
+    Fixture,
+    Metadata,
+    TestSuite,
+)
+from rasa.e2e_test.stub_custom_action import StubCustomAction
+from rasa.e2e_test.utils.io import (
+    check_beta_feature_flag_for_custom_actions_stubs,
+    extract_fixtures,
+    extract_metadata,
+    extract_stub_custom_actions,
+    extract_test_case_from_path,
+    is_test_case_file,
+    parse_raw_yaml,
+    validate_path_to_test_cases,
+    validate_test_case,
+)
+from rasa.shared.core.flows import FlowsList
+from rasa.shared.utils.yaml import (
+    read_schema_file,
+    validate_yaml_content_using_schema,
+    write_yaml,
 )
-from rasa.e2e_test.e2e_test_case import TestSuite
 
+if TYPE_CHECKING:
+    from rasa.dialogue_understanding.commands import Command
 
-def read_test_suite(test_case_path: str) -> TestSuite:
+
+def read_du_test_schema() -> Union[List[Any], Dict[str, Any]]:
+    """Read the schema for the dialogue understanding test cases.
+
+    Returns:
+        Dict[str, Any]: Schema for the dialogue understanding test cases.
+    """
+    return read_schema_file(SCHEMA_FILE_PATH)
+
+
+def read_test_suite(
+    test_case_path: str,
+    flows: FlowsList,
+    custom_command_classes: List["Command"] = [],
+    remove_default_commands: List[str] = [],
+) -> TestSuite:
     """Read the test cases from the given test case path.
 
     Args:
         test_case_path: Path to the test cases.
+        flows: List of flows.
+        custom_command_classes: Custom command classes to use in the test cases.
+        remove_default_commands: Default commands to remove from the test cases.
 
     Returns:
         TestSuite: Test suite containing the dialogue understanding test cases.
     """
-    return TestSuite([], [], [], {})
+
+    def _extract_test_cases(
+        test_file_content: dict,
+        test_case_name: str,
+        test_file: str,
+        flows: FlowsList,
+        custom_command_classes: List["Command"] = [],
+        remove_default_commands: List[str] = [],
+    ) -> List[DialogueUnderstandingTestCase]:
+        """Extract test cases from the test file content.
+
+        Args:
+            test_file_content: Content of the test file.
+            test_case_name: Name of the test case to extract.
+            test_file: Path to the test file.
+            flows: List of flows.
+
+        Returns:
+            List of test cases.
+        """
+        test_cases_content = test_file_content.get(KEY_TEST_CASES) or []
+        if test_case_name:
+            return [
+                DialogueUnderstandingTestCase.from_dict(
+                    test_case_dict,
+                    flows,
+                    file=test_file,
+                    custom_command_classes=custom_command_classes,
+                    remove_default_commands=remove_default_commands,
+                )
+                for test_case_dict in test_cases_content
+                if test_case_name == test_case_dict.get(KEY_TEST_CASE)
+            ]
+        return [
+            DialogueUnderstandingTestCase.from_dict(
+                test_case_dict,
+                flows,
+                file=test_file,
+                custom_command_classes=custom_command_classes,
+                remove_default_commands=remove_default_commands,
+            )
+            for test_case_dict in test_cases_content
+        ]
+
+    # Extract test case path and name
+    test_case_path, test_case_name = extract_test_case_from_path(test_case_path)
+    validate_path_to_test_cases(test_case_path)
+
+    # Load test files and schema
+    test_files = rasa.shared.data.get_data_files([test_case_path], is_test_case_file)
+    test_schema = read_du_test_schema()
+
+    # Initialize containers
+    input_test_cases = []
+    fixtures: Dict[str, Fixture] = {}
+    metadata: Dict[str, Metadata] = {}
+    stub_custom_actions: Dict[str, StubCustomAction] = {}
+
+    # Process each test file
+    for test_file in test_files:
+        test_file_content = parse_raw_yaml(Path(test_file).read_text(encoding="utf-8"))
+
+        # Validate YAML content using the provided function
+        validate_yaml_content_using_schema(test_file_content, test_schema)
+
+        # Parse test cases, fixtures, metadata, and stub custom actions
+        test_cases = _extract_test_cases(
+            test_file_content,
+            test_case_name,
+            test_file,
+            flows,
+            custom_command_classes,
+            remove_default_commands,
+        )
+        fixtures.update(extract_fixtures(test_file_content, fixtures))
+        metadata.update(extract_metadata(test_file_content, metadata))
+        stub_custom_actions.update(
+            extract_stub_custom_actions(test_file_content, test_file)
+        )
+        input_test_cases.extend(test_cases)
+
+    validate_test_case(test_case_name, input_test_cases, fixtures, metadata)
+    if stub_custom_actions:
+        check_beta_feature_flag_for_custom_actions_stubs()
+
+    return TestSuite(
+        input_test_cases,
+        list(fixtures.values()),
+        list(metadata.values()),
+        stub_custom_actions,
+    )
 
 
 def write_test_results_to_file(
-    failed_tests: List[DialogueUnderstandingTestResult],
-    passed_tests: List[DialogueUnderstandingTestResult],
-    command_metrics: Dict[str, Dict[str, float]],
+    test_suite_result: DialogueUnderstandingTestSuiteResult,
     output_file: str,
     output_prompt: bool,
 ) -> None:
     """Write the test results to the given output file.
 
     Args:
-        failed_tests: Failed test cases.
-        passed_tests: Passed test cases.
-        command_metrics: Metrics for the commands.
+        test_suite_result: Test results suite containing the test results.
         output_file: Path to the output file.
         output_prompt: Whether to log the prompt or not.
     """
-    pass
+    write_yaml(
+        test_suite_result.to_dict(output_prompt),
+        target=output_file,
+        should_preserve_key_order=True,
+    )
+
+    print(f"Results written to '{output_file}'.")
 
 
 def print_test_results(
-    failed_tests: List[DialogueUnderstandingTestResult],
-    passed_tests: List[DialogueUnderstandingTestResult],
-    command_metrics: Dict[str, Dict[str, float]],
+    test_suite_result: DialogueUnderstandingTestSuiteResult,
     output_prompt: bool,
 ) -> None:
-    """Print the test results to console.
+    """Print the result of the test run.
+
+    Example output (truncated for brevity):
+        =================================================================== FAILURES
+        ===================================================================
+
+        ---------------- test_case:
+        /Users/milos/Projects/rasa-calm-demo/dialogue_understanding_tests/
+        immediate_cancellation_and_start_of_new_flow.yml::
+        user immediately cancels and starts new flow -----------------
+
+        == failure starting at user message 'I want to send money'.
+
+        -- COMMAND GENERATOR(s) --
+        SingleStepLLMCommandGenerator
+
+        -- CONVERSATION --
+        user: I want to send money
+        ---EXPECTED---                               | ---PREDICTED---
+        StartFlow(transfer_money)                    | StartFlow(transfer_money)
+        SetSlot(transfer_money_amount_of_money, 878) |
+
+        ...
+
+        set slot (2 commands in total):
+          tp: 0 fp: 1 fn: 2
+          precision: 0.0000
+          recall   : 0.0000
+          f1       : 0.0000
+
+        cancel flow (1 commands in total):
+          tp: 1 fp: 0 fn: 0
+          precision: 1.0000
+          recall   : 1.0000
+          f1       : 1.0000
+
+        =================================== 1 failed test cases, 0 passed test cases
+        ===================================
+        ======== 2 failed user steps, 1 passed user steps (accuracy: 0.3333) ========
 
     Args:
-        failed_tests: Failed test cases.
-        passed_tests: Passed test cases.
-        command_metrics: Metrics for the commands.
+        test_suite_result: Test results suite containing the test results.
         output_prompt: Whether to log the prompt or not.
     """
-    pass
+    if (
+        test_suite_result.number_of_passed_tests
+        + test_suite_result.number_of_failed_tests
+        == 0
+    ):
+        # no tests were run, print error
+        rasa.shared.utils.cli.print_error(
+            rasa.shared.utils.cli.pad("No test cases found.", char="!")
+        )
+        return
+
+    if test_suite_result.number_of_failed_tests > 0:
+        # print failure headline
+        print()
+        rich.print(
+            f"[bold][red3]"
+            f"{rasa.shared.utils.cli.pad('FAILURES', char='=')}"
+            f"[/red3][/bold]"
+        )
+
+        # print failed test steps
+        print_failed_cases(test_suite_result, output_prompt=output_prompt)
+
+    print_command_summary(test_suite_result.command_metrics)
+    print_final_line(test_suite_result)
+
+
+def print_failed_cases(
+    test_suite_result: DialogueUnderstandingTestSuiteResult,
+    output_prompt: bool,
+) -> None:
+    """Print the details of a failed test case."""
+    # Group the failed test steps by test case
+    step_groups = defaultdict(list)
+    for step in test_suite_result.failed_test_steps:
+        key = f"{step.file}::{step.test_case_name}"
+        step_groups[key].append(step)
+
+    for failed_test_case, failed_test_steps in step_groups.items():
+        fail_headline = f"test_case: {failed_test_case}"
+        print()
+        rasa.shared.utils.cli.print_error(
+            f"{rasa.shared.utils.cli.pad(fail_headline, char='-')}\n"
+        )
+        print(f"Number of failed steps: {len(failed_test_steps)}")
+        for step in failed_test_steps:
+            print()
+            rasa.shared.utils.cli.print_info(
+                f"== failure starting at user message "
+                f"'{step.failed_user_utterance}'. "
+            )
+
+            rich.print("\n[red3]-- COMMAND GENERATOR(s) --[/red3]")
+            rich.print("\n".join(step.command_generators))
+            if output_prompt:
+                print_prompt(step)
+            rich.print("\n[red3]-- CONVERSATION --[/red3]")
+            rich.print("\n".join(step.conversation_with_diff))
+
+
+def print_prompt(step: FailedTestStep) -> None:
+    if step.prompt is None:
+        return
+    prompt_data = step.prompt
+
+    rich.print("\n[red3]-- PROMPT(s) --[/red3]")
+    for component, prompts in prompt_data.items():
+        rich.print(f"[bold]{component}[/bold]")
+        for subcomponent, prompt in prompts:
+            rich.print(f"    [bold]{subcomponent}[/bold]")
+            rich.print(f"        {prompt}")
+
+
+def print_command_summary(metrics: Dict[str, CommandMetrics]) -> None:
+    """Print the command summary.
+
+    Args:
+        metrics: Dict of command to precision, recall, f1, etc. scores
+    """
+    print()
+    rasa.shared.utils.cli.print_info(rasa.shared.utils.cli.pad("COMMAND METRICS"))
+
+    for command_name, command_metric in metrics.items():
+        rasa.shared.utils.cli.print_info(
+            f"{command_name} " f"({command_metric.total_count} commands in total):"
+        )
+        rasa.shared.utils.cli.print_info(
+            f"  tp: {command_metric.tp} "
+            f"fp: {command_metric.fp} "
+            f"fn: {command_metric.fn}"
+        )
+        rasa.shared.utils.cli.print_info(
+            f"  precision: {command_metric.get_precision():.4f}"
+        )
+        rasa.shared.utils.cli.print_info(
+            f"  recall   : {command_metric.get_recall():.4f}"
+        )
+        rasa.shared.utils.cli.print_info(
+            f"  f1       : {command_metric.get_f1_score():.4f}"
+        )
+
+
+def print_final_line(test_suite_result: DialogueUnderstandingTestSuiteResult) -> None:
+    """Print the final line of the test output.
+
+    Args:
+        test_suite_result: Test suite result.
+    """
+    has_failed = test_suite_result.number_of_failed_tests > 0
+
+    final_line_color = "green3" if not has_failed else "red3"
+
+    def _print_accuracy_line(number_passed: int, number_failed: int, type: str) -> None:
+        if number_passed > 0:
+            line = rasa.shared.utils.cli.pad(
+                f"[bold red3]{number_failed} failed {type}[/bold red3]"
+                f"[bright_white], [/bright_white]"
+                f"[bold green3]{number_passed} passed {type}[/bold green3]"
+                f"[bright_white] "
+                f"(accuracy:  {number_passed / (number_passed + number_failed):.4f})"
+                f"[/bright_white]"
+            )
+        else:
+            line = rasa.shared.utils.cli.pad(
+                f"[bold red3]{number_failed} failed {type}[/bold red3]"
+                f"[bright_white], [/bright_white]"
+                f"[bold green3]{number_passed} passed {type}[/bold green3]"
+            )
+        rich.print(f"[{final_line_color}]{line}[/{final_line_color}]")
+
+    _print_accuracy_line(
+        test_suite_result.number_of_passed_tests,
+        test_suite_result.number_of_failed_tests,
+        "test cases",
+    )
+
+    _print_accuracy_line(
+        test_suite_result.number_of_passed_user_utterances,
+        test_suite_result.number_of_failed_user_utterances,
+        "user steps",
+    )

rasa/dialogue_understanding_test/test_case_simulation/exception.py

@@ -0,0 +1,28 @@
+from typing import Any, Dict, Optional
+
+from rasa.shared.exceptions import RasaException
+
+
+class TestCaseTrackerSimulatorException(RasaException):
+    def __init__(
+        self,
+        test_case_name: str,
+        user_message: Optional[str] = None,
+        failure_reason: Optional[str] = None,
+        original_exception: Optional[Dict[str, Any]] = None,
+    ):
+        super().__init__("An error occurred while simulating a conversation.")
+        self.test_case_name = test_case_name
+        self.user_message = user_message
+        self.original_exception = original_exception
+        self.failure_reason = failure_reason
+
+    def __str__(self) -> str:
+        s = f"{self.__class__.__name__}:"
+        s += f"\nTest case: {self.test_case_name}"
+        if self.user_message is not None:
+            s += f"\nUser message: {self.user_message}"
+        if self.failure_reason is not None:
+            s += f"\nFailure reason: {self.failure_reason}"
+        s += f"\nOriginal error: {self.original_exception}\n"
+        return s