rasa-pro 3.12.0.dev1__py3-none-any.whl

rasa/dialogue_understanding/stack/utils.py

@@ -0,0 +1,211 @@
+from typing import List, Optional, Set, Tuple
+import typing
+from rasa.dialogue_understanding.patterns.collect_information import (
+    CollectInformationPatternFlowStackFrame,
+)
+from rasa.dialogue_understanding.stack.frames import BaseFlowStackFrame
+from rasa.dialogue_understanding.stack.dialogue_stack import DialogueStack
+from rasa.dialogue_understanding.stack.frames import UserFlowStackFrame
+from rasa.dialogue_understanding.stack.frames.flow_stack_frame import FlowStackFrameType
+from rasa.dialogue_understanding.stack.frames.pattern_frame import PatternFlowStackFrame
+from rasa.shared.core.flows.steps.collect import CollectInformationFlowStep
+from rasa.shared.core.flows.steps.constants import END_STEP
+from rasa.shared.core.flows.steps.continuation import ContinueFlowStep
+from rasa.shared.core.flows import FlowsList
+
+if typing.TYPE_CHECKING:
+    from rasa.shared.core.trackers import DialogueStateTracker
+
+
+def top_flow_frame(
+    dialogue_stack: DialogueStack,
+    ignore_collect_information_pattern: bool = True,
+    ignore_call_frames: bool = True,
+) -> Optional[BaseFlowStackFrame]:
+    """Returns the topmost flow frame from the tracker.
+
+    By default, the topmost flow frame is ignored if it is the
+    `pattern_collect_information`. This is because the `pattern_collect_information`
+    is a special flow frame that is used to collect information from the user
+    and commonly, is not what you are looking for when you want the topmost frame.
+
+    Also excludes frames created by a call step. They are treated as if they are
+    directly part of the flow that called them.
+
+    Args:
+        dialogue_stack: The dialogue stack to use.
+        ignore_collect_information_pattern: Whether to ignore the
+            `pattern_collect_information` frame.
+        ignore_call_frames: Whether to ignore user frames of type `call`
+
+    Returns:
+        The topmost flow frame from the tracker. `None` if there
+        is no frame on the stack.
+    """
+    for frame in reversed(dialogue_stack.frames):
+        if ignore_collect_information_pattern and isinstance(
+            frame, CollectInformationPatternFlowStackFrame
+        ):
+            continue
+        if ignore_call_frames and (
+            isinstance(frame, UserFlowStackFrame)
+            and frame.frame_type == FlowStackFrameType.CALL
+        ):
+            continue
+        if isinstance(frame, BaseFlowStackFrame):
+            return frame
+    return None
+
+
+def top_user_flow_frame(dialogue_stack: DialogueStack) -> Optional[UserFlowStackFrame]:
+    """Returns the topmost user flow frame from the tracker.
+
+    User flows are flows that are created by developers of an assistant and
+    describe tasks the assistant can fulfil. The topmost user flow frame is
+    the topmost frame on the stack for one of these user flows. While looking
+    for this topmost user flow, other frame types such as PatternFrames,
+    SearchFrames, ChitchatFrames, and user flows started through call steps
+    are skipped. The latter are treated as part of the calling user flow.
+
+    Args:
+        dialogue_stack: The dialogue stack to use.
+
+    Returns:
+    The topmost user flow frame from the tracker.
+    """
+    for frame in reversed(dialogue_stack.frames):
+        if (
+            isinstance(frame, UserFlowStackFrame)
+            and frame.frame_type != FlowStackFrameType.CALL
+        ):
+            return frame
+    return None
+
+
+def filled_slots_for_active_flow(
+    tracker: "DialogueStateTracker", all_flows: FlowsList
+) -> Tuple[Set[str], Optional[str]]:
+    """Get all slots that have been filled for the 'current user flow'.
+
+    All patterns that sit ontop of that user flow as well as
+    flows called by this flow, are also included.
+
+    Any collect information step that is part of a pattern on top of the current
+    user flow are also included.
+
+    Args:
+        tracker: The tracker to get the filled slots from.
+        all_flows: All flows.
+
+    Returns:
+    All slots that have been filled for the current flow and the id of the currently
+    active flow.
+    """
+    stack = tracker.stack
+    user_frame = top_user_flow_frame(stack)
+    active_flow = user_frame.flow_id if user_frame else None
+
+    filled_slots = set()
+    for collect_step, _ in previous_collect_steps_for_active_flow(tracker, all_flows):
+        filled_slots.add(collect_step.collect)
+
+    return filled_slots, active_flow
+
+
+def previous_collect_steps_for_active_flow(
+    tracker: "DialogueStateTracker", all_flows: FlowsList
+) -> List[Tuple[CollectInformationFlowStep, str]]:
+    stack = tracker.stack
+    user_frame = top_user_flow_frame(stack)
+
+    if not user_frame:
+        return []
+
+    collect_steps: List[Tuple[CollectInformationFlowStep, str]] = []
+
+    active_frames = {frame.frame_id for frame in stack.frames}
+
+    for previous_stack in tracker.previous_stack_states():
+        active_frame = top_user_flow_frame(previous_stack)
+        if not active_frame or active_frame.frame_id != user_frame.frame_id:
+            continue
+
+        top_frame = previous_stack.top()
+        if not isinstance(top_frame, BaseFlowStackFrame):
+            continue
+
+        step = top_frame.step(all_flows)
+        if not isinstance(step, CollectInformationFlowStep):
+            continue
+
+        if isinstance(top_frame, UserFlowStackFrame):
+            collect_steps.append((step, top_frame.flow_id))
+        elif (
+            isinstance(top_frame, PatternFlowStackFrame)
+            and top_frame.frame_id in active_frames
+        ):
+            # if this is a pattern, it is only relevant if it is still
+            # active in the current state of the conversation.
+            # completed patterns in the past are not relevant
+            collect_steps.append((step, top_frame.flow_id))
+
+    return collect_steps
+
+
+def user_flows_on_the_stack(dialogue_stack: DialogueStack) -> Set[str]:
+    """Get all user flows that are currently on the stack.
+
+    Args:
+        dialogue_stack: The dialogue stack.
+
+    Returns:
+    All user flows that are currently on the stack.
+    """
+    return {
+        f.flow_id for f in dialogue_stack.frames if isinstance(f, UserFlowStackFrame)
+    }
+
+
+def end_top_user_flow(stack: DialogueStack) -> DialogueStack:
+    """Ends all frames on top of the stack including the topmost user frame.
+
+    Ends all flows until the next user flow is reached. This is useful
+    if you want to end all flows that are currently on the stack and
+    the user flow that triggered them.
+
+    Args:
+        stack: The dialogue stack.
+    """
+    updated_stack = stack.copy()
+
+    for frame in reversed(updated_stack.frames):
+        if isinstance(frame, BaseFlowStackFrame):
+            frame.step_id = ContinueFlowStep.continue_step_for_id(END_STEP)
+            if isinstance(frame, UserFlowStackFrame):
+                break
+    return updated_stack
+
+
+def get_collect_steps_excluding_ask_before_filling_for_active_flow(
+    dialogue_stack: DialogueStack, all_flows: FlowsList
+) -> Set[str]:
+    """Get all collect steps that are part of the current flow, without
+    considering the collect steps that has to be asked before filling.
+
+    Args:
+        dialogue_stack: The dialogue stack.
+        all_flows: All flows.
+
+    Returns:
+        All collect steps that are part of the current active flow,
+        excluding the collect steps that have to be asked before filling.
+    """
+    active_frame = top_user_flow_frame(dialogue_stack)
+    if active_frame is None:
+        return set()
+    active_flow = active_frame.flow(all_flows)
+    return set(
+        step.collect
+        for step in active_flow.get_collect_steps()
+        if not step.ask_before_filling
+    )

rasa/dialogue_understanding/utils.py

@@ -0,0 +1,14 @@
+from contextlib import contextmanager
+from typing import Generator
+
+record_commands_and_prompts = False
+
+
+@contextmanager
+def set_record_commands_and_prompts() -> Generator:
+    global record_commands_and_prompts
+    record_commands_and_prompts = True
+    try:
+        yield
+    finally:
+        record_commands_and_prompts = False

rasa/dialogue_understanding_test/command_metric_calculation.py

@@ -0,0 +1,12 @@
+from typing import List, Dict
+
+from rasa.dialogue_understanding_test.du_test_result import (
+    DialogueUnderstandingTestResult,
+)
+
+
+def calculate_command_metrics(
+    test_results: List[DialogueUnderstandingTestResult],
+) -> Dict[str, Dict[str, float]]:
+    """Calculate the metrics for the commands."""
+    return {}

rasa/dialogue_understanding_test/constants.py

@@ -0,0 +1,17 @@
+import rasa.e2e_test.constants as constants
+
+KEY_FIXTURES = constants.KEY_FIXTURES
+KEY_METADATA = constants.KEY_METADATA
+KEY_STUB_CUSTOM_ACTIONS = constants.KEY_STUB_CUSTOM_ACTIONS
+KEY_USER_INPUT = constants.KEY_USER_INPUT
+KEY_BOT_INPUT = constants.KEY_BOT_INPUT
+KEY_BOT_UTTERED = constants.KEY_BOT_UTTERED
+KEY_STEPS = constants.KEY_STEPS
+KEY_TEST_CASE = constants.KEY_TEST_CASE
+KEY_TEST_CASES = constants.KEY_TEST_CASES
+KEY_COMMANDS = "commands"
+
+ACTOR_USER = "user"
+ACTOR_BOT = "bot"
+
+DEFAULT_INPUT_TESTS_PATH = "du_tests/"

rasa/dialogue_understanding_test/du_test_case.py

@@ -0,0 +1,118 @@
+from typing import List, Optional, Dict, Any
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from rasa.dialogue_understanding.commands import Command
+from rasa.dialogue_understanding_test.constants import (
+    ACTOR_USER,
+    KEY_COMMANDS,
+    ACTOR_BOT,
+    KEY_TEST_CASE,
+    KEY_STEPS,
+    KEY_FIXTURES,
+    KEY_METADATA,
+    KEY_USER_INPUT,
+    KEY_BOT_INPUT,
+    KEY_BOT_UTTERED,
+)
+
+
+class DialogueUnderstandingOutput(BaseModel):
+    """Output containing prompts and generated commands by component.
+
+    Example of commands:
+        {
+            "MultiStepLLMCommandGenerator": [
+                {"command": "set_slot", "name": "slot_name", "value": "slot_value"},
+            ],
+            "NLUCommandAdapter": [
+                {"command": "start_flow", "name": "test_flow"},
+            ]
+        }
+
+    Example of prompts:
+        {
+            "MultiStepLLMCommandGenerator": [
+                (
+                    "fill_slots_prompt",
+                    {
+                        "user_prompt": "<prompt content>",
+                        "system_prompt": "<prompt content>"
+                    }
+                ),
+                (
+                    "handle_flows_prompt",
+                    {
+                        "user_prompt": "<prompt content>",
+                        "system_prompt": "<prompt content>"
+                    }
+                ),
+            ],
+        }
+    """
+
+    prompts: Dict[str, tuple[str, Dict[str, str]]]
+    commands: Dict[str, List[Command]]
+
+    model_config = ConfigDict(frozen=True)
+
+    def get_component_data(
+        self, component_name: str
+    ) -> tuple[Optional[tuple[str, Dict[str, str]]], List[Command]]:
+        """Get both the prompts and commands for a specific component."""
+        return self.prompts.get(component_name), self.commands.get(component_name, [])
+
+
+class DialogueUnderstandingTestStep(BaseModel):
+    actor: str
+    text: Optional[str] = None
+    template: Optional[str] = None
+    line: Optional[int] = None
+    metadata_name: Optional[str] = None
+    commands: Optional[List[Command]] = None
+    dialogue_understanding_output: Optional[DialogueUnderstandingOutput] = None
+
+    def as_dict(self) -> Dict[str, Any]:
+        if self.actor == ACTOR_USER:
+            if self.commands:
+                return {
+                    KEY_USER_INPUT: self.text,
+                    # TODO: The command should be converted into our DSL
+                    KEY_COMMANDS: [command.as_dict() for command in self.commands],
+                }
+            return {ACTOR_USER: self.text}
+        elif self.actor == ACTOR_BOT:
+            if self.text is not None:
+                return {KEY_BOT_INPUT: self.text}
+            elif self.template is not None:
+                return {KEY_BOT_UTTERED: self.template}
+
+        return {}
+
+
+class DialogueUnderstandingTestCase(BaseModel):
+    name: str
+    steps: list[DialogueUnderstandingTestStep] = Field(min_length=1)
+    file: Optional[str] = None
+    line: Optional[int] = None
+    fixture_names: Optional[List[str]] = None
+    metadata_name: Optional[str] = None
+
+    def full_name(self) -> str:
+        return f"{self.file}::{self.name}"
+
+    def as_dict(self) -> Dict[str, Any]:
+        result = {
+            KEY_TEST_CASE: self.name,
+            KEY_STEPS: [step.as_dict() for step in self.steps],
+        }
+        if self.fixture_names:
+            result[KEY_FIXTURES] = self.fixture_names
+        if self.metadata_name:
+            result[KEY_METADATA] = self.metadata_name
+        return result
+
+
+# Update forward references
+DialogueUnderstandingTestStep.model_rebuild()
+DialogueUnderstandingTestCase.model_rebuild()

rasa/dialogue_understanding_test/du_test_result.py

@@ -0,0 +1,11 @@
+from typing import Optional
+
+from pydantic import BaseModel
+
+from rasa.dialogue_understanding_test.du_test_case import DialogueUnderstandingTestCase
+
+
+class DialogueUnderstandingTestResult(BaseModel):
+    test_case: DialogueUnderstandingTestCase
+    passed: bool
+    error_line: Optional[int] = None

rasa/dialogue_understanding_test/du_test_runner.py

@@ -0,0 +1,93 @@
+import asyncio
+from typing import Dict, Optional, Text, Union, List
+
+import structlog
+
+from rasa.core.exceptions import AgentNotReady
+from rasa.core.persistor import StorageType
+from rasa.core.utils import AvailableEndpoints
+from rasa.dialogue_understanding_test.du_test_case import DialogueUnderstandingTestCase
+from rasa.dialogue_understanding_test.du_test_result import (
+    DialogueUnderstandingTestResult,
+)
+from rasa.e2e_test.e2e_test_case import (
+    KEY_STUB_CUSTOM_ACTIONS,
+    ActualStepOutput,
+    TestStep,
+    Fixture,
+    Metadata,
+)
+from rasa.e2e_test.e2e_test_runner import E2ETestRunner
+from rasa.utils.endpoints import EndpointConfig
+
+structlogger = structlog.get_logger()
+
+TEST_TURNS_TYPE = Dict[int, Union[TestStep, ActualStepOutput]]
+
+
+class DialogueUnderstandingTestRunner:
+    """Dialogue Understanding test suite runner."""
+
+    def __init__(
+        self,
+        model_path: Optional[Text] = None,
+        model_server: Optional[EndpointConfig] = None,
+        remote_storage: Optional[StorageType] = None,
+        endpoints: Optional[AvailableEndpoints] = None,
+    ) -> None:
+        """Initializes the Dialogue Understanding test suite runner.
+
+        Args:
+            model_path: Path to the model.
+            model_server: Model server configuration.
+            remote_storage: Remote storage to use for model retrieval.
+            endpoints: Endpoints configuration.
+        """
+        import rasa.core.agent
+
+        self._check_action_server(endpoints)
+
+        self.agent = asyncio.run(
+            rasa.core.agent.load_agent(
+                model_path=model_path,
+                model_server=model_server,
+                remote_storage=remote_storage,
+                endpoints=endpoints,
+            )
+        )
+        if not self.agent.is_ready():
+            raise AgentNotReady(
+                "Agent needs to be prepared before usage. "
+                "Please check that the agent was able to "
+                "load the trained model."
+            )
+
+    def _check_action_server(self, endpoints: AvailableEndpoints) -> None:
+        """Check if the action server is reachable."""
+        are_custom_actions_stubbed = (
+            endpoints
+            and endpoints.action
+            and endpoints.action.kwargs.get(KEY_STUB_CUSTOM_ACTIONS)
+        )
+        if endpoints and not are_custom_actions_stubbed:
+            E2ETestRunner._action_server_is_reachable(
+                endpoints, "dialogue_understanding_test_runner"
+            )
+
+    async def run_tests(
+        self,
+        test_cases: List[DialogueUnderstandingTestCase],
+        fixtures: List[Fixture],
+        metadata: List[Metadata],
+    ) -> List[DialogueUnderstandingTestResult]:
+        """Run the dialogue understanding tests.
+
+        Args:
+            test_cases: List of test cases.
+            fixtures: List of fixtures.
+            metadata: List of metadata.
+
+        Returns:
+            List[DialogueUnderstandingTestResult]: List of test results.
+        """
+        return []

rasa/dialogue_understanding_test/io.py

@@ -0,0 +1,54 @@
+from typing import List, Dict
+
+from rasa.dialogue_understanding_test.du_test_result import (
+    DialogueUnderstandingTestResult,
+)
+from rasa.e2e_test.e2e_test_case import TestSuite
+
+
+def read_test_suite(test_case_path: str) -> TestSuite:
+    """Read the test cases from the given test case path.
+
+    Args:
+        test_case_path: Path to the test cases.
+
+    Returns:
+        TestSuite: Test suite containing the dialogue understanding test cases.
+    """
+    return TestSuite([], [], [], {})
+
+
+def write_test_results_to_file(
+    failed_tests: List[DialogueUnderstandingTestResult],
+    passed_tests: List[DialogueUnderstandingTestResult],
+    command_metrics: Dict[str, Dict[str, float]],
+    output_file: str,
+    output_prompt: bool,
+) -> None:
+    """Write the test results to the given output file.
+
+    Args:
+        failed_tests: Failed test cases.
+        passed_tests: Passed test cases.
+        command_metrics: Metrics for the commands.
+        output_file: Path to the output file.
+        output_prompt: Whether to log the prompt or not.
+    """
+    pass
+
+
+def print_test_results(
+    failed_tests: List[DialogueUnderstandingTestResult],
+    passed_tests: List[DialogueUnderstandingTestResult],
+    command_metrics: Dict[str, Dict[str, float]],
+    output_prompt: bool,
+) -> None:
+    """Print the test results to console.
+
+    Args:
+        failed_tests: Failed test cases.
+        passed_tests: Passed test cases.
+        command_metrics: Metrics for the commands.
+        output_prompt: Whether to log the prompt or not.
+    """
+    pass

rasa/dialogue_understanding_test/validation.py

@@ -0,0 +1,22 @@
+import argparse
+from typing import List
+
+from rasa.dialogue_understanding_test.du_test_case import DialogueUnderstandingTestCase
+
+
+def validate_cli_arguments(args: argparse.Namespace) -> None:
+    """Validate the CLI arguments for the dialogue understanding test.
+
+    Args:
+        args: Commandline arguments.
+    """
+    pass
+
+
+def validate_test_cases(test_cases: List[DialogueUnderstandingTestCase]) -> None:
+    """Validate the dialogue understanding test cases.
+
+    Args:
+        test_cases: Test cases to validate.
+    """
+    pass