rasa-pro 3.11.3a1.dev2__py3-none-any.whl

rasa/dialogue_understanding/stack/utils.py

@@ -0,0 +1,211 @@
+from typing import List, Optional, Set, Tuple
+import typing
+from rasa.dialogue_understanding.patterns.collect_information import (
+    CollectInformationPatternFlowStackFrame,
+)
+from rasa.dialogue_understanding.stack.frames import BaseFlowStackFrame
+from rasa.dialogue_understanding.stack.dialogue_stack import DialogueStack
+from rasa.dialogue_understanding.stack.frames import UserFlowStackFrame
+from rasa.dialogue_understanding.stack.frames.flow_stack_frame import FlowStackFrameType
+from rasa.dialogue_understanding.stack.frames.pattern_frame import PatternFlowStackFrame
+from rasa.shared.core.flows.steps.collect import CollectInformationFlowStep
+from rasa.shared.core.flows.steps.constants import END_STEP
+from rasa.shared.core.flows.steps.continuation import ContinueFlowStep
+from rasa.shared.core.flows import FlowsList
+
+if typing.TYPE_CHECKING:
+    from rasa.shared.core.trackers import DialogueStateTracker
+
+
+def top_flow_frame(
+    dialogue_stack: DialogueStack,
+    ignore_collect_information_pattern: bool = True,
+    ignore_call_frames: bool = True,
+) -> Optional[BaseFlowStackFrame]:
+    """Returns the topmost flow frame from the tracker.
+
+    By default, the topmost flow frame is ignored if it is the
+    `pattern_collect_information`. This is because the `pattern_collect_information`
+    is a special flow frame that is used to collect information from the user
+    and commonly, is not what you are looking for when you want the topmost frame.
+
+    Also excludes frames created by a call step. They are treated as if they are
+    directly part of the flow that called them.
+
+    Args:
+        dialogue_stack: The dialogue stack to use.
+        ignore_collect_information_pattern: Whether to ignore the
+            `pattern_collect_information` frame.
+        ignore_call_frames: Whether to ignore user frames of type `call`
+
+    Returns:
+        The topmost flow frame from the tracker. `None` if there
+        is no frame on the stack.
+    """
+    for frame in reversed(dialogue_stack.frames):
+        if ignore_collect_information_pattern and isinstance(
+            frame, CollectInformationPatternFlowStackFrame
+        ):
+            continue
+        if ignore_call_frames and (
+            isinstance(frame, UserFlowStackFrame)
+            and frame.frame_type == FlowStackFrameType.CALL
+        ):
+            continue
+        if isinstance(frame, BaseFlowStackFrame):
+            return frame
+    return None
+
+
+def top_user_flow_frame(dialogue_stack: DialogueStack) -> Optional[UserFlowStackFrame]:
+    """Returns the topmost user flow frame from the tracker.
+
+    User flows are flows that are created by developers of an assistant and
+    describe tasks the assistant can fulfil. The topmost user flow frame is
+    the topmost frame on the stack for one of these user flows. While looking
+    for this topmost user flow, other frame types such as PatternFrames,
+    SearchFrames, ChitchatFrames, and user flows started through call steps
+    are skipped. The latter are treated as part of the calling user flow.
+
+    Args:
+        dialogue_stack: The dialogue stack to use.
+
+    Returns:
+    The topmost user flow frame from the tracker.
+    """
+    for frame in reversed(dialogue_stack.frames):
+        if (
+            isinstance(frame, UserFlowStackFrame)
+            and frame.frame_type != FlowStackFrameType.CALL
+        ):
+            return frame
+    return None
+
+
+def filled_slots_for_active_flow(
+    tracker: "DialogueStateTracker", all_flows: FlowsList
+) -> Tuple[Set[str], Optional[str]]:
+    """Get all slots that have been filled for the 'current user flow'.
+
+    All patterns that sit ontop of that user flow as well as
+    flows called by this flow, are also included.
+
+    Any collect information step that is part of a pattern on top of the current
+    user flow are also included.
+
+    Args:
+        tracker: The tracker to get the filled slots from.
+        all_flows: All flows.
+
+    Returns:
+    All slots that have been filled for the current flow and the id of the currently
+    active flow.
+    """
+    stack = tracker.stack
+    user_frame = top_user_flow_frame(stack)
+    active_flow = user_frame.flow_id if user_frame else None
+
+    filled_slots = set()
+    for collect_step, _ in previous_collect_steps_for_active_flow(tracker, all_flows):
+        filled_slots.add(collect_step.collect)
+
+    return filled_slots, active_flow
+
+
+def previous_collect_steps_for_active_flow(
+    tracker: "DialogueStateTracker", all_flows: FlowsList
+) -> List[Tuple[CollectInformationFlowStep, str]]:
+    stack = tracker.stack
+    user_frame = top_user_flow_frame(stack)
+
+    if not user_frame:
+        return []
+
+    collect_steps: List[Tuple[CollectInformationFlowStep, str]] = []
+
+    active_frames = {frame.frame_id for frame in stack.frames}
+
+    for previous_stack in tracker.previous_stack_states():
+        active_frame = top_user_flow_frame(previous_stack)
+        if not active_frame or active_frame.frame_id != user_frame.frame_id:
+            continue
+
+        top_frame = previous_stack.top()
+        if not isinstance(top_frame, BaseFlowStackFrame):
+            continue
+
+        step = top_frame.step(all_flows)
+        if not isinstance(step, CollectInformationFlowStep):
+            continue
+
+        if isinstance(top_frame, UserFlowStackFrame):
+            collect_steps.append((step, top_frame.flow_id))
+        elif (
+            isinstance(top_frame, PatternFlowStackFrame)
+            and top_frame.frame_id in active_frames
+        ):
+            # if this is a pattern, it is only relevant if it is still
+            # active in the current state of the conversation.
+            # completed patterns in the past are not relevant
+            collect_steps.append((step, top_frame.flow_id))
+
+    return collect_steps
+
+
+def user_flows_on_the_stack(dialogue_stack: DialogueStack) -> Set[str]:
+    """Get all user flows that are currently on the stack.
+
+    Args:
+        dialogue_stack: The dialogue stack.
+
+    Returns:
+    All user flows that are currently on the stack.
+    """
+    return {
+        f.flow_id for f in dialogue_stack.frames if isinstance(f, UserFlowStackFrame)
+    }
+
+
+def end_top_user_flow(stack: DialogueStack) -> DialogueStack:
+    """Ends all frames on top of the stack including the topmost user frame.
+
+    Ends all flows until the next user flow is reached. This is useful
+    if you want to end all flows that are currently on the stack and
+    the user flow that triggered them.
+
+    Args:
+        stack: The dialogue stack.
+    """
+    updated_stack = stack.copy()
+
+    for frame in reversed(updated_stack.frames):
+        if isinstance(frame, BaseFlowStackFrame):
+            frame.step_id = ContinueFlowStep.continue_step_for_id(END_STEP)
+            if isinstance(frame, UserFlowStackFrame):
+                break
+    return updated_stack
+
+
+def get_collect_steps_excluding_ask_before_filling_for_active_flow(
+    dialogue_stack: DialogueStack, all_flows: FlowsList
+) -> Set[str]:
+    """Get all collect steps that are part of the current flow, without
+    considering the collect steps that has to be asked before filling.
+
+    Args:
+        dialogue_stack: The dialogue stack.
+        all_flows: All flows.
+
+    Returns:
+        All collect steps that are part of the current active flow,
+        excluding the collect steps that have to be asked before filling.
+    """
+    active_frame = top_user_flow_frame(dialogue_stack)
+    if active_frame is None:
+        return set()
+    active_flow = active_frame.flow(all_flows)
+    return set(
+        step.collect
+        for step in active_flow.get_collect_steps()
+        if not step.ask_before_filling
+    )

rasa/dialogue_understanding/utils.py

@@ -0,0 +1,14 @@
+from contextlib import contextmanager
+from typing import Generator
+
+record_commands_and_prompts = False
+
+
+@contextmanager
+def set_record_commands_and_prompts() -> Generator:
+    global record_commands_and_prompts
+    record_commands_and_prompts = True
+    try:
+        yield
+    finally:
+        record_commands_and_prompts = False

rasa/dialogue_understanding_test/constants.py

@@ -0,0 +1,15 @@
+import rasa.e2e_test.constants as constants
+
+KEY_FIXTURES = constants.KEY_FIXTURES
+KEY_METADATA = constants.KEY_METADATA
+KEY_STUB_CUSTOM_ACTIONS = constants.KEY_STUB_CUSTOM_ACTIONS
+KEY_USER_INPUT = constants.KEY_USER_INPUT
+KEY_BOT_INPUT = constants.KEY_BOT_INPUT
+KEY_BOT_UTTERED = constants.KEY_BOT_UTTERED
+KEY_STEPS = constants.KEY_STEPS
+KEY_TEST_CASE = constants.KEY_TEST_CASE
+KEY_TEST_CASES = constants.KEY_TEST_CASES
+KEY_COMMANDS = "commands"
+
+ACTOR_USER = "user"
+ACTOR_BOT = "bot"

rasa/dialogue_understanding_test/du_test_case.py

@@ -0,0 +1,118 @@
+from typing import List, Optional, Dict, Any
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from rasa.dialogue_understanding.commands import Command
+from rasa.dialogue_understanding_test.constants import (
+    ACTOR_USER,
+    KEY_COMMANDS,
+    ACTOR_BOT,
+    KEY_TEST_CASE,
+    KEY_STEPS,
+    KEY_FIXTURES,
+    KEY_METADATA,
+    KEY_USER_INPUT,
+    KEY_BOT_INPUT,
+    KEY_BOT_UTTERED,
+)
+
+
+class DialogueUnderstandingOutput(BaseModel):
+    """Output containing prompts and generated commands by component.
+
+    Example of commands:
+        {
+            "MultiStepLLMCommandGenerator": [
+                {"command": "set_slot", "name": "slot_name", "value": "slot_value"},
+            ],
+            "NLUCommandAdapter": [
+                {"command": "start_flow", "name": "test_flow"},
+            ]
+        }
+
+    Example of prompts:
+        {
+            "MultiStepLLMCommandGenerator": [
+                (
+                    "fill_slots_prompt",
+                    {
+                        "user_prompt": "<prompt content>",
+                        "system_prompt": "<prompt content>"
+                    }
+                ),
+                (
+                    "handle_flows_prompt",
+                    {
+                        "user_prompt": "<prompt content>",
+                        "system_prompt": "<prompt content>"
+                    }
+                ),
+            ],
+        }
+    """
+
+    prompts: Dict[str, tuple[str, Dict[str, str]]]
+    commands: Dict[str, List[Command]]
+
+    model_config = ConfigDict(frozen=True)
+
+    def get_component_data(
+        self, component_name: str
+    ) -> tuple[Optional[tuple[str, Dict[str, str]]], List[Command]]:
+        """Get both the prompts and commands for a specific component."""
+        return self.prompts.get(component_name), self.commands.get(component_name, [])
+
+
+class DialogueUnderstandingTestStep(BaseModel):
+    actor: str
+    text: Optional[str] = None
+    template: Optional[str] = None
+    line: Optional[int] = None
+    metadata_name: Optional[str] = None
+    commands: Optional[List[Command]] = None
+    dialogue_understanding_output: Optional[DialogueUnderstandingOutput] = None
+
+    def as_dict(self) -> Dict[str, Any]:
+        if self.actor == ACTOR_USER:
+            if self.commands:
+                return {
+                    KEY_USER_INPUT: self.text,
+                    # TODO: The command should be converted into our DSL
+                    KEY_COMMANDS: [command.as_dict() for command in self.commands],
+                }
+            return {ACTOR_USER: self.text}
+        elif self.actor == ACTOR_BOT:
+            if self.text is not None:
+                return {KEY_BOT_INPUT: self.text}
+            elif self.template is not None:
+                return {KEY_BOT_UTTERED: self.template}
+
+        return {}
+
+
+class DialogueUnderstandingTestCase(BaseModel):
+    name: str
+    steps: list[DialogueUnderstandingTestStep] = Field(min_length=1)
+    file: Optional[str] = None
+    line: Optional[int] = None
+    fixture_names: Optional[List[str]] = None
+    metadata_name: Optional[str] = None
+
+    def full_name(self) -> str:
+        return f"{self.file}::{self.name}"
+
+    def as_dict(self) -> Dict[str, Any]:
+        result = {
+            KEY_TEST_CASE: self.name,
+            KEY_STEPS: [step.as_dict() for step in self.steps],
+        }
+        if self.fixture_names:
+            result[KEY_FIXTURES] = self.fixture_names
+        if self.metadata_name:
+            result[KEY_METADATA] = self.metadata_name
+        return result
+
+
+# Update forward references
+DialogueUnderstandingTestStep.model_rebuild()
+DialogueUnderstandingTestCase.model_rebuild()

rasa/dialogue_understanding_test/du_test_result.py

@@ -0,0 +1,11 @@
+from typing import Optional
+
+from pydantic import BaseModel
+
+from rasa.dialogue_understanding_test.du_test_case import DialogueUnderstandingTestCase
+
+
+class DialogueUnderstandingTestResult(BaseModel):
+    test_case: DialogueUnderstandingTestCase
+    passed: bool
+    error_line: Optional[int] = None

rasa/e2e_test/aggregate_test_stats_calculator.py

@@ -0,0 +1,134 @@
+from dataclasses import dataclass
+from typing import List, Optional, Set, TYPE_CHECKING
+
+import structlog
+
+from rasa.e2e_test.assertions import _get_all_assertion_subclasses
+
+if TYPE_CHECKING:
+    from rasa.e2e_test.assertions import Assertion
+    from rasa.e2e_test.e2e_test_case import TestCase
+    from rasa.e2e_test.e2e_test_result import TestResult
+
+structlogger = structlog.get_logger()
+
+
+@dataclass
+class AccuracyCalculation:
+    """Data class for storing the accuracy calculation for an assertion type."""
+
+    assertion_type: str
+    accuracy: float
+
+
+class AggregateTestStatsCalculator:
+    """Class for calculating the aggregate test statistics for assertion types."""
+
+    def __init__(
+        self,
+        passed_results: List["TestResult"],
+        failed_results: List["TestResult"],
+        test_cases: List["TestCase"],
+    ) -> None:
+        self.passed_results = passed_results
+        self.failed_results = failed_results
+        self.test_cases = test_cases
+
+        self.failed_assertion_set: Set["Assertion"] = set()
+        self.failed_test_cases_without_assertion_failure: Set[str] = set()
+        self.passed_count_mapping = {
+            subclass_type: 0
+            for subclass_type in _get_all_assertion_subclasses().keys()
+            if subclass_type != ""
+        }
+        self.failed_count_mapping = {
+            subclass_type: 0
+            for subclass_type in _get_all_assertion_subclasses().keys()
+            if subclass_type != ""
+        }
+
+    def calculate(self) -> List[AccuracyCalculation]:
+        """Calculates the aggregate test statistics for assertion types."""
+        self._update_failed_count_mapping()
+        self._update_passed_count_mapping()
+
+        accuracy_calculations = []
+
+        for assertion_type in self.passed_count_mapping.keys():
+            accuracy = self._calculate_accuracy(assertion_type)
+            if accuracy is None:
+                continue
+
+            accuracy_calculations.append(AccuracyCalculation(assertion_type, accuracy))
+
+        return accuracy_calculations
+
+    def _calculate_accuracy(self, assertion_type: str) -> Optional[float]:
+        """Calculates the accuracy for the given assertion type."""
+        passed_count = self.passed_count_mapping[assertion_type]
+        failed_count = self.failed_count_mapping[assertion_type]
+
+        total_count = passed_count + failed_count
+
+        if total_count == 0:
+            structlogger.debug(
+                "aggregate_test_stats.calculate_accuracy."
+                "no_test_cases_for_assertion_type",
+                assertion_type=assertion_type,
+            )
+            return None
+
+        return passed_count / total_count
+
+    def _update_passed_count_mapping(self) -> None:
+        """Updates the passed count mapping based on the passed results.
+
+        We only count the assertions of passed tests and those
+        that are not in the failed assertion set.
+        We also do not count the assertions that follow a failed assertion.
+        """
+        passed_test_case_names = [
+            passed.test_case.name for passed in self.passed_results
+        ]
+        # We filter out test cases that failed without an assertion failure
+        filtered_test_cases = [
+            test_case
+            for test_case in self.test_cases
+            if test_case.name not in self.failed_test_cases_without_assertion_failure
+        ]
+
+        for test_case in filtered_test_cases:
+            if test_case.name in passed_test_case_names:
+                for step in test_case.steps:
+                    if step.assertions is None:
+                        continue
+
+                    for assertion in step.assertions:
+                        self.passed_count_mapping[assertion.type()] += 1
+            else:
+                for step in test_case.steps:
+                    if step.assertions is None:
+                        continue
+
+                    for assertion in step.assertions:
+                        if assertion not in self.failed_assertion_set:
+                            self.passed_count_mapping[assertion.type()] += 1
+                        else:
+                            break
+
+    def _update_failed_count_mapping(self) -> None:
+        """Updates the failed count mapping based on the failed results."""
+        for failed in self.failed_results:
+            if failed.assertion_failure is None:
+                structlogger.debug(
+                    "aggregate_test_stats.calculate."
+                    "no_assertion_failure_in_failed_result",
+                    test_case=failed.test_case.name,
+                )
+                self.failed_test_cases_without_assertion_failure.add(
+                    failed.test_case.name
+                )
+                continue
+
+            self.failed_assertion_set.add(failed.assertion_failure.assertion)
+            self.failed_count_mapping[failed.assertion_failure.assertion.type()] += 1