rasa-pro 3.10.15__py3-none-any.whl → 3.11.0__py3-none-any.whl

rasa/e2e_test/assertions.py

@@ -71,6 +71,7 @@ class AssertionType(Enum):
     SLOT_WAS_SET = "slot_was_set"
     SLOT_WAS_NOT_SET = "slot_was_not_set"
     BOT_UTTERED = "bot_uttered"
+    BOT_DID_NOT_UTTER = "bot_did_not_utter"
     GENERATIVE_RESPONSE_IS_RELEVANT = "generative_response_is_relevant"
     GENERATIVE_RESPONSE_IS_GROUNDED = "generative_response_is_grounded"
 
@@ -452,11 +453,6 @@ class ActionExecutedAssertion(Assertion):
         **kwargs: Any,
     ) -> Tuple[Optional[AssertionFailure], Optional[Event]]:
         """Run the action executed assertion on the given events for that user turn."""
-        step_index = kwargs.get("step_index")
-        original_turn_events, turn_events = _get_turn_events_based_on_step_index(
-            step_index, turn_events, prior_events
-        )
-
         try:
             matching_event = next(
                 event
@@ -469,7 +465,7 @@ class ActionExecutedAssertion(Assertion):
             error_message += assertion_order_error_message
 
             return self._generate_assertion_failure(
-                error_message, prior_events, original_turn_events, self.line
+                error_message, prior_events, turn_events, self.line
             )
 
         return None, matching_event
@@ -524,11 +520,6 @@ class SlotWasSetAssertion(Assertion):
         """Run the slot_was_set assertion on the given events for that user turn."""
         matching_event = None
 
-        step_index = kwargs.get("step_index")
-        original_turn_events, turn_events = _get_turn_events_based_on_step_index(
-            step_index, turn_events, prior_events
-        )
-
         for slot in self.slots:
             matching_events = [
                 event
@@ -567,7 +558,7 @@ class SlotWasSetAssertion(Assertion):
                 error_message += assertion_order_error_message
 
                 return self._generate_assertion_failure(
-                    error_message, prior_events, original_turn_events, slot.line
+                    error_message, prior_events, turn_events, slot.line
                 )
 
         return None, matching_event
@@ -605,11 +596,6 @@ class SlotWasNotSetAssertion(Assertion):
         """Run the slot_was_not_set assertion on the given events for that user turn."""
         matching_event = None
 
-        step_index = kwargs.get("step_index")
-        original_turn_events, turn_events = _get_turn_events_based_on_step_index(
-            step_index, turn_events, prior_events
-        )
-
         for slot in self.slots:
             matching_events = [
                 event
@@ -645,7 +631,7 @@ class SlotWasNotSetAssertion(Assertion):
                 error_message += assertion_order_error_message
 
                 return self._generate_assertion_failure(
-                    error_message, prior_events, original_turn_events, slot.line
+                    error_message, prior_events, turn_events, slot.line
                 )
 
         return None, matching_event
@@ -737,11 +723,7 @@ class BotUtteredAssertion(Assertion):
     ) -> Tuple[Optional[AssertionFailure], Optional[Event]]:
         """Run the bot_uttered assertion on the given events for that user turn."""
         matching_event = None
-
-        step_index = kwargs.get("step_index")
-        original_turn_events, turn_events = _get_turn_events_based_on_step_index(
-            step_index, turn_events, prior_events
-        )
+        error_messages = []
 
         if self.utter_name is not None:
             try:
@@ -752,11 +734,8 @@ class BotUtteredAssertion(Assertion):
                     and event.metadata.get("utter_action") == self.utter_name
                 )
             except StopIteration:
-                error_message = f"Bot did not utter '{self.utter_name}' response."
-                error_message += assertion_order_error_message
-
-                return self._generate_assertion_failure(
-                    error_message, prior_events, original_turn_events, self.line
+                error_messages.append(
+                    f"Bot did not utter '{self.utter_name}' response."
                 )
 
         if self.text_matches is not None:
@@ -768,16 +747,11 @@ class BotUtteredAssertion(Assertion):
                     if isinstance(event, BotUttered) and pattern.search(event.text)
                 )
             except StopIteration:
-                error_message = (
+                error_messages.append(
                     f"Bot did not utter any response which "
                     f"matches the provided text pattern "
                     f"'{self.text_matches}'."
                 )
-                error_message += assertion_order_error_message
-
-                return self._generate_assertion_failure(
-                    error_message, prior_events, original_turn_events, self.line
-                )
 
         if self.buttons:
             try:
@@ -787,13 +761,16 @@ class BotUtteredAssertion(Assertion):
                     if isinstance(event, BotUttered) and self._buttons_match(event)
                 )
             except StopIteration:
-                error_message = (
+                error_messages.append(
                     "Bot did not utter any response with the expected buttons."
                 )
-                error_message += assertion_order_error_message
-                return self._generate_assertion_failure(
-                    error_message, prior_events, original_turn_events, self.line
-                )
+
+        if error_messages:
+            error_message = " ".join(error_messages)
+            error_message += assertion_order_error_message
+            return self._generate_assertion_failure(
+                error_message, prior_events, turn_events, self.line
+            )
 
         return None, matching_event
 
@@ -823,6 +800,126 @@ class BotUtteredAssertion(Assertion):
         return hash(json.dumps(self.as_dict()))
 
 
+@dataclass
+class BotDidNotUtterAssertion(Assertion):
+    """Class for the 'bot_did_not_utter' assertion."""
+
+    utter_name: Optional[str] = None
+    text_matches: Optional[str] = None
+    buttons: Optional[List[AssertedButton]] = None
+    line: Optional[int] = None
+
+    @classmethod
+    def type(cls) -> str:
+        return AssertionType.BOT_DID_NOT_UTTER.value
+
+    @staticmethod
+    def from_dict(assertion_dict: Dict[Text, Any]) -> BotDidNotUtterAssertion:
+        """Creates a BotDidNotUtterAssertion from a dictionary."""
+        assertion_dict = assertion_dict.get(AssertionType.BOT_DID_NOT_UTTER.value, {})
+        utter_name = assertion_dict.get("utter_name")
+        text_matches = assertion_dict.get("text_matches")
+        buttons = [
+            AssertedButton.from_dict(button)
+            for button in assertion_dict.get("buttons", [])
+        ]
+
+        if not utter_name and not text_matches and not buttons:
+            raise RasaException(
+                "A 'bot_did_not_utter' assertion is empty. "
+                "It should contain at least one of the allowed properties: "
+                "'utter_name', 'text_matches', or 'buttons'."
+            )
+
+        return BotDidNotUtterAssertion(
+            utter_name=utter_name,
+            text_matches=text_matches,
+            buttons=buttons,
+            line=assertion_dict.lc.line + 1 if hasattr(assertion_dict, "lc") else None,
+        )
+
+    def run(
+        self,
+        turn_events: List[Event],
+        prior_events: List[Event],
+        assertion_order_error_message: str = "",
+        **kwargs: Any,
+    ) -> Tuple[Optional[AssertionFailure], Optional[Event]]:
+        """Checks that the bot did not utter the specified messages or buttons."""
+        for event in turn_events:
+            if isinstance(event, BotUttered):
+                error_messages = []
+                if self._utter_name_matches(event):
+                    error_messages.append(
+                        f"Bot uttered a forbidden utterance '{self.utter_name}'."
+                    )
+                if self._text_matches(event):
+                    error_messages.append(
+                        f"Bot uttered a forbidden message matching "
+                        f"the pattern '{self.text_matches}'."
+                    )
+                if self._buttons_match(event):
+                    error_messages.append(
+                        "Bot uttered a forbidden response with specified buttons."
+                    )
+
+                if error_messages:
+                    error_message = " ".join(error_messages)
+                    error_message += assertion_order_error_message
+                    return self._generate_assertion_failure(
+                        error_message, prior_events, turn_events, self.line
+                    )
+        return None, None
+
+    def _utter_name_matches(self, event: BotUttered) -> bool:
+        if self.utter_name is not None:
+            if event.metadata.get("utter_action") == self.utter_name:
+                return True
+        return False
+
+    def _text_matches(self, event: BotUttered) -> bool:
+        if self.text_matches is not None:
+            pattern = re.compile(self.text_matches)
+            if pattern.search(event.text):
+                return True
+        return False
+
+    def _buttons_match(self, event: BotUttered) -> bool:
+        """Check if the bot response contains any of the forbidden buttons."""
+        if self.buttons is None:
+            return False
+
+        actual_buttons = event.data.get("buttons", [])
+        if not actual_buttons:
+            return False
+
+        for actual_button in actual_buttons:
+            if any(
+                self._is_forbidden_button(actual_button, forbidden_button)
+                for forbidden_button in self.buttons
+            ):
+                return True
+        return False
+
+    @staticmethod
+    def _is_forbidden_button(
+        actual_button: Dict[str, Any], forbidden_button: AssertedButton
+    ) -> bool:
+        """Check if the button matches any of the forbidden buttons."""
+        actual_title = actual_button.get("title")
+        actual_payload = actual_button.get("payload")
+
+        title_matches = forbidden_button.title == actual_title
+        payload_matches = forbidden_button.payload == actual_payload
+        if title_matches and payload_matches:
+            return True
+        return False
+
+    def __hash__(self) -> int:
+        """Hash method to ensure the assertion is hashable."""
+        return hash(json.dumps(self.as_dict()))
+
+
 @dataclass
 class GenerativeResponseMixin(Assertion):
     """Mixin class for storing generative response assertions."""
@@ -1199,25 +1296,3 @@ def _find_matching_generative_events(turn_events: List[Event]) -> List[BotUttere
         and event.metadata.get(UTTER_SOURCE_METADATA_KEY)
         in ELIGIBLE_UTTER_SOURCE_METADATA
     ]
-
-
-def _get_turn_events_based_on_step_index(
-    step_index: int, turn_events: List[Event], prior_events: List[Event]
-) -> Tuple[List[Event], List[Event]]:
-    """Get the turn events based on the step index.
-
-    For the first step, we need to include the prior events as well
-    in the same user turn. For the subsequent steps, we only need the
-    events that follow the user uttered event on which the tracker
-    was originally sliced by.
-
-    Returns:
-        List[Event]: The copy of turn_events
-        List[Event]: The turn events based on the step index
-
-    """
-    original_turn_events = turn_events[:]
-    if step_index == 0:
-        return original_turn_events, prior_events + turn_events
-
-    return original_turn_events, turn_events

rasa/e2e_test/assertions_schema.yml

@@ -83,6 +83,29 @@ schema;assertions:
            text_matches:
              type: str
              nullable: false
+       bot_did_not_utter:
+         type: map
+         nullable: false
+         mapping:
+           utter_name:
+             type: str
+             nullable: false
+           buttons:
+             type: seq
+             nullable: false
+             matching: "all"
+             sequence:
+               - type: map
+                 mapping:
+                   title:
+                     type: str
+                     nullable: false
+                   payload:
+                     type: str
+                     nullable: false
+           text_matches:
+             type: str
+             nullable: false
        generative_response_is_relevant:
          type: map
          mapping:

rasa/e2e_test/e2e_test_case.py

@@ -1,8 +1,10 @@
-import logging
 from collections import OrderedDict
+from collections import defaultdict
 from dataclasses import dataclass
 from typing import Any, Dict, List, Optional, Text, Union
 
+import structlog
+
 from rasa.e2e_test.assertions import Assertion
 from rasa.e2e_test.constants import (
     KEY_ASSERTIONS,
@@ -20,10 +22,11 @@ from rasa.e2e_test.constants import (
     KEY_USER_INPUT,
 )
 from rasa.e2e_test.stub_custom_action import StubCustomAction
+from rasa.shared.constants import DOCS_BASE_URL
 from rasa.shared.core.events import BotUttered, SlotSet, UserUttered
 from rasa.shared.exceptions import RasaException
 
-logger = logging.getLogger(__name__)
+structlogger = structlog.get_logger(__name__)
 
 
 @dataclass(frozen=True)
@@ -343,9 +346,10 @@ class ActualStepOutput:
             try:
                 return self.user_uttered_events[0]
             except IndexError:
-                logger.debug(
-                    f"Could not find `UserUttered` event in the ActualStepOutput: "
-                    f"{self}"
+                structlogger.debug(
+                    "e2e_test_case.get_user_uttered_event.no_user_uttered_event",
+                    event_info=f"Could not find `UserUttered` event in the "
+                    f"ActualStepOutput: {self}",
                 )
                 return None
         return None
@@ -395,7 +399,7 @@ class TestCase:
             else:
                 steps.append(TestStep.from_dict(step))
 
-        return TestCase(
+        test_case = TestCase(
             name=input_test_case.get(KEY_TEST_CASE, "default"),
             steps=steps,
             file=file,
@@ -405,6 +409,81 @@ class TestCase:
             fixture_names=input_test_case.get(KEY_FIXTURES),
             metadata_name=input_test_case.get(KEY_METADATA),
         )
+        test_case.validate()
+        return test_case
+
+    def validate(self) -> None:
+        """Validates the test case.
+
+        This method calls all validation methods required for the test case.
+        """
+        if self.uses_assertions():
+            self.validate_duplicate_user_messages_metadata()
+
+    def validate_duplicate_user_messages_metadata(self) -> None:
+        """Validates that duplicate user messages use metadata correctly.
+
+        Ensures that each duplicate user message uses unique metadata.
+
+        Raises warnings if any issues are found.
+        """
+        docs_link = (
+            f"{DOCS_BASE_URL}/testing/e2e-testing-assertions/assertions-how-to-guide/"
+            "#how-to-handle-duplicate-user-text-messages-in-the-same-test-case"
+        )
+        no_metadata_event_info = (
+            "Test case '{name}' has duplicate user steps with text '{text}', "
+            "and user step at line {line} lacks metadata. When using "
+            "duplicate user messages, metadata should be set on each step to ensure "
+            f"correct processing. Please refer to the documentation: {docs_link}"
+        )
+        non_unique_metadata_event_info = (
+            "Test case '{name}' has duplicate user steps with text '{text}', "
+            "and user step at line {line} has duplicate metadata "
+            "name '{metadata_name}'. Metadata names should be unique for each user "
+            "step among duplicates. This may cause issues in processing "
+            f"user messages. Please refer to the documentation: {docs_link}"
+        )
+
+        # Use dict[str, list] structure to group steps by user message text to easily
+        # identify and validate instances with duplicate messages and their metadata.
+        message_steps = defaultdict(list)
+
+        # Collect user steps by text
+        for step in self.steps:
+            if step.actor == KEY_USER_INPUT and step.text:
+                message_steps[step.text].append(step)
+
+        # Check for duplicate messages
+        for text, steps in message_steps.items():
+            if len(steps) <= 1:
+                continue
+
+            metadata_names_used = set()
+            for step in steps:
+                if not step.metadata_name:
+                    structlogger.warning(
+                        "e2e_test_case.validate_duplicate_user_messages_metadata.no_metadata",
+                        event_info=no_metadata_event_info.format(
+                            name=self.name,
+                            text=text,
+                            line=step.line,
+                        ),
+                    )
+                    break
+                elif step.metadata_name in metadata_names_used:
+                    structlogger.warning(
+                        "e2e_test_case.validate_duplicate_user_messages_metadata.non_unique_metadata",
+                        event_info=non_unique_metadata_event_info.format(
+                            name=self.name,
+                            text=text,
+                            line=step.line,
+                            metadata_name=step.metadata_name,
+                        ),
+                    )
+                    break
+                else:
+                    metadata_names_used.add(step.metadata_name)
 
     def as_dict(self) -> Dict[Text, Any]:
         """Returns the test case as a dictionary."""

rasa/e2e_test/e2e_test_runner.py

@@ -16,6 +16,7 @@ import rasa.shared.utils.io
 from rasa.core.channels import CollectingOutputChannel, UserMessage
 from rasa.core.constants import ACTIVE_FLOW_METADATA_KEY, STEP_ID_METADATA_KEY
 from rasa.core.exceptions import AgentNotReady
+from rasa.core.persistor import StorageType
 from rasa.core.utils import AvailableEndpoints
 from rasa.e2e_test.constants import TEST_CASE_NAME, TEST_FILE_NAME
 from rasa.e2e_test.e2e_config import create_llm_judge_config
@@ -34,7 +35,6 @@ from rasa.e2e_test.e2e_test_result import (
     TestResult,
 )
 from rasa.llm_fine_tuning.conversations import Conversation
-from rasa.nlu.persistor import StorageType
 from rasa.shared.constants import RASA_DEFAULT_FLOW_PATTERN_PREFIX
 from rasa.shared.core.events import (
     ActionExecuted,
@@ -442,7 +442,7 @@ class E2ETestRunner:
         assertion_failure_found = False
         input_metadata = input_metadata if input_metadata else []
 
-        for index, step in enumerate(test_case.steps):
+        for step in test_case.steps:
             if not step.assertions:
                 structlogger.debug(
                     "e2e_test_runner.run_assertions.no_assertions.skipping_step",
@@ -490,7 +490,6 @@ class E2ETestRunner:
                     assertion_order_error_message=assertion_order_error_msg,
                     llm_judge_config=self.llm_judge_config,
                     step_text=step.text,
-                    step_index=index,
                 )
 
                 if assertion_failure:

rasa/engine/graph.py

@@ -634,4 +634,3 @@ class GraphModelConfiguration:
     language: Optional[Text]
     core_target: Optional[Text]
     nlu_target: Optional[Text]
-    spaces: Optional[Dict[Text, Text]] = None

rasa/engine/loader.py

@@ -4,6 +4,10 @@ from typing import Tuple, Type
 from rasa.engine.graph import ExecutionContext
 from rasa.engine.runner.interface import GraphRunner
 from rasa.engine.storage.storage import ModelMetadata, ModelStorage
+from rasa.engine.validation import (
+    validate_model_client_configuration_setup_during_inference_time,
+    validate_model_group_configuration_setup,
+)
 
 
 def load_predict_graph_runner(
@@ -26,6 +30,14 @@ def load_predict_graph_runner(
     model_storage, model_metadata = model_storage_class.from_model_archive(
         storage_path=storage_path, model_archive_path=model_archive_path
     )
+
+    # Components using LLMs or embeddings can reference model groups defined in
+    # the endpoints.yml file for their client configurations. To ensure they will work
+    # properly validate model group references before loading
+    # the components.
+    validate_model_group_configuration_setup()
+    validate_model_client_configuration_setup_during_inference_time(model_metadata)
+
     runner = graph_runner_class.create(
         graph_schema=model_metadata.predict_schema,
         model_storage=model_storage,

rasa/engine/recipes/config_files/default_config.yml

@@ -35,9 +35,6 @@ pipeline:
 policies:
   - name: MemoizationPolicy
   - name: RulePolicy
-  - name: UnexpecTEDIntentPolicy
-    max_history: 5
-    epochs: 100
   - name: TEDPolicy
     max_history: 5
     epochs: 100

rasa/engine/recipes/default_recipe.py

@@ -233,7 +233,6 @@ class DefaultV1Recipe(Recipe):
             training_type=training_type,
             assistant_id=config.get(ASSISTANT_ID_KEY),
             language=config.get("language"),
-            spaces=config.get("spaces"),
             core_target=core_target,
             nlu_target=f"run_{RegexMessageHandler.__name__}",
         )

rasa/engine/recipes/graph_recipe.py

@@ -73,7 +73,6 @@ class GraphV1Recipe(Recipe):
             training_type=training_type,
             assistant_id=config.get(ASSISTANT_ID_KEY),
             language=config.get("language"),
-            spaces=config.get("spaces"),
             core_target=core_target,
             nlu_target=nlu_target,
         )

rasa/engine/runner/dask.py

@@ -220,7 +220,7 @@ async def execute_dask_graph(dsk: Dict[str, Any], result: List[str]) -> Any:
     # if start_state_from_dask fails, we will have something
     # to pass to the final block.
     state = {}
-    keyorder = dask.local.order(dsk)  # type:ignore[no-untyped-call]
+    keyorder = dask.local.order(dsk)
 
     state = dask.local.start_state_from_dask(dsk, cache=cache, sortkey=keyorder.get)  # type:ignore[no-untyped-call]
 
@@ -235,7 +235,7 @@ async def execute_dask_graph(dsk: Dict[str, Any], result: List[str]) -> Any:
         # Notify task is running
         state["running"].add(key)
 
-        dependencies = dask.local.get_dependencies(dsk, key)  # type:ignore[no-untyped-call]
+        dependencies = dask.local.get_dependencies(dsk, key)
         # Prep args to send
         data = {dep: state["cache"][dep] for dep in dependencies}