rasa-pro 3.11.0rc3__py3-none-any.whl → 3.11.2__py3-none-any.whl

rasa/core/utils.py

@@ -233,7 +233,9 @@ class AvailableEndpoints:
         self.model_groups = model_groups
 
     @classmethod
-    def get_instance(cls, endpoint_file: Optional[Text] = None) -> "AvailableEndpoints":
+    def get_instance(
+        cls, endpoint_file: Optional[Text] = DEFAULT_ENDPOINTS_PATH
+    ) -> "AvailableEndpoints":
         """Get the singleton instance of AvailableEndpoints."""
         # Ensure that the instance is initialized only once.
         if cls._instance is None:

rasa/e2e_test/aggregate_test_stats_calculator.py

@@ -35,6 +35,7 @@ class AggregateTestStatsCalculator:
         self.test_cases = test_cases
 
         self.failed_assertion_set: Set["Assertion"] = set()
+        self.failed_test_cases_without_assertion_failure: Set[str] = set()
         self.passed_count_mapping = {
             subclass_type: 0
             for subclass_type in _get_all_assertion_subclasses().keys()
@@ -89,8 +90,14 @@ class AggregateTestStatsCalculator:
         passed_test_case_names = [
             passed.test_case.name for passed in self.passed_results
         ]
+        # We filter out test cases that failed without an assertion failure
+        filtered_test_cases = [
+            test_case
+            for test_case in self.test_cases
+            if test_case.name not in self.failed_test_cases_without_assertion_failure
+        ]
 
-        for test_case in self.test_cases:
+        for test_case in filtered_test_cases:
             if test_case.name in passed_test_case_names:
                 for step in test_case.steps:
                     if step.assertions is None:
@@ -118,6 +125,9 @@ class AggregateTestStatsCalculator:
                     "no_assertion_failure_in_failed_result",
                     test_case=failed.test_case.name,
                 )
+                self.failed_test_cases_without_assertion_failure.add(
+                    failed.test_case.name
+                )
                 continue
 
             self.failed_assertion_set.add(failed.assertion_failure.assertion)

rasa/e2e_test/assertions.py

@@ -71,6 +71,7 @@ class AssertionType(Enum):
     SLOT_WAS_SET = "slot_was_set"
     SLOT_WAS_NOT_SET = "slot_was_not_set"
     BOT_UTTERED = "bot_uttered"
+    BOT_DID_NOT_UTTER = "bot_did_not_utter"
     GENERATIVE_RESPONSE_IS_RELEVANT = "generative_response_is_relevant"
     GENERATIVE_RESPONSE_IS_GROUNDED = "generative_response_is_grounded"
 
@@ -452,6 +453,11 @@ class ActionExecutedAssertion(Assertion):
         **kwargs: Any,
     ) -> Tuple[Optional[AssertionFailure], Optional[Event]]:
         """Run the action executed assertion on the given events for that user turn."""
+        step_index = kwargs.get("step_index")
+        original_turn_events, turn_events = _get_turn_events_based_on_step_index(
+            step_index, turn_events, prior_events
+        )
+
         try:
             matching_event = next(
                 event
@@ -464,7 +470,7 @@ class ActionExecutedAssertion(Assertion):
             error_message += assertion_order_error_message
 
             return self._generate_assertion_failure(
-                error_message, prior_events, turn_events, self.line
+                error_message, prior_events, original_turn_events, self.line
             )
 
         return None, matching_event
@@ -519,6 +525,11 @@ class SlotWasSetAssertion(Assertion):
         """Run the slot_was_set assertion on the given events for that user turn."""
         matching_event = None
 
+        step_index = kwargs.get("step_index")
+        original_turn_events, turn_events = _get_turn_events_based_on_step_index(
+            step_index, turn_events, prior_events
+        )
+
         for slot in self.slots:
             matching_events = [
                 event
@@ -557,7 +568,7 @@ class SlotWasSetAssertion(Assertion):
                 error_message += assertion_order_error_message
 
                 return self._generate_assertion_failure(
-                    error_message, prior_events, turn_events, slot.line
+                    error_message, prior_events, original_turn_events, slot.line
                 )
 
         return None, matching_event
@@ -595,6 +606,11 @@ class SlotWasNotSetAssertion(Assertion):
         """Run the slot_was_not_set assertion on the given events for that user turn."""
         matching_event = None
 
+        step_index = kwargs.get("step_index")
+        original_turn_events, turn_events = _get_turn_events_based_on_step_index(
+            step_index, turn_events, prior_events
+        )
+
         for slot in self.slots:
             matching_events = [
                 event
@@ -630,7 +646,7 @@ class SlotWasNotSetAssertion(Assertion):
                 error_message += assertion_order_error_message
 
                 return self._generate_assertion_failure(
-                    error_message, prior_events, turn_events, slot.line
+                    error_message, prior_events, original_turn_events, slot.line
                 )
 
         return None, matching_event
@@ -722,6 +738,12 @@ class BotUtteredAssertion(Assertion):
     ) -> Tuple[Optional[AssertionFailure], Optional[Event]]:
         """Run the bot_uttered assertion on the given events for that user turn."""
         matching_event = None
+        error_messages = []
+
+        step_index = kwargs.get("step_index")
+        original_turn_events, turn_events = _get_turn_events_based_on_step_index(
+            step_index, turn_events, prior_events
+        )
 
         if self.utter_name is not None:
             try:
@@ -732,11 +754,8 @@ class BotUtteredAssertion(Assertion):
                     and event.metadata.get("utter_action") == self.utter_name
                 )
             except StopIteration:
-                error_message = f"Bot did not utter '{self.utter_name}' response."
-                error_message += assertion_order_error_message
-
-                return self._generate_assertion_failure(
-                    error_message, prior_events, turn_events, self.line
+                error_messages.append(
+                    f"Bot did not utter '{self.utter_name}' response."
                 )
 
         if self.text_matches is not None:
@@ -748,16 +767,11 @@ class BotUtteredAssertion(Assertion):
                     if isinstance(event, BotUttered) and pattern.search(event.text)
                 )
             except StopIteration:
-                error_message = (
+                error_messages.append(
                     f"Bot did not utter any response which "
                     f"matches the provided text pattern "
                     f"'{self.text_matches}'."
                 )
-                error_message += assertion_order_error_message
-
-                return self._generate_assertion_failure(
-                    error_message, prior_events, turn_events, self.line
-                )
 
         if self.buttons:
             try:
@@ -767,13 +781,16 @@ class BotUtteredAssertion(Assertion):
                     if isinstance(event, BotUttered) and self._buttons_match(event)
                 )
             except StopIteration:
-                error_message = (
+                error_messages.append(
                     "Bot did not utter any response with the expected buttons."
                 )
-                error_message += assertion_order_error_message
-                return self._generate_assertion_failure(
-                    error_message, prior_events, turn_events, self.line
-                )
+
+        if error_messages:
+            error_message = " ".join(error_messages)
+            error_message += assertion_order_error_message
+            return self._generate_assertion_failure(
+                error_message, prior_events, original_turn_events, self.line
+            )
 
         return None, matching_event
 
@@ -803,6 +820,131 @@ class BotUtteredAssertion(Assertion):
         return hash(json.dumps(self.as_dict()))
 
 
+@dataclass
+class BotDidNotUtterAssertion(Assertion):
+    """Class for the 'bot_did_not_utter' assertion."""
+
+    utter_name: Optional[str] = None
+    text_matches: Optional[str] = None
+    buttons: Optional[List[AssertedButton]] = None
+    line: Optional[int] = None
+
+    @classmethod
+    def type(cls) -> str:
+        return AssertionType.BOT_DID_NOT_UTTER.value
+
+    @staticmethod
+    def from_dict(assertion_dict: Dict[Text, Any]) -> BotDidNotUtterAssertion:
+        """Creates a BotDidNotUtterAssertion from a dictionary."""
+        assertion_dict = assertion_dict.get(AssertionType.BOT_DID_NOT_UTTER.value, {})
+        utter_name = assertion_dict.get("utter_name")
+        text_matches = assertion_dict.get("text_matches")
+        buttons = [
+            AssertedButton.from_dict(button)
+            for button in assertion_dict.get("buttons", [])
+        ]
+
+        if not utter_name and not text_matches and not buttons:
+            raise RasaException(
+                "A 'bot_did_not_utter' assertion is empty. "
+                "It should contain at least one of the allowed properties: "
+                "'utter_name', 'text_matches', or 'buttons'."
+            )
+
+        return BotDidNotUtterAssertion(
+            utter_name=utter_name,
+            text_matches=text_matches,
+            buttons=buttons,
+            line=assertion_dict.lc.line + 1 if hasattr(assertion_dict, "lc") else None,
+        )
+
+    def run(
+        self,
+        turn_events: List[Event],
+        prior_events: List[Event],
+        assertion_order_error_message: str = "",
+        **kwargs: Any,
+    ) -> Tuple[Optional[AssertionFailure], Optional[Event]]:
+        """Checks that the bot did not utter the specified messages or buttons."""
+        step_index = kwargs.get("step_index")
+        original_turn_events, turn_events = _get_turn_events_based_on_step_index(
+            step_index, turn_events, prior_events
+        )
+
+        for event in turn_events:
+            if isinstance(event, BotUttered):
+                error_messages = []
+                if self._utter_name_matches(event):
+                    error_messages.append(
+                        f"Bot uttered a forbidden utterance '{self.utter_name}'."
+                    )
+                if self._text_matches(event):
+                    error_messages.append(
+                        f"Bot uttered a forbidden message matching "
+                        f"the pattern '{self.text_matches}'."
+                    )
+                if self._buttons_match(event):
+                    error_messages.append(
+                        "Bot uttered a forbidden response with specified buttons."
+                    )
+
+                if error_messages:
+                    error_message = " ".join(error_messages)
+                    error_message += assertion_order_error_message
+                    return self._generate_assertion_failure(
+                        error_message, prior_events, original_turn_events, self.line
+                    )
+        return None, None
+
+    def _utter_name_matches(self, event: BotUttered) -> bool:
+        if self.utter_name is not None:
+            if event.metadata.get("utter_action") == self.utter_name:
+                return True
+        return False
+
+    def _text_matches(self, event: BotUttered) -> bool:
+        if self.text_matches is not None:
+            pattern = re.compile(self.text_matches)
+            if pattern.search(event.text):
+                return True
+        return False
+
+    def _buttons_match(self, event: BotUttered) -> bool:
+        """Check if the bot response contains any of the forbidden buttons."""
+        if self.buttons is None:
+            return False
+
+        actual_buttons = event.data.get("buttons", [])
+        if not actual_buttons:
+            return False
+
+        for actual_button in actual_buttons:
+            if any(
+                self._is_forbidden_button(actual_button, forbidden_button)
+                for forbidden_button in self.buttons
+            ):
+                return True
+        return False
+
+    @staticmethod
+    def _is_forbidden_button(
+        actual_button: Dict[str, Any], forbidden_button: AssertedButton
+    ) -> bool:
+        """Check if the button matches any of the forbidden buttons."""
+        actual_title = actual_button.get("title")
+        actual_payload = actual_button.get("payload")
+
+        title_matches = forbidden_button.title == actual_title
+        payload_matches = forbidden_button.payload == actual_payload
+        if title_matches and payload_matches:
+            return True
+        return False
+
+    def __hash__(self) -> int:
+        """Hash method to ensure the assertion is hashable."""
+        return hash(json.dumps(self.as_dict()))
+
+
 @dataclass
 class GenerativeResponseMixin(Assertion):
     """Mixin class for storing generative response assertions."""
@@ -1179,3 +1321,25 @@ def _find_matching_generative_events(turn_events: List[Event]) -> List[BotUttere
         and event.metadata.get(UTTER_SOURCE_METADATA_KEY)
         in ELIGIBLE_UTTER_SOURCE_METADATA
     ]
+
+
+def _get_turn_events_based_on_step_index(
+    step_index: int, turn_events: List[Event], prior_events: List[Event]
+) -> Tuple[List[Event], List[Event]]:
+    """Get the turn events based on the step index.
+
+    For the first step, we need to include the prior events as well
+    in the same user turn. For the subsequent steps, we only need the
+    events that follow the user uttered event on which the tracker
+    was originally sliced by.
+
+    Returns:
+        List[Event]: The copy of turn_events
+        List[Event]: The turn events based on the step index
+
+    """
+    original_turn_events = turn_events[:]
+    if step_index == 0:
+        return original_turn_events, prior_events + turn_events
+
+    return original_turn_events, turn_events

rasa/e2e_test/assertions_schema.yml

@@ -83,6 +83,29 @@ schema;assertions:
            text_matches:
              type: str
              nullable: false
+       bot_did_not_utter:
+         type: map
+         nullable: false
+         mapping:
+           utter_name:
+             type: str
+             nullable: false
+           buttons:
+             type: seq
+             nullable: false
+             matching: "all"
+             sequence:
+               - type: map
+                 mapping:
+                   title:
+                     type: str
+                     nullable: false
+                   payload:
+                     type: str
+                     nullable: false
+           text_matches:
+             type: str
+             nullable: false
        generative_response_is_relevant:
          type: map
          mapping:

rasa/e2e_test/e2e_test_runner.py

@@ -136,7 +136,7 @@ class E2ETestRunner:
             return turns
 
         tracker = await self.agent.processor.fetch_tracker_with_initial_session(
-            sender_id
+            sender_id, output_channel=collector
         )
         # turn -1 i used to contain events that happen during
         # the start of the session and before the first user message
@@ -442,7 +442,7 @@ class E2ETestRunner:
         assertion_failure_found = False
         input_metadata = input_metadata if input_metadata else []
 
-        for step in test_case.steps:
+        for index, step in enumerate(test_case.steps):
             if not step.assertions:
                 structlogger.debug(
                     "e2e_test_runner.run_assertions.no_assertions.skipping_step",
@@ -490,6 +490,7 @@ class E2ETestRunner:
                     assertion_order_error_message=assertion_order_error_msg,
                     llm_judge_config=self.llm_judge_config,
                     step_text=step.text,
+                    step_index=index,
                 )
 
                 if assertion_failure:
@@ -826,7 +827,7 @@ class E2ETestRunner:
             return
 
         tracker = await self.agent.processor.fetch_tracker_with_initial_session(
-            sender_id
+            sender_id, output_channel=CollectingOutputChannel()
         )
 
         for fixture in fixtures:

rasa/engine/graph.py

@@ -67,6 +67,14 @@ class SchemaNode:
     is_input: bool = False
     resource: Optional[Resource] = None
 
+    def matches_type(self, node_type: Type, include_subtypes: bool = True) -> bool:
+        """Checks if schema node's 'uses' is of specified node type.
+        By default, it also checks for subtypes of the specified node type.
+        """
+        return (self.uses is node_type) or (
+            include_subtypes and issubclass(self.uses, node_type)
+        )
+
 
 @dataclass
 class GraphSchema:
@@ -166,9 +174,7 @@ class GraphSchema:
         By default, it also checks for subtypes of the specified node type.
         """
         for node in self.nodes.values():
-            if (node.uses is node_type) or (
-                include_subtypes and issubclass(node.uses, node_type)
-            ):
+            if node.matches_type(node_type, include_subtypes):
                 return True
         return False
 

rasa/engine/loader.py

@@ -4,6 +4,10 @@ from typing import Tuple, Type
 from rasa.engine.graph import ExecutionContext
 from rasa.engine.runner.interface import GraphRunner
 from rasa.engine.storage.storage import ModelMetadata, ModelStorage
+from rasa.engine.validation import (
+    validate_model_client_configuration_setup_during_inference_time,
+    validate_model_group_configuration_setup,
+)
 
 
 def load_predict_graph_runner(
@@ -26,6 +30,14 @@ def load_predict_graph_runner(
     model_storage, model_metadata = model_storage_class.from_model_archive(
         storage_path=storage_path, model_archive_path=model_archive_path
     )
+
+    # Components using LLMs or embeddings can reference model groups defined in
+    # the endpoints.yml file for their client configurations. To ensure they will work
+    # properly validate model group references before loading
+    # the components.
+    validate_model_group_configuration_setup()
+    validate_model_client_configuration_setup_during_inference_time(model_metadata)
+
     runner = graph_runner_class.create(
         graph_schema=model_metadata.predict_schema,
         model_storage=model_storage,