rasa-pro 3.10.14__tar.gz → 3.10.15__tar.gz

{rasa_pro-3.10.14 → rasa_pro-3.10.15}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: rasa-pro
-Version: 3.10.14
+Version: 3.10.15
 Summary: State-of-the-art open-core Conversational AI framework for Enterprises that natively leverages generative AI for effortless assistant development.
 Home-page: https://rasa.com
 Keywords: nlp,machine-learning,machine-learning-library,bot,bots,botkit,rasa conversational-agents,conversational-ai,chatbot,chatbot-framework,bot-framework

{rasa_pro-3.10.14 → rasa_pro-3.10.15}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "rasa-pro"
-version = "3.10.14"
+version = "3.10.15"
 description = "State-of-the-art open-core Conversational AI framework for Enterprises that natively leverages generative AI for effortless assistant development."
 authors = [ "Rasa Technologies GmbH <hi@rasa.com>",]
 maintainers = [ "Tom Bocklisch <tom@rasa.com>",]
@@ -189,7 +189,7 @@ diskcache = "~5.6.3"
 skops = "~0.10.0"
 safetensors = "~0.4.5"
 [[tool.poetry.dependencies.tensorflow-io-gcs-filesystem]]
-version = "==0.31" # newer versions doesn't provide wheels for Windows
+version = "==0.31" # newer versions don't provide wheels for Windows
 markers = "sys_platform == 'win32'"
 
 [[tool.poetry.dependencies.tensorflow-io-gcs-filesystem]]

{rasa_pro-3.10.14 → rasa_pro-3.10.15}/rasa/api.py

@@ -41,7 +41,7 @@ def run(
     from rasa.shared.constants import DOCS_BASE_URL
     from rasa.shared.utils.cli import print_warning
 
-    _endpoints = AvailableEndpoints.read_endpoints(endpoints)
+    _endpoints = AvailableEndpoints.get_instance(endpoints)
 
     if not connector and not credentials:
         connector = "rest"

{rasa_pro-3.10.14 → rasa_pro-3.10.15}/rasa/cli/e2e_test.py

@@ -164,7 +164,7 @@ def execute_e2e_tests(args: argparse.Namespace) -> None:
     args.endpoints = rasa.cli.utils.get_validated_path(
         args.endpoints, "endpoints", DEFAULT_ENDPOINTS_PATH, True
     )
-    endpoints = AvailableEndpoints.read_endpoints(args.endpoints)
+    endpoints = AvailableEndpoints.get_instance(args.endpoints)
 
     # Ignore all endpoints apart from action server, model, nlu and nlg
     # to ensure InMemoryTrackerStore is being used instead of production

{rasa_pro-3.10.14 → rasa_pro-3.10.15}/rasa/cli/evaluate.py

@@ -217,6 +217,6 @@ def _create_tracker_loader(
         A MarkerTrackerLoader object configured with the specified strategy against
         the configured tracker store.
     """
-    endpoints = AvailableEndpoints.read_endpoints(endpoint_config)
+    endpoints = AvailableEndpoints.get_instance(endpoint_config)
     tracker_store = TrackerStore.create(endpoints.tracker_store, domain=domain)
     return MarkerTrackerLoader(tracker_store, strategy, count, seed)

{rasa_pro-3.10.14 → rasa_pro-3.10.15}/rasa/cli/llm_fine_tuning.py

@@ -352,7 +352,7 @@ def get_valid_endpoints(endpoints_file: str) -> AvailableEndpoints:
     validated_endpoints_file = rasa.cli.utils.get_validated_path(
         endpoints_file, "endpoints", DEFAULT_ENDPOINTS_PATH, True
     )
-    endpoints = AvailableEndpoints.read_endpoints(validated_endpoints_file)
+    endpoints = AvailableEndpoints.get_instance(validated_endpoints_file)
 
     # Ignore all endpoints apart from action server, model, nlu and nlg
     # to ensure InMemoryTrackerStore is being used instead of production

{rasa_pro-3.10.14 → rasa_pro-3.10.15}/rasa/cli/run.py

@@ -106,7 +106,7 @@ def run(args: argparse.Namespace) -> None:
         return
 
     # start server if model server is configured
-    endpoints = AvailableEndpoints.read_endpoints(args.endpoints)
+    endpoints = AvailableEndpoints.get_instance(args.endpoints)
     model_server = endpoints.model if endpoints and endpoints.model else None
     if model_server is not None:
         rasa_run(**vars(args))

{rasa_pro-3.10.14 → rasa_pro-3.10.15}/rasa/cli/train.py

@@ -14,8 +14,10 @@ import rasa.core.utils
 import rasa.utils.common
 from rasa.api import train as train_all
 from rasa.cli import SubParsersAction
+from rasa.core import ContextualResponseRephraser
 from rasa.core.nlg.generator import NaturalLanguageGenerator
 from rasa.core.train import do_compare_training
+from rasa.engine.validation import validate_api_type_config_key_usage
 from rasa.nlu.persistor import get_persistor
 from rasa.shared.constants import (
     CONFIG_MANDATORY_KEYS,
@@ -23,6 +25,7 @@ from rasa.shared.constants import (
     CONFIG_MANDATORY_KEYS_NLU,
     DEFAULT_DATA_PATH,
     DEFAULT_DOMAIN_PATHS,
+    LLM_CONFIG_KEY,
 )
 from rasa.shared.exceptions import RasaException
 from rasa.shared.importers.importer import TrainingDataImporter
@@ -75,6 +78,12 @@ def add_subparser(
 def _check_nlg_endpoint_validity(endpoint: Union[Path, str]) -> None:
     try:
         endpoints = rasa.core.utils.read_endpoints_from_path(endpoint)
+        if endpoints.nlg is not None:
+            validate_api_type_config_key_usage(
+                endpoints.nlg.kwargs,
+                LLM_CONFIG_KEY,
+                ContextualResponseRephraser.__name__,
+            )
         NaturalLanguageGenerator.create(endpoints.nlg)
     except Exception as e:
         structlogger.error(

{rasa_pro-3.10.14 → rasa_pro-3.10.15}/rasa/cli/x.py

@@ -179,7 +179,7 @@ def run_in_enterprise_connection_mode(args: argparse.Namespace) -> None:
     print_success("Starting a Rasa server in Rasa Enterprise connection mode... 🚀")
 
     credentials_path, endpoints_path = _get_credentials_and_endpoints_paths(args)
-    endpoints = AvailableEndpoints.read_endpoints(endpoints_path)
+    endpoints = AvailableEndpoints.get_instance(endpoints_path)
 
     _rasa_service(args, endpoints, None, credentials_path)
 

{rasa_pro-3.10.14 → rasa_pro-3.10.15}/rasa/core/policies/enterprise_search_policy.py

@@ -76,6 +76,7 @@ from rasa.shared.utils.llm import (
     sanitize_message_for_prompt,
     tracker_as_readable_transcript,
     try_instantiate_llm_client,
+    try_instantiate_embedder,
 )
 from rasa.core.information_retrieval.faiss import FAISS_Store
 from rasa.core.information_retrieval import (
@@ -661,6 +662,18 @@ class EnterpriseSearchPolicy(Policy):
         execution_context: ExecutionContext,
         **kwargs: Any,
     ) -> "EnterpriseSearchPolicy":
+        try_instantiate_llm_client(
+            config.get(LLM_CONFIG_KEY),
+            DEFAULT_LLM_CONFIG,
+            "enterprise_search_policy.load",
+            EnterpriseSearchPolicy.__name__,
+        )
+        try_instantiate_embedder(
+            config.get(EMBEDDINGS_CONFIG_KEY),
+            DEFAULT_EMBEDDINGS_CONFIG,
+            "enterprise_search_policy.load",
+            EnterpriseSearchPolicy.__name__,
+        )
         """Loads a trained policy (see parent class for full docstring)."""
         prompt_template = None
         store_type = config.get(VECTOR_STORE_PROPERTY, {}).get(
@@ -695,7 +708,6 @@ class EnterpriseSearchPolicy(Policy):
             logger.warning(
                 "enterprise_search_policy.load.failed", error=e, resource=resource.name
             )
-
         return cls(
             config,
             model_storage,

{rasa_pro-3.10.14 → rasa_pro-3.10.15}/rasa/core/policies/flows/flow_executor.py

@@ -484,7 +484,8 @@ def validate_collect_step(
     step: CollectInformationFlowStep,
     stack: DialogueStack,
     available_actions: List[str],
-    slots: Dict[Text, Slot],
+    slots: Dict[str, Slot],
+    flow_name: str,
 ) -> bool:
     """Validate that a collect step can be executed.
 
@@ -507,12 +508,12 @@ def validate_collect_step(
         slot_name=step.collect,
     )
 
-    cancel_flow_and_push_internal_error(stack)
+    cancel_flow_and_push_internal_error(stack, flow_name)
 
     return False
 
 
-def cancel_flow_and_push_internal_error(stack: DialogueStack) -> None:
+def cancel_flow_and_push_internal_error(stack: DialogueStack, flow_name: str) -> None:
     """Cancel the top user flow and push the internal error pattern."""
     top_frame = stack.top()
 
@@ -524,7 +525,7 @@ def cancel_flow_and_push_internal_error(stack: DialogueStack) -> None:
         canceled_frames = CancelFlowCommand.select_canceled_frames(stack)
         stack.push(
             CancelPatternFlowStackFrame(
-                canceled_name=top_frame.flow_id,
+                canceled_name=flow_name,
                 canceled_frames=canceled_frames,
             )
         )
@@ -536,6 +537,7 @@ def validate_custom_slot_mappings(
     stack: DialogueStack,
     tracker: DialogueStateTracker,
     available_actions: List[str],
+    flow_name: str,
 ) -> bool:
     """Validate a slot with custom mappings.
 
@@ -556,7 +558,7 @@ def validate_custom_slot_mappings(
                     action=step.collect_action,
                     collect=step.collect,
                 )
-                cancel_flow_and_push_internal_error(stack)
+                cancel_flow_and_push_internal_error(stack, flow_name)
                 return False
 
     return True
@@ -596,7 +598,12 @@ def run_step(
 
     if isinstance(step, CollectInformationFlowStep):
         return _run_collect_information_step(
-            available_actions, initial_events, stack, step, tracker
+            available_actions,
+            initial_events,
+            stack,
+            step,
+            tracker,
+            flow.readable_name(),
         )
 
     elif isinstance(step, ActionFlowStep):
@@ -716,15 +723,18 @@ def _run_collect_information_step(
     stack: DialogueStack,
     step: CollectInformationFlowStep,
     tracker: DialogueStateTracker,
+    flow_name: str,
 ) -> FlowStepResult:
-    is_step_valid = validate_collect_step(step, stack, available_actions, tracker.slots)
+    is_step_valid = validate_collect_step(
+        step, stack, available_actions, tracker.slots, flow_name
+    )
 
     if not is_step_valid:
         # if we return any other FlowStepResult, the assistant will stay silent
         # instead of triggering the internal error pattern
         return ContinueFlowWithNextStep(events=initial_events)
     is_mapping_valid = validate_custom_slot_mappings(
-        step, stack, tracker, available_actions
+        step, stack, tracker, available_actions, flow_name
     )
 
     if not is_mapping_valid:

{rasa_pro-3.10.14 → rasa_pro-3.10.15}/rasa/core/policies/intentless_policy.py

@@ -71,6 +71,7 @@ from rasa.shared.utils.llm import (
     sanitize_message_for_prompt,
     tracker_as_readable_transcript,
     try_instantiate_llm_client,
+    try_instantiate_embedder,
 )
 from rasa.utils.ml_utils import (
     extract_ai_response_examples,
@@ -918,6 +919,18 @@ class IntentlessPolicy(Policy):
         **kwargs: Any,
     ) -> "IntentlessPolicy":
         """Loads a trained policy (see parent class for full docstring)."""
+        try_instantiate_llm_client(
+            config.get(LLM_CONFIG_KEY),
+            DEFAULT_LLM_CONFIG,
+            "intentless_policy.load",
+            IntentlessPolicy.__name__,
+        )
+        try_instantiate_embedder(
+            config.get(EMBEDDINGS_CONFIG_KEY),
+            DEFAULT_EMBEDDINGS_CONFIG,
+            "intentless_policy.load",
+            IntentlessPolicy.__name__,
+        )
         responses_docsearch = None
         samples_docsearch = None
         prompt_template = None
@@ -943,7 +956,6 @@ class IntentlessPolicy(Policy):
             structlogger.warning(
                 "intentless_policy.load.failed", error=e, resource_name=resource.name
             )
-
         return cls(
             config,
             model_storage,

{rasa_pro-3.10.14 → rasa_pro-3.10.15}/rasa/core/processor.py

@@ -1254,11 +1254,13 @@ class MessageProcessor:
             tracker.update(events[0])
             return self.should_predict_another_action(action.name())
         except Exception:
-            logger.exception(
-                f"Encountered an exception while running action '{action.name()}'."
-                "Bot will continue, but the actions events are lost. "
-                "Please check the logs of your action server for "
-                "more information."
+            structlogger.exception(
+                "rasa.core.processor.run_action.exception",
+                event_info=f"Encountered an exception while "
+                f"running action '{action.name()}'."
+                f"Bot will continue, but the actions events are lost. "
+                f"Please check the logs of your action server for "
+                f"more information.",
             )
             events = []
 

{rasa_pro-3.10.14 → rasa_pro-3.10.15}/rasa/core/training/interactive.py

@@ -1688,7 +1688,7 @@ def run_interactive_learning(
         p = None
 
     app = run.configure_app(port=port, conversation_id="default", enable_api=True)
-    endpoints = AvailableEndpoints.read_endpoints(server_args.get("endpoints"))
+    endpoints = AvailableEndpoints.get_instance(server_args.get("endpoints"))
 
     # before_server_start handlers make sure the agent is loaded before the
     # interactive learning IO starts

{rasa_pro-3.10.14 → rasa_pro-3.10.15}/rasa/core/utils.py

@@ -171,6 +171,8 @@ def is_limit_reached(num_messages: int, limit: Optional[int]) -> bool:
 class AvailableEndpoints:
     """Collection of configured endpoints."""
 
+    _instance = None
+
     @classmethod
     def read_endpoints(cls, endpoint_file: Text) -> "AvailableEndpoints":
         """Read the different endpoints from a yaml file."""
@@ -217,6 +219,14 @@ class AvailableEndpoints:
         self.event_broker = event_broker
         self.vector_store = vector_store
 
+    @classmethod
+    def get_instance(cls, endpoint_file: Optional[Text] = None) -> "AvailableEndpoints":
+        """Get the singleton instance of AvailableEndpoints."""
+        # Ensure that the instance is initialized only once.
+        if cls._instance is None:
+            cls._instance = cls.read_endpoints(endpoint_file)
+        return cls._instance
+
 
 def read_endpoints_from_path(
     endpoints_path: Optional[Union[Path, Text]] = None,
@@ -234,7 +244,7 @@ def read_endpoints_from_path(
     endpoints_config_path = cli_utils.get_validated_path(
         endpoints_path, "endpoints", DEFAULT_ENDPOINTS_PATH, True
     )
-    return AvailableEndpoints.read_endpoints(endpoints_config_path)
+    return AvailableEndpoints.get_instance(endpoints_config_path)
 
 
 def _lock_store_is_multi_worker_compatible(

{rasa_pro-3.10.14 → rasa_pro-3.10.15}/rasa/dialogue_understanding/coexistence/llm_based_router.py

@@ -161,7 +161,14 @@ class LLMBasedRouter(GraphComponent):
                 "llm_based_router.load.failed", error=e, resource=resource.name
             )
 
-        return cls(config, model_storage, resource, prompt_template=prompt_template)
+        router = cls(config, model_storage, resource, prompt_template=prompt_template)
+        try_instantiate_llm_client(
+            router.config.get(LLM_CONFIG_KEY),
+            DEFAULT_LLM_CONFIG,
+            "llm_based_router.load",
+            LLMBasedRouter.__name__,
+        )
+        return router
 
     @classmethod
     def create(

{rasa_pro-3.10.14 → rasa_pro-3.10.15}/rasa/dialogue_understanding/generator/flow_retrieval.py

@@ -50,6 +50,7 @@ from rasa.shared.utils.llm import (
     USER,
     get_prompt_template,
     allowed_values_for_slot,
+    try_instantiate_embedder,
 )
 
 DEFAULT_FLOW_DOCUMENT_TEMPLATE = importlib.resources.read_text(
@@ -142,6 +143,12 @@ class FlowRetrieval:
         """Load flow retrieval with previously populated FAISS vector store."""
         # initialize base flow retrieval
         flow_retrieval = FlowRetrieval(config, model_storage, resource)
+        try_instantiate_embedder(
+            flow_retrieval.config.get(EMBEDDINGS_CONFIG_KEY),
+            DEFAULT_EMBEDDINGS_CONFIG,
+            "flow_retrieval.load",
+            FlowRetrieval.__name__,
+        )
         # load vector store
         vector_store = cls._load_vector_store(
             flow_retrieval.config, model_storage, resource

{rasa_pro-3.10.14 → rasa_pro-3.10.15}/rasa/dialogue_understanding/generator/multi_step/multi_step_llm_command_generator.py

@@ -24,6 +24,7 @@ from rasa.dialogue_understanding.generator.constants import (
     LLM_CONFIG_KEY,
     USER_INPUT_CONFIG_KEY,
     FLOW_RETRIEVAL_KEY,
+    DEFAULT_LLM_CONFIG,
 )
 from rasa.dialogue_understanding.generator.flow_retrieval import FlowRetrieval
 from rasa.dialogue_understanding.generator.llm_based_command_generator import (
@@ -53,6 +54,7 @@ from rasa.shared.utils.llm import (
     tracker_as_readable_transcript,
     sanitize_message_for_prompt,
     allowed_values_for_slot,
+    try_instantiate_llm_client,
 )
 
 # multistep template keys
@@ -141,6 +143,12 @@ class MultiStepLLMCommandGenerator(LLMBasedCommandGenerator):
         prompts = cls._load_prompt_templates(model_storage, resource)
         # init base command generator
         command_generator = cls(config, model_storage, resource, prompts)
+        try_instantiate_llm_client(
+            command_generator.config.get(LLM_CONFIG_KEY),
+            DEFAULT_LLM_CONFIG,
+            "multi_step_llm_command_generator.load",
+            MultiStepLLMCommandGenerator.__name__,
+        )
         # load flow retrieval if enabled
         if command_generator.enabled_flow_retrieval:
             command_generator.flow_retrieval = cls.load_flow_retrival(

{rasa_pro-3.10.14 → rasa_pro-3.10.15}/rasa/dialogue_understanding/generator/single_step/single_step_llm_command_generator.py

@@ -21,6 +21,7 @@ from rasa.dialogue_understanding.generator.constants import (
     LLM_CONFIG_KEY,
     USER_INPUT_CONFIG_KEY,
     FLOW_RETRIEVAL_KEY,
+    DEFAULT_LLM_CONFIG,
 )
 from rasa.dialogue_understanding.generator.flow_retrieval import (
     FlowRetrieval,
@@ -48,6 +49,7 @@ from rasa.shared.utils.llm import (
     get_prompt_template,
     tracker_as_readable_transcript,
     sanitize_message_for_prompt,
+    try_instantiate_llm_client,
 )
 from rasa.utils.log_utils import log_llm
 
@@ -136,6 +138,12 @@ class SingleStepLLMCommandGenerator(LLMBasedCommandGenerator):
         )
         # init base command generator
         command_generator = cls(config, model_storage, resource, prompt_template)
+        try_instantiate_llm_client(
+            command_generator.config.get(LLM_CONFIG_KEY),
+            DEFAULT_LLM_CONFIG,
+            "single_step_llm_command_generator.load",
+            SingleStepLLMCommandGenerator.__name__,
+        )
         # load flow retrieval if enabled
         if command_generator.enabled_flow_retrieval:
             command_generator.flow_retrieval = cls.load_flow_retrival(

{rasa_pro-3.10.14 → rasa_pro-3.10.15}/rasa/e2e_test/aggregate_test_stats_calculator.py

@@ -35,6 +35,7 @@ class AggregateTestStatsCalculator:
         self.test_cases = test_cases
 
         self.failed_assertion_set: Set["Assertion"] = set()
+        self.failed_test_cases_without_assertion_failure: Set[str] = set()
         self.passed_count_mapping = {
             subclass_type: 0
             for subclass_type in _get_all_assertion_subclasses().keys()
@@ -89,8 +90,14 @@ class AggregateTestStatsCalculator:
         passed_test_case_names = [
             passed.test_case.name for passed in self.passed_results
         ]
+        # We filter out test cases that failed without an assertion failure
+        filtered_test_cases = [
+            test_case
+            for test_case in self.test_cases
+            if test_case.name not in self.failed_test_cases_without_assertion_failure
+        ]
 
-        for test_case in self.test_cases:
+        for test_case in filtered_test_cases:
             if test_case.name in passed_test_case_names:
                 for step in test_case.steps:
                     if step.assertions is None:
@@ -118,6 +125,9 @@ class AggregateTestStatsCalculator:
                     "no_assertion_failure_in_failed_result",
                     test_case=failed.test_case.name,
                 )
+                self.failed_test_cases_without_assertion_failure.add(
+                    failed.test_case.name
+                )
                 continue
 
             self.failed_assertion_set.add(failed.assertion_failure.assertion)

{rasa_pro-3.10.14 → rasa_pro-3.10.15}/rasa/e2e_test/assertions.py

@@ -452,6 +452,11 @@ class ActionExecutedAssertion(Assertion):
         **kwargs: Any,
     ) -> Tuple[Optional[AssertionFailure], Optional[Event]]:
         """Run the action executed assertion on the given events for that user turn."""
+        step_index = kwargs.get("step_index")
+        original_turn_events, turn_events = _get_turn_events_based_on_step_index(
+            step_index, turn_events, prior_events
+        )
+
         try:
             matching_event = next(
                 event
@@ -464,7 +469,7 @@ class ActionExecutedAssertion(Assertion):
             error_message += assertion_order_error_message
 
             return self._generate_assertion_failure(
-                error_message, prior_events, turn_events, self.line
+                error_message, prior_events, original_turn_events, self.line
             )
 
         return None, matching_event
@@ -519,6 +524,11 @@ class SlotWasSetAssertion(Assertion):
         """Run the slot_was_set assertion on the given events for that user turn."""
         matching_event = None
 
+        step_index = kwargs.get("step_index")
+        original_turn_events, turn_events = _get_turn_events_based_on_step_index(
+            step_index, turn_events, prior_events
+        )
+
         for slot in self.slots:
             matching_events = [
                 event
@@ -557,7 +567,7 @@ class SlotWasSetAssertion(Assertion):
                 error_message += assertion_order_error_message
 
                 return self._generate_assertion_failure(
-                    error_message, prior_events, turn_events, slot.line
+                    error_message, prior_events, original_turn_events, slot.line
                 )
 
         return None, matching_event
@@ -595,6 +605,11 @@ class SlotWasNotSetAssertion(Assertion):
         """Run the slot_was_not_set assertion on the given events for that user turn."""
         matching_event = None
 
+        step_index = kwargs.get("step_index")
+        original_turn_events, turn_events = _get_turn_events_based_on_step_index(
+            step_index, turn_events, prior_events
+        )
+
         for slot in self.slots:
             matching_events = [
                 event
@@ -630,7 +645,7 @@ class SlotWasNotSetAssertion(Assertion):
                 error_message += assertion_order_error_message
 
                 return self._generate_assertion_failure(
-                    error_message, prior_events, turn_events, slot.line
+                    error_message, prior_events, original_turn_events, slot.line
                 )
 
         return None, matching_event
@@ -723,6 +738,11 @@ class BotUtteredAssertion(Assertion):
         """Run the bot_uttered assertion on the given events for that user turn."""
         matching_event = None
 
+        step_index = kwargs.get("step_index")
+        original_turn_events, turn_events = _get_turn_events_based_on_step_index(
+            step_index, turn_events, prior_events
+        )
+
         if self.utter_name is not None:
             try:
                 matching_event = next(
@@ -736,7 +756,7 @@ class BotUtteredAssertion(Assertion):
                 error_message += assertion_order_error_message
 
                 return self._generate_assertion_failure(
-                    error_message, prior_events, turn_events, self.line
+                    error_message, prior_events, original_turn_events, self.line
                 )
 
         if self.text_matches is not None:
@@ -756,7 +776,7 @@ class BotUtteredAssertion(Assertion):
                 error_message += assertion_order_error_message
 
                 return self._generate_assertion_failure(
-                    error_message, prior_events, turn_events, self.line
+                    error_message, prior_events, original_turn_events, self.line
                 )
 
         if self.buttons:
@@ -772,7 +792,7 @@ class BotUtteredAssertion(Assertion):
                 )
                 error_message += assertion_order_error_message
                 return self._generate_assertion_failure(
-                    error_message, prior_events, turn_events, self.line
+                    error_message, prior_events, original_turn_events, self.line
                 )
 
         return None, matching_event
@@ -1179,3 +1199,25 @@ def _find_matching_generative_events(turn_events: List[Event]) -> List[BotUttere
         and event.metadata.get(UTTER_SOURCE_METADATA_KEY)
         in ELIGIBLE_UTTER_SOURCE_METADATA
     ]
+
+
+def _get_turn_events_based_on_step_index(
+    step_index: int, turn_events: List[Event], prior_events: List[Event]
+) -> Tuple[List[Event], List[Event]]:
+    """Get the turn events based on the step index.
+
+    For the first step, we need to include the prior events as well
+    in the same user turn. For the subsequent steps, we only need the
+    events that follow the user uttered event on which the tracker
+    was originally sliced by.
+
+    Returns:
+        List[Event]: The copy of turn_events
+        List[Event]: The turn events based on the step index
+
+    """
+    original_turn_events = turn_events[:]
+    if step_index == 0:
+        return original_turn_events, prior_events + turn_events
+
+    return original_turn_events, turn_events

{rasa_pro-3.10.14 → rasa_pro-3.10.15}/rasa/e2e_test/e2e_test_runner.py

@@ -136,7 +136,7 @@ class E2ETestRunner:
             return turns
 
         tracker = await self.agent.processor.fetch_tracker_with_initial_session(
-            sender_id
+            sender_id, output_channel=collector
         )
         # turn -1 i used to contain events that happen during
         # the start of the session and before the first user message
@@ -442,7 +442,7 @@ class E2ETestRunner:
         assertion_failure_found = False
         input_metadata = input_metadata if input_metadata else []
 
-        for step in test_case.steps:
+        for index, step in enumerate(test_case.steps):
             if not step.assertions:
                 structlogger.debug(
                     "e2e_test_runner.run_assertions.no_assertions.skipping_step",
@@ -490,6 +490,7 @@ class E2ETestRunner:
                     assertion_order_error_message=assertion_order_error_msg,
                     llm_judge_config=self.llm_judge_config,
                     step_text=step.text,
+                    step_index=index,
                 )
 
                 if assertion_failure:
@@ -826,7 +827,7 @@ class E2ETestRunner:
             return
 
         tracker = await self.agent.processor.fetch_tracker_with_initial_session(
-            sender_id
+            sender_id, output_channel=CollectingOutputChannel()
         )
 
         for fixture in fixtures: