rasa-pro 3.11.3a1.dev7__py3-none-any.whl → 3.12.0.dev2__py3-none-any.whl

rasa/core/policies/flows/flow_executor.py

@@ -82,7 +82,6 @@ from rasa.shared.core.flows.steps import (
     NoOperationFlowStep,
 )
 from rasa.shared.core.flows.steps.collect import SlotRejection
-from rasa.shared.core.flows.steps.constants import START_STEP
 from rasa.shared.core.slots import Slot
 from rasa.shared.core.trackers import (
     DialogueStateTracker,
@@ -317,7 +316,7 @@ def reset_scoped_slots(
     def _reset_slot(slot_name: Text, dialogue_tracker: DialogueStateTracker) -> None:
         slot = dialogue_tracker.slots.get(slot_name, None)
         initial_value = slot.initial_value if slot else None
-        events.append(SlotSet(slot_name, initial_value, metadata={"reset": True}))
+        events.append(SlotSet(slot_name, initial_value))
 
     if (
         isinstance(current_frame, UserFlowStackFrame)
@@ -449,7 +448,6 @@ def advance_flows_until_next_action(
                     tracker,
                     available_actions,
                     flows,
-                    previous_step_id,
                 )
                 new_events = step_result.events
                 if (
@@ -466,9 +464,6 @@ def advance_flows_until_next_action(
                     new_events.insert(
                         idx, FlowCompleted(active_frame.flow_id, previous_step_id)
                     )
-                attach_stack_metadata_to_events(
-                    next_step.id, current_flow.id, new_events
-                )
                 tracker.update_stack(step_stack)
                 tracker.update_with_events(new_events)
 
@@ -572,17 +567,6 @@ def validate_custom_slot_mappings(
     return True
 
 
-def attach_stack_metadata_to_events(
-    step_id: str,
-    flow_id: str,
-    events: List[Event],
-) -> None:
-    """Attach the stack metadata to the events."""
-    for event in events:
-        event.metadata[STEP_ID_METADATA_KEY] = step_id
-        event.metadata[ACTIVE_FLOW_METADATA_KEY] = flow_id
-
-
 def run_step(
     step: FlowStep,
     flow: Flow,
@@ -590,7 +574,6 @@ def run_step(
     tracker: DialogueStateTracker,
     available_actions: List[str],
     flows: FlowsList,
-    previous_step_id: str,
 ) -> FlowStepResult:
     """Run a single step of a flow.
 
@@ -608,19 +591,12 @@ def run_step(
         tracker: The tracker to run the step on.
         available_actions: The actions that are available in the domain.
         flows: All flows.
-        previous_step_id: The ID of the previous step.
 
     Returns:
     A result of running the step describing where to transition to.
     """
     initial_events: List[Event] = []
-    if previous_step_id == START_STEP:
-        # if the previous step id is the start step, we need to add a flow
-        # started event to the initial events.
-        # we can't use the current step to check this, as the current step is the
-        # first step in the flow -> other steps might link to this flow, so the
-        # only reliable way to check if we are starting a new flow is checking for
-        # the START_STEP meta step
+    if step == flow.first_step_in_flow():
         initial_events.append(FlowStarted(flow.id, metadata=stack.current_context()))
 
     if isinstance(step, CollectInformationFlowStep):

rasa/core/processor.py

@@ -818,9 +818,8 @@ class MessageProcessor:
         return parse_data
 
     def _sanitize_message(self, message: UserMessage) -> UserMessage:
-        """Sanitize user messages.
-
-        Removes prepended slashes before the actual content.
+        """Sanitize user message by removing prepended slashes before the
+        actual content.
         """
         # Regex pattern to match leading slashes and any whitespace before
         # actual content
@@ -922,7 +921,9 @@ class MessageProcessor:
         return [command.as_dict() for command in commands]
 
     def _contains_undefined_intent(self, message: Message) -> bool:
-        """Checks if the message contains an undefined intent."""
+        """Checks if the message contains an intent that is undefined
+        in the domain.
+        """
         intent_name = message.get(INTENT, {}).get("name")
         return intent_name is not None and intent_name not in self.domain.intents
 
@@ -986,8 +987,6 @@ class MessageProcessor:
         if parse_data["entities"]:
             self._log_slots(tracker)
 
-        plugin_manager().hook.after_new_user_message(tracker=tracker)
-
         logger.debug(
             f"Logged UserUtterance - tracker now has {len(tracker.events)} events."
         )
@@ -1306,7 +1305,7 @@ class MessageProcessor:
             self._log_slots(tracker)
 
         await self.execute_side_effects(events, tracker, output_channel)
-        plugin_manager().hook.after_action_executed(tracker=tracker)
+
         return self.should_predict_another_action(action.name())
 
     def _log_action_on_tracker(
@@ -1442,10 +1441,8 @@ class MessageProcessor:
         return len(filtered_commands) > 0
 
     def _is_calm_assistant(self) -> bool:
-        """Inspects the nodes of the graph schema to decide if we are in CALM.
-
-        To determine whether we are in CALM mode, we check if any node is
-        associated with the `FlowPolicy`, which is indicative of a
+        """Inspects the nodes of the graph schema to determine whether
+        any node is associated with the `FlowPolicy`, which is indicative of a
         CALM assistant setup.
 
         Returns:

rasa/dialogue_understanding/generator/command_generator.py

@@ -26,12 +26,8 @@ from rasa.shared.nlu.constants import (
     PROMPTS,
     KEY_USER_PROMPT,
     KEY_SYSTEM_PROMPT,
-    KEY_LLM_RESPONSE_METADATA,
-    KEY_PROMPT_NAME,
-    KEY_COMPONENT_NAME,
 )
 from rasa.shared.nlu.training_data.message import Message
-from rasa.shared.providers.llm.llm_response import LLMResponse
 from rasa.shared.utils.llm import DEFAULT_MAX_USER_INPUT_CHARACTERS
 
 structlogger = structlog.get_logger()
@@ -403,56 +399,66 @@ class CommandGenerator:
         prompt_name: str,
         user_prompt: str,
         system_prompt: Optional[str] = None,
-        llm_response: Optional[LLMResponse] = None,
     ) -> None:
         """Add prompt to the message parse data.
 
         Prompt is only added in case the flag 'record_commands_and_prompts' is set.
         Example of prompts in the message parse data:
             Message(data={
-              PROMPTS: [
-                  {
-                    "component_name": "MultiStepLLMCommandGenerator",
-                    "prompt_name": "fill_slots_prompt",
-                    "user_prompt": "...",
-                    "system_prompt": "...",
-                    "llm_response_metadata": { ... }
-                  },
-                  {
-                    "component_name": "MultiStepLLMCommandGenerator",
-                    "prompt_name": "handle_flows_prompt",
-                    "user_prompt": "...",
-                    "system_prompt": "...",
-                    "llm_response_metadata": { ... }
-                  },
-                  {
-                    "component_name": "SingleStepLLMCommandGenerator",
-                    "prompt_name": "prompt_template",
-                    "user_prompt": "...",
-                    "system_prompt": "...",
-                    "llm_response_metadata": { ... }
-                  }
+              PROMPTS: {
+                "MultiStepLLMCommandGenerator": [
+                    (
+                        "fill_slots_prompt",
+                        {
+                            "user_prompt": <prompt content>",
+                            "system_prompt": <prompt content>"
+                        }
+                    ),
+                    (
+                        "handle_flows_prompt",
+                        {
+                            "user_prompt": <prompt content>",
+                            "system_prompt": <prompt content>"
+                        }
+                    ),
+                ],
+                "SingleStepLLMCommandGenerator": [
+                    (
+                        "prompt_template",
+                        {
+                            "user_prompt": <prompt content>",
+                            "system_prompt": <prompt content>"
+                        }
+                    ),
                 ]
+              }
             })
         """
         from rasa.dialogue_understanding.utils import record_commands_and_prompts
 
-        # Only set prompt if the flag "record_commands_and_prompts" is set to True.
+        # only set prompt if the flag "record_commands_and_prompts" is set to True
         if not record_commands_and_prompts:
             return
 
-        # Construct the dictionary with prompt details.
-        prompt_data: Dict[Text, Any] = {
-            KEY_COMPONENT_NAME: component_name,
-            KEY_PROMPT_NAME: prompt_name,
-            KEY_USER_PROMPT: user_prompt,
-            KEY_LLM_RESPONSE_METADATA: llm_response.to_dict() if llm_response else None,
-            **({KEY_SYSTEM_PROMPT: system_prompt} if system_prompt else {}),
-        }
-
-        # Get or create a top-level "prompts" list.
-        prompts = message.get(PROMPTS) or []
-        prompts.append(prompt_data)
-
-        # Update the message with the new prompts list.
-        message.set(PROMPTS, prompts, add_to_output=True)
+        prompt_tuple = (
+            prompt_name,
+            {
+                KEY_USER_PROMPT: user_prompt,
+                **({KEY_SYSTEM_PROMPT: system_prompt} if system_prompt else {}),
+            },
+        )
+
+        if message.get(PROMPTS) is not None:
+            prompts = message.get(PROMPTS)
+            if component_name in prompts:
+                prompts[component_name].append(prompt_tuple)
+            else:
+                prompts[component_name] = [prompt_tuple]
+        else:
+            prompts = {component_name: [prompt_tuple]}
+
+        message.set(
+            PROMPTS,
+            prompts,
+            add_to_output=True,
+        )

rasa/dialogue_understanding/generator/llm_based_command_generator.py

@@ -32,7 +32,6 @@ from rasa.shared.exceptions import ProviderClientAPIException
 from rasa.shared.nlu.constants import FLOWS_IN_PROMPT
 from rasa.shared.nlu.training_data.message import Message
 from rasa.shared.nlu.training_data.training_data import TrainingData
-from rasa.shared.providers.llm.llm_response import LLMResponse
 from rasa.shared.utils.health_check.llm_health_check_mixin import LLMHealthCheckMixin
 from rasa.shared.utils.llm import (
     allowed_values_for_slot,
@@ -305,21 +304,22 @@ class LLMBasedCommandGenerator(
         )
         return filtered_flows
 
-    async def invoke_llm(self, prompt: Text) -> Optional[LLMResponse]:
+    async def invoke_llm(self, prompt: Text) -> Optional[Text]:
         """Use LLM to generate a response.
 
         Args:
             prompt: The prompt to send to the LLM.
 
         Returns:
-            An LLMResponse object.
+            The generated text.
 
         Raises:
-            ProviderClientAPIException: If an error occurs during the LLM API call.
+            ProviderClientAPIException if an error during API call.
         """
         llm = llm_factory(self.config.get(LLM_CONFIG_KEY), DEFAULT_LLM_CONFIG)
         try:
-            return await llm.acompletion(prompt)
+            llm_response = await llm.acompletion(prompt)
+            return llm_response.choices[0]
         except Exception as e:
             # unfortunately, langchain does not wrap LLM exceptions which means
             # we have to catch all exceptions here

rasa/dialogue_understanding/generator/llm_command_generator.py

@@ -10,7 +10,6 @@ from rasa.engine.recipes.default_recipe import DefaultV1Recipe
 from rasa.engine.storage.resource import Resource
 from rasa.engine.storage.storage import ModelStorage
 from rasa.shared.exceptions import ProviderClientAPIException
-from rasa.shared.providers.llm.llm_response import LLMResponse
 from rasa.shared.utils.io import raise_deprecation_warning
 
 structlogger = structlog.get_logger()
@@ -54,7 +53,7 @@ class LLMCommandGenerator(SingleStepLLMCommandGenerator):
             **kwargs,
         )
 
-    async def invoke_llm(self, prompt: Text) -> Optional[LLMResponse]:
+    async def invoke_llm(self, prompt: Text) -> Optional[Text]:
         try:
             return await super().invoke_llm(prompt)
         except ProviderClientAPIException:

rasa/dialogue_understanding/generator/multi_step/multi_step_llm_command_generator.py

@@ -51,7 +51,6 @@ from rasa.shared.core.trackers import DialogueStateTracker
 from rasa.shared.exceptions import ProviderClientAPIException
 from rasa.shared.nlu.constants import TEXT
 from rasa.shared.nlu.training_data.message import Message
-from rasa.shared.providers.llm.llm_response import LLMResponse
 from rasa.shared.utils.io import deep_container_fingerprint
 from rasa.shared.utils.llm import (
     get_prompt_template,
@@ -536,12 +535,7 @@ class MultiStepLLMCommandGenerator(LLMBasedCommandGenerator):
             prompt=prompt,
         )
 
-        response = await self.invoke_llm(prompt)
-        llm_response = LLMResponse.ensure_llm_response(response)
-        actions = None
-        if llm_response and llm_response.choices:
-            actions = llm_response.choices[0]
-
+        actions = await self.invoke_llm(prompt)
         structlogger.debug(
             "multi_step_llm_command_generator"
             ".predict_commands_for_active_flow"
@@ -553,11 +547,10 @@ class MultiStepLLMCommandGenerator(LLMBasedCommandGenerator):
 
         if commands:
             self._add_prompt_to_message_parse_data(
-                message=message,
-                component_name=MultiStepLLMCommandGenerator.__name__,
-                prompt_name="fill_slots_for_active_flow_prompt",
-                user_prompt=prompt,
-                llm_response=llm_response,
+                message,
+                MultiStepLLMCommandGenerator.__name__,
+                "fill_slots_for_active_flow_prompt",
+                prompt,
             )
 
         return commands
@@ -591,12 +584,7 @@ class MultiStepLLMCommandGenerator(LLMBasedCommandGenerator):
             prompt=prompt,
         )
 
-        response = await self.invoke_llm(prompt)
-        llm_response = LLMResponse.ensure_llm_response(response)
-        actions = None
-        if llm_response and llm_response.choices:
-            actions = llm_response.choices[0]
-
+        actions = await self.invoke_llm(prompt)
         structlogger.debug(
             "multi_step_llm_command_generator"
             ".predict_commands_for_handling_flows"
@@ -610,11 +598,10 @@ class MultiStepLLMCommandGenerator(LLMBasedCommandGenerator):
 
         if commands:
             self._add_prompt_to_message_parse_data(
-                message=message,
-                component_name=MultiStepLLMCommandGenerator.__name__,
-                prompt_name="handle_flows_prompt",
-                user_prompt=prompt,
-                llm_response=llm_response,
+                message,
+                MultiStepLLMCommandGenerator.__name__,
+                "handle_flows_prompt",
+                prompt,
             )
 
         return commands
@@ -681,12 +668,7 @@ class MultiStepLLMCommandGenerator(LLMBasedCommandGenerator):
             prompt=prompt,
         )
 
-        response = await self.invoke_llm(prompt)
-        llm_response = LLMResponse.ensure_llm_response(response)
-        actions = None
-        if llm_response and llm_response.choices:
-            actions = llm_response.choices[0]
-
+        actions = await self.invoke_llm(prompt)
         structlogger.debug(
             "multi_step_llm_command_generator"
             ".predict_commands_for_newly_started_flow"
@@ -713,11 +695,10 @@ class MultiStepLLMCommandGenerator(LLMBasedCommandGenerator):
 
         if commands:
             self._add_prompt_to_message_parse_data(
-                message=message,
-                component_name=MultiStepLLMCommandGenerator.__name__,
-                prompt_name="fill_slots_for_new_flow_prompt",
-                user_prompt=prompt,
-                llm_response=llm_response,
+                message,
+                MultiStepLLMCommandGenerator.__name__,
+                "fill_slots_for_new_flow_prompt",
+                prompt,
             )
 
         return commands

rasa/dialogue_understanding/generator/single_step/single_step_llm_command_generator.py

@@ -46,7 +46,6 @@ from rasa.shared.core.trackers import DialogueStateTracker
 from rasa.shared.exceptions import ProviderClientAPIException
 from rasa.shared.nlu.constants import TEXT, LLM_COMMANDS, LLM_PROMPT
 from rasa.shared.nlu.training_data.message import Message
-from rasa.shared.providers.llm.llm_response import LLMResponse
 from rasa.shared.utils.io import deep_container_fingerprint
 from rasa.shared.utils.llm import (
     get_prompt_template,
@@ -265,16 +264,13 @@ class SingleStepLLMCommandGenerator(LLMBasedCommandGenerator):
             prompt=flow_prompt,
         )
 
-        response = await self.invoke_llm(flow_prompt)
-        llm_response = LLMResponse.ensure_llm_response(response)
+        action_list = await self.invoke_llm(flow_prompt)
         # The check for 'None' maintains compatibility with older versions
         # of LLMCommandGenerator. In previous implementations, 'invoke_llm'
         # might return 'None' to indicate a failure to generate actions.
-        if llm_response is None or not llm_response.choices:
+        if action_list is None:
             return [ErrorCommand()]
 
-        action_list = llm_response.choices[0]
-
         log_llm(
             logger=structlogger,
             log_module="SingleStepLLMCommandGenerator",
@@ -289,11 +285,10 @@ class SingleStepLLMCommandGenerator(LLMBasedCommandGenerator):
             message, SingleStepLLMCommandGenerator.__name__, commands
         )
         self._add_prompt_to_message_parse_data(
-            message=message,
-            component_name=SingleStepLLMCommandGenerator.__name__,
-            prompt_name="command_generator_prompt",
-            user_prompt=flow_prompt,
-            llm_response=llm_response,
+            message,
+            SingleStepLLMCommandGenerator.__name__,
+            "command_generator_prompt",
+            flow_prompt,
         )
 
         return commands

rasa/dialogue_understanding/utils.py

@@ -1,14 +1,7 @@
 from contextlib import contextmanager
 from typing import Generator
 
-from rasa.dialogue_understanding.constants import (
-    RASA_RECORD_COMMANDS_AND_PROMPTS_ENV_VAR_NAME,
-)
-from rasa.utils.common import get_bool_env_variable
-
-record_commands_and_prompts = get_bool_env_variable(
-    RASA_RECORD_COMMANDS_AND_PROMPTS_ENV_VAR_NAME, False
-)
+record_commands_and_prompts = False
 
 
 @contextmanager

rasa/dialogue_understanding_test/command_metric_calculation.py

@@ -0,0 +1,12 @@
+from typing import List, Dict
+
+from rasa.dialogue_understanding_test.du_test_result import (
+    DialogueUnderstandingTestResult,
+)
+
+
+def calculate_command_metrics(
+    test_results: List[DialogueUnderstandingTestResult],
+) -> Dict[str, Dict[str, float]]:
+    """Calculate the metrics for the commands."""
+    return {}

rasa/dialogue_understanding_test/constants.py

@@ -13,3 +13,5 @@ KEY_COMMANDS = "commands"
 
 ACTOR_USER = "user"
 ACTOR_BOT = "bot"
+
+DEFAULT_INPUT_TESTS_PATH = "du_tests/"

rasa/dialogue_understanding_test/du_test_runner.py

@@ -0,0 +1,93 @@
+import asyncio
+from typing import Dict, Optional, Text, Union, List
+
+import structlog
+
+from rasa.core.exceptions import AgentNotReady
+from rasa.core.persistor import StorageType
+from rasa.core.utils import AvailableEndpoints
+from rasa.dialogue_understanding_test.du_test_case import DialogueUnderstandingTestCase
+from rasa.dialogue_understanding_test.du_test_result import (
+    DialogueUnderstandingTestResult,
+)
+from rasa.e2e_test.e2e_test_case import (
+    KEY_STUB_CUSTOM_ACTIONS,
+    ActualStepOutput,
+    TestStep,
+    Fixture,
+    Metadata,
+)
+from rasa.e2e_test.e2e_test_runner import E2ETestRunner
+from rasa.utils.endpoints import EndpointConfig
+
+structlogger = structlog.get_logger()
+
+TEST_TURNS_TYPE = Dict[int, Union[TestStep, ActualStepOutput]]
+
+
+class DialogueUnderstandingTestRunner:
+    """Dialogue Understanding test suite runner."""
+
+    def __init__(
+        self,
+        model_path: Optional[Text] = None,
+        model_server: Optional[EndpointConfig] = None,
+        remote_storage: Optional[StorageType] = None,
+        endpoints: Optional[AvailableEndpoints] = None,
+    ) -> None:
+        """Initializes the Dialogue Understanding test suite runner.
+
+        Args:
+            model_path: Path to the model.
+            model_server: Model server configuration.
+            remote_storage: Remote storage to use for model retrieval.
+            endpoints: Endpoints configuration.
+        """
+        import rasa.core.agent
+
+        self._check_action_server(endpoints)
+
+        self.agent = asyncio.run(
+            rasa.core.agent.load_agent(
+                model_path=model_path,
+                model_server=model_server,
+                remote_storage=remote_storage,
+                endpoints=endpoints,
+            )
+        )
+        if not self.agent.is_ready():
+            raise AgentNotReady(
+                "Agent needs to be prepared before usage. "
+                "Please check that the agent was able to "
+                "load the trained model."
+            )
+
+    def _check_action_server(self, endpoints: AvailableEndpoints) -> None:
+        """Check if the action server is reachable."""
+        are_custom_actions_stubbed = (
+            endpoints
+            and endpoints.action
+            and endpoints.action.kwargs.get(KEY_STUB_CUSTOM_ACTIONS)
+        )
+        if endpoints and not are_custom_actions_stubbed:
+            E2ETestRunner._action_server_is_reachable(
+                endpoints, "dialogue_understanding_test_runner"
+            )
+
+    async def run_tests(
+        self,
+        test_cases: List[DialogueUnderstandingTestCase],
+        fixtures: List[Fixture],
+        metadata: List[Metadata],
+    ) -> List[DialogueUnderstandingTestResult]:
+        """Run the dialogue understanding tests.
+
+        Args:
+            test_cases: List of test cases.
+            fixtures: List of fixtures.
+            metadata: List of metadata.
+
+        Returns:
+            List[DialogueUnderstandingTestResult]: List of test results.
+        """
+        return []

rasa/dialogue_understanding_test/io.py

@@ -0,0 +1,54 @@
+from typing import List, Dict
+
+from rasa.dialogue_understanding_test.du_test_result import (
+    DialogueUnderstandingTestResult,
+)
+from rasa.e2e_test.e2e_test_case import TestSuite
+
+
+def read_test_suite(test_case_path: str) -> TestSuite:
+    """Read the test cases from the given test case path.
+
+    Args:
+        test_case_path: Path to the test cases.
+
+    Returns:
+        TestSuite: Test suite containing the dialogue understanding test cases.
+    """
+    return TestSuite([], [], [], {})
+
+
+def write_test_results_to_file(
+    failed_tests: List[DialogueUnderstandingTestResult],
+    passed_tests: List[DialogueUnderstandingTestResult],
+    command_metrics: Dict[str, Dict[str, float]],
+    output_file: str,
+    output_prompt: bool,
+) -> None:
+    """Write the test results to the given output file.
+
+    Args:
+        failed_tests: Failed test cases.
+        passed_tests: Passed test cases.
+        command_metrics: Metrics for the commands.
+        output_file: Path to the output file.
+        output_prompt: Whether to log the prompt or not.
+    """
+    pass
+
+
+def print_test_results(
+    failed_tests: List[DialogueUnderstandingTestResult],
+    passed_tests: List[DialogueUnderstandingTestResult],
+    command_metrics: Dict[str, Dict[str, float]],
+    output_prompt: bool,
+) -> None:
+    """Print the test results to console.
+
+    Args:
+        failed_tests: Failed test cases.
+        passed_tests: Passed test cases.
+        command_metrics: Metrics for the commands.
+        output_prompt: Whether to log the prompt or not.
+    """
+    pass

rasa/dialogue_understanding_test/validation.py

@@ -0,0 +1,22 @@
+import argparse
+from typing import List
+
+from rasa.dialogue_understanding_test.du_test_case import DialogueUnderstandingTestCase
+
+
+def validate_cli_arguments(args: argparse.Namespace) -> None:
+    """Validate the CLI arguments for the dialogue understanding test.
+
+    Args:
+        args: Commandline arguments.
+    """
+    pass
+
+
+def validate_test_cases(test_cases: List[DialogueUnderstandingTestCase]) -> None:
+    """Validate the dialogue understanding test cases.
+
+    Args:
+        test_cases: Test cases to validate.
+    """
+    pass