rasa-pro 3.12.6.dev2__py3-none-any.whl → 3.13.0.dev2__py3-none-any.whl

rasa/dialogue_understanding/generator/prompt_templates/command_prompt_v2_gpt_4o_2024_11_20_template.jinja2

@@ -16,7 +16,9 @@ Use the following structured data:
 * `set slot slot_name slot_value`: Slot setting. For example, `set slot transfer_money_recipient Freddy`. Can be used to correct and change previously set values.
 * `cancel flow`: Cancelling the current flow.
 * `disambiguate flows flow_name1 flow_name2 ... flow_name_n`: Disambiguate which flow should be started when user input is ambiguous by listing the potential flows as options. For example, `disambiguate flows list_contacts add_contact remove_contact ...` if the user just wrote "contacts".
-* `provide info`: Responding to the user's questions by supplying relevant information, such as answering FAQs or explaining services.
+{%- if relevant_documents.results %}
+* `search and reply`: Responding to the user's message by using the relevant FAQs (included in this prompt) retrieved from the knowledge base.
+{%- endif %}
 * `offtopic reply`: Responding to casual or social user messages that are unrelated to any flows, engaging in friendly conversation and addressing off-topic remarks.
 * `hand over`: Handing over to a human, in case the user seems frustrated or explicitly asks to speak to one.
 
@@ -27,16 +29,36 @@ Use the following structured data:
 * For categorical slots try to match the user message with allowed slot values. Use "other" if you cannot match it.
 * Set the boolean slots based on the user response. Map positive responses to `True`, and negative to `False`.
 * Extract text slot values exactly as provided by the user. Avoid assumptions, format changes, or partial extractions.
-* Only use information provided by the user.
 * Use clarification in ambiguous cases.
+* Use `disambiguate flows` only when multiple flows could fit the same message (e.g., "card" could mean `block_card` or `replace_card`).
+{%- if relevant_documents.results %}
+* A user asking a question does not automatically imply that they want `search and reply`. The objective is to help them complete a business process if its possible to do so via a flow.
+* **Flow Priority**: If a user message can be addressed by starting a flow (even if it looks like a general question), ALWAYS start the flow first. Example: If the user says "How do I activate my card?", use `start flow activate_card` instead of `search and reply`. Only use `search and reply` if no flow matches the request.
+{%- endif %}
+* Only use information provided by the user.
 * Multiple flows can be started. If a user wants to digress into a second flow, you do not need to cancel the current flow.
 * Do not cancel the flow unless the user explicitly requests it.
 * Strictly adhere to the provided action format.
 * Focus on the last message and take it one step at a time.
 * Use the previous conversation steps only to aid understanding.
 
+{%- if relevant_documents.results %}
+
+---
+
+## Relevant FAQs from the knowledge base
+```json
+{"documents":[{% for document in relevant_documents.results %}{"Q":"{{ document.text }}","A":"{{ document.metadata.answer }}"},{% endfor %}]}
+```
+
+---
+
+{% else %}
+
 ---
 
+{% endif -%}
+
 ## Current State
 {% if current_flow != None %}Use the following structured data:
 ```json

rasa/dialogue_understanding/generator/single_step/compact_llm_command_generator.py

@@ -4,6 +4,7 @@ from typing import Any, Dict, List, Optional, Text
 import structlog
 
 import rasa.shared.utils.io
+from rasa.core.information_retrieval import SearchResultList
 from rasa.dialogue_understanding.commands import (
     CannotHandleCommand,
     Command,
@@ -38,6 +39,7 @@ from rasa.dialogue_understanding.utils import (
     add_commands_to_message_parse_data,
     add_prompt_to_message_parse_data,
 )
+from rasa.document_retrieval.constants import POST_PROCESSED_DOCUMENTS_KEY
 from rasa.engine.graph import ExecutionContext
 from rasa.engine.recipes.default_recipe import DefaultV1Recipe
 from rasa.engine.storage.resource import Resource
@@ -47,10 +49,6 @@ from rasa.shared.constants import (
     AWS_BEDROCK_PROVIDER,
     AZURE_OPENAI_PROVIDER,
     EMBEDDINGS_CONFIG_KEY,
-    LANGFUSE_CUSTOM_METADATA_DICT,
-    LANGFUSE_METADATA_SESSION_ID,
-    LANGFUSE_METADATA_USER_ID,
-    LANGFUSE_TAGS,
     MAX_TOKENS_CONFIG_KEY,
     PROMPT_TEMPLATE_CONFIG_KEY,
     ROUTE_TO_CALM_SLOT,
@@ -366,14 +364,7 @@ class CompactLLMCommandGenerator(LLMBasedCommandGenerator):
             prompt=flow_prompt,
         )
 
-        metadata = {
-            LANGFUSE_METADATA_USER_ID: self.user_id,
-            LANGFUSE_METADATA_SESSION_ID: tracker.sender_id if tracker else "",
-            LANGFUSE_CUSTOM_METADATA_DICT: {"component": self.__class__.__name__},
-            LANGFUSE_TAGS: [self.__class__.__name__],
-        }
-
-        response = await self.invoke_llm(flow_prompt, metadata)
+        response = await self.invoke_llm(flow_prompt)
         llm_response = LLMResponse.ensure_llm_response(response)
         # The check for 'None' maintains compatibility with older versions
         # of LLMCommandGenerator. In previous implementations, 'invoke_llm'
@@ -504,6 +495,10 @@ class CompactLLMCommandGenerator(LLMBasedCommandGenerator):
         latest_user_message = sanitize_message_for_prompt(message.get(TEXT))
         current_conversation += f"\nUSER: {latest_user_message}"
 
+        relevant_documents = SearchResultList.from_dict(
+            message.get(POST_PROCESSED_DOCUMENTS_KEY, [])
+        )
+
         inputs = {
             "available_flows": self.prepare_flows_for_template(
                 startable_flows, tracker
@@ -517,6 +512,7 @@ class CompactLLMCommandGenerator(LLMBasedCommandGenerator):
             "current_slot_allowed_values": current_slot_allowed_values,
             "user_message": latest_user_message,
             "is_repeat_command_enabled": self.repeat_command_enabled,
+            "relevant_documents": relevant_documents,
         }
 
         return self.compile_template(self.prompt_template).render(**inputs)

rasa/dialogue_understanding/patterns/default_flows_for_patterns.yml

@@ -1,17 +1,6 @@
 version: "3.1"
 responses:
 
-  utter_ask_continue_previous_flow:
-    - text: "Confirm if you would like to continue with the initial topic: {{context.interrupted_flow_id}}?"
-      metadata:
-        rephrase: True
-        template: jinja
-      buttons:
-        - title: Continue with the previous topic.
-          payload: /SetSlots(continue_previous_flow=True)
-        - title: Switch to new topic.
-          payload: /SetSlots(continue_previous_flow=False)
-
   utter_ask_rephrase:
     - text: I’m sorry I am unable to understand you, could you please rephrase?
 
@@ -20,20 +9,6 @@ responses:
       metadata:
         rephrase: True
 
-  utter_block_digressions:
-    - text: "We can look into {{ context.interrupting_flow_id }} later. Let's focus on the current topic: {{ context.interrupted_flow_id }}."
-      metadata:
-        rephrase: True
-        template: jinja
-    - text: "Let's continue with the current topic: {{ context.interrupted_flow_id }}."
-      condition:
-        - type: slot
-          name: continue_previous_flow
-          value: True
-      metadata:
-        rephrase: True
-        template: jinja
-
   utter_boolean_slot_rejection:
     - text: "Sorry, the value you provided, `{{value}}`, is not valid. Please respond with a valid value."
       metadata:
@@ -60,12 +35,6 @@ responses:
         rephrase: True
         template: jinja
 
-  utter_continue_interruption:
-    - text: "Let's continue with the chosen topic instead: {{ context.interrupting_flow_id }}."
-      metadata:
-        rephrase: True
-        template: jinja
-
   utter_corrected_previous_input:
     - text: "Ok, I am updating {{ context.corrected_slots.keys()|join(', ') }} to {{ context.new_slot_values | join(', ') }} respectively."
       metadata:
@@ -150,10 +119,6 @@ slots:
     type: float
     initial_value: 0.0
     max_value: 1000000
-  continue_previous_flow:
-    type: bool
-    mappings:
-      - type: from_llm
 
 flows:
   pattern_cancel_flow:
@@ -197,7 +162,6 @@ flows:
     steps:
       - action: action_clarify_flows
       - action: utter_clarification_options_rasa
-      - action: action_listen
 
   pattern_code_change:
     description: Conversation repair flow for cleaning the stack after an assistant update
@@ -247,31 +211,6 @@ flows:
                 next: END
           - else: END
 
-  pattern_handle_digressions:
-    description: Conversation repair flow for handling digressions
-    name: pattern handle digressions
-    steps:
-      - noop: true
-        id: branching
-        next:
-          - if: context.ask_confirm_digressions contains context.interrupting_flow_id
-            then: continue_previous_flow
-          - if: context.block_digressions contains context.interrupting_flow_id
-            then: block_digression
-          - else: continue_digression
-      - id: continue_previous_flow
-        collect: continue_previous_flow
-        next:
-          - if: slots.continue_previous_flow
-            then: block_digression
-          - else: continue_digression
-      - id: block_digression
-        action: action_block_digression
-        next: END
-      - id: continue_digression
-        action: action_continue_digression
-        next: END
-
   pattern_human_handoff:
     description: Conversation repair flow for switching users to a human agent if their request can't be handled
     name: pattern human handoff

rasa/dialogue_understanding/processor/command_processor.py

@@ -18,9 +18,6 @@ from rasa.dialogue_understanding.commands import (
 from rasa.dialogue_understanding.commands.handle_code_change_command import (
     HandleCodeChangeCommand,
 )
-from rasa.dialogue_understanding.commands.handle_digressions_command import (
-    HandleDigressionsCommand,
-)
 from rasa.dialogue_understanding.commands.set_slot_command import SetSlotExtractor
 from rasa.dialogue_understanding.commands.utils import (
     create_validate_frames_from_slot_set_events,
@@ -454,21 +451,7 @@ def clean_up_commands(
                 )
                 continue
 
-            if should_add_handle_digressions_command(tracker, all_flows, top_flow_id):
-                handle_digression_command = HandleDigressionsCommand(flow=command.flow)
-                if handle_digression_command in clean_commands:
-                    structlogger.debug(
-                        "command_processor.clean_up_commands.skip_handle_digressions.command_already_present",
-                        command=handle_digression_command,
-                    )
-                    continue
-                clean_commands.append(handle_digression_command)
-                structlogger.debug(
-                    "command_processor.clean_up_commands.push_handle_digressions",
-                    command=command,
-                )
-            else:
-                clean_commands.append(command)
+            clean_commands.append(command)
 
         # handle chitchat command differently from other free-form answer commands
         elif isinstance(command, ChitChatAnswerCommand):
@@ -503,21 +486,9 @@ def clean_up_commands(
     # when coexistence is enabled, by default there will be a SetSlotCommand
     # for the ROUTE_TO_CALM_SLOT slot.
     if tracker.has_coexistence_routing_slot and len(clean_commands) > 2:
-        clean_commands = filter_cannot_handle_command_for_skipped_slots(clean_commands)
+        clean_commands = filter_cannot_handle_command(clean_commands)
     elif not tracker.has_coexistence_routing_slot and len(clean_commands) > 1:
-        clean_commands = filter_cannot_handle_command_for_skipped_slots(clean_commands)
-
-    # remove cancel flow when there is a handle digression command
-    # otherwise the cancel command will cancel the active flow which defined a specific
-    # behavior for the digression
-    if contains_command(clean_commands, HandleDigressionsCommand) and contains_command(
-        clean_commands, CancelFlowCommand
-    ):
-        clean_commands = [
-            command
-            for command in clean_commands
-            if not isinstance(command, CancelFlowCommand)
-        ]
+        clean_commands = filter_cannot_handle_command(clean_commands)
 
     clean_commands = ensure_max_number_of_command_type(
         clean_commands, RepeatBotMessagesCommand, 1
@@ -857,12 +828,12 @@ def should_slot_be_set(
     return True
 
 
-def filter_cannot_handle_command_for_skipped_slots(
+def filter_cannot_handle_command(
     clean_commands: List[Command],
 ) -> List[Command]:
-    """Filter out a 'cannot handle' command for skipped slots.
+    """Filter out a 'cannot handle' command.
 
-    This is used to filter out a 'cannot handle' command for skipped slots
+    This is used to filter out a 'cannot handle' command
     in case other commands are present.
 
     Returns:
@@ -871,34 +842,5 @@ def filter_cannot_handle_command_for_skipped_slots(
     return [
         command
         for command in clean_commands
-        if not (
-            isinstance(command, CannotHandleCommand)
-            and command.reason
-            and CANNOT_HANDLE_REASON == command.reason
-        )
+        if not isinstance(command, CannotHandleCommand)
     ]
-
-
-def should_add_handle_digressions_command(
-    tracker: DialogueStateTracker, all_flows: FlowsList, top_flow_id: str
-) -> bool:
-    """Check if a handle digressions command should be added to the commands.
-
-    The command should replace a StartFlow command only if we are at a collect step of
-    a flow and a new flow is predicted by the command generator to start.
-    """
-    current_flow = all_flows.flow_by_id(top_flow_id)
-    current_flow_condition = current_flow and (
-        current_flow.ask_confirm_digressions or current_flow.block_digressions
-    )
-
-    collect_info = get_current_collect_step(tracker.stack, all_flows)
-
-    if collect_info and (
-        collect_info.ask_confirm_digressions
-        or collect_info.block_digressions
-        or current_flow_condition
-    ):
-        return True
-
-    return False

rasa/dialogue_understanding/stack/utils.py

@@ -4,9 +4,6 @@ from typing import List, Optional, Set, Tuple
 from rasa.dialogue_understanding.patterns.collect_information import (
     CollectInformationPatternFlowStackFrame,
 )
-from rasa.dialogue_understanding.patterns.continue_interrupted import (
-    ContinueInterruptedPatternFlowStackFrame,
-)
 from rasa.dialogue_understanding.stack.dialogue_stack import DialogueStack
 from rasa.dialogue_understanding.stack.frames import (
     BaseFlowStackFrame,
@@ -221,38 +218,3 @@ def get_collect_steps_excluding_ask_before_filling_for_active_flow(
         for step in active_flow.get_collect_steps()
         if not step.ask_before_filling
     )
-
-
-def remove_digression_from_stack(stack: DialogueStack, flow_id: str) -> DialogueStack:
-    """Remove a specific flow frame from the stack and other frames that reference it.
-
-    The main use-case is to prevent duplicate digressions from being added to the stack.
-
-    Args:
-        stack: The dialogue stack.
-        flow_id: The flow to remove.
-
-    Returns:
-        The updated dialogue stack.
-    """
-    updated_stack = stack.copy()
-    original_frames = updated_stack.frames[:]
-    found_digression_index = -1
-    for index, frame in enumerate(original_frames):
-        if isinstance(frame, BaseFlowStackFrame) and frame.flow_id == flow_id:
-            updated_stack.frames.pop(index)
-            found_digression_index = index
-
-        # we also need to remove the `ContinueInterruptedPatternFlowStackFrame`
-        elif (
-            isinstance(frame, ContinueInterruptedPatternFlowStackFrame)
-            and frame.previous_flow_name == flow_id
-            and found_digression_index + 1 == index
-        ):
-            # we know that this frame is always added after the digressing flow frame
-            # that was blocked previously by action_block_digressions,
-            # so this check would occur after the digressing flow was popped.
-            # Therefore, we need to update the index dynamically before popping.
-            updated_stack.frames.pop(index - 1)
-
-    return updated_stack

rasa/dialogue_understanding_test/command_metric_calculation.py

@@ -1,54 +1,21 @@
+import typing
 from collections import defaultdict
 from typing import Dict, List
 
-from pydantic import BaseModel
-
 from rasa.dialogue_understanding.commands import Command
 from rasa.dialogue_understanding_test.command_comparison import (
     is_command_present_in_list,
 )
-from rasa.dialogue_understanding_test.du_test_result import (
-    DialogueUnderstandingTestResult,
-)
-
-
-class CommandMetrics(BaseModel):
-    tp: int
-    fp: int
-    fn: int
-    total_count: int
-
-    @staticmethod
-    def _safe_divide(numerator: float, denominator: float) -> float:
-        """Safely perform division, returning 0.0 if the denominator is zero."""
-        return numerator / denominator if denominator > 0 else 0.0
+from rasa.dialogue_understanding_test.command_metrics import CommandMetrics
 
-    def get_precision(self) -> float:
-        return self._safe_divide(self.tp, self.tp + self.fp)
-
-    def get_recall(self) -> float:
-        return self._safe_divide(self.tp, self.tp + self.fn)
-
-    def get_f1_score(self) -> float:
-        precision = self.get_precision()
-        recall = self.get_recall()
-
-        return self._safe_divide(2 * precision * recall, precision + recall)
-
-    def as_dict(self) -> Dict[str, float]:
-        return {
-            "tp": self.tp,
-            "fp": self.fp,
-            "fn": self.fn,
-            "precision": self.get_precision(),
-            "recall": self.get_recall(),
-            "f1_score": self.get_f1_score(),
-            "total_count": self.total_count,
-        }
+if typing.TYPE_CHECKING:
+    from rasa.dialogue_understanding_test.du_test_result import (
+        DialogueUnderstandingTestResult,
+    )
 
 
 def calculate_command_metrics(
-    test_results: List[DialogueUnderstandingTestResult],
+    test_results: List["DialogueUnderstandingTestResult"],
 ) -> Dict[str, CommandMetrics]:
     """Calculate the command metrics for the test result."""
     metrics: Dict[str, CommandMetrics] = defaultdict(

rasa/dialogue_understanding_test/command_metrics.py

@@ -0,0 +1,38 @@
+from typing import Dict
+
+from pydantic import BaseModel
+
+
+class CommandMetrics(BaseModel):
+    tp: int
+    fp: int
+    fn: int
+    total_count: int
+
+    @staticmethod
+    def _safe_divide(numerator: float, denominator: float) -> float:
+        """Safely perform division, returning 0.0 if the denominator is zero."""
+        return numerator / denominator if denominator > 0 else 0.0
+
+    def get_precision(self) -> float:
+        return self._safe_divide(self.tp, self.tp + self.fp)
+
+    def get_recall(self) -> float:
+        return self._safe_divide(self.tp, self.tp + self.fn)
+
+    def get_f1_score(self) -> float:
+        precision = self.get_precision()
+        recall = self.get_recall()
+
+        return self._safe_divide(2 * precision * recall, precision + recall)
+
+    def as_dict(self) -> Dict[str, float]:
+        return {
+            "tp": self.tp,
+            "fp": self.fp,
+            "fn": self.fn,
+            "precision": self.get_precision(),
+            "recall": self.get_recall(),
+            "f1_score": self.get_f1_score(),
+            "total_count": self.total_count,
+        }

rasa/dialogue_understanding_test/du_test_case.py

@@ -1,7 +1,11 @@
+from collections import defaultdict
 from typing import Any, Dict, Iterator, List, Optional, Tuple
 
 from pydantic import BaseModel, Field
 
+from rasa.core import IntentlessPolicy
+from rasa.core.nlg.contextual_response_rephraser import ContextualResponseRephraser
+from rasa.core.policies.enterprise_search_policy import EnterpriseSearchPolicy
 from rasa.dialogue_understanding.commands.prompt_command import PromptCommand
 from rasa.dialogue_understanding.generator.command_parser import parse_commands
 from rasa.dialogue_understanding_test.command_comparison import are_command_lists_equal
@@ -69,6 +73,8 @@ class DialogueUnderstandingOutput(BaseModel):
     commands: Dict[str, List[PromptCommand]]
     # List of prompts
     prompts: Optional[List[Dict[str, Any]]] = None
+    # Latency of the full message roundtrip
+    latency: Optional[float] = None
 
     class Config:
         """Skip validation for PromptCommand protocol as pydantic does not know how to
@@ -88,27 +94,41 @@ class DialogueUnderstandingOutput(BaseModel):
     def get_component_names_that_predicted_commands_or_have_llm_response(
         self,
     ) -> List[str]:
-        """Get all component names that have predicted commands or recieved
+        """Get all relevant component names.
+
+        Components are relevant if they have predicted commands or received a
         non-empty response from LLM.
         """
+        # Exclude components that are not related to Dialogue Understanding
+        component_names_to_exclude = [
+            EnterpriseSearchPolicy.__name__,
+            IntentlessPolicy.__name__,
+            ContextualResponseRephraser.__name__,
+        ]
+
         component_names_that_predicted_commands = (
             [
                 component_name
                 for component_name, predicted_commands in self.commands.items()
                 if predicted_commands
+                and component_name not in component_names_to_exclude
             ]
             if self.commands
             else []
         )
+
         components_with_prompts = (
             [
                 str(prompt.get(KEY_COMPONENT_NAME, None))
                 for prompt in self.prompts
                 if prompt.get(KEY_LLM_RESPONSE_METADATA, None)
+                and prompt.get(KEY_COMPONENT_NAME, None)
+                not in component_names_to_exclude
             ]
             if self.prompts
             else []
         )
+
         return list(
             set(component_names_that_predicted_commands + components_with_prompts)
         )
@@ -290,41 +310,54 @@ class DialogueUnderstandingTestStep(BaseModel):
 
         return ""
 
-    def get_latencies(self) -> List[float]:
+    def get_latencies(self) -> Dict[str, List[float]]:
         if self.dialogue_understanding_output is None:
-            return []
+            return {}
 
-        prompts = self.dialogue_understanding_output.get_component_name_to_prompt_info()
+        component_name_to_prompt_info = (
+            self.dialogue_understanding_output.get_component_name_to_prompt_info()
+        )
 
-        return [
-            prompt_data.get(KEY_LATENCY, 0.0)
-            for prompt in prompts.values()
-            for prompt_data in prompt
-        ]
+        latencies = defaultdict(list)
+        for component_name, prompt_info_list in component_name_to_prompt_info.items():
+            for prompt_info in prompt_info_list:
+                latencies[component_name].append(prompt_info.get(KEY_LATENCY, 0.0))
 
-    def get_completion_tokens(self) -> List[int]:
+        return latencies
+
+    def get_completion_tokens(self) -> Dict[str, List[float]]:
         if self.dialogue_understanding_output is None:
-            return []
+            return {}
 
-        prompts = self.dialogue_understanding_output.get_component_name_to_prompt_info()
+        component_name_to_prompt_info = (
+            self.dialogue_understanding_output.get_component_name_to_prompt_info()
+        )
 
-        return [
-            prompt_data.get(KEY_COMPLETION_TOKENS, 0)
-            for prompt in prompts.values()
-            for prompt_data in prompt
-        ]
+        completion_tokens = defaultdict(list)
+        for component_name, prompt_info_list in component_name_to_prompt_info.items():
+            for prompt_info in prompt_info_list:
+                completion_tokens[component_name].append(
+                    prompt_info.get(KEY_COMPLETION_TOKENS, 0.0)
+                )
+
+        return completion_tokens
 
-    def get_prompt_tokens(self) -> List[int]:
+    def get_prompt_tokens(self) -> Dict[str, List[float]]:
         if self.dialogue_understanding_output is None:
-            return []
+            return {}
 
-        prompts = self.dialogue_understanding_output.get_component_name_to_prompt_info()
+        component_name_to_prompt_info = (
+            self.dialogue_understanding_output.get_component_name_to_prompt_info()
+        )
 
-        return [
-            prompt_data.get(KEY_PROMPT_TOKENS, 0)
-            for prompt in prompts.values()
-            for prompt_data in prompt
-        ]
+        prompt_tokens = defaultdict(list)
+        for component_name, prompt_info_list in component_name_to_prompt_info.items():
+            for prompt_info in prompt_info_list:
+                prompt_tokens[component_name].append(
+                    prompt_info.get(KEY_PROMPT_TOKENS, 0.0)
+                )
+
+        return prompt_tokens
 
 
 class DialogueUnderstandingTestCase(BaseModel):