rasa-pro 3.12.18.dev1__py3-none-any.whl → 3.13.0a1.dev2__py3-none-any.whl

rasa/dialogue_understanding/utils.py

@@ -1,7 +1,9 @@
 from contextlib import contextmanager
 from typing import Any, Dict, Generator, List, Optional, Text
 
-from rasa.dialogue_understanding.commands import Command
+import structlog
+
+from rasa.dialogue_understanding.commands import Command, NoopCommand, SetSlotCommand
 from rasa.dialogue_understanding.constants import (
     RASA_RECORD_COMMANDS_AND_PROMPTS_ENV_VAR_NAME,
 )
@@ -16,7 +18,6 @@ from rasa.shared.nlu.constants import (
     KEY_USER_PROMPT,
     PREDICTED_COMMANDS,
     PROMPTS,
-    SET_SLOT_COMMAND,
 )
 from rasa.shared.nlu.training_data.message import Message
 from rasa.shared.providers.llm.llm_response import LLMResponse
@@ -26,6 +27,8 @@ record_commands_and_prompts = get_bool_env_variable(
     RASA_RECORD_COMMANDS_AND_PROMPTS_ENV_VAR_NAME, False
 )
 
+structlogger = structlog.get_logger()
+
 
 @contextmanager
 def set_record_commands_and_prompts() -> Generator:
@@ -144,21 +147,74 @@ def _handle_via_nlu_in_coexistence(
     if not tracker:
         return False
 
+    commands = message.get(COMMANDS, [])
+
+    # If coexistence routing slot is not active, this setup doesn't
+    # support dual routing -> default to CALM
     if not tracker.has_coexistence_routing_slot:
+        structlogger.debug(
+            "utils.handle_via_nlu_in_coexistence"
+            ".tracker_missing_route_session_to_calm_slot",
+            event_info=(
+                f"Tracker doesn't have the '{ROUTE_TO_CALM_SLOT}' slot."
+                f"Routing to CALM."
+            ),
+            route_session_to_calm=commands,
+        )
         return False
 
+    # Check if the routing decision is stored in the tracker slot
+    # If slot is true -> route to CALM
+    # If slot is false -> route to DM1
     value = tracker.get_slot(ROUTE_TO_CALM_SLOT)
     if value is not None:
+        structlogger.debug(
+            "utils.handle_via_nlu_in_coexistence"
+            ".tracker_route_session_to_calm_slot_value",
+            event_info=(
+                f"Tracker slot '{ROUTE_TO_CALM_SLOT}' set to '{value}'. "
+                f"Routing to "
+                f"{'CALM' if value else 'NLU system'}."
+            ),
+            route_session_to_calm_value_in_tracker=value,
+        )
         return not value
 
-    # routing slot has been reset so we need to check
-    # the command issued by the Router component
-    if message.get(COMMANDS):
-        for command in message.get(COMMANDS):
-            if (
-                command.get("command") == SET_SLOT_COMMAND
-                and command.get("name") == ROUTE_TO_CALM_SLOT
-            ):
-                return not command.get("value")
-
+    # Non-sticky routing to DM1 is only allowed if NoopCommand is the sole predicted
+    # command. In that case, route to DM1
+    if len(commands) == 1 and commands[0].get("command") == NoopCommand.command():
+        structlogger.debug(
+            "utils.handle_via_nlu_in_coexistence.noop_command_detected",
+            event_info="NoopCommand found. Routing to NLU system non-sticky.",
+            commands=commands,
+        )
+        return True
+
+    # If the slot was reset (e.g. new session), try to infer routing from
+    # attached commands. Look for a SetSlotCommand targeting the ROUTE_TO_CALM_SLOT
+    for command in message.get(COMMANDS, []):
+        # If slot is true -> route to CALM
+        # If slot is false -> route to DM1
+        if (
+            command.get("command") == SetSlotCommand.command()
+            and command.get("name") == ROUTE_TO_CALM_SLOT
+        ):
+            structlogger.debug(
+                "utils.handle_via_nlu_in_coexistence.set_slot_command_detected",
+                event_info=(
+                    f"SetSlotCommand setting the '{ROUTE_TO_CALM_SLOT}' to "
+                    f"'{command.get('value')}'. "
+                    f"Routing to "
+                    f"{'CALM' if command.get('value') else 'NLU system'}."
+                ),
+                commands=commands,
+            )
+            return not command.get("value")
+
+    # If no routing info is available -> default to CALM
+    structlogger.debug(
+        "utils.handle_via_nlu_in_coexistence.no_routing_info_available",
+        event_info="No routing info available. Routing to CALM.",
+        commands=commands,
+    )
     return False

rasa/dialogue_understanding_test/command_metric_calculation.py

@@ -1,54 +1,21 @@
+import typing
 from collections import defaultdict
 from typing import Dict, List
 
-from pydantic import BaseModel
-
 from rasa.dialogue_understanding.commands import Command
 from rasa.dialogue_understanding_test.command_comparison import (
     is_command_present_in_list,
 )
-from rasa.dialogue_understanding_test.du_test_result import (
-    DialogueUnderstandingTestResult,
-)
-
-
-class CommandMetrics(BaseModel):
-    tp: int
-    fp: int
-    fn: int
-    total_count: int
-
-    @staticmethod
-    def _safe_divide(numerator: float, denominator: float) -> float:
-        """Safely perform division, returning 0.0 if the denominator is zero."""
-        return numerator / denominator if denominator > 0 else 0.0
+from rasa.dialogue_understanding_test.command_metrics import CommandMetrics
 
-    def get_precision(self) -> float:
-        return self._safe_divide(self.tp, self.tp + self.fp)
-
-    def get_recall(self) -> float:
-        return self._safe_divide(self.tp, self.tp + self.fn)
-
-    def get_f1_score(self) -> float:
-        precision = self.get_precision()
-        recall = self.get_recall()
-
-        return self._safe_divide(2 * precision * recall, precision + recall)
-
-    def as_dict(self) -> Dict[str, float]:
-        return {
-            "tp": self.tp,
-            "fp": self.fp,
-            "fn": self.fn,
-            "precision": self.get_precision(),
-            "recall": self.get_recall(),
-            "f1_score": self.get_f1_score(),
-            "total_count": self.total_count,
-        }
+if typing.TYPE_CHECKING:
+    from rasa.dialogue_understanding_test.du_test_result import (
+        DialogueUnderstandingTestResult,
+    )
 
 
 def calculate_command_metrics(
-    test_results: List[DialogueUnderstandingTestResult],
+    test_results: List["DialogueUnderstandingTestResult"],
 ) -> Dict[str, CommandMetrics]:
     """Calculate the command metrics for the test result."""
     metrics: Dict[str, CommandMetrics] = defaultdict(

rasa/dialogue_understanding_test/command_metrics.py

@@ -0,0 +1,38 @@
+from typing import Dict
+
+from pydantic import BaseModel
+
+
+class CommandMetrics(BaseModel):
+    tp: int
+    fp: int
+    fn: int
+    total_count: int
+
+    @staticmethod
+    def _safe_divide(numerator: float, denominator: float) -> float:
+        """Safely perform division, returning 0.0 if the denominator is zero."""
+        return numerator / denominator if denominator > 0 else 0.0
+
+    def get_precision(self) -> float:
+        return self._safe_divide(self.tp, self.tp + self.fp)
+
+    def get_recall(self) -> float:
+        return self._safe_divide(self.tp, self.tp + self.fn)
+
+    def get_f1_score(self) -> float:
+        precision = self.get_precision()
+        recall = self.get_recall()
+
+        return self._safe_divide(2 * precision * recall, precision + recall)
+
+    def as_dict(self) -> Dict[str, float]:
+        return {
+            "tp": self.tp,
+            "fp": self.fp,
+            "fn": self.fn,
+            "precision": self.get_precision(),
+            "recall": self.get_recall(),
+            "f1_score": self.get_f1_score(),
+            "total_count": self.total_count,
+        }

rasa/dialogue_understanding_test/du_test_case.py

@@ -1,7 +1,11 @@
+from collections import defaultdict
 from typing import Any, Dict, Iterator, List, Optional, Tuple
 
 from pydantic import BaseModel, Field
 
+from rasa.core.nlg.contextual_response_rephraser import ContextualResponseRephraser
+from rasa.core.policies.enterprise_search_policy import EnterpriseSearchPolicy
+from rasa.core.policies.intentless_policy import IntentlessPolicy
 from rasa.dialogue_understanding.commands.prompt_command import PromptCommand
 from rasa.dialogue_understanding.generator.command_parser import parse_commands
 from rasa.dialogue_understanding_test.command_comparison import are_command_lists_equal
@@ -69,6 +73,8 @@ class DialogueUnderstandingOutput(BaseModel):
     commands: Dict[str, List[PromptCommand]]
     # List of prompts
     prompts: Optional[List[Dict[str, Any]]] = None
+    # Latency of the full message roundtrip
+    latency: Optional[float] = None
 
     class Config:
         """Skip validation for PromptCommand protocol as pydantic does not know how to
@@ -88,27 +94,41 @@ class DialogueUnderstandingOutput(BaseModel):
     def get_component_names_that_predicted_commands_or_have_llm_response(
         self,
     ) -> List[str]:
-        """Get all component names that have predicted commands or recieved
+        """Get all relevant component names.
+
+        Components are relevant if they have predicted commands or received a
         non-empty response from LLM.
         """
+        # Exclude components that are not related to Dialogue Understanding
+        component_names_to_exclude = [
+            EnterpriseSearchPolicy.__name__,
+            IntentlessPolicy.__name__,
+            ContextualResponseRephraser.__name__,
+        ]
+
         component_names_that_predicted_commands = (
             [
                 component_name
                 for component_name, predicted_commands in self.commands.items()
                 if predicted_commands
+                and component_name not in component_names_to_exclude
             ]
             if self.commands
             else []
         )
+
         components_with_prompts = (
             [
                 str(prompt.get(KEY_COMPONENT_NAME, None))
                 for prompt in self.prompts
                 if prompt.get(KEY_LLM_RESPONSE_METADATA, None)
+                and prompt.get(KEY_COMPONENT_NAME, None)
+                not in component_names_to_exclude
             ]
             if self.prompts
             else []
         )
+
         return list(
             set(component_names_that_predicted_commands + components_with_prompts)
         )
@@ -298,41 +318,54 @@ class DialogueUnderstandingTestStep(BaseModel):
 
         return ""
 
-    def get_latencies(self) -> List[float]:
+    def get_latencies(self) -> Dict[str, List[float]]:
         if self.dialogue_understanding_output is None:
-            return []
+            return {}
 
-        prompts = self.dialogue_understanding_output.get_component_name_to_prompt_info()
+        component_name_to_prompt_info = (
+            self.dialogue_understanding_output.get_component_name_to_prompt_info()
+        )
 
-        return [
-            prompt_data.get(KEY_LATENCY, 0.0)
-            for prompt in prompts.values()
-            for prompt_data in prompt
-        ]
+        latencies = defaultdict(list)
+        for component_name, prompt_info_list in component_name_to_prompt_info.items():
+            for prompt_info in prompt_info_list:
+                latencies[component_name].append(prompt_info.get(KEY_LATENCY, 0.0))
 
-    def get_completion_tokens(self) -> List[int]:
+        return latencies
+
+    def get_completion_tokens(self) -> Dict[str, List[float]]:
         if self.dialogue_understanding_output is None:
-            return []
+            return {}
 
-        prompts = self.dialogue_understanding_output.get_component_name_to_prompt_info()
+        component_name_to_prompt_info = (
+            self.dialogue_understanding_output.get_component_name_to_prompt_info()
+        )
 
-        return [
-            prompt_data.get(KEY_COMPLETION_TOKENS, 0)
-            for prompt in prompts.values()
-            for prompt_data in prompt
-        ]
+        completion_tokens = defaultdict(list)
+        for component_name, prompt_info_list in component_name_to_prompt_info.items():
+            for prompt_info in prompt_info_list:
+                completion_tokens[component_name].append(
+                    prompt_info.get(KEY_COMPLETION_TOKENS, 0.0)
+                )
+
+        return completion_tokens
 
-    def get_prompt_tokens(self) -> List[int]:
+    def get_prompt_tokens(self) -> Dict[str, List[float]]:
         if self.dialogue_understanding_output is None:
-            return []
+            return {}
 
-        prompts = self.dialogue_understanding_output.get_component_name_to_prompt_info()
+        component_name_to_prompt_info = (
+            self.dialogue_understanding_output.get_component_name_to_prompt_info()
+        )
 
-        return [
-            prompt_data.get(KEY_PROMPT_TOKENS, 0)
-            for prompt in prompts.values()
-            for prompt_data in prompt
-        ]
+        prompt_tokens = defaultdict(list)
+        for component_name, prompt_info_list in component_name_to_prompt_info.items():
+            for prompt_info in prompt_info_list:
+                prompt_tokens[component_name].append(
+                    prompt_info.get(KEY_PROMPT_TOKENS, 0.0)
+                )
+
+        return prompt_tokens
 
 
 class DialogueUnderstandingTestCase(BaseModel):