rasa-pro 3.12.5__py3-none-any.whl → 3.13.0.dev1__py3-none-any.whl

rasa/dialogue_understanding_test/command_metric_calculation.py

@@ -1,54 +1,21 @@
+import typing
 from collections import defaultdict
 from typing import Dict, List
 
-from pydantic import BaseModel
-
 from rasa.dialogue_understanding.commands import Command
 from rasa.dialogue_understanding_test.command_comparison import (
     is_command_present_in_list,
 )
-from rasa.dialogue_understanding_test.du_test_result import (
-    DialogueUnderstandingTestResult,
-)
-
-
-class CommandMetrics(BaseModel):
-    tp: int
-    fp: int
-    fn: int
-    total_count: int
-
-    @staticmethod
-    def _safe_divide(numerator: float, denominator: float) -> float:
-        """Safely perform division, returning 0.0 if the denominator is zero."""
-        return numerator / denominator if denominator > 0 else 0.0
+from rasa.dialogue_understanding_test.command_metrics import CommandMetrics
 
-    def get_precision(self) -> float:
-        return self._safe_divide(self.tp, self.tp + self.fp)
-
-    def get_recall(self) -> float:
-        return self._safe_divide(self.tp, self.tp + self.fn)
-
-    def get_f1_score(self) -> float:
-        precision = self.get_precision()
-        recall = self.get_recall()
-
-        return self._safe_divide(2 * precision * recall, precision + recall)
-
-    def as_dict(self) -> Dict[str, float]:
-        return {
-            "tp": self.tp,
-            "fp": self.fp,
-            "fn": self.fn,
-            "precision": self.get_precision(),
-            "recall": self.get_recall(),
-            "f1_score": self.get_f1_score(),
-            "total_count": self.total_count,
-        }
+if typing.TYPE_CHECKING:
+    from rasa.dialogue_understanding_test.du_test_result import (
+        DialogueUnderstandingTestResult,
+    )
 
 
 def calculate_command_metrics(
-    test_results: List[DialogueUnderstandingTestResult],
+    test_results: List["DialogueUnderstandingTestResult"],
 ) -> Dict[str, CommandMetrics]:
     """Calculate the command metrics for the test result."""
     metrics: Dict[str, CommandMetrics] = defaultdict(

rasa/dialogue_understanding_test/command_metrics.py

@@ -0,0 +1,38 @@
+from typing import Dict
+
+from pydantic import BaseModel
+
+
+class CommandMetrics(BaseModel):
+    tp: int
+    fp: int
+    fn: int
+    total_count: int
+
+    @staticmethod
+    def _safe_divide(numerator: float, denominator: float) -> float:
+        """Safely perform division, returning 0.0 if the denominator is zero."""
+        return numerator / denominator if denominator > 0 else 0.0
+
+    def get_precision(self) -> float:
+        return self._safe_divide(self.tp, self.tp + self.fp)
+
+    def get_recall(self) -> float:
+        return self._safe_divide(self.tp, self.tp + self.fn)
+
+    def get_f1_score(self) -> float:
+        precision = self.get_precision()
+        recall = self.get_recall()
+
+        return self._safe_divide(2 * precision * recall, precision + recall)
+
+    def as_dict(self) -> Dict[str, float]:
+        return {
+            "tp": self.tp,
+            "fp": self.fp,
+            "fn": self.fn,
+            "precision": self.get_precision(),
+            "recall": self.get_recall(),
+            "f1_score": self.get_f1_score(),
+            "total_count": self.total_count,
+        }

rasa/dialogue_understanding_test/du_test_case.py

@@ -1,7 +1,11 @@
+from collections import defaultdict
 from typing import Any, Dict, Iterator, List, Optional, Tuple
 
 from pydantic import BaseModel, Field
 
+from rasa.core import IntentlessPolicy
+from rasa.core.nlg.contextual_response_rephraser import ContextualResponseRephraser
+from rasa.core.policies.enterprise_search_policy import EnterpriseSearchPolicy
 from rasa.dialogue_understanding.commands.prompt_command import PromptCommand
 from rasa.dialogue_understanding.generator.command_parser import parse_commands
 from rasa.dialogue_understanding_test.command_comparison import are_command_lists_equal
@@ -69,6 +73,8 @@ class DialogueUnderstandingOutput(BaseModel):
     commands: Dict[str, List[PromptCommand]]
     # List of prompts
     prompts: Optional[List[Dict[str, Any]]] = None
+    # Latency of the full message roundtrip
+    latency: Optional[float] = None
 
     class Config:
         """Skip validation for PromptCommand protocol as pydantic does not know how to
@@ -88,27 +94,41 @@ class DialogueUnderstandingOutput(BaseModel):
     def get_component_names_that_predicted_commands_or_have_llm_response(
         self,
     ) -> List[str]:
-        """Get all component names that have predicted commands or recieved
+        """Get all relevant component names.
+
+        Components are relevant if they have predicted commands or received a
         non-empty response from LLM.
         """
+        # Exclude components that are not related to Dialogue Understanding
+        component_names_to_exclude = [
+            EnterpriseSearchPolicy.__name__,
+            IntentlessPolicy.__name__,
+            ContextualResponseRephraser.__name__,
+        ]
+
         component_names_that_predicted_commands = (
             [
                 component_name
                 for component_name, predicted_commands in self.commands.items()
                 if predicted_commands
+                and component_name not in component_names_to_exclude
             ]
             if self.commands
             else []
         )
+
         components_with_prompts = (
             [
                 str(prompt.get(KEY_COMPONENT_NAME, None))
                 for prompt in self.prompts
                 if prompt.get(KEY_LLM_RESPONSE_METADATA, None)
+                and prompt.get(KEY_COMPONENT_NAME, None)
+                not in component_names_to_exclude
             ]
             if self.prompts
             else []
         )
+
         return list(
             set(component_names_that_predicted_commands + components_with_prompts)
         )
@@ -290,41 +310,54 @@ class DialogueUnderstandingTestStep(BaseModel):
 
         return ""
 
-    def get_latencies(self) -> List[float]:
+    def get_latencies(self) -> Dict[str, List[float]]:
         if self.dialogue_understanding_output is None:
-            return []
+            return {}
 
-        prompts = self.dialogue_understanding_output.get_component_name_to_prompt_info()
+        component_name_to_prompt_info = (
+            self.dialogue_understanding_output.get_component_name_to_prompt_info()
+        )
 
-        return [
-            prompt_data.get(KEY_LATENCY, 0.0)
-            for prompt in prompts.values()
-            for prompt_data in prompt
-        ]
+        latencies = defaultdict(list)
+        for component_name, prompt_info_list in component_name_to_prompt_info.items():
+            for prompt_info in prompt_info_list:
+                latencies[component_name].append(prompt_info.get(KEY_LATENCY, 0.0))
 
-    def get_completion_tokens(self) -> List[int]:
+        return latencies
+
+    def get_completion_tokens(self) -> Dict[str, List[float]]:
         if self.dialogue_understanding_output is None:
-            return []
+            return {}
 
-        prompts = self.dialogue_understanding_output.get_component_name_to_prompt_info()
+        component_name_to_prompt_info = (
+            self.dialogue_understanding_output.get_component_name_to_prompt_info()
+        )
 
-        return [
-            prompt_data.get(KEY_COMPLETION_TOKENS, 0)
-            for prompt in prompts.values()
-            for prompt_data in prompt
-        ]
+        completion_tokens = defaultdict(list)
+        for component_name, prompt_info_list in component_name_to_prompt_info.items():
+            for prompt_info in prompt_info_list:
+                completion_tokens[component_name].append(
+                    prompt_info.get(KEY_COMPLETION_TOKENS, 0.0)
+                )
+
+        return completion_tokens
 
-    def get_prompt_tokens(self) -> List[int]:
+    def get_prompt_tokens(self) -> Dict[str, List[float]]:
         if self.dialogue_understanding_output is None:
-            return []
+            return {}
 
-        prompts = self.dialogue_understanding_output.get_component_name_to_prompt_info()
+        component_name_to_prompt_info = (
+            self.dialogue_understanding_output.get_component_name_to_prompt_info()
+        )
 
-        return [
-            prompt_data.get(KEY_PROMPT_TOKENS, 0)
-            for prompt in prompts.values()
-            for prompt_data in prompt
-        ]
+        prompt_tokens = defaultdict(list)
+        for component_name, prompt_info_list in component_name_to_prompt_info.items():
+            for prompt_info in prompt_info_list:
+                prompt_tokens[component_name].append(
+                    prompt_info.get(KEY_PROMPT_TOKENS, 0.0)
+                )
+
+        return prompt_tokens
 
 
 class DialogueUnderstandingTestCase(BaseModel):