rasa-pro 3.12.0.dev4__py3-none-any.whl → 3.12.0.dev6__py3-none-any.whl

rasa/core/policies/enterprise_search_policy.py

@@ -76,7 +76,7 @@ from rasa.shared.providers.embedding._langchain_embedding_client_adapter import
     _LangchainEmbeddingClientAdapter,
 )
 from rasa.shared.providers.llm.llm_client import LLMClient
-from rasa.shared.providers.llm.llm_response import LLMResponse
+from rasa.shared.providers.llm.llm_response import LLMResponse, measure_llm_latency
 from rasa.shared.utils.cli import print_error_and_exit
 from rasa.shared.utils.health_check.embeddings_health_check_mixin import (
     EmbeddingsHealthCheckMixin,
@@ -639,6 +639,7 @@ class EnterpriseSearchPolicy(LLMHealthCheckMixin, EmbeddingsHealthCheckMixin, Po
         )
         return prompt
 
+    @measure_llm_latency
     async def _generate_llm_answer(
         self, llm: LLMClient, prompt: Text
     ) -> Optional[LLMResponse]:

rasa/core/processor.py

@@ -205,6 +205,8 @@ class MessageProcessor:
             tracker = await self.run_action_extract_slots(
                 message.output_channel, tracker
             )
+        else:
+            tracker = await self.run_command_processor(tracker)
 
         await self._run_prediction_loop(message.output_channel, tracker)
 
@@ -683,7 +685,7 @@ class MessageProcessor:
         )
 
         tracker = await self.run_action_extract_slots(output_channel, tracker)
-
+        tracker = await self.run_command_processor(tracker)
         await self._run_prediction_loop(output_channel, tracker)
         # save tracker state to continue conversation from this state
         await self.save_tracker(tracker)
@@ -1091,8 +1093,6 @@ class MessageProcessor:
         # keep taking actions decided by the policy until it chooses to 'listen'
         should_predict_another_action = True
 
-        tracker = await self.run_command_processor(tracker)
-
         # action loop. predicts actions until we hit action listen
         while should_predict_another_action and self._should_handle_message(tracker):
             # this actually just calls the policy's method by the same name

rasa/dialogue_understanding/generator/llm_based_command_generator.py

@@ -30,7 +30,7 @@ from rasa.shared.exceptions import FileIOException, ProviderClientAPIException
 from rasa.shared.nlu.constants import FLOWS_IN_PROMPT
 from rasa.shared.nlu.training_data.message import Message
 from rasa.shared.nlu.training_data.training_data import TrainingData
-from rasa.shared.providers.llm.llm_response import LLMResponse
+from rasa.shared.providers.llm.llm_response import LLMResponse, measure_llm_latency
 from rasa.shared.utils.health_check.llm_health_check_mixin import LLMHealthCheckMixin
 from rasa.shared.utils.llm import (
     allowed_values_for_slot,
@@ -303,6 +303,7 @@ class LLMBasedCommandGenerator(
         )
         return filtered_flows
 
+    @measure_llm_latency
     async def invoke_llm(self, prompt: Text) -> Optional[LLMResponse]:
         """Use LLM to generate a response.
 

rasa/dialogue_understanding_test/command_metric_calculation.py

@@ -79,12 +79,16 @@ def calculate_command_metrics(
     return metrics
 
 
+def _get_command_name(command: Command) -> str:
+    return command.command().replace(" ", "_")
+
+
 def _increase_total_count(
     commands: List[Command],
     metrics: Dict[str, CommandMetrics],
 ) -> None:
     for command in commands:
-        metrics[command.command()].total_count += 1
+        metrics[_get_command_name(command)].total_count += 1
 
 
 def _increase_tp(
@@ -92,7 +96,7 @@ def _increase_tp(
     metrics: Dict[str, CommandMetrics],
 ) -> None:
     for command in commands:
-        metrics[command.command()].tp += 1
+        metrics[_get_command_name(command)].tp += 1
 
 
 def _update_metrics_true_positive_and_false_negative(
@@ -101,7 +105,7 @@ def _update_metrics_true_positive_and_false_negative(
     metrics: Dict[str, CommandMetrics],
 ) -> None:
     for expected_command in expected_commands:
-        command_name = expected_command.command()
+        command_name = _get_command_name(expected_command)
         if is_command_present_in_list(expected_command, predicted_commands):
             metrics[command_name].tp += 1
         else:
@@ -115,4 +119,4 @@ def _update_metrics_false_positive(
 ) -> None:
     for predicted_command in predicted_commands:
         if not is_command_present_in_list(predicted_command, expected_commands):
-            metrics[predicted_command.command()].fp += 1
+            metrics[_get_command_name(predicted_command)].fp += 1

rasa/dialogue_understanding_test/du_test_result.py

@@ -27,6 +27,7 @@ OUTPUT_NUMBER_OF_FAILED_USER_UTTERANCES = "number_of_failed_user_utterances"
 OUTPUT_COMMAND_METRICS = "command_metrics"
 OUTPUT_NAMES_OF_FAILED_TESTS = "names_of_failed_tests"
 OUTPUT_NAMES_OF_PASSED_TESTS = "names_of_passed_tests"
+OUTPUT_LLM_COMMAND_GENERATOR_CONFIG = "llm_command_generator_config"
 
 
 class DialogueUnderstandingTestResult(BaseModel):
@@ -151,6 +152,7 @@ class DialogueUnderstandingTestSuiteResult:
         self.names_of_failed_tests: List[str] = []
         self.names_of_passed_tests: List[str] = []
         self.failed_test_steps: List[FailedTestStep] = []
+        self.llm_config: Optional[Dict[str, Any]] = None
 
     @classmethod
     def from_results(
@@ -158,6 +160,7 @@ class DialogueUnderstandingTestSuiteResult:
         failing_test_results: List[DialogueUnderstandingTestResult],
         passing_test_results: List[DialogueUnderstandingTestResult],
         command_metrics: Dict[str, "CommandMetrics"],
+        llm_config: Optional[Dict[str, Any]],
     ) -> "DialogueUnderstandingTestSuiteResult":
         """Create a DialogueUnderstandingTestSuiteResult object from the test results.
 
@@ -171,6 +174,7 @@ class DialogueUnderstandingTestSuiteResult:
                 representing the test cases that passed.
             command_metrics: A dictionary of command-specific performance metrics, keyed
                 by command name.
+            llm_config: A dictionary containing the command generator configuration.
 
         Returns:
             A DialogueUnderstandingTestSuiteResult object containing aggregated test
@@ -202,6 +206,8 @@ class DialogueUnderstandingTestSuiteResult:
             failing_test_results
         )
 
+        instance.llm_config = llm_config
+
         return instance
 
     def _set_user_utterance_metrics(
@@ -297,4 +303,7 @@ class DialogueUnderstandingTestSuiteResult:
 
         result_dict["failed_test_steps"] = failed_steps_list
 
+        if self.llm_config:
+            result_dict[OUTPUT_LLM_COMMAND_GENERATOR_CONFIG] = self.llm_config
+
         return result_dict