rasa-pro 3.11.3a1.dev4__tar.gz → 3.11.3a1.dev6__tar.gz

{rasa_pro-3.11.3a1.dev4 → rasa_pro-3.11.3a1.dev6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: rasa-pro
-Version: 3.11.3a1.dev4
+Version: 3.11.3a1.dev6
 Summary: State-of-the-art open-core Conversational AI framework for Enterprises that natively leverages generative AI for effortless assistant development.
 Home-page: https://rasa.com
 Keywords: nlp,machine-learning,machine-learning-library,bot,bots,botkit,rasa conversational-agents,conversational-ai,chatbot,chatbot-framework,bot-framework

{rasa_pro-3.11.3a1.dev4 → rasa_pro-3.11.3a1.dev6}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "rasa-pro"
-version = "3.11.3a1.dev4"
+version = "3.11.3a1.dev6"
 description = "State-of-the-art open-core Conversational AI framework for Enterprises that natively leverages generative AI for effortless assistant development."
 authors = [ "Rasa Technologies GmbH <hi@rasa.com>",]
 maintainers = [ "Tom Bocklisch <tom@rasa.com>",]

{rasa_pro-3.11.3a1.dev4 → rasa_pro-3.11.3a1.dev6}/rasa/core/actions/action.py

@@ -17,6 +17,15 @@ from jsonschema import Draft202012Validator
 
 import rasa.core
 import rasa.shared.utils.io
+from rasa.core.actions.constants import (
+    TEXT,
+    ELEMENTS,
+    QUICK_REPLIES,
+    BUTTONS,
+    ATTACHMENT,
+    IMAGE,
+    CUSTOM,
+)
 from rasa.core.actions.custom_action_executor import (
     CustomActionExecutor,
     NoEndpointCustomActionExecutor,
@@ -93,6 +102,7 @@ from rasa.shared.nlu.constants import (
     INTENT_NAME_KEY,
     INTENT_RANKING_KEY,
 )
+from rasa.shared.nlu.constants import PROMPTS
 from rasa.shared.utils.io import raise_warning
 from rasa.shared.utils.schemas.events import EVENTS_SCHEMA
 from rasa.utils.endpoints import ClientResponseError, EndpointConfig
@@ -255,18 +265,19 @@ def action_for_name_or_text(
 def create_bot_utterance(message: Dict[Text, Any]) -> BotUttered:
     """Create BotUttered event from message."""
     bot_message = BotUttered(
-        text=message.pop("text", None),
+        text=message.pop(TEXT, None),
         data={
-            "elements": message.pop("elements", None),
-            "quick_replies": message.pop("quick_replies", None),
-            "buttons": message.pop("buttons", None),
+            ELEMENTS: message.pop(ELEMENTS, None),
+            QUICK_REPLIES: message.pop(QUICK_REPLIES, None),
+            BUTTONS: message.pop(BUTTONS, None),
             # for legacy / compatibility reasons we need to set the image
             # to be the attachment if there is no other attachment (the
             # `.get` is intentional - no `pop` as we still need the image`
             # property to set it in the following line)
-            "attachment": message.pop("attachment", None) or message.get("image", None),
-            "image": message.pop("image", None),
-            "custom": message.pop("custom", None),
+            ATTACHMENT: message.pop(ATTACHMENT, None) or message.get(IMAGE, None),
+            IMAGE: message.pop(IMAGE, None),
+            CUSTOM: message.pop(CUSTOM, None),
+            PROMPTS: message.pop(PROMPTS, None),
         },
         metadata=message,
     )

{rasa_pro-3.11.3a1.dev4 → rasa_pro-3.11.3a1.dev6}/rasa/core/actions/constants.py

@@ -3,3 +3,11 @@ SELECTIVE_DOMAIN = "enable_selective_domain"
 
 SSL_CLIENT_CERT_FIELD = "ssl_client_cert"
 SSL_CLIENT_KEY_FIELD = "ssl_client_key"
+
+TEXT = "text"
+ELEMENTS = "elements"
+QUICK_REPLIES = "quick_replies"
+BUTTONS = "buttons"
+ATTACHMENT = "attachment"
+IMAGE = "image"
+CUSTOM = "custom"

{rasa_pro-3.11.3a1.dev4 → rasa_pro-3.11.3a1.dev6}/rasa/core/channels/socketio.py

@@ -218,7 +218,7 @@ class SocketIOInput(InputChannel):
             return None
         return SocketIOOutput(self.sio, self.bot_message_evt)
 
-    async def on_new_tracker_dump(self, sender_id: str, tracker_dump: str):
+    async def on_new_tracker_dump(self, sender_id: str, tracker_dump: str) -> None:
         if self.sio:
             await self.sio.emit("tracker", tracker_dump, room=sender_id)
 

{rasa_pro-3.11.3a1.dev4 → rasa_pro-3.11.3a1.dev6}/rasa/core/nlg/contextual_response_rephraser.py

@@ -2,6 +2,7 @@ from typing import Any, Dict, Optional, Text
 
 import structlog
 from jinja2 import Template
+
 from rasa import telemetry
 from rasa.core.nlg.response import TemplatedNaturalLanguageGenerator
 from rasa.core.nlg.summarize import summarize_conversation
@@ -18,6 +19,12 @@ from rasa.shared.constants import (
 from rasa.shared.core.domain import KEY_RESPONSES_TEXT, Domain
 from rasa.shared.core.events import BotUttered, UserUttered
 from rasa.shared.core.trackers import DialogueStateTracker
+from rasa.shared.nlu.constants import (
+    PROMPTS,
+    KEY_USER_PROMPT,
+    KEY_LLM_RESPONSE_METADATA,
+)
+from rasa.shared.providers.llm.llm_response import LLMResponse
 from rasa.shared.utils.health_check.llm_health_check_mixin import LLMHealthCheckMixin
 from rasa.shared.utils.llm import (
     DEFAULT_OPENAI_GENERATE_MODEL_NAME,
@@ -124,6 +131,48 @@ class ContextualResponseRephraser(
             ContextualResponseRephraser.__name__,
         )
 
+    @classmethod
+    def _add_prompt_and_llm_metadata_to_response(
+        cls,
+        response: Dict[str, Any],
+        prompt_name: str,
+        user_prompt: str,
+        llm_response: Optional["LLMResponse"] = None,
+    ) -> Dict[str, Any]:
+        """Stores the prompt and LLMResponse metadata to response.
+
+        Args:
+            response: The response to add the prompt and LLMResponse metadata to.
+            prompt_name: A name identifying prompt usage.
+            user_prompt: The user prompt that was sent to the LLM.
+            llm_response: The response object from the LLM (None if no response).
+        """
+        from rasa.dialogue_understanding.utils import record_commands_and_prompts
+
+        if not record_commands_and_prompts:
+            return response
+
+        prompt_data: Dict[Text, Any] = {
+            KEY_USER_PROMPT: user_prompt,
+        }
+
+        if llm_response is not None:
+            prompt_data[KEY_LLM_RESPONSE_METADATA] = llm_response.to_dict()
+        else:
+            prompt_data[KEY_LLM_RESPONSE_METADATA] = None
+
+        prompt_tuple = (prompt_name, prompt_data)
+
+        component_name = cls.__name__
+        component_prompts = response.get(PROMPTS, {})
+        if component_name in component_prompts:
+            component_prompts[component_name].append(prompt_tuple)
+        else:
+            component_prompts[component_name] = [prompt_tuple]
+
+        response[PROMPTS] = component_prompts
+        return response
+
     def _last_message_if_human(self, tracker: DialogueStateTracker) -> Optional[str]:
         """Returns the latest message from the tracker.
 
@@ -142,20 +191,21 @@ class ContextualResponseRephraser(
                 return None
         return None
 
-    async def _generate_llm_response(self, prompt: str) -> Optional[str]:
-        """Use LLM to generate a response.
+    async def _generate_llm_response(self, prompt: str) -> Optional[LLMResponse]:
+        """
+        Use LLM to generate a response, returning an LLMResponse object
+        containing both the generated text (choices) and metadata.
 
         Args:
-            prompt: the prompt to send to the LLM
+            prompt: The prompt to send to the LLM.
 
         Returns:
-            generated text
+            An LLMResponse object if successful, otherwise None.
         """
         llm = llm_factory(self.llm_config, DEFAULT_LLM_CONFIG)
 
         try:
-            llm_response = await llm.acompletion(prompt)
-            return llm_response.choices[0]
+            return await llm.acompletion(prompt)
         except Exception as e:
             # unfortunately, langchain does not wrap LLM exceptions which means
             # we have to catch all exceptions here
@@ -255,11 +305,21 @@ class ContextualResponseRephraser(
             or self.llm_property(MODEL_NAME_CONFIG_KEY),
             llm_model_group_id=self.llm_property(MODEL_GROUP_ID_CONFIG_KEY),
         )
-        if not (updated_text := await self._generate_llm_response(prompt)):
-            # If the LLM fails to generate a response, we
-            # return the original response.
+        llm_response = await self._generate_llm_response(prompt)
+        llm_response = LLMResponse.ensure_llm_response(llm_response)
+
+        response = self._add_prompt_and_llm_metadata_to_response(
+            response=response,
+            prompt_name="rephrase_prompt",
+            user_prompt=prompt,
+            llm_response=llm_response,
+        )
+
+        if llm_response is None or not llm_response.choices:
+            # If the LLM fails to generate a response, return the original response.
             return response
 
+        updated_text = llm_response.choices[0]
         structlogger.debug(
             "nlg.rewrite.complete",
             response_text=response_text,

{rasa_pro-3.11.3a1.dev4 → rasa_pro-3.11.3a1.dev6}/rasa/core/policies/enterprise_search_policy.py

@@ -2,6 +2,7 @@ import importlib.resources
 import json
 import re
 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Text
+
 import dotenv
 import structlog
 from jinja2 import Template
@@ -63,11 +64,17 @@ from rasa.shared.core.events import Event, UserUttered, BotUttered
 from rasa.shared.core.generator import TrackerWithCachedStates
 from rasa.shared.core.trackers import DialogueStateTracker, EventVerbosity
 from rasa.shared.exceptions import RasaException, FileIOException
+from rasa.shared.nlu.constants import (
+    PROMPTS,
+    KEY_USER_PROMPT,
+    KEY_LLM_RESPONSE_METADATA,
+)
 from rasa.shared.nlu.training_data.training_data import TrainingData
 from rasa.shared.providers.embedding._langchain_embedding_client_adapter import (
     _LangchainEmbeddingClientAdapter,
 )
 from rasa.shared.providers.llm.llm_client import LLMClient
+from rasa.shared.providers.llm.llm_response import LLMResponse
 from rasa.shared.utils.cli import print_error_and_exit
 from rasa.shared.utils.health_check.embeddings_health_check_mixin import (
     EmbeddingsHealthCheckMixin,
@@ -272,6 +279,50 @@ class EnterpriseSearchPolicy(LLMHealthCheckMixin, EmbeddingsHealthCheckMixin, Po
         # Wrap the embedding client in the adapter
         return _LangchainEmbeddingClientAdapter(client)
 
+    @classmethod
+    def _add_prompt_and_llm_response_to_latest_message(
+        cls,
+        tracker: DialogueStateTracker,
+        prompt_name: str,
+        user_prompt: str,
+        llm_response: Optional[LLMResponse] = None,
+    ) -> None:
+        """Stores the prompt and LLMResponse metadata in the tracker.
+
+        Args:
+            tracker: The DialogueStateTracker containing the current conversation state.
+            prompt_name: A name identifying prompt usage.
+            user_prompt: The user prompt that was sent to the LLM.
+            llm_response: The response object from the LLM (None if no response).
+        """
+        from rasa.dialogue_understanding.utils import record_commands_and_prompts
+
+        if not record_commands_and_prompts:
+            return
+
+        if not tracker.latest_message:
+            return
+
+        parse_data = tracker.latest_message.parse_data
+        if parse_data is not None and PROMPTS not in parse_data:
+            parse_data[PROMPTS] = {}  # type: ignore[literal-required]
+
+        component_name = cls.__name__
+        component_prompts = parse_data[PROMPTS].get(component_name, [])  # type: ignore[literal-required]
+
+        prompt_data: Dict[Text, Any] = {
+            KEY_USER_PROMPT: user_prompt,
+        }
+
+        if llm_response is not None:
+            prompt_data[KEY_LLM_RESPONSE_METADATA] = llm_response.to_dict()
+        else:
+            prompt_data[KEY_LLM_RESPONSE_METADATA] = None
+
+        prompt_tuple = (prompt_name, prompt_data)
+        component_prompts.append(prompt_tuple)
+        parse_data[PROMPTS][component_name] = component_prompts  # type: ignore[literal-required]
+
     def train(  # type: ignore[override]
         self,
         training_trackers: List[TrackerWithCachedStates],
@@ -498,13 +549,27 @@ class EnterpriseSearchPolicy(LLMHealthCheckMixin, EmbeddingsHealthCheckMixin, Po
 
         if self.use_llm:
             prompt = self._render_prompt(tracker, documents.results)
-            llm_answer = await self._generate_llm_answer(llm, prompt)
+            response = await self._generate_llm_answer(llm, prompt)
+            llm_response = LLMResponse.ensure_llm_response(response)
+
+            self._add_prompt_and_llm_response_to_latest_message(
+                tracker=tracker,
+                prompt_name="enterprise_search_prompt",
+                user_prompt=prompt,
+                llm_response=llm_response,
+            )
+
+            if llm_response is None or not llm_response.choices:
+                logger.debug(f"{logger_key}.no_llm_response")
+                response = None
+            else:
+                llm_answer = llm_response.choices[0]
 
-            if self.citation_enabled:
-                llm_answer = self.post_process_citations(llm_answer)
+                if self.citation_enabled:
+                    llm_answer = self.post_process_citations(llm_answer)
 
-            logger.debug(f"{logger_key}.llm_answer", llm_answer=llm_answer)
-            response = llm_answer
+                logger.debug(f"{logger_key}.llm_answer", llm_answer=llm_answer)
+                response = llm_answer
         else:
             response = documents.results[0].metadata.get("answer", None)
             if not response:
@@ -516,7 +581,6 @@ class EnterpriseSearchPolicy(LLMHealthCheckMixin, EmbeddingsHealthCheckMixin, Po
                 "enterprise_search_policy.predict_action_probabilities.no_llm",
                 search_results=documents,
             )
-
         if response is None:
             return self._create_prediction_internal_error(domain, tracker)
 
@@ -581,10 +645,18 @@ class EnterpriseSearchPolicy(LLMHealthCheckMixin, EmbeddingsHealthCheckMixin, Po
 
     async def _generate_llm_answer(
         self, llm: LLMClient, prompt: Text
-    ) -> Optional[Text]:
+    ) -> Optional[LLMResponse]:
+        """Fetches an LLM completion for the provided prompt.
+
+        Args:
+            llm: The LLM client used to get the completion.
+            prompt: The prompt text to send to the model.
+
+        Returns:
+            An LLMResponse object, or None if the call fails.
+        """
         try:
-            llm_response = await llm.acompletion(prompt)
-            llm_answer = llm_response.choices[0]
+            return await llm.acompletion(prompt)
         except Exception as e:
             # unfortunately, langchain does not wrap LLM exceptions which means
             # we have to catch all exceptions here
@@ -592,9 +664,7 @@ class EnterpriseSearchPolicy(LLMHealthCheckMixin, EmbeddingsHealthCheckMixin, Po
                 "enterprise_search_policy._generate_llm_answer.llm_error",
                 error=e,
             )
-            llm_answer = None
-
-        return llm_answer
+            return None
 
     def _create_prediction(
         self,

rasa_pro-3.11.3a1.dev6/rasa/dialogue_understanding/constants.py

@@ -0,0 +1 @@
+RASA_RECORD_COMMANDS_AND_PROMPTS_ENV_VAR_NAME = "RASA_RECORD_COMMANDS_AND_PROMPTS"

{rasa_pro-3.11.3a1.dev4 → rasa_pro-3.11.3a1.dev6}/rasa/dialogue_understanding/generator/command_generator.py

@@ -26,8 +26,10 @@ from rasa.shared.nlu.constants import (
     PROMPTS,
     KEY_USER_PROMPT,
     KEY_SYSTEM_PROMPT,
+    KEY_LLM_RESPONSE_METADATA,
 )
 from rasa.shared.nlu.training_data.message import Message
+from rasa.shared.providers.llm.llm_response import LLMResponse
 from rasa.shared.utils.llm import DEFAULT_MAX_USER_INPUT_CHARACTERS
 
 structlogger = structlog.get_logger()
@@ -399,6 +401,7 @@ class CommandGenerator:
         prompt_name: str,
         user_prompt: str,
         system_prompt: Optional[str] = None,
+        llm_response: Optional[LLMResponse] = None,
     ) -> None:
         """Add prompt to the message parse data.
 
@@ -411,14 +414,16 @@ class CommandGenerator:
                         "fill_slots_prompt",
                         {
                             "user_prompt": <prompt content>",
-                            "system_prompt": <prompt content>"
+                            "system_prompt": <prompt content>",
+                            "llm_response_metadata": <metadata dict from LLMResponse>
                         }
                     ),
                     (
                         "handle_flows_prompt",
                         {
                             "user_prompt": <prompt content>",
-                            "system_prompt": <prompt content>"
+                            "system_prompt": <prompt content>",
+                            "llm_response_metadata": <metadata dict from LLMResponse>
                         }
                     ),
                 ],
@@ -427,7 +432,8 @@ class CommandGenerator:
                         "prompt_template",
                         {
                             "user_prompt": <prompt content>",
-                            "system_prompt": <prompt content>"
+                            "system_prompt": <prompt content>",
+                            "llm_response_metadata": <metadata dict from LLMResponse>
                         }
                     ),
                 ]
@@ -440,13 +446,17 @@ class CommandGenerator:
         if not record_commands_and_prompts:
             return
 
-        prompt_tuple = (
-            prompt_name,
-            {
-                KEY_USER_PROMPT: user_prompt,
-                **({KEY_SYSTEM_PROMPT: system_prompt} if system_prompt else {}),
-            },
-        )
+        prompt_data: Dict[Text, Any] = {
+            KEY_USER_PROMPT: user_prompt,
+            **({KEY_SYSTEM_PROMPT: system_prompt} if system_prompt else {}),
+        }
+
+        if llm_response is not None:
+            prompt_data[KEY_LLM_RESPONSE_METADATA] = llm_response.to_dict()
+        else:
+            prompt_data[KEY_LLM_RESPONSE_METADATA] = None
+
+        prompt_tuple = (prompt_name, prompt_data)
 
         if message.get(PROMPTS) is not None:
             prompts = message.get(PROMPTS)

{rasa_pro-3.11.3a1.dev4 → rasa_pro-3.11.3a1.dev6}/rasa/dialogue_understanding/generator/llm_based_command_generator.py

@@ -32,6 +32,7 @@ from rasa.shared.exceptions import ProviderClientAPIException
 from rasa.shared.nlu.constants import FLOWS_IN_PROMPT
 from rasa.shared.nlu.training_data.message import Message
 from rasa.shared.nlu.training_data.training_data import TrainingData
+from rasa.shared.providers.llm.llm_response import LLMResponse
 from rasa.shared.utils.health_check.llm_health_check_mixin import LLMHealthCheckMixin
 from rasa.shared.utils.llm import (
     allowed_values_for_slot,
@@ -304,22 +305,21 @@ class LLMBasedCommandGenerator(
         )
         return filtered_flows
 
-    async def invoke_llm(self, prompt: Text) -> Optional[Text]:
+    async def invoke_llm(self, prompt: Text) -> Optional[LLMResponse]:
         """Use LLM to generate a response.
 
         Args:
             prompt: The prompt to send to the LLM.
 
         Returns:
-            The generated text.
+            An LLMResponse object.
 
         Raises:
-            ProviderClientAPIException if an error during API call.
+            ProviderClientAPIException: If an error occurs during the LLM API call.
         """
         llm = llm_factory(self.config.get(LLM_CONFIG_KEY), DEFAULT_LLM_CONFIG)
         try:
-            llm_response = await llm.acompletion(prompt)
-            return llm_response.choices[0]
+            return await llm.acompletion(prompt)
         except Exception as e:
             # unfortunately, langchain does not wrap LLM exceptions which means
             # we have to catch all exceptions here

{rasa_pro-3.11.3a1.dev4 → rasa_pro-3.11.3a1.dev6}/rasa/dialogue_understanding/generator/llm_command_generator.py

@@ -10,6 +10,7 @@ from rasa.engine.recipes.default_recipe import DefaultV1Recipe
 from rasa.engine.storage.resource import Resource
 from rasa.engine.storage.storage import ModelStorage
 from rasa.shared.exceptions import ProviderClientAPIException
+from rasa.shared.providers.llm.llm_response import LLMResponse
 from rasa.shared.utils.io import raise_deprecation_warning
 
 structlogger = structlog.get_logger()
@@ -53,7 +54,7 @@ class LLMCommandGenerator(SingleStepLLMCommandGenerator):
             **kwargs,
         )
 
-    async def invoke_llm(self, prompt: Text) -> Optional[Text]:
+    async def invoke_llm(self, prompt: Text) -> Optional[LLMResponse]:
         try:
             return await super().invoke_llm(prompt)
         except ProviderClientAPIException:

{rasa_pro-3.11.3a1.dev4 → rasa_pro-3.11.3a1.dev6}/rasa/dialogue_understanding/generator/multi_step/multi_step_llm_command_generator.py

@@ -51,6 +51,7 @@ from rasa.shared.core.trackers import DialogueStateTracker
 from rasa.shared.exceptions import ProviderClientAPIException
 from rasa.shared.nlu.constants import TEXT
 from rasa.shared.nlu.training_data.message import Message
+from rasa.shared.providers.llm.llm_response import LLMResponse
 from rasa.shared.utils.io import deep_container_fingerprint
 from rasa.shared.utils.llm import (
     get_prompt_template,
@@ -535,7 +536,12 @@ class MultiStepLLMCommandGenerator(LLMBasedCommandGenerator):
             prompt=prompt,
         )
 
-        actions = await self.invoke_llm(prompt)
+        response = await self.invoke_llm(prompt)
+        llm_response = LLMResponse.ensure_llm_response(response)
+        actions = None
+        if llm_response and llm_response.choices:
+            actions = llm_response.choices[0]
+
         structlogger.debug(
             "multi_step_llm_command_generator"
             ".predict_commands_for_active_flow"
@@ -547,10 +553,11 @@ class MultiStepLLMCommandGenerator(LLMBasedCommandGenerator):
 
         if commands:
             self._add_prompt_to_message_parse_data(
-                message,
-                MultiStepLLMCommandGenerator.__name__,
-                "fill_slots_for_active_flow_prompt",
-                prompt,
+                message=message,
+                component_name=MultiStepLLMCommandGenerator.__name__,
+                prompt_name="fill_slots_for_active_flow_prompt",
+                user_prompt=prompt,
+                llm_response=llm_response,
             )
 
         return commands
@@ -584,7 +591,12 @@ class MultiStepLLMCommandGenerator(LLMBasedCommandGenerator):
             prompt=prompt,
         )
 
-        actions = await self.invoke_llm(prompt)
+        response = await self.invoke_llm(prompt)
+        llm_response = LLMResponse.ensure_llm_response(response)
+        actions = None
+        if llm_response and llm_response.choices:
+            actions = llm_response.choices[0]
+
         structlogger.debug(
             "multi_step_llm_command_generator"
             ".predict_commands_for_handling_flows"
@@ -598,10 +610,11 @@ class MultiStepLLMCommandGenerator(LLMBasedCommandGenerator):
 
         if commands:
             self._add_prompt_to_message_parse_data(
-                message,
-                MultiStepLLMCommandGenerator.__name__,
-                "handle_flows_prompt",
-                prompt,
+                message=message,
+                component_name=MultiStepLLMCommandGenerator.__name__,
+                prompt_name="handle_flows_prompt",
+                user_prompt=prompt,
+                llm_response=llm_response,
             )
 
         return commands
@@ -668,7 +681,12 @@ class MultiStepLLMCommandGenerator(LLMBasedCommandGenerator):
             prompt=prompt,
         )
 
-        actions = await self.invoke_llm(prompt)
+        response = await self.invoke_llm(prompt)
+        llm_response = LLMResponse.ensure_llm_response(response)
+        actions = None
+        if llm_response and llm_response.choices:
+            actions = llm_response.choices[0]
+
         structlogger.debug(
             "multi_step_llm_command_generator"
             ".predict_commands_for_newly_started_flow"
@@ -695,10 +713,11 @@ class MultiStepLLMCommandGenerator(LLMBasedCommandGenerator):
 
         if commands:
             self._add_prompt_to_message_parse_data(
-                message,
-                MultiStepLLMCommandGenerator.__name__,
-                "fill_slots_for_new_flow_prompt",
-                prompt,
+                message=message,
+                component_name=MultiStepLLMCommandGenerator.__name__,
+                prompt_name="fill_slots_for_new_flow_prompt",
+                user_prompt=prompt,
+                llm_response=llm_response,
             )
 
         return commands

{rasa_pro-3.11.3a1.dev4 → rasa_pro-3.11.3a1.dev6}/rasa/dialogue_understanding/generator/single_step/single_step_llm_command_generator.py

@@ -46,6 +46,7 @@ from rasa.shared.core.trackers import DialogueStateTracker
 from rasa.shared.exceptions import ProviderClientAPIException
 from rasa.shared.nlu.constants import TEXT, LLM_COMMANDS, LLM_PROMPT
 from rasa.shared.nlu.training_data.message import Message
+from rasa.shared.providers.llm.llm_response import LLMResponse
 from rasa.shared.utils.io import deep_container_fingerprint
 from rasa.shared.utils.llm import (
     get_prompt_template,
@@ -264,13 +265,16 @@ class SingleStepLLMCommandGenerator(LLMBasedCommandGenerator):
             prompt=flow_prompt,
         )
 
-        action_list = await self.invoke_llm(flow_prompt)
+        response = await self.invoke_llm(flow_prompt)
+        llm_response = LLMResponse.ensure_llm_response(response)
         # The check for 'None' maintains compatibility with older versions
         # of LLMCommandGenerator. In previous implementations, 'invoke_llm'
         # might return 'None' to indicate a failure to generate actions.
-        if action_list is None:
+        if llm_response is None or not llm_response.choices:
             return [ErrorCommand()]
 
+        action_list = llm_response.choices[0]
+
         log_llm(
             logger=structlogger,
             log_module="SingleStepLLMCommandGenerator",
@@ -285,10 +289,11 @@ class SingleStepLLMCommandGenerator(LLMBasedCommandGenerator):
             message, SingleStepLLMCommandGenerator.__name__, commands
         )
         self._add_prompt_to_message_parse_data(
-            message,
-            SingleStepLLMCommandGenerator.__name__,
-            "command_generator_prompt",
-            flow_prompt,
+            message=message,
+            component_name=SingleStepLLMCommandGenerator.__name__,
+            prompt_name="command_generator_prompt",
+            user_prompt=flow_prompt,
+            llm_response=llm_response,
         )
 
         return commands

{rasa_pro-3.11.3a1.dev4 → rasa_pro-3.11.3a1.dev6}/rasa/dialogue_understanding/utils.py

@@ -1,7 +1,14 @@
 from contextlib import contextmanager
 from typing import Generator
 
-record_commands_and_prompts = False
+from rasa.dialogue_understanding.constants import (
+    RASA_RECORD_COMMANDS_AND_PROMPTS_ENV_VAR_NAME,
+)
+from rasa.utils.common import get_bool_env_variable
+
+record_commands_and_prompts = get_bool_env_variable(
+    RASA_RECORD_COMMANDS_AND_PROMPTS_ENV_VAR_NAME, False
+)
 
 
 @contextmanager