rasa-pro 3.11.0a4.dev3__py3-none-any.whl → 3.11.0rc2__py3-none-any.whl

rasa/dialogue_understanding/generator/flow_retrieval.py

@@ -1,5 +1,4 @@
-"""
-The module is primarily centered around the `FlowRetrieval` class which handles the
+"""The module is primarily centered around the `FlowRetrieval` class which handles the
 initialization, configuration validation, vector store management, and flow retrieval
 logic. It integrates components for managing embeddings, vector stores, and
 flow-specific templates, facilitating semantic search functionalities.
@@ -27,8 +26,10 @@ from langchain.docstore.document import Document
 from langchain.schema.embeddings import Embeddings
 from langchain_community.vectorstores.faiss import FAISS
 from langchain_community.vectorstores.utils import DistanceStrategy
+
 from rasa.engine.storage.resource import Resource
 from rasa.engine.storage.storage import ModelStorage
+import rasa.shared.utils.io
 from rasa.shared.constants import (
     EMBEDDINGS_CONFIG_KEY,
     PROVIDER_CONFIG_KEY,
@@ -37,12 +38,15 @@ from rasa.shared.constants import (
 from rasa.shared.core.domain import Domain
 from rasa.shared.core.flows import FlowsList
 from rasa.shared.core.trackers import DialogueStateTracker
+from rasa.shared.exceptions import ProviderClientAPIException
 from rasa.shared.nlu.constants import TEXT, FLOWS_FROM_SEMANTIC_SEARCH
 from rasa.shared.nlu.training_data.message import Message
-from rasa.shared.exceptions import ProviderClientAPIException
 from rasa.shared.providers.embedding._langchain_embedding_client_adapter import (
     _LangchainEmbeddingClientAdapter,
 )
+from rasa.shared.utils.health_check.embeddings_health_check_mixin import (
+    EmbeddingsHealthCheckMixin,
+)
 from rasa.shared.utils.llm import (
     tracker_as_readable_transcript,
     embedder_factory,
@@ -50,12 +54,15 @@ from rasa.shared.utils.llm import (
     USER,
     get_prompt_template,
     allowed_values_for_slot,
+    resolve_model_client_config,
 )
 
 DEFAULT_FLOW_DOCUMENT_TEMPLATE = importlib.resources.read_text(
     "rasa.dialogue_understanding.generator", "flow_document_template.jinja2"
 )
 
+FLOW_RETRIEVAL_CONFIG_FILE_NAME = "flow_retrieval_config.json"
+
 DEFAULT_EMBEDDINGS_CONFIG = {
     PROVIDER_CONFIG_KEY: OPENAI_PROVIDER,
     "model": DEFAULT_OPENAI_EMBEDDING_MODEL_NAME,
@@ -73,7 +80,7 @@ DEFAULT_SHOULD_EMBED_SLOTS = True
 structlogger = structlog.get_logger()
 
 
-class FlowRetrieval:
+class FlowRetrieval(EmbeddingsHealthCheckMixin):
     @classmethod
     def get_default_config(cls) -> Dict[str, Any]:
         """The default config for the flow retrieval."""
@@ -92,6 +99,9 @@ class FlowRetrieval:
     ):
         config = {**self.get_default_config(), **config}
         self.config = self.validate_config(config)
+        self.config[EMBEDDINGS_CONFIG_KEY] = resolve_model_client_config(
+            self.config.get(EMBEDDINGS_CONFIG_KEY), FlowRetrieval.__name__
+        )
         self.vector_store: Optional[FAISS] = None
         self.flow_document_template = get_prompt_template(
             None, DEFAULT_FLOW_DOCUMENT_TEMPLATE
@@ -140,6 +150,18 @@ class FlowRetrieval:
         **kwargs: Any,
     ) -> "FlowRetrieval":
         """Load flow retrieval with previously populated FAISS vector store."""
+
+        # Perform health check on resolved embedding client config
+        embeddings_config = resolve_model_client_config(
+            config.get(EMBEDDINGS_CONFIG_KEY, {})
+        )
+        cls.perform_embeddings_health_check(
+            embeddings_config,
+            DEFAULT_EMBEDDINGS_CONFIG,
+            "flow_retrieval.load",
+            FlowRetrieval.__name__,
+        )
+
         # initialize base flow retrieval
         flow_retrieval = FlowRetrieval(config, model_storage, resource)
         # load vector store
@@ -147,6 +169,7 @@ class FlowRetrieval:
             flow_retrieval.config, model_storage, resource
         )
         flow_retrieval.vector_store = vector_store
+
         return flow_retrieval
 
     @classmethod
@@ -178,13 +201,21 @@ class FlowRetrieval:
         Returns:
             The embedder.
         """
+        # Copy the config so original config is not modified
+        config = config.copy()
+        # Resolve config and instantiate the embedding client
+        config[EMBEDDINGS_CONFIG_KEY] = resolve_model_client_config(
+            config.get(EMBEDDINGS_CONFIG_KEY), FlowRetrieval.__name__
+        )
         client = embedder_factory(
             config.get(EMBEDDINGS_CONFIG_KEY), DEFAULT_EMBEDDINGS_CONFIG
         )
+        # Wrap the embedding client in the adapter
         return _LangchainEmbeddingClientAdapter(client)
 
     def persist(self) -> None:
         self._persist_vector_store()
+        self._persist_config()
 
     def _persist_vector_store(self) -> None:
         """Persists the FAISS vector store."""
@@ -197,6 +228,12 @@ class FlowRetrieval:
                 event_info="Vector store is None, not persisted.",
             )
 
+    def _persist_config(self) -> None:
+        with self._model_storage.write_to(self._resource) as path:
+            rasa.shared.utils.io.dump_obj_as_json_to_file(
+                path / FLOW_RETRIEVAL_CONFIG_FILE_NAME, self.config
+            )
+
     def populate(self, flows: FlowsList, domain: Domain) -> None:
         """Populates the vector store with embeddings generated from
         documents based on the flow descriptions, and flow slots
@@ -206,6 +243,14 @@ class FlowRetrieval:
             flows: List of flows to populate the vector store with.
             domain: The domain containing relevant slot information.
         """
+        # Perform health check before populating the vector store with flows
+        self.perform_embeddings_health_check(
+            self.config.get(EMBEDDINGS_CONFIG_KEY),
+            DEFAULT_EMBEDDINGS_CONFIG,
+            "flow_retrieval.train",
+            FlowRetrieval.__name__,
+        )
+
         flows_to_embedd = flows.exclude_link_only_flows()
         embeddings = self._create_embedder(self.config)
         documents = self._generate_flow_documents(flows_to_embedd, domain)

rasa/dialogue_understanding/generator/llm_based_command_generator.py

@@ -2,7 +2,6 @@ from abc import ABC, abstractmethod
 from functools import lru_cache
 from typing import Dict, Any, List, Optional, Tuple, Union, Text
 
-import os
 import structlog
 from jinja2 import Template
 
@@ -17,13 +16,13 @@ from rasa.dialogue_understanding.generator.constants import (
     LLM_CONFIG_KEY,
     FLOW_RETRIEVAL_KEY,
     FLOW_RETRIEVAL_ACTIVE_KEY,
+    FLOW_RETRIEVAL_FLOW_THRESHOLD,
 )
 from rasa.dialogue_understanding.generator.flow_retrieval import FlowRetrieval
 from rasa.engine.graph import GraphComponent, ExecutionContext
 from rasa.engine.recipes.default_recipe import DefaultV1Recipe
 from rasa.engine.storage.resource import Resource
 from rasa.engine.storage.storage import ModelStorage
-from rasa.shared.constants import LLM_API_HEALTH_CHECK_ENV_VAR
 from rasa.shared.core.domain import Domain
 from rasa.shared.core.flows import FlowStep, Flow, FlowsList
 from rasa.shared.core.flows.steps.collect import CollectInformationFlowStep
@@ -33,11 +32,11 @@ from rasa.shared.exceptions import ProviderClientAPIException
 from rasa.shared.nlu.constants import FLOWS_IN_PROMPT
 from rasa.shared.nlu.training_data.message import Message
 from rasa.shared.nlu.training_data.training_data import TrainingData
+from rasa.shared.utils.health_check.llm_health_check_mixin import LLMHealthCheckMixin
 from rasa.shared.utils.llm import (
     allowed_values_for_slot,
-    llm_api_health_check,
     llm_factory,
-    try_instantiate_llm_client,
+    resolve_model_client_config,
 )
 from rasa.utils.log_utils import log_llm
 
@@ -50,7 +49,9 @@ structlogger = structlog.get_logger()
     ],
     is_trainable=True,
 )
-class LLMBasedCommandGenerator(GraphComponent, CommandGenerator, ABC):
+class LLMBasedCommandGenerator(
+    LLMHealthCheckMixin, GraphComponent, CommandGenerator, ABC
+):
     """An abstract class defining interface and common functionality
     of an LLM-based command generators.
     """
@@ -64,6 +65,9 @@ class LLMBasedCommandGenerator(GraphComponent, CommandGenerator, ABC):
     ) -> None:
         super().__init__(config)
         self.config = {**self.get_default_config(), **config}
+        self.config[LLM_CONFIG_KEY] = resolve_model_client_config(
+            self.config.get(LLM_CONFIG_KEY), LLMBasedCommandGenerator.__name__
+        )
         self._model_storage = model_storage
         self._resource = resource
         self.flow_retrieval: Optional[FlowRetrieval]
@@ -73,17 +77,9 @@ class LLMBasedCommandGenerator(GraphComponent, CommandGenerator, ABC):
                 self.config[FLOW_RETRIEVAL_KEY], model_storage, resource
             )
             structlogger.info("llm_based_command_generator.flow_retrieval.enabled")
+            self.config[FLOW_RETRIEVAL_KEY] = self.flow_retrieval.config
         else:
             self.flow_retrieval = None
-            structlogger.warn(
-                "llm_based_command_generator.flow_retrieval.disabled",
-                event_info=(
-                    "Disabling flow retrieval can cause issues when there are a "
-                    "large number of flows to be included in the prompt. For more"
-                    "information see:\n"
-                    "https://rasa.com/docs/rasa-pro/concepts/dialogue-understanding#how-the-llmcommandgenerator-works"
-                ),
-            )
 
     ### Abstract methods
     @staticmethod
@@ -171,18 +167,32 @@ class LLMBasedCommandGenerator(GraphComponent, CommandGenerator, ABC):
         """Train the llm based command generator. Stores all flows into a vector
         store.
         """
-        # Validate llm configuration
-        llm_client = try_instantiate_llm_client(
+        self.perform_llm_health_check(
             self.config.get(LLM_CONFIG_KEY),
             DEFAULT_LLM_CONFIG,
             "llm_based_command_generator.train",
             LLMBasedCommandGenerator.__name__,
         )
-        if os.getenv(LLM_API_HEALTH_CHECK_ENV_VAR, "true").lower() == "true":
-            llm_api_health_check(
-                llm_client,
-                "llm_based_command_generator.train",
-                LLMBasedCommandGenerator.__name__,
+
+        if (
+            self.flow_retrieval is None
+            and len(flows.user_flows) > FLOW_RETRIEVAL_FLOW_THRESHOLD
+        ):
+            structlogger.warn(
+                "llm_based_command_generator.flow_retrieval.disabled",
+                event_info=(
+                    f"You have {len(flows.user_flows)} user flows but flow "
+                    f"retrieval is disabled. "
+                    f"It is recommended to enable flow retrieval if the "
+                    f"total number of user flows exceed "
+                    f"{FLOW_RETRIEVAL_FLOW_THRESHOLD}. "
+                    f"Keeping it disabled can result in deterioration of "
+                    f"command generator's functional "
+                    f"performance and higher costs because of increased "
+                    f"number of tokens in the prompt. For more"
+                    "information see:\n"
+                    "https://rasa.com/docs/rasa-pro/concepts/dialogue-understanding#how-the-llmcommandgenerator-works"
+                ),
             )
 
         # flow retrieval is populated with only user-defined flows
@@ -192,10 +202,11 @@ class LLMBasedCommandGenerator(GraphComponent, CommandGenerator, ABC):
         except Exception as e:
             structlogger.error(
                 "llm_based_command_generator.train.failed",
-                event_info=("Flow retrieval store isinaccessible."),
+                event_info="Flow retrieval store is inaccessible.",
                 error=e,
             )
             raise
+
         self.persist()
         return self._resource
 
@@ -233,7 +244,10 @@ class LLMBasedCommandGenerator(GraphComponent, CommandGenerator, ABC):
 
     @classmethod
     def load_flow_retrival(
-        cls, config: Dict[Text, Any], model_storage: ModelStorage, resource: Resource
+        cls,
+        config: Dict[str, Any],
+        model_storage: ModelStorage,
+        resource: Resource,
     ) -> Optional[FlowRetrieval]:
         """Load the FlowRetrieval component if it is enabled in the configuration."""
         enable_flow_retrieval = config.get(FLOW_RETRIEVAL_KEY, {}).get(

rasa/dialogue_understanding/generator/multi_step/multi_step_llm_command_generator.py

@@ -24,6 +24,7 @@ from rasa.dialogue_understanding.generator.constants import (
     LLM_CONFIG_KEY,
     USER_INPUT_CONFIG_KEY,
     FLOW_RETRIEVAL_KEY,
+    DEFAULT_LLM_CONFIG,
 )
 from rasa.dialogue_understanding.generator.flow_retrieval import FlowRetrieval
 from rasa.dialogue_understanding.generator.llm_based_command_generator import (
@@ -39,7 +40,10 @@ from rasa.engine.graph import ExecutionContext
 from rasa.engine.recipes.default_recipe import DefaultV1Recipe
 from rasa.engine.storage.resource import Resource
 from rasa.engine.storage.storage import ModelStorage
-from rasa.shared.constants import RASA_PATTERN_CANNOT_HANDLE_NOT_SUPPORTED
+from rasa.shared.constants import (
+    RASA_PATTERN_CANNOT_HANDLE_NOT_SUPPORTED,
+    EMBEDDINGS_CONFIG_KEY,
+)
 from rasa.shared.constants import ROUTE_TO_CALM_SLOT
 from rasa.shared.core.flows import FlowStep, Flow, FlowsList
 from rasa.shared.core.flows.steps.collect import CollectInformationFlowStep
@@ -53,6 +57,7 @@ from rasa.shared.utils.llm import (
     tracker_as_readable_transcript,
     sanitize_message_for_prompt,
     allowed_values_for_slot,
+    resolve_model_client_config,
 )
 
 # multistep template keys
@@ -70,6 +75,7 @@ DEFAULT_HANDLE_FLOWS_TEMPLATE = importlib.resources.read_text(
 DEFAULT_FILL_SLOTS_TEMPLATE = importlib.resources.read_text(
     "rasa.dialogue_understanding.generator.multi_step", "fill_slots_prompt.jinja2"
 ).strip()
+MULTI_STEP_LLM_COMMAND_GENERATOR_CONFIG_FILE = "config.json"
 
 # dictionary of template names and associated file names and default values
 PROMPT_TEMPLATES = {
@@ -138,7 +144,18 @@ class MultiStepLLMCommandGenerator(LLMBasedCommandGenerator):
         **kwargs: Any,
     ) -> "MultiStepLLMCommandGenerator":
         """Loads trained component (see parent class for full docstring)."""
+
+        # Perform health check of the LLM client config
+        llm_config = resolve_model_client_config(config.get(LLM_CONFIG_KEY, {}))
+        cls.perform_llm_health_check(
+            llm_config,
+            DEFAULT_LLM_CONFIG,
+            "multi_step_llm_command_generator.load",
+            MultiStepLLMCommandGenerator.__name__,
+        )
+
         prompts = cls._load_prompt_templates(model_storage, resource)
+
         # init base command generator
         command_generator = cls(config, model_storage, resource, prompts)
         # load flow retrieval if enabled
@@ -146,13 +163,13 @@ class MultiStepLLMCommandGenerator(LLMBasedCommandGenerator):
             command_generator.flow_retrieval = cls.load_flow_retrival(
                 command_generator.config, model_storage, resource
             )
+
         return command_generator
 
     def persist(self) -> None:
         """Persist this component to disk for future loading."""
-        # persist prompt template
         self._persist_prompt_templates()
-        # persist flow retrieval
+        self._persist_config()
         if self.flow_retrieval is not None:
             self.flow_retrieval.persist()
 
@@ -229,9 +246,9 @@ class MultiStepLLMCommandGenerator(LLMBasedCommandGenerator):
         commands: List[Command] = []
 
         slot_set_re = re.compile(
-            r"""SetSlot\((\"?[a-zA-Z_][a-zA-Z0-9_-]*?\"?), ?(.*)\)"""
+            r"""SetSlot\(['"]?([a-zA-Z_][a-zA-Z0-9_-]*)['"]?, ?['"]?(.*)['"]?\)"""
         )
-        start_flow_re = re.compile(r"StartFlow\(([a-zA-Z0-9_-]+?)\)")
+        start_flow_re = re.compile(r"StartFlow\(['\"]?([a-zA-Z0-9_-]+)['\"]?\)")
         change_flow_re = re.compile(r"ChangeFlow\(\)")
         cancel_flow_re = re.compile(r"CancelFlow\(\)")
         chitchat_re = re.compile(r"ChitChat\(\)")
@@ -280,9 +297,19 @@ class MultiStepLLMCommandGenerator(LLMBasedCommandGenerator):
                 commands.append(HumanHandoffCommand())
             elif match := clarify_re.search(action):
                 options = sorted([opt.strip() for opt in match.group(1).split(",")])
+                # Remove surrounding quotes if present
+                cleaned_options = []
+                for flow in options:
+                    if (flow.startswith('"') and flow.endswith('"')) or (
+                        flow.startswith("'") and flow.endswith("'")
+                    ):
+                        cleaned_options.append(flow[1:-1])
+                    else:
+                        cleaned_options.append(flow)
+                # check if flow is valid
                 valid_options = [
                     flow
-                    for flow in options
+                    for flow in cleaned_options
                     if flow in flows.user_flow_ids
                     and flow not in user_flows_on_the_stack(tracker.stack)
                 ]
@@ -293,6 +320,13 @@ class MultiStepLLMCommandGenerator(LLMBasedCommandGenerator):
             elif change_flow_re.search(action):
                 commands.append(ChangeFlowCommand())
 
+        if not commands:
+            structlogger.debug(
+                "multi_step_llm_command_generator.parse_commands",
+                message="No commands were parsed from the LLM actions.",
+                actions=actions,
+            )
+
         return commands
 
     ### Helper methods
@@ -368,6 +402,13 @@ class MultiStepLLMCommandGenerator(LLMBasedCommandGenerator):
                 file_path = path / file_name
                 rasa.shared.utils.io.write_text_file(template, file_path)
 
+    def _persist_config(self) -> None:
+        """Persist config as a source of truth for resolved clients."""
+        with self._model_storage.write_to(self._resource) as path:
+            rasa.shared.utils.io.dump_obj_as_json_to_file(
+                path / MULTI_STEP_LLM_COMMAND_GENERATOR_CONFIG_FILE, self.config
+            )
+
     async def _predict_commands_with_multi_step(
         self,
         message: Message,
@@ -761,11 +802,17 @@ class MultiStepLLMCommandGenerator(LLMBasedCommandGenerator):
             .get(FILE_PATH_KEY),
             DEFAULT_FILL_SLOTS_TEMPLATE,
         )
+
+        llm_config = resolve_model_client_config(
+            config.get(LLM_CONFIG_KEY), MultiStepLLMCommandGenerator.__name__
+        )
+        embedding_config = resolve_model_client_config(
+            config.get(FLOW_RETRIEVAL_KEY, {}).get(EMBEDDINGS_CONFIG_KEY),
+            FlowRetrieval.__name__,
+        )
+
         return deep_container_fingerprint(
-            [
-                handle_flows_template,
-                fill_slots_template,
-            ]
+            [handle_flows_template, fill_slots_template, llm_config, embedding_config]
         )
 
     @staticmethod

rasa/dialogue_understanding/generator/nlu_command_adapter.py

@@ -19,6 +19,7 @@ from rasa.engine.storage.storage import ModelStorage
 from rasa.shared.constants import ROUTE_TO_CALM_SLOT
 from rasa.shared.core.domain import Domain
 from rasa.shared.core.flows.flows_list import FlowsList
+from rasa.shared.core.flows.steps import CollectInformationFlowStep
 from rasa.shared.core.slot_mappings import (
     SlotFillingManager,
     extract_slot_value,
@@ -217,7 +218,24 @@ def _issue_set_slot_commands(
     commands: List[Command] = []
     domain = domain if domain else Domain.empty()
     slot_filling_manager = SlotFillingManager(domain, tracker, message)
-    available_slot_names = flows.available_slot_names()
+
+    # only use slots that don't have ask_before_filling set to True
+    available_slot_names = flows.available_slot_names(ask_before_filling=False)
+
+    # check if the current step is a CollectInformationFlowStep
+    # in case it has ask_before_filling set to True, we need to add the
+    # slot to the available_slot_names
+    if tracker.active_flow:
+        flow = flows.flow_by_id(tracker.active_flow)
+        step_id = tracker.current_step_id
+        if flow is not None:
+            current_step = flow.step_by_id(step_id)
+            if (
+                current_step
+                and isinstance(current_step, CollectInformationFlowStep)
+                and current_step.ask_before_filling
+            ):
+                available_slot_names.add(current_step.collect)
 
     for _, slot in tracker.slots.items():
         # if a slot is not collected in available flows,

rasa/dialogue_understanding/generator/single_step/single_step_llm_command_generator.py

@@ -22,6 +22,7 @@ from rasa.dialogue_understanding.generator.constants import (
     LLM_CONFIG_KEY,
     USER_INPUT_CONFIG_KEY,
     FLOW_RETRIEVAL_KEY,
+    DEFAULT_LLM_CONFIG,
 )
 from rasa.dialogue_understanding.generator.flow_retrieval import (
     FlowRetrieval,
@@ -38,6 +39,7 @@ from rasa.shared.constants import (
     ROUTE_TO_CALM_SLOT,
     PROMPT_CONFIG_KEY,
     PROMPT_TEMPLATE_CONFIG_KEY,
+    EMBEDDINGS_CONFIG_KEY,
 )
 from rasa.shared.core.flows import FlowsList
 from rasa.shared.core.trackers import DialogueStateTracker
@@ -49,9 +51,10 @@ from rasa.shared.utils.llm import (
     get_prompt_template,
     tracker_as_readable_transcript,
     sanitize_message_for_prompt,
+    resolve_model_client_config,
 )
-from rasa.utils.log_utils import log_llm
 from rasa.utils.beta import ensure_beta_feature_is_enabled, BetaNotEnabledException
+from rasa.utils.log_utils import log_llm
 
 COMMAND_PROMPT_FILE_NAME = "command_prompt.jinja2"
 
@@ -59,6 +62,7 @@ DEFAULT_COMMAND_PROMPT_TEMPLATE = importlib.resources.read_text(
     "rasa.dialogue_understanding.generator.single_step",
     "command_prompt_template.jinja2",
 )
+SINGLE_STEP_LLM_COMMAND_GENERATOR_CONFIG_FILE = "config.json"
 
 structlogger = structlog.get_logger()
 
@@ -132,10 +136,21 @@ class SingleStepLLMCommandGenerator(LLMBasedCommandGenerator):
         **kwargs: Any,
     ) -> "SingleStepLLMCommandGenerator":
         """Loads trained component (see parent class for full docstring)."""
+
+        # Perform health check of the LLM API endpoint
+        llm_config = resolve_model_client_config(config.get(LLM_CONFIG_KEY, {}))
+        cls.perform_llm_health_check(
+            llm_config,
+            DEFAULT_LLM_CONFIG,
+            "single_step_llm_command_generator.load",
+            SingleStepLLMCommandGenerator.__name__,
+        )
+
         # load prompt template from the model storage.
         prompt_template = cls.load_prompt_template_from_model_storage(
             model_storage, resource, COMMAND_PROMPT_FILE_NAME
         )
+
         # init base command generator
         command_generator = cls(config, model_storage, resource, prompt_template)
         # load flow retrieval if enabled
@@ -143,18 +158,29 @@ class SingleStepLLMCommandGenerator(LLMBasedCommandGenerator):
             command_generator.flow_retrieval = cls.load_flow_retrival(
                 command_generator.config, model_storage, resource
             )
+
         return command_generator
 
     def persist(self) -> None:
         """Persist this component to disk for future loading."""
-        # persist prompt template
+        self._persist_prompt_template()
+        self._persist_config()
+        if self.flow_retrieval is not None:
+            self.flow_retrieval.persist()
+
+    def _persist_prompt_template(self) -> None:
+        """Persist prompt template for future loading."""
         with self._model_storage.write_to(self._resource) as path:
             rasa.shared.utils.io.write_text_file(
                 self.prompt_template, path / COMMAND_PROMPT_FILE_NAME
             )
-        # persist flow retrieval
-        if self.flow_retrieval is not None:
-            self.flow_retrieval.persist()
+
+    def _persist_config(self) -> None:
+        """Persist config as a source of truth for resolved clients."""
+        with self._model_storage.write_to(self._resource) as path:
+            rasa.shared.utils.io.dump_obj_as_json_to_file(
+                path / SINGLE_STEP_LLM_COMMAND_GENERATOR_CONFIG_FILE, self.config
+            )
 
     async def predict_commands(
         self,
@@ -187,6 +213,12 @@ class SingleStepLLMCommandGenerator(LLMBasedCommandGenerator):
 
         if not commands:
             # no commands are parsed or there's an invalid command
+            structlogger.warning(
+                "single_step_llm_command_generator.predict_commands",
+                message="No commands were predicted as the LLM response could "
+                "not be parsed or the LLM responded with an invalid command."
+                "Returning a CannotHandleCommand instead.",
+            )
             commands = [CannotHandleCommand()]
 
         if tracker.has_coexistence_routing_slot:
@@ -287,14 +319,16 @@ class SingleStepLLMCommandGenerator(LLMBasedCommandGenerator):
 
         commands: List[Command] = []
 
-        slot_set_re = re.compile(r"""SetSlot\(([a-zA-Z_][a-zA-Z0-9_-]*?), ?(.*)\)""")
-        start_flow_re = re.compile(r"StartFlow\(([a-zA-Z0-9_-]+?)\)")
+        slot_set_re = re.compile(
+            r"""SetSlot\(['"]?([a-zA-Z_][a-zA-Z0-9_-]*)['"]?, ?['"]?(.*)['"]?\)"""
+        )
+        start_flow_re = re.compile(r"StartFlow\(['\"]?([a-zA-Z0-9_-]+)['\"]?\)")
         cancel_flow_re = re.compile(r"CancelFlow\(\)")
         chitchat_re = re.compile(r"ChitChat\(\)")
         skip_question_re = re.compile(r"SkipQuestion\(\)")
         knowledge_re = re.compile(r"SearchAndReply\(\)")
         humand_handoff_re = re.compile(r"HumanHandoff\(\)")
-        clarify_re = re.compile(r"Clarify\(([a-zA-Z0-9_, ]+)\)")
+        clarify_re = re.compile(r"Clarify\(([\"\'a-zA-Z0-9_, ]+)\)")
         repeat_re = re.compile(r"RepeatLastBotMessages\(\)")
 
         for action in actions.strip().splitlines():
@@ -326,19 +360,36 @@ class SingleStepLLMCommandGenerator(LLMBasedCommandGenerator):
                 commands.append(RepeatBotMessagesCommand())
             elif match := clarify_re.search(action):
                 options = sorted([opt.strip() for opt in match.group(1).split(",")])
+                # Remove surrounding quotes if present
+                cleaned_options = []
+                for flow in options:
+                    if (flow.startswith('"') and flow.endswith('"')) or (
+                        flow.startswith("'") and flow.endswith("'")
+                    ):
+                        cleaned_options.append(flow[1:-1])
+                    else:
+                        cleaned_options.append(flow)
+                # check if flow is valid
                 valid_options = [
-                    flow for flow in options if flow in flows.user_flow_ids
+                    flow for flow in cleaned_options if flow in flows.user_flow_ids
                 ]
                 if len(set(valid_options)) == 1:
                     commands.extend(cls.start_flow_by_name(valid_options[0], flows))
                 elif len(valid_options) > 1:
                     commands.append(ClarifyCommand(valid_options))
 
+        if not commands:
+            structlogger.debug(
+                "single_step_llm_command_generator.parse_commands",
+                message="No commands were parsed from the LLM actions.",
+                actions=actions,
+            )
+
         return commands
 
     @classmethod
     def fingerprint_addon(cls: Any, config: Dict[str, Any]) -> Optional[str]:
-        """Add a fingerprint of the knowledge base for the graph."""
+        """Add a fingerprint for the graph."""
         config_prompt = (
             config.get(PROMPT_CONFIG_KEY)
             or config.get(PROMPT_TEMPLATE_CONFIG_KEY)
@@ -348,7 +399,16 @@ class SingleStepLLMCommandGenerator(LLMBasedCommandGenerator):
             config_prompt,
             DEFAULT_COMMAND_PROMPT_TEMPLATE,
         )
-        return deep_container_fingerprint(prompt_template)
+        llm_config = resolve_model_client_config(
+            config.get(LLM_CONFIG_KEY), SingleStepLLMCommandGenerator.__name__
+        )
+        embedding_config = resolve_model_client_config(
+            config.get(FLOW_RETRIEVAL_KEY, {}).get(EMBEDDINGS_CONFIG_KEY),
+            FlowRetrieval.__name__,
+        )
+        return deep_container_fingerprint(
+            [prompt_template, llm_config, embedding_config]
+        )
 
     ### Helper methods
     def render_template(