rasa-pro 3.11.0a4.dev3__py3-none-any.whl → 3.11.0rc1__py3-none-any.whl

rasa/dialogue_understanding/coexistence/llm_based_router.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 import importlib
-import os
 from typing import Any, Dict, List, Optional
 
 import structlog
@@ -16,13 +15,15 @@ from rasa.dialogue_understanding.coexistence.constants import (
 )
 from rasa.dialogue_understanding.commands import Command, SetSlotCommand
 from rasa.dialogue_understanding.commands.noop_command import NoopCommand
-from rasa.dialogue_understanding.generator.constants import LLM_CONFIG_KEY
+from rasa.dialogue_understanding.generator.constants import (
+    LLM_CONFIG_KEY,
+    TRAINED_MODEL_NAME_CONFIG_KEY,
+)
 from rasa.engine.graph import ExecutionContext, GraphComponent
 from rasa.engine.recipes.default_recipe import DefaultV1Recipe
 from rasa.engine.storage.resource import Resource
 from rasa.engine.storage.storage import ModelStorage
 from rasa.shared.constants import (
-    LLM_API_HEALTH_CHECK_ENV_VAR,
     ROUTE_TO_CALM_SLOT,
     PROMPT_CONFIG_KEY,
     PROVIDER_CONFIG_KEY,
@@ -35,12 +36,16 @@ from rasa.shared.exceptions import InvalidConfigException, FileIOException
 from rasa.shared.nlu.constants import COMMANDS, TEXT
 from rasa.shared.nlu.training_data.message import Message
 from rasa.shared.nlu.training_data.training_data import TrainingData
+from rasa.shared.utils.io import deep_container_fingerprint
 from rasa.shared.utils.llm import (
     DEFAULT_OPENAI_CHAT_MODEL_NAME,
     get_prompt_template,
-    llm_api_health_check,
     llm_factory,
-    try_instantiate_llm_client,
+    resolve_model_client_config,
+)
+from rasa.shared.utils.health_check import (
+    perform_training_time_llm_health_check,
+    perform_inference_time_llm_health_check,
 )
 from rasa.utils.log_utils import log_llm
 
@@ -48,6 +53,7 @@ LLM_BASED_ROUTER_PROMPT_FILE_NAME = "llm_based_router_prompt.jinja2"
 DEFAULT_COMMAND_PROMPT_TEMPLATE = importlib.resources.read_text(
     "rasa.dialogue_understanding.coexistence", "router_template.jinja2"
 )
+LLM_BASED_ROUTER_CONFIG_FILE_NAME = "config.json"
 
 # Token ids for gpt 3.5 and gpt 4 corresponding to space + capitalized Letter
 A_TO_C_TOKEN_IDS_CHATGPT = [
@@ -96,6 +102,9 @@ class LLMBasedRouter(GraphComponent):
         prompt_template: Optional[str] = None,
     ) -> None:
         self.config = {**self.get_default_config(), **config}
+        self.config[LLM_CONFIG_KEY] = resolve_model_client_config(
+            self.config.get(LLM_CONFIG_KEY), LLMBasedRouter.__name__
+        )
 
         self.prompt_template = (
             prompt_template
@@ -129,20 +138,20 @@ class LLMBasedRouter(GraphComponent):
             rasa.shared.utils.io.write_text_file(
                 self.prompt_template, path / LLM_BASED_ROUTER_PROMPT_FILE_NAME
             )
+            rasa.shared.utils.io.dump_obj_as_json_to_file(
+                path / LLM_BASED_ROUTER_CONFIG_FILE_NAME, self.config
+            )
 
     def train(self, training_data: TrainingData) -> Resource:
         """Train the intent classifier on a data set."""
-        # Validate llm configuration
-        llm_client = try_instantiate_llm_client(
-            self.config.get(LLM_CONFIG_KEY),
-            DEFAULT_LLM_CONFIG,
-            "llm_based_router.train",
-            LLMBasedRouter.__name__,
-        )
-        if os.getenv(LLM_API_HEALTH_CHECK_ENV_VAR, "true").lower() == "true":
-            llm_api_health_check(
-                llm_client, "llm_based_router.train", LLMBasedRouter.__name__
+        self.config[TRAINED_MODEL_NAME_CONFIG_KEY] = (
+            perform_training_time_llm_health_check(
+                self.config.get(LLM_CONFIG_KEY),
+                DEFAULT_LLM_CONFIG,
+                "llm_based_router.train",
+                LLMBasedRouter.__name__,
             )
+        )
 
         self.persist()
         return self._resource
@@ -158,17 +167,36 @@ class LLMBasedRouter(GraphComponent):
     ) -> "LLMBasedRouter":
         """Loads trained component (see parent class for full docstring)."""
         prompt_template = None
+        persisted_config = None
         try:
             with model_storage.read_from(resource) as path:
                 prompt_template = rasa.shared.utils.io.read_file(
                     path / LLM_BASED_ROUTER_PROMPT_FILE_NAME
                 )
+                persisted_config = rasa.shared.utils.io.read_json_file(
+                    path / LLM_BASED_ROUTER_CONFIG_FILE_NAME
+                )
         except (FileNotFoundError, FileIOException) as e:
             structlogger.warning(
                 "llm_based_router.load.failed", error=e, resource=resource.name
             )
 
-        return cls(config, model_storage, resource, prompt_template=prompt_template)
+        router = cls(config, model_storage, resource, prompt_template=prompt_template)
+
+        train_model_name = (
+            persisted_config.get(TRAINED_MODEL_NAME_CONFIG_KEY, None)
+            if persisted_config
+            else None
+        )
+        perform_inference_time_llm_health_check(
+            router.config.get(LLM_CONFIG_KEY),
+            DEFAULT_LLM_CONFIG,
+            train_model_name,
+            "llm_based_router.load",
+            LLMBasedRouter.__name__,
+        )
+
+        return router
 
     @classmethod
     def create(
@@ -298,3 +326,17 @@ class LLMBasedRouter(GraphComponent):
             # we have to catch all exceptions here
             structlogger.error("llm_based_router.llm.error", error=e)
             return None
+
+    @classmethod
+    def fingerprint_addon(cls, config: Dict[str, Any]) -> Optional[str]:
+        """Add a fingerprint of llm based router for the graph."""
+        prompt_template = get_prompt_template(
+            config.get(PROMPT_CONFIG_KEY),
+            DEFAULT_COMMAND_PROMPT_TEMPLATE,
+        )
+
+        llm_config = resolve_model_client_config(
+            config.get(LLM_CONFIG_KEY), LLMBasedRouter.__name__
+        )
+
+        return deep_container_fingerprint([prompt_template, llm_config])

rasa/dialogue_understanding/commands/change_flow_command.py

@@ -36,3 +36,9 @@ class ChangeFlowCommand(Command):
         # the change flow command is not actually pushing anything to the tracker,
         # but it is predicted by the MultiStepLLMCommandGenerator and used internally
         return []
+
+    def __eq__(self, other: Any) -> bool:
+        return isinstance(other, ChangeFlowCommand)
+
+    def __hash__(self) -> int:
+        return hash(self.command())

rasa/dialogue_understanding/commands/user_silence_command.py

@@ -0,0 +1,59 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Dict, List
+from rasa.dialogue_understanding.commands import Command
+from rasa.dialogue_understanding.patterns.user_silence import (
+    UserSilencePatternFlowStackFrame,
+)
+from rasa.shared.core.events import Event
+from rasa.shared.core.flows import FlowsList
+from rasa.shared.core.trackers import DialogueStateTracker
+
+
+@dataclass
+class UserSilenceCommand(Command):
+    """A command to indicate user silence."""
+
+    @classmethod
+    def command(cls) -> str:
+        """Returns the command type."""
+        return "user silence"
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> UserSilenceCommand:
+        """Converts the dictionary to a command.
+
+        Returns:
+            The converted dictionary.
+        """
+        return UserSilenceCommand()
+
+    def run_command_on_tracker(
+        self,
+        tracker: DialogueStateTracker,
+        all_flows: FlowsList,
+        original_tracker: DialogueStateTracker,
+    ) -> List[Event]:
+        """Runs the command on the tracker.
+
+        Args:
+            tracker: The tracker to run the command on.
+            all_flows: All flows in the assistant.
+            original_tracker: The tracker before any command was executed.
+
+        Returns:
+            The events to apply to the tracker.
+        """
+        stack = tracker.stack
+        stack.push(UserSilencePatternFlowStackFrame())
+        return tracker.create_stack_updated_events(stack)
+
+    def __hash__(self) -> int:
+        return hash(self.command())
+
+    def __eq__(self, other: object) -> bool:
+        if not isinstance(other, UserSilenceCommand):
+            return False
+
+        return True

rasa/dialogue_understanding/commands/utils.py

@@ -11,6 +11,7 @@ from rasa.dialogue_understanding.commands import (
     SkipQuestionCommand,
     RestartCommand,
 )
+from rasa.dialogue_understanding.commands.user_silence_command import UserSilenceCommand
 from rasa.dialogue_understanding.patterns.cancel import CancelPatternFlowStackFrame
 from rasa.dialogue_understanding.patterns.cannot_handle import (
     CannotHandlePatternFlowStackFrame,
@@ -27,9 +28,13 @@ from rasa.dialogue_understanding.patterns.session_start import (
 from rasa.dialogue_understanding.patterns.skip_question import (
     SkipQuestionPatternFlowStackFrame,
 )
+from rasa.dialogue_understanding.patterns.user_silence import (
+    UserSilencePatternFlowStackFrame,
+)
 
 triggerable_pattern_to_command_class: Dict[str, Type[Command]] = {
     SessionStartPatternFlowStackFrame.flow_id: SessionStartCommand,
+    UserSilencePatternFlowStackFrame.flow_id: UserSilenceCommand,
     CancelPatternFlowStackFrame.flow_id: CancelFlowCommand,
     ChitchatPatternFlowStackFrame.flow_id: ChitChatAnswerCommand,
     HumanHandoffPatternFlowStackFrame.flow_id: HumanHandoffCommand,

rasa/dialogue_understanding/generator/constants.py

@@ -18,8 +18,12 @@ DEFAULT_LLM_CONFIG = {
 }
 
 LLM_CONFIG_KEY = "llm"
+TRAINED_MODEL_NAME_CONFIG_KEY = "trained_llm_model_name"
+TRAINED_EMBEDDINGS_CONFIG_KEY = "trained_embeddings_model_name"
 USER_INPUT_CONFIG_KEY = "user_input"
 
 FLOW_RETRIEVAL_KEY = "flow_retrieval"
 FLOW_RETRIEVAL_ACTIVE_KEY = "active"
 FLOW_RETRIEVAL_EMBEDDINGS_CONFIG_KEY = "embeddings"
+
+FLOW_RETRIEVAL_FLOW_THRESHOLD = 20

rasa/dialogue_understanding/generator/flow_retrieval.py

@@ -1,5 +1,4 @@
-"""
-The module is primarily centered around the `FlowRetrieval` class which handles the
+"""The module is primarily centered around the `FlowRetrieval` class which handles the
 initialization, configuration validation, vector store management, and flow retrieval
 logic. It integrates components for managing embeddings, vector stores, and
 flow-specific templates, facilitating semantic search functionalities.
@@ -27,8 +26,13 @@ from langchain.docstore.document import Document
 from langchain.schema.embeddings import Embeddings
 from langchain_community.vectorstores.faiss import FAISS
 from langchain_community.vectorstores.utils import DistanceStrategy
+
+from rasa.dialogue_understanding.generator.constants import (
+    TRAINED_EMBEDDINGS_CONFIG_KEY,
+)
 from rasa.engine.storage.resource import Resource
 from rasa.engine.storage.storage import ModelStorage
+
 from rasa.shared.constants import (
     EMBEDDINGS_CONFIG_KEY,
     PROVIDER_CONFIG_KEY,
@@ -37,9 +41,9 @@ from rasa.shared.constants import (
 from rasa.shared.core.domain import Domain
 from rasa.shared.core.flows import FlowsList
 from rasa.shared.core.trackers import DialogueStateTracker
+from rasa.shared.exceptions import ProviderClientAPIException, FileIOException
 from rasa.shared.nlu.constants import TEXT, FLOWS_FROM_SEMANTIC_SEARCH
 from rasa.shared.nlu.training_data.message import Message
-from rasa.shared.exceptions import ProviderClientAPIException
 from rasa.shared.providers.embedding._langchain_embedding_client_adapter import (
     _LangchainEmbeddingClientAdapter,
 )
@@ -50,12 +54,20 @@ from rasa.shared.utils.llm import (
     USER,
     get_prompt_template,
     allowed_values_for_slot,
+    resolve_model_client_config,
+)
+from rasa.shared.utils.health_check import (
+    perform_training_time_embeddings_health_check,
+    perform_inference_time_embeddings_health_check,
 )
+from rasa.shared.utils.io import dump_obj_as_json_to_file, read_json_file
 
 DEFAULT_FLOW_DOCUMENT_TEMPLATE = importlib.resources.read_text(
     "rasa.dialogue_understanding.generator", "flow_document_template.jinja2"
 )
 
+FLOW_RETRIEVAL_CONFIG_FILE_NAME = "flow_retrieval_config.json"
+
 DEFAULT_EMBEDDINGS_CONFIG = {
     PROVIDER_CONFIG_KEY: OPENAI_PROVIDER,
     "model": DEFAULT_OPENAI_EMBEDDING_MODEL_NAME,
@@ -82,6 +94,7 @@ class FlowRetrieval:
             MAX_FLOWS_FROM_SEMANTIC_SEARCH_KEY: DEFAULT_MAX_FLOWS_FROM_SEMANTIC_SEARCH,
             TURNS_TO_EMBED_KEY: DEFAULT_TURNS_TO_EMBED,
             SHOULD_EMBED_SLOTS_KEY: DEFAULT_SHOULD_EMBED_SLOTS,
+            TRAINED_EMBEDDINGS_CONFIG_KEY: None,
         }
 
     def __init__(
@@ -92,6 +105,9 @@ class FlowRetrieval:
     ):
         config = {**self.get_default_config(), **config}
         self.config = self.validate_config(config)
+        self.config[EMBEDDINGS_CONFIG_KEY] = resolve_model_client_config(
+            self.config.get(EMBEDDINGS_CONFIG_KEY), FlowRetrieval.__name__
+        )
         self.vector_store: Optional[FAISS] = None
         self.flow_document_template = get_prompt_template(
             None, DEFAULT_FLOW_DOCUMENT_TEMPLATE
@@ -131,6 +147,16 @@ class FlowRetrieval:
 
         return config
 
+    def train(self) -> None:
+        self.config[TRAINED_EMBEDDINGS_CONFIG_KEY] = (
+            perform_training_time_embeddings_health_check(
+                self.config.get(EMBEDDINGS_CONFIG_KEY),
+                DEFAULT_EMBEDDINGS_CONFIG,
+                "flow_retrieval.train",
+                FlowRetrieval.__name__,
+            )
+        )
+
     @classmethod
     def load(
         cls,
@@ -147,6 +173,31 @@ class FlowRetrieval:
             flow_retrieval.config, model_storage, resource
         )
         flow_retrieval.vector_store = vector_store
+
+        persisted_config = None
+        try:
+            with model_storage.read_from(resource) as path:
+                persisted_config = read_json_file(
+                    path / FLOW_RETRIEVAL_CONFIG_FILE_NAME
+                )
+        except (FileNotFoundError, FileIOException) as e:
+            structlogger.warning(
+                "flow_retrieval.load.failed", error=e, resource=resource.name
+            )
+
+        train_embeddings_name = (
+            persisted_config.get(TRAINED_EMBEDDINGS_CONFIG_KEY, None)
+            if persisted_config
+            else None
+        )
+        perform_inference_time_embeddings_health_check(
+            flow_retrieval.config.get(EMBEDDINGS_CONFIG_KEY),
+            DEFAULT_EMBEDDINGS_CONFIG,
+            train_embeddings_name,
+            "flow_retrieval.load",
+            FlowRetrieval.__name__,
+        )
+
         return flow_retrieval
 
     @classmethod
@@ -178,13 +229,24 @@ class FlowRetrieval:
         Returns:
             The embedder.
         """
+        # Copy the config so original config is not modified
+        config = config.copy()
+        # Resolve config and instantiate the embedding client
+        config[EMBEDDINGS_CONFIG_KEY] = resolve_model_client_config(
+            config.get(EMBEDDINGS_CONFIG_KEY), FlowRetrieval.__name__
+        )
         client = embedder_factory(
             config.get(EMBEDDINGS_CONFIG_KEY), DEFAULT_EMBEDDINGS_CONFIG
         )
+        # Wrap the embedding client in the adapter
         return _LangchainEmbeddingClientAdapter(client)
 
     def persist(self) -> None:
         self._persist_vector_store()
+        with self._model_storage.write_to(self._resource) as path:
+            dump_obj_as_json_to_file(
+                path / FLOW_RETRIEVAL_CONFIG_FILE_NAME, self.config
+            )
 
     def _persist_vector_store(self) -> None:
         """Persists the FAISS vector store."""

rasa/dialogue_understanding/generator/llm_based_command_generator.py

@@ -2,7 +2,6 @@ from abc import ABC, abstractmethod
 from functools import lru_cache
 from typing import Dict, Any, List, Optional, Tuple, Union, Text
 
-import os
 import structlog
 from jinja2 import Template
 
@@ -17,13 +16,14 @@ from rasa.dialogue_understanding.generator.constants import (
     LLM_CONFIG_KEY,
     FLOW_RETRIEVAL_KEY,
     FLOW_RETRIEVAL_ACTIVE_KEY,
+    FLOW_RETRIEVAL_FLOW_THRESHOLD,
+    TRAINED_MODEL_NAME_CONFIG_KEY,
 )
 from rasa.dialogue_understanding.generator.flow_retrieval import FlowRetrieval
 from rasa.engine.graph import GraphComponent, ExecutionContext
 from rasa.engine.recipes.default_recipe import DefaultV1Recipe
 from rasa.engine.storage.resource import Resource
 from rasa.engine.storage.storage import ModelStorage
-from rasa.shared.constants import LLM_API_HEALTH_CHECK_ENV_VAR
 from rasa.shared.core.domain import Domain
 from rasa.shared.core.flows import FlowStep, Flow, FlowsList
 from rasa.shared.core.flows.steps.collect import CollectInformationFlowStep
@@ -35,15 +35,18 @@ from rasa.shared.nlu.training_data.message import Message
 from rasa.shared.nlu.training_data.training_data import TrainingData
 from rasa.shared.utils.llm import (
     allowed_values_for_slot,
-    llm_api_health_check,
     llm_factory,
-    try_instantiate_llm_client,
+    resolve_model_client_config,
 )
+from rasa.shared.utils.health_check import perform_training_time_llm_health_check
 from rasa.utils.log_utils import log_llm
 
 structlogger = structlog.get_logger()
 
 
+LLM_BASED_COMMAND_GENERATOR_CONFIG_FILE = "config.json"
+
+
 @DefaultV1Recipe.register(
     [
         DefaultV1Recipe.ComponentType.COMMAND_GENERATOR,
@@ -64,6 +67,9 @@ class LLMBasedCommandGenerator(GraphComponent, CommandGenerator, ABC):
     ) -> None:
         super().__init__(config)
         self.config = {**self.get_default_config(), **config}
+        self.config[LLM_CONFIG_KEY] = resolve_model_client_config(
+            self.config.get(LLM_CONFIG_KEY), LLMBasedCommandGenerator.__name__
+        )
         self._model_storage = model_storage
         self._resource = resource
         self.flow_retrieval: Optional[FlowRetrieval]
@@ -73,17 +79,9 @@ class LLMBasedCommandGenerator(GraphComponent, CommandGenerator, ABC):
                 self.config[FLOW_RETRIEVAL_KEY], model_storage, resource
             )
             structlogger.info("llm_based_command_generator.flow_retrieval.enabled")
+            self.config[FLOW_RETRIEVAL_KEY] = self.flow_retrieval.config
         else:
             self.flow_retrieval = None
-            structlogger.warn(
-                "llm_based_command_generator.flow_retrieval.disabled",
-                event_info=(
-                    "Disabling flow retrieval can cause issues when there are a "
-                    "large number of flows to be included in the prompt. For more"
-                    "information see:\n"
-                    "https://rasa.com/docs/rasa-pro/concepts/dialogue-understanding#how-the-llmcommandgenerator-works"
-                ),
-            )
 
     ### Abstract methods
     @staticmethod
@@ -108,7 +106,11 @@ class LLMBasedCommandGenerator(GraphComponent, CommandGenerator, ABC):
     @abstractmethod
     def persist(self) -> None:
         """Persist the component to disk for future loading."""
-        pass
+        # persist the config to store the resolved llm and embedding config
+        with self._model_storage.write_to(self._resource) as path:
+            rasa.shared.utils.io.dump_obj_as_json_to_file(
+                path / LLM_BASED_COMMAND_GENERATOR_CONFIG_FILE, self.config
+            )
 
     @abstractmethod
     async def predict_commands(
@@ -171,19 +173,35 @@ class LLMBasedCommandGenerator(GraphComponent, CommandGenerator, ABC):
         """Train the llm based command generator. Stores all flows into a vector
         store.
         """
-        # Validate llm configuration
-        llm_client = try_instantiate_llm_client(
-            self.config.get(LLM_CONFIG_KEY),
-            DEFAULT_LLM_CONFIG,
-            "llm_based_command_generator.train",
-            LLMBasedCommandGenerator.__name__,
-        )
-        if os.getenv(LLM_API_HEALTH_CHECK_ENV_VAR, "true").lower() == "true":
-            llm_api_health_check(
-                llm_client,
+        self.config[TRAINED_MODEL_NAME_CONFIG_KEY] = (
+            perform_training_time_llm_health_check(
+                self.config.get(LLM_CONFIG_KEY),
+                DEFAULT_LLM_CONFIG,
                 "llm_based_command_generator.train",
                 LLMBasedCommandGenerator.__name__,
             )
+        )
+
+        if (
+            self.flow_retrieval is None
+            and len(flows.user_flows) > FLOW_RETRIEVAL_FLOW_THRESHOLD
+        ):
+            structlogger.warn(
+                "llm_based_command_generator.flow_retrieval.disabled",
+                event_info=(
+                    f"You have {len(flows.user_flows)} user flows but flow "
+                    f"retrieval is disabled. "
+                    f"It is recommended to enable flow retrieval if the "
+                    f"total number of user flows exceed "
+                    f"{FLOW_RETRIEVAL_FLOW_THRESHOLD}. "
+                    f"Keeping it disabled can result in deterioration of "
+                    f"command generator's functional "
+                    f"performance and higher costs because of increased "
+                    f"number of tokens in the prompt. For more"
+                    "information see:\n"
+                    "https://rasa.com/docs/rasa-pro/concepts/dialogue-understanding#how-the-llmcommandgenerator-works"
+                ),
+            )
 
         # flow retrieval is populated with only user-defined flows
         try:
@@ -192,10 +210,12 @@ class LLMBasedCommandGenerator(GraphComponent, CommandGenerator, ABC):
         except Exception as e:
             structlogger.error(
                 "llm_based_command_generator.train.failed",
-                event_info=("Flow retrieval store isinaccessible."),
+                event_info="Flow retrieval store isinaccessible.",
                 error=e,
             )
             raise
+        if self.flow_retrieval is not None:
+            self.flow_retrieval.train()
         self.persist()
         return self._resource
 
@@ -231,9 +251,31 @@ class LLMBasedCommandGenerator(GraphComponent, CommandGenerator, ABC):
             )
         return None
 
+    @classmethod
+    def load_config_from_model_storage(
+        cls,
+        model_storage: ModelStorage,
+        resource: Resource,
+    ) -> Optional[Text]:
+        try:
+            with model_storage.read_from(resource) as path:
+                return rasa.shared.utils.io.read_json_file(
+                    path / LLM_BASED_COMMAND_GENERATOR_CONFIG_FILE
+                )
+        except (FileNotFoundError, FileIOException) as e:
+            structlogger.warning(
+                "llm_based_command_generator.load_config.failed",
+                error=e,
+                resource=resource.name,
+            )
+        return None
+
     @classmethod
     def load_flow_retrival(
-        cls, config: Dict[Text, Any], model_storage: ModelStorage, resource: Resource
+        cls,
+        config: Dict[str, Any],
+        model_storage: ModelStorage,
+        resource: Resource,
     ) -> Optional[FlowRetrieval]:
         """Load the FlowRetrieval component if it is enabled in the configuration."""
         enable_flow_retrieval = config.get(FLOW_RETRIEVAL_KEY, {}).get(