rasa-pro 3.9.18__py3-none-any.whl → 3.10.16__py3-none-any.whl

rasa/engine/recipes/default_components.py

@@ -12,7 +12,6 @@ from rasa.nlu.classifiers.keyword_intent_classifier import KeywordIntentClassifi
 from rasa.dialogue_understanding.generator import (
     LLMCommandGenerator,
 )
-from rasa.nlu.classifiers.llm_intent_classifier import LLMIntentClassifier
 from rasa.nlu.classifiers.logistic_regression_classifier import (
     LogisticRegressionClassifier,
 )
@@ -60,7 +59,6 @@ DEFAULT_COMPONENTS = [
     NLUCommandAdapter,
     LLMCommandGenerator,
     LLMBasedRouter,
-    LLMIntentClassifier,
     IntentBasedRouter,
     # Response Selectors
     ResponseSelector,

rasa/engine/recipes/default_recipe.py

@@ -405,7 +405,7 @@ class DefaultV1Recipe(Recipe):
             return {}
 
         def resolver_name_from_parameter(parameter: str) -> str:
-            # we got a couple special cases to handle wher the parameter name
+            # we got a couple special cases to handle where the parameter name
             # doesn't match the provider name
             if "training_trackers" == parameter:
                 return "training_tracker_provider"
@@ -597,7 +597,7 @@ class DefaultV1Recipe(Recipe):
             needs={"importer": "finetuning_validator"},
             uses=StoryGraphProvider,
             constructor_name="create",
-            fn="provide",
+            fn="provide_train",
             config={"exclusion_percentage": cli_parameters.get("exclusion_percentage")},
             is_input=True,
         )
@@ -882,6 +882,14 @@ class DefaultV1Recipe(Recipe):
             config={},
             resource=Resource("domain_provider"),
         )
+        predict_nodes["story_graph_provider"] = SchemaNode(
+            **DEFAULT_PREDICT_KWARGS,
+            needs={},
+            uses=StoryGraphProvider,
+            fn="provide_inference",
+            config={},
+            resource=Resource("story_graph_provider"),
+        )
         predict_nodes["flows_provider"] = SchemaNode(
             **DEFAULT_PREDICT_KWARGS,
             needs={},

rasa/engine/storage/local_model_storage.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import logging
 import shutil
 import sys
 import tempfile
@@ -8,19 +7,21 @@ import uuid
 from contextlib import contextmanager
 from datetime import datetime
 from pathlib import Path
-from tarsafe import TarSafe
 from typing import Generator, Optional, Text, Tuple, Union
 
-import rasa.utils.common
+import structlog
+from tarsafe import TarSafe
+
+import rasa.model
 import rasa.shared.utils.io
-from rasa.engine.storage.storage import ModelMetadata, ModelStorage
+import rasa.utils.common
 from rasa.engine.graph import GraphModelConfiguration
 from rasa.engine.storage.resource import Resource
+from rasa.engine.storage.storage import ModelMetadata, ModelStorage
 from rasa.exceptions import UnsupportedModelVersionError
 from rasa.shared.core.domain import Domain
-import rasa.model
 
-logger = logging.getLogger(__name__)
+structlogger = structlog.get_logger()
 
 # Paths within model archive
 MODEL_ARCHIVE_COMPONENTS_DIR = "components"
@@ -86,7 +87,13 @@ class LocalModelStorage(ModelStorage):
             cls._extract_archive_to_directory(
                 model_archive_path, temporary_directory_path
             )
-            logger.debug(f"Extracted model to '{temporary_directory_path}'.")
+            structlogger.debug(
+                "local_model_storage.from_model_archive",
+                event_info=(
+                    f"Extracted model '{model_archive_path}' to "
+                    f"'{temporary_directory_path}'."
+                ),
+            )
 
             cls._initialize_model_storage_from_model_archive(
                 temporary_directory_path, storage_path
@@ -143,6 +150,10 @@ class LocalModelStorage(ModelStorage):
         temporary_directory: Path, storage_path: Path
     ) -> None:
         for path in (temporary_directory / MODEL_ARCHIVE_COMPONENTS_DIR).glob("*"):
+            structlogger.debug(
+                "local_model_storage._initialize_model_storage_from_model_archive",
+                event_info=f"Moving '{path}' to '{storage_path}'.",
+            )
             shutil.move(str(path), str(storage_path))
 
     @staticmethod
@@ -156,7 +167,10 @@ class LocalModelStorage(ModelStorage):
     @contextmanager
     def write_to(self, resource: Resource) -> Generator[Path, None, None]:
         """Persists data for a resource (see parent class for full docstring)."""
-        logger.debug(f"Resource '{resource.name}' was requested for writing.")
+        structlogger.debug(
+            "local_model_storage.write_to.resource_write_requested",
+            event_info=f"Resource '{resource.name}' was requested for writing.",
+        )
         directory = self._directory_for_resource(resource)
 
         if not directory.exists():
@@ -164,7 +178,10 @@ class LocalModelStorage(ModelStorage):
 
         yield directory
 
-        logger.debug(f"Resource '{resource.name}' was persisted.")
+        structlogger.debug(
+            "local_model_storage.write_to.resource_persisted",
+            event_info=f"Resource '{resource.name}' was persisted.",
+        )
 
     def _directory_for_resource(self, resource: Resource) -> Path:
         return self._storage_path / resource.name
@@ -172,7 +189,10 @@ class LocalModelStorage(ModelStorage):
     @contextmanager
     def read_from(self, resource: Resource) -> Generator[Path, None, None]:
         """Provides the data of a `Resource` (see parent class for full docstring)."""
-        logger.debug(f"Resource '{resource.name}' was requested for reading.")
+        structlogger.debug(
+            "local_model_storage.read_from",
+            event_info=f"Resource '{resource.name}' was requested for reading.",
+        )
         directory = self._directory_for_resource(resource)
 
         if not directory.exists():
@@ -194,7 +214,12 @@ class LocalModelStorage(ModelStorage):
         domain: Domain,
     ) -> ModelMetadata:
         """Creates model package (see parent class for full docstring)."""
-        logger.debug(f"Start to created model package for path '{model_archive_path}'.")
+        structlogger.debug(
+            "local_model_storage.create_model_package.started",
+            event_info=(
+                f"Start to created model " f"package for path '{model_archive_path}'.",
+            ),
+        )
 
         with windows_safe_temporary_directory() as temp_dir:
             temporary_directory = Path(temp_dir)
@@ -215,7 +240,10 @@ class LocalModelStorage(ModelStorage):
             with TarSafe.open(model_archive_path, "w:gz") as tar:
                 tar.add(temporary_directory, arcname="")
 
-        logger.debug(f"Model package created in path '{model_archive_path}'.")
+        structlogger.debug(
+            "local_model_storage.create_model_package.finished",
+            event_info=f"Model package created in path '{model_archive_path}'.",
+        )
 
         return model_metadata
 

rasa/engine/validation.py

@@ -16,6 +16,7 @@ from typing import (
     Union,
     TypeVar,
     List,
+    Literal,
 )
 
 import structlog
@@ -34,6 +35,7 @@ from rasa.dialogue_understanding.coexistence.constants import (
 from rasa.dialogue_understanding.generator import (
     LLMBasedCommandGenerator,
 )
+from rasa.dialogue_understanding.generator.constants import FLOW_RETRIEVAL_KEY
 from rasa.dialogue_understanding.patterns.chitchat import FLOW_PATTERN_CHITCHAT
 from rasa.engine.constants import RESERVED_PLACEHOLDERS
 from rasa.engine.exceptions import GraphSchemaValidationException
@@ -47,7 +49,15 @@ from rasa.engine.graph import (
 from rasa.engine.storage.resource import Resource
 from rasa.engine.storage.storage import ModelStorage
 from rasa.engine.training.fingerprinting import Fingerprintable
-from rasa.shared.constants import DOCS_URL_GRAPH_COMPONENTS, ROUTE_TO_CALM_SLOT
+from rasa.shared.constants import (
+    DOCS_URL_GRAPH_COMPONENTS,
+    ROUTE_TO_CALM_SLOT,
+    API_TYPE_CONFIG_KEY,
+    VALID_PROVIDERS_FOR_API_TYPE_CONFIG_KEY,
+    PROVIDER_CONFIG_KEY,
+    LLM_CONFIG_KEY,
+    EMBEDDINGS_CONFIG_KEY,
+)
 from rasa.shared.core.constants import ACTION_RESET_ROUTING, ACTION_TRIGGER_CHITCHAT
 from rasa.shared.core.domain import Domain
 from rasa.shared.core.flows import FlowsList, Flow
@@ -871,3 +881,70 @@ def validate_command_generator_setup(
 ) -> None:
     schema = model_configuration.predict_schema
     validate_command_generator_exclusivity(schema)
+
+
+def validate_model_client_configuration_setup(config: Dict[str, Any]) -> None:
+    """Validates the model client configuration setup.
+
+    Validation fails, if
+    - the LLM/embeddings provider is defined using 'api_type' key for providers other
+    than 'openai' or 'azure'
+
+    Args:
+        config: The config dictionary
+    """
+    for outer_key in ["pipeline", "policies"]:
+        if outer_key not in config or config[outer_key] is None:
+            continue
+
+        for component_config in config[outer_key]:
+            for key in [LLM_CONFIG_KEY, EMBEDDINGS_CONFIG_KEY]:
+                validate_api_type_config_key_usage(component_config, key)
+
+            # as flow retrieval is not a component itself, we need to
+            # check it separately
+            if (
+                FLOW_RETRIEVAL_KEY in component_config
+                and EMBEDDINGS_CONFIG_KEY in component_config[FLOW_RETRIEVAL_KEY]
+            ):
+                validate_api_type_config_key_usage(
+                    component_config[FLOW_RETRIEVAL_KEY],
+                    EMBEDDINGS_CONFIG_KEY,
+                    component_config["name"] + "." + FLOW_RETRIEVAL_KEY,
+                )
+
+
+def validate_api_type_config_key_usage(
+    component_config: Dict[str, Any],
+    key: Literal["llm", "embeddings"],
+    component_name: Optional[str] = None,
+) -> None:
+    """Validate the LLM/embeddings configuration of a component.
+
+    Validation fails, if
+    - the LLM/embeddings provider is defined using 'api_type' key for providers other
+    than 'openai' or 'azure'
+
+    Args:
+        component_config: The config of the component
+        key: either 'llm' or 'embeddings'
+        component_name: the name of the component
+    """
+    if component_config is None or key not in component_config:
+        return
+
+    if API_TYPE_CONFIG_KEY in component_config[key]:
+        api_type = component_config[key][API_TYPE_CONFIG_KEY]
+        if api_type not in VALID_PROVIDERS_FOR_API_TYPE_CONFIG_KEY:
+            structlogger.error(
+                "validation.component.api_type_config_key_invalid",
+                event_info=(
+                    f"You specified '{API_TYPE_CONFIG_KEY}: {api_type}' for "
+                    f"'{component_name or component_config['name']}', which is not "
+                    f"allowed. "
+                    f"The '{API_TYPE_CONFIG_KEY}' key can only be used for the "
+                    f"following providers: {VALID_PROVIDERS_FOR_API_TYPE_CONFIG_KEY}. "
+                    f"For other providers, please use the '{PROVIDER_CONFIG_KEY}' key."
+                ),
+            )
+            sys.exit(1)

rasa/env.py

@@ -1,5 +1,14 @@
+# A central place to define all environment variables used by Rasa
+
 AUTH_TOKEN_ENV = "AUTH_TOKEN"
 JWT_SECRET_ENV = "JWT_SECRET"
 JWT_METHOD_ENV = "JWT_METHOD"
 DEFAULT_JWT_METHOD = "HS256"
 JWT_PRIVATE_KEY_ENV = "JWT_PRIVATE_KEY"
+
+REMOTE_STORAGE_PATH_ENV = "REMOTE_STORAGE_PATH"
+BUCKET_NAME_ENV = "BUCKET_NAME"
+AWS_ENDPOINT_URL_ENV = "AWS_ENDPOINT_URL"
+AZURE_CONTAINER_ENV = "AZURE_CONTAINER"
+AZURE_ACCOUNT_NAME_ENV = "AZURE_ACCOUNT_NAME"
+AZURE_ACCOUNT_KEY_ENV = "AZURE_ACCOUNT_KEY"

rasa/graph_components/providers/story_graph_provider.py

@@ -1,19 +1,37 @@
 from __future__ import annotations
-from typing import Dict, Text, Any
+from typing import Dict, Text, Any, List
 
 from rasa.engine.graph import GraphComponent, ExecutionContext
 from rasa.engine.storage.resource import Resource
 from rasa.engine.storage.storage import ModelStorage
-from rasa.shared.core.training_data.structures import StoryGraph
+from rasa.shared.core.training_data.structures import StoryGraph, StoryStep
 from rasa.shared.importers.importer import TrainingDataImporter
+from rasa.shared.core.training_data.story_writer.yaml_story_writer import (
+    YAMLStoryWriter,
+)
+from rasa.shared.core.training_data.story_reader.yaml_story_reader import (
+    YAMLStoryReader,
+)
+
+
+STORIES_PERSISTENCE_FILE_NAME = "stories.yml"
 
 
 class StoryGraphProvider(GraphComponent):
     """Provides the training data from stories."""
 
-    def __init__(self, config: Dict[Text, Any]) -> None:
+    def __init__(
+        self,
+        config: Dict[Text, Any],
+        model_storage: ModelStorage,
+        resource: Resource,
+        stories: StoryGraph = None,
+    ) -> None:
         """Creates provider from config."""
         self._config = config
+        self._model_storage = model_storage
+        self._resource = resource
+        self._stories = stories
 
     @staticmethod
     def get_default_config() -> Dict[Text, Any]:
@@ -29,9 +47,36 @@ class StoryGraphProvider(GraphComponent):
         execution_context: ExecutionContext,
     ) -> StoryGraphProvider:
         """Creates component (see parent class for full docstring)."""
-        return cls(config)
+        return cls(config, model_storage, resource)
 
-    def provide(self, importer: TrainingDataImporter) -> StoryGraph:
+    @classmethod
+    def load(
+        cls,
+        config: Dict[Text, Any],
+        model_storage: ModelStorage,
+        resource: Resource,
+        execution_context: ExecutionContext,
+        **kwargs: Any,
+    ) -> StoryGraphProvider:
+        """Creates provider using a persisted version of itself."""
+        with model_storage.read_from(resource) as resource_directory:
+            reader = YAMLStoryReader()
+            story_steps = reader.read_from_file(
+                resource_directory / STORIES_PERSISTENCE_FILE_NAME
+            )
+        stories = StoryGraph(story_steps)
+        return cls(config, model_storage, resource, stories)
+
+    def _persist(self, story_steps: List[StoryStep]) -> None:
+        """Persists flows to model storage."""
+        with self._model_storage.write_to(self._resource) as resource_directory:
+            writer = YAMLStoryWriter()
+            writer.dump(
+                resource_directory / STORIES_PERSISTENCE_FILE_NAME,
+                story_steps,
+            )
+
+    def provide_train(self, importer: TrainingDataImporter) -> StoryGraph:
         """Provides the story graph from the training data.
 
         Args:
@@ -40,4 +85,12 @@ class StoryGraphProvider(GraphComponent):
         Returns:
             The story graph containing stories and rules used for training.
         """
-        return importer.get_stories(**self._config)
+        stories = importer.get_stories(**self._config)
+        self._persist(stories.story_steps)
+        return stories
+
+    def provide_inference(self) -> StoryGraph:
+        """Provides the stories configuration during inference."""
+        if self._stories is None:
+            self._stories = StoryGraph([])
+        return self._stories

rasa/llm_fine_tuning/annotation_module.py

@@ -0,0 +1,241 @@
+import asyncio
+from contextlib import contextmanager
+from typing import List, Generator, Optional, Tuple, Union
+
+import structlog
+
+from rasa.dialogue_understanding.commands import Command
+from rasa.e2e_test.e2e_test_case import TestSuite, TestCase, ActualStepOutput, TestStep
+from rasa.e2e_test.e2e_test_runner import E2ETestRunner, TEST_TURNS_TYPE
+from rasa.llm_fine_tuning.conversations import Conversation, ConversationStep
+from rasa.llm_fine_tuning.storage import StorageContext
+from rasa.shared.core.constants import USER
+from rasa.shared.core.trackers import DialogueStateTracker
+from rasa.shared.nlu.constants import LLM_PROMPT, LLM_COMMANDS
+from rasa.shared.utils.llm import tracker_as_readable_transcript
+
+ANNOTATION_MODULE_STORAGE_LOCATION = "1_command_annotations"
+
+preparing_fine_tuning_data = False
+
+structlogger = structlog.get_logger()
+
+
+@contextmanager
+def set_preparing_fine_tuning_data() -> Generator:
+    global preparing_fine_tuning_data
+    preparing_fine_tuning_data = True
+    try:
+        yield
+    finally:
+        preparing_fine_tuning_data = False
+
+
+def annotate_e2e_tests(
+    e2e_test_runner: E2ETestRunner,
+    test_suite: TestSuite,
+    storage_context: StorageContext,
+) -> List[Conversation]:
+    with set_preparing_fine_tuning_data():
+        converations = asyncio.run(
+            e2e_test_runner.run_tests_for_fine_tuning(
+                test_suite.test_cases,
+                test_suite.fixtures,
+                test_suite.metadata,
+            )
+        )
+
+    storage_context.write_conversations(
+        converations, ANNOTATION_MODULE_STORAGE_LOCATION
+    )
+
+    return converations
+
+
+def _get_previous_actual_step_output(
+    test_turns: TEST_TURNS_TYPE, i: int
+) -> Optional[ActualStepOutput]:
+    while i > 0:
+        i = i - 1
+        if isinstance(test_turns[i], ActualStepOutput):
+            return test_turns[i]  # type:ignore[return-value]
+    return None
+
+
+def generate_conversation(
+    test_turns: TEST_TURNS_TYPE,
+    test_case: TestCase,
+    tracker: DialogueStateTracker,
+    assertions_used: bool = False,
+) -> Optional[Conversation]:
+    """Generates a conversation object in case of e2e test passing.
+
+    Args:
+        test_turns: the turns that happened when running the test case or test step.
+        test_case: the `TestCase` instance.
+        tracker: the dialogue state tracker.
+        assertions_used: if True the e2e test format with assertions was used.
+
+    Returns:
+        Conversation.
+    """
+    steps = []
+
+    if assertions_used:
+        # we only have user steps, extract the bot response from the bot uttered
+        # events of the test turn
+        for i, original_step in enumerate(test_case.steps):
+            previous_turn = _get_previous_actual_step_output(test_turns, i)
+            steps.append(
+                _convert_to_conversation_step(
+                    original_step, test_turns[i], test_case.name, previous_turn
+                )
+            )
+            steps.extend(_create_bot_test_steps(test_turns[i]))
+    else:
+        for i, original_step in enumerate(test_case.steps):
+            if original_step.actor == USER:
+                previous_turn = _get_previous_actual_step_output(test_turns, i)
+                steps.append(
+                    _convert_to_conversation_step(
+                        original_step, test_turns[i], test_case.name, previous_turn
+                    )
+                )
+            else:
+                steps.append(original_step)
+
+    # Some messages in an e2e test case could be mapped to commands via
+    # 'NLUCommandAdapter', e.g. the message will not be annotated with a prompt and
+    # commands pair. Only convert steps that have a prompt and commands present into a
+    # ConversationStep.
+    # The conversation needs to have at least one 'ConversationStep' to be valid for
+    # fine-tuning.
+    if not any([isinstance(step, ConversationStep) for step in steps]):
+        structlogger.warning(
+            "annotation_module.skip_test_case.missing_llm_commands_and_prompts",
+            test_case=test_case.name,
+            file=test_case.file,
+        )
+        return None
+
+    transcript = tracker_as_readable_transcript(tracker, max_turns=None)
+
+    return Conversation(test_case.name, test_case, steps, transcript)
+
+
+def _create_bot_test_steps(current_turn: ActualStepOutput) -> List[TestStep]:
+    test_steps = []
+    for bot_event in current_turn.bot_uttered_events:
+        template = None
+        if "utter_action" in bot_event.metadata:
+            template = bot_event.metadata["utter_action"]
+
+        test_steps.append(TestStep(actor="bot", text=bot_event.text, template=template))
+
+    return test_steps
+
+
+def _convert_to_conversation_step(
+    current_step: TestStep,
+    current_turn: ActualStepOutput,
+    test_case_name: str,
+    previous_turn: Optional[ActualStepOutput],
+) -> Union[TestStep, ConversationStep]:
+    if not current_step.text == current_turn.text or not isinstance(
+        current_turn, ActualStepOutput
+    ):
+        # There should be a one to one mapping between test steps (steps read from file)
+        # and test turns (test result of e2e test). Verify that the current step is
+        # aligned with the current turn.
+        structlogger.debug(
+            "annotation_module.convert_to_conversation_step.skip_user_message",
+            test_case=test_case_name,
+            user_message=current_step.text,
+        )
+        return current_step
+
+    llm_prompt, llm_commands = _extract_llm_prompt_and_commands(current_turn)
+    if not llm_commands or not llm_prompt:
+        # If no commands or no prompt is present we cannot create a data point
+        # for fine-tuning, skipping this step.
+        structlogger.debug(
+            "annotation_module.convert_to_conversation_step.skip_user_message",
+            test_case=test_case_name,
+            user_message=current_step.text,
+            message="No commands/prompt associated with the message.",
+        )
+        return current_step
+
+    commands = [Command.command_from_json(data) for data in llm_commands]
+    rephrase = _should_be_rephrased(current_turn.text, previous_turn, test_case_name)
+
+    return ConversationStep(current_step, commands, llm_prompt, rephrase=rephrase)
+
+
+def _should_be_rephrased(
+    current_user_message: str,
+    previous_turn: Optional[ActualStepOutput],
+    test_case_name: str,
+) -> bool:
+    """Checks if the current user message should be rephrased or not.
+
+    A user message should not be rephrased in case the user message comes from a button
+    payload, i.e. the user clicked on a button.
+
+    Args:
+        current_user_message: The current user message.
+        previous_turn: The previous turn containing the bot uttered event that came
+            before.
+        test_case_name: The name of the test case.
+
+    Returns:
+        True, in case the user message should be rephrased, False otherwise.
+    """
+    # there is no previous turn, we are at the beginning of the conversation
+    if not previous_turn:
+        return True
+
+    buttons_present = (
+        previous_turn.bot_uttered_events
+        and "buttons" in previous_turn.bot_uttered_events[-1].data
+        and previous_turn.bot_uttered_events[-1].data["buttons"] is not None
+    )
+
+    if not buttons_present:
+        return True
+
+    # if the user utterance comes from a button payload we should not rephrase
+    # the user utterance in later steps
+    button_data = previous_turn.bot_uttered_events[-1].data["buttons"]
+    button_payloads = [data["payload"].lower() for data in button_data]
+    if current_user_message.lower() in button_payloads:
+        structlogger.debug(
+            "annotation_module.user_message_should_not_be_rephrased",
+            rephrase=False,
+            user_message=current_user_message,
+            test_case_name=test_case_name,
+        )
+        return False
+
+    return True
+
+
+def _extract_llm_prompt_and_commands(
+    turn: ActualStepOutput,
+) -> Tuple[Optional[str], Optional[str]]:
+    # There should be exactly one 'UserUttered' event
+    if not turn.user_uttered_events or len(turn.user_uttered_events) != 1:
+        return None, None
+
+    # Check if 'parse_data' contains the prompt and the commands
+    if (
+        not turn.user_uttered_events[0].parse_data
+        or LLM_PROMPT not in turn.user_uttered_events[0].parse_data
+        or LLM_COMMANDS not in turn.user_uttered_events[0].parse_data
+    ):
+        return None, None
+
+    return (
+        turn.user_uttered_events[0].parse_data[LLM_PROMPT],
+        turn.user_uttered_events[0].parse_data[LLM_COMMANDS],
+    )