rasa-pro 3.13.0.dev20250612__py3-none-any.whl → 3.13.0.dev20250613__py3-none-any.whl

rasa/llm_fine_tuning/paraphrasing_module.py

@@ -1,11 +1,9 @@
-from typing import Any, Dict, List, Tuple, Type
+from typing import Any, Dict, List, Tuple
 
 import structlog
 from tqdm import tqdm
 
-from rasa.dialogue_understanding.generator.llm_based_command_generator import (
-    LLMBasedCommandGenerator,
-)
+from rasa.core.agent import Agent
 from rasa.llm_fine_tuning.conversations import Conversation
 from rasa.llm_fine_tuning.paraphrasing.conversation_rephraser import (
     ConversationRephraser,
@@ -28,8 +26,7 @@ async def create_paraphrased_conversations(
     rephrase_config: Dict[str, Any],
     num_rephrases: int,
     flows: FlowsList,
-    llm_command_generator: Type[LLMBasedCommandGenerator],
-    llm_command_generator_config: Dict[str, Any],
+    agent: Agent,
     storage_context: StorageContext,
 ) -> Tuple[List[Conversation], Dict[str, Any]]:
     """Create paraphrased conversations.
@@ -42,7 +39,7 @@ async def create_paraphrased_conversations(
         rephrase_config: The path to the rephrase configuration file.
         num_rephrases: The number of rephrases to produce per user message.
         flows: All flows.
-        llm_command_generator_config: The configuration of the trained model.
+        agent: The Rasa agent.
         storage_context: The storage context.
 
     Returns:
@@ -50,7 +47,7 @@ async def create_paraphrased_conversations(
         rephrasing.
     """
     rephraser = ConversationRephraser(rephrase_config)
-    validator = RephraseValidator(llm_command_generator_config, flows)
+    validator = RephraseValidator(flows)
 
     if num_rephrases <= 0:
         structlogger.info(
@@ -64,18 +61,19 @@ async def create_paraphrased_conversations(
     rephrased_conversations: List[Conversation] = []
     for i in tqdm(range(len(conversations))):
         current_conversation = conversations[i]
-
         try:
             # rephrase all user messages even if rephrase=False is set
             # to not confuse the LLM and get valid output
             rephrasings = await rephraser.rephrase_conversation(
-                conversations[i], num_rephrases
+                current_conversation, num_rephrases
             )
             # filter out the rephrasings for user messages that have rephrase=False set
-            rephrasings = _filter_rephrasings(rephrasings, conversations[i])
+            rephrasings = _filter_rephrasings(rephrasings, current_conversation)
             # check if the rephrasings are still producing the same commands
             rephrasings = await validator.validate_rephrasings(
-                rephrasings, current_conversation, llm_command_generator
+                agent,
+                rephrasings,
+                current_conversation,
             )
         except ProviderClientAPIException as e:
             structlogger.error(

rasa/llm_fine_tuning/storage.py

@@ -96,9 +96,9 @@ class FileStorageStrategy(StorageStrategy):
         file_path = self._get_file_path(storage_location)
         self._create_output_dir(file_path)
 
-        with open(str(file_path), "w") as outfile:
+        with open(str(file_path), "w", encoding="utf-8") as outfile:
             for example in llm_data:
-                json.dump(example.as_dict(), outfile)
+                json.dump(example.as_dict(), outfile, ensure_ascii=False)
                 outfile.write("\n")
 
     def write_formatted_finetuning_data(
@@ -110,9 +110,9 @@ class FileStorageStrategy(StorageStrategy):
         file_path = self._get_file_path(module_storage_location, file_name)
         self._create_output_dir(file_path)
 
-        with open(str(file_path), "w") as file:
+        with open(str(file_path), "w", encoding="utf-8") as file:
             for example in formatted_data:
-                json.dump(example.as_dict(), file)
+                json.dump(example.as_dict(), file, ensure_ascii=False)
                 file.write("\n")
 
     def write_e2e_test_suite_to_yaml_file(

rasa/llm_fine_tuning/utils.py

@@ -1,7 +1,69 @@
-from typing import List
+from contextlib import contextmanager
+from datetime import datetime
+from typing import Callable, Generator, List, Union
+
+import structlog
 
 from rasa.dialogue_understanding.commands.prompt_command import PromptCommand
+from rasa.dialogue_understanding.generator import LLMBasedCommandGenerator
+from rasa.shared.providers.llm.llm_response import LLMResponse
+
+structlogger = structlog.get_logger()
 
 
 def commands_as_string(commands: List[PromptCommand], delimiter: str = "\n") -> str:
     return delimiter.join([command.to_dsl() for command in commands])
+
+
+def make_mock_invoke_llm(commands: str) -> Callable:
+    """Capture the `commands` in a closure so the resulting async function
+    can use it as its response.
+
+    Args:
+        commands: The commands to return from the mock LLM call.
+    """
+
+    async def _mock_invoke_llm(
+        self: LLMBasedCommandGenerator, prompt: Union[List[dict], List[str], str]
+    ) -> LLMResponse:
+        structlogger.debug(
+            f"LLM call intercepted, response mocked. "
+            f"Responding with the following commands: '{commands}' "
+            f"to the prompt: {prompt}"
+        )
+        fake_response_dict = {
+            "id": "",
+            "choices": [commands],
+            "created": int(datetime.now().timestamp()),
+            "model": "mocked-llm",
+        }
+        return LLMResponse.from_dict(fake_response_dict)
+
+    return _mock_invoke_llm
+
+
+@contextmanager
+def patch_invoke_llm_in_generators(mock_impl: Callable) -> Generator:
+    """Replace CommandGenerator.invoke_llm in the base class AND in all
+    current subclasses (recursively).  Everything is restored on exit.
+    """
+    originals = {}
+
+    def collect(cls: type[LLMBasedCommandGenerator]) -> None:
+        # store current attribute, then recurse
+        originals[cls] = cls.invoke_llm
+        for sub in cls.__subclasses__():
+            collect(sub)
+
+    # collect every existing subclass of CommandGenerator
+    collect(LLMBasedCommandGenerator)  # type: ignore[type-abstract]
+
+    try:
+        # apply the monkey-patch everywhere
+        for cls in originals:
+            cls.invoke_llm = mock_impl  # type: ignore[assignment]
+        yield
+    finally:
+        # restore originals (even if an exception happened)
+        for cls, orig in originals.items():
+            cls.invoke_llm = orig  # type: ignore[assignment]

rasa/model_manager/model_api.py

@@ -7,12 +7,16 @@ from typing import Any, Callable, Dict, Optional, Union
 import dotenv
 import psutil
 import structlog
+from ruamel.yaml import YAMLError
 from sanic import Blueprint, Sanic, response
 from sanic.exceptions import NotFound
 from sanic.request import Request
 from sanic.response import json
 from socketio import AsyncServer
 
+import rasa
+from rasa.cli.project_templates.defaults import get_rasa_defaults
+from rasa.cli.scaffold import ProjectTemplateName, scaffold_path
 from rasa.constants import MODEL_ARCHIVE_EXTENSION
 from rasa.exceptions import ModelNotFound
 from rasa.model_manager import config
@@ -45,6 +49,10 @@ from rasa.model_manager.warm_rasa_process import (
     initialize_warm_rasa_process,
     shutdown_warm_rasa_processes,
 )
+from rasa.server import ErrorResponse
+from rasa.shared.exceptions import InvalidConfigException
+from rasa.shared.utils.yaml import dump_obj_as_yaml_to_string
+from rasa.studio.upload import build_calm_import_parts
 
 dotenv.load_dotenv()
 
@@ -476,6 +484,86 @@ def internal_blueprint() -> Blueprint:
         except ModelNotFound:
             return response.raw(b"", status=404)
 
+    @bp.post("/defaults")
+    async def get_defaults(request: Request) -> response.HTTPResponse:
+        """Returns the system defaults like prompts, patterns, etc."""
+        body = request.json or {}
+        config_yaml = body.get("config")
+        if config_yaml is None:
+            exc = ErrorResponse(
+                HTTPStatus.BAD_REQUEST,
+                "BadRequest",
+                "Missing `config` key in request body.",
+            )
+            return response.json(exc.error_info, status=exc.status)
+
+        endpoints_yaml = body.get("endpoints")
+        if endpoints_yaml is None:
+            exc = ErrorResponse(
+                HTTPStatus.BAD_REQUEST,
+                "BadRequest",
+                "Missing `endpoints` key in request body.",
+            )
+            return response.json(exc.error_info, status=exc.status)
+
+        try:
+            defaults = get_rasa_defaults(config_yaml, endpoints_yaml)
+        except (YAMLError, InvalidConfigException) as e:
+            exc = ErrorResponse(
+                HTTPStatus.INTERNAL_SERVER_ERROR,
+                "InitDataError",
+                f"Failed to load defaults. Error: {e!s}",
+            )
+            return response.json(exc.error_info, status=exc.status)
+        return response.json(defaults.model_dump(exclude_none=True))
+
+    @bp.get("/project_template")
+    async def get_project_template(request: Request) -> response.HTTPResponse:
+        """Return initial project template data."""
+        template = request.args.get("template", ProjectTemplateName.DEFAULT.value)
+
+        try:
+            template_enum = ProjectTemplateName(template)
+        except ValueError:
+            valid_templates = ", ".join([t.value for t in ProjectTemplateName])
+            exc = ErrorResponse(
+                HTTPStatus.BAD_REQUEST,
+                "BadRequest",
+                f"Unknown template '{template}'. Valid templates: "
+                f"{valid_templates}",
+            )
+            return response.json(exc.error_info, status=exc.status)
+
+        template_dir = scaffold_path(template_enum)
+        if not os.path.isdir(template_dir):
+            exc = ErrorResponse(
+                HTTPStatus.INTERNAL_SERVER_ERROR,
+                "InitDataError",
+                f"Template directory '{template_dir}' not found.",
+            )
+            return response.json(exc.error_info, status=exc.status)
+
+        assistant_name, parts = build_calm_import_parts(
+            data_path=f"{template_dir}/data",
+            domain_path=f"{template_dir}/domain",
+            config_path=f"{template_dir}/config.yml",
+            endpoints_path=f"{template_dir}/endpoints.yml",
+            assistant_name=template_enum.value,
+        )
+
+        defaults = get_rasa_defaults(
+            config_yaml=dump_obj_as_yaml_to_string(parts.config),
+            endpoints_yaml=dump_obj_as_yaml_to_string(parts.endpoints),
+        )
+        return response.json(
+            {
+                **parts.model_dump(exclude_none=True),
+                "assistantName": assistant_name,
+                "defaults": defaults.model_dump(exclude_none=True),
+                "version": rasa.__version__,
+            }
+        )
+
     return bp
 
 

rasa/model_manager/trainer_service.py

@@ -7,7 +7,7 @@ from typing import Any, Dict, Optional
 import structlog
 from pydantic import BaseModel, ConfigDict
 
-from rasa.constants import MODEL_ARCHIVE_EXTENSION
+from rasa.constants import MODEL_ARCHIVE_EXTENSION, RASA_DIR_NAME
 from rasa.model_manager import config
 from rasa.model_manager.utils import (
     ensure_base_directory_exists,
@@ -171,7 +171,7 @@ def seed_training_directory_with_rasa_cache(
             training_base_path=training_base_path,
         )
         # copy the cache to the training directory
-        shutil.copytree(src=cache_path, dst=subpath(training_base_path, ".rasa"))
+        shutil.copytree(src=cache_path, dst=subpath(training_base_path, RASA_DIR_NAME))
 
 
 def persist_rasa_cache(assistant_id: str, training_base_path: str) -> None:
@@ -184,12 +184,12 @@ def persist_rasa_cache(assistant_id: str, training_base_path: str) -> None:
     cache_path = cache_for_assistant_path(assistant_id)
 
     # if the training failed and didn't create a cache, skip this step
-    if not os.path.exists(subpath(training_base_path, ".rasa")):
+    if not os.path.exists(subpath(training_base_path, RASA_DIR_NAME)):
         return
 
     # clean up the cache directory first
     shutil.rmtree(cache_path, ignore_errors=True)
-    shutil.copytree(src=subpath(training_base_path, ".rasa"), dst=cache_path)
+    shutil.copytree(src=subpath(training_base_path, RASA_DIR_NAME), dst=cache_path)
 
 
 def write_training_data_to_files(

rasa/plugin.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 import argparse
 import functools
 import sys
-from typing import TYPE_CHECKING, Any, List, Optional, Text, Union
+from typing import TYPE_CHECKING, List, Optional, Text, Union
 
 import pluggy
 
@@ -69,16 +69,6 @@ def create_tracker_store(  # type: ignore[empty-body]
     """Hook specification for wrapping with AuthRetryTrackerStore."""
 
 
-@hookspec(firstresult=True)  # type: ignore[misc]
-def init_anonymization_pipeline(endpoints_file: Optional[Text]) -> None:
-    """Hook specification for initialising the anonymization pipeline."""
-
-
-@hookspec(firstresult=True)  # type: ignore[misc]
-def get_anonymization_pipeline() -> Optional[Any]:
-    """Hook specification for getting the anonymization pipeline."""
-
-
 @hookspec  # type: ignore[misc]
 def after_server_stop() -> None:
     """Hook specification for stopping the server.

rasa/privacy/constants.py

@@ -0,0 +1,83 @@
+PRIVACY_CONFIG_SCHEMA = "privacy/privacy_config_schema.json"
+REDACTION_CHAR_KEY = "redaction_char"
+KEEP_LEFT_KEY = "keep_left"
+KEEP_RIGHT_KEY = "keep_right"
+DELETION_KEY = "deletion"
+ANONYMIZATION_KEY = "anonymization"
+TRACKER_STORE_SETTINGS = "tracker_store_settings"
+SLOT_KEY = "slot"
+TEXT_KEY = "text"
+ENTITIES_KEY = "entities"
+VALUE_KEY = "value"
+ENTITY_LABEL_KEY = "label"
+
+USER_CHAT_INACTIVITY_IN_MINUTES_ENV_VAR_NAME = "USER_CHAT_INACTIVITY_IN_MINUTES"
+GLINER_MODEL_PATH_ENV_VAR_NAME = "GLINER_MODEL_PATH"
+HUGGINGFACE_CACHE_DIR_ENV_VAR_NAME = "HUGGINGFACE_HUB_CACHE_DIR"
+
+DEFAULT_PII_MODEL = "urchade/gliner_multi_pii-v1"
+GLINER_LABELS = [
+    "person",
+    "organization",
+    "company",
+    "phone number",
+    "address",
+    "full address",
+    "postcode",
+    "zip code",
+    "passport number",
+    "email",
+    "credit card number",
+    "social security number",
+    "health insurance id number",
+    "date of birth",
+    "mobile phone number",
+    "bank account number",
+    "medication",
+    "cpf",
+    "driver's license number",
+    "tax identification number",
+    "medical condition",
+    "identity card number",
+    "national id number",
+    "ip address",
+    "email address",
+    "iban",
+    "credit card expiration date",
+    "username",
+    "health insurance number",
+    "registration number",
+    "student id number",
+    "insurance number",
+    "membership number",
+    "booking number",
+    "landline phone number",
+    "blood type",
+    "cvv",
+    "reservation number",
+    "digital signature",
+    "social media handle",
+    "license plate number",
+    "cnpj",
+    "postal code",
+    "passport_number",
+    "serial number",
+    "vehicle registration number",
+    "fax number",
+    "visa number",
+    "insurance company",
+    "identity document number",
+    "transaction number",
+    "national health insurance number",
+    "cvc",
+    "birth certificate number",
+    "train ticket number",
+    "passport expiration date",
+    "social_security_number",
+    "personally identifiable information",
+    "banking routing number",
+    "sort code",
+    "routing number",
+    "tax number",
+    "swift code",
+]

rasa/privacy/event_broker_utils.py

@@ -0,0 +1,77 @@
+from typing import TYPE_CHECKING, List, Optional
+
+import structlog
+
+from rasa.core.brokers.broker import EventBroker
+from rasa.utils.endpoints import EndpointConfig
+
+if TYPE_CHECKING:
+    from asyncio import AbstractEventLoop
+
+structlogger = structlog.get_logger(__name__)
+
+
+async def create_event_brokers(
+    event_broker_endpoint: Optional[EndpointConfig],
+    event_loop: Optional["AbstractEventLoop"] = None,
+) -> List[EventBroker]:
+    """Create EventBroker objects for each anonymization topic or queue."""
+    if event_broker_endpoint is None or event_broker_endpoint.type is None:
+        structlogger.debug(
+            "rasa.privacy_filtering.create_event_broker.no_event_broker_type",
+        )
+        return []
+
+    if event_broker_endpoint.type == "kafka":
+        event_collection = event_broker_endpoint.kwargs.get("anonymization_topics", [])
+        event_collection_type = "topic"
+    elif event_broker_endpoint.type == "pika":
+        event_collection = event_broker_endpoint.kwargs.get("anonymization_queues", [])
+        event_collection_type = "queues"
+    else:
+        structlogger.debug(
+            "rasa.privacy_filtering.create_event_broker.unsupported_event_broker_type",
+            event_broker_type=event_broker_endpoint.type,
+        )
+        return []
+
+    if not event_collection:
+        structlogger.debug(
+            f"rasa.privacy_filtering.create_event_broker.no_anonymization_{event_collection_type}",
+            event_collection_type=event_collection_type,
+        )
+        return []
+
+    return await _create_event_brokers(
+        event_broker_endpoint, event_collection, event_collection_type, event_loop
+    )
+
+
+async def _create_event_brokers(
+    event_broker_endpoint: EndpointConfig,
+    event_collection: List[str],
+    event_collection_type: str,
+    event_loop: Optional["AbstractEventLoop"] = None,
+) -> List[EventBroker]:
+    """Create event brokers."""
+    event_brokers = []
+    for item in event_collection:
+        event_broker_endpoint.kwargs[event_collection_type] = (
+            item if event_collection_type == "topic" else [item]
+        )
+        structlogger.debug(
+            "rasa.privacy_filtering.create_event_broker",
+            event_info=f"Setting anonymized event streaming to '{item}'.",
+        )
+
+        event_broker = await EventBroker.create(event_broker_endpoint, event_loop)
+        if event_broker is None:
+            structlogger.debug(
+                "rasa.privacy_filtering.create_event_broker.no_event_broker_created",
+                event_info=f"No event broker created for publishing to '{item}'.",
+            )
+            continue
+
+        event_brokers.append(event_broker)
+
+    return event_brokers