rasa-pro 3.14.0.dev20250901__py3-none-any.whl → 3.14.0.dev20250922__py3-none-any.whl

rasa/builder/jobs.py

@@ -16,10 +16,13 @@ from rasa.builder.models import (
     JobStatusEvent,
 )
 from rasa.builder.project_generator import ProjectGenerator
-from rasa.builder.training_service import train_and_load_agent
+from rasa.builder.training_service import (
+    train_and_load_agent,
+    try_load_existing_agent,
+    update_agent,
+)
 from rasa.builder.validation_service import validate_project
 from rasa.cli.scaffold import ProjectTemplateName
-from rasa.core.channels.studio_chat import StudioChatInput
 
 structlogger = structlog.get_logger()
 
@@ -45,7 +48,6 @@ async def run_prompt_to_bot_job(
         prompt: The natural language prompt for bot generation.
     """
     project_generator: ProjectGenerator = app.ctx.project_generator
-    input_channel: StudioChatInput = app.ctx.input_channel
 
     await push_job_status_event(job, JobStatus.received)
 
@@ -60,10 +62,8 @@ async def run_prompt_to_bot_job(
 
         # 2. Training
         await push_job_status_event(job, JobStatus.training)
-        app.ctx.agent = await train_and_load_agent(
-            project_generator.get_training_input()
-        )
-        input_channel.agent = app.ctx.agent
+        agent = await train_and_load_agent(project_generator.get_training_input())
+        update_agent(agent, app)
         await push_job_status_event(job, JobStatus.train_success)
 
         structlogger.info(
@@ -128,8 +128,7 @@ async def run_template_to_bot_job(
         job: The job information instance.
         template_name: The name of the template to use for bot generation.
     """
-    project_generator = app.ctx.project_generator
-    input_channel = app.ctx.input_channel
+    project_generator: ProjectGenerator = app.ctx.project_generator
 
     await push_job_status_event(job, JobStatus.received)
 
@@ -142,10 +141,14 @@ async def run_template_to_bot_job(
 
         # 2) Training
         await push_job_status_event(job, JobStatus.training)
-        app.ctx.agent = await train_and_load_agent(
-            project_generator.get_training_input()
-        )
-        input_channel.agent = app.ctx.agent
+        agent = await try_load_existing_agent(project_generator.project_folder)
+        if agent is None:
+            agent = await train_and_load_agent(project_generator.get_training_input())
+        else:
+            structlogger.info(
+                "bot_builder_service.template_to_bot.agent_loaded_from_cache",
+            )
+        update_agent(agent, app)
         await push_job_status_event(job, JobStatus.train_success)
 
         # 3) Done
@@ -204,17 +207,26 @@ async def run_template_to_bot_job(
         job_manager.mark_done(job, error=str(exc))
 
 
-async def run_update_files_job(
+async def run_replace_all_files_job(
     app: "Sanic",
     job: JobInfo,
     bot_files: dict,
 ) -> None:
+    """Run the replace-all-files job in the background.
+
+    This replaces all bot files with the provided files and deletes any files
+    not included in the request (excluding .rasa/ and models/ directories).
+
+    Args:
+        app: The Sanic application instance.
+        job: The job information instance.
+        bot_files: Dictionary of file names to content for replacement.
+    """
     project_generator = app.ctx.project_generator
-    input_channel = app.ctx.input_channel
     await push_job_status_event(job, JobStatus.received)
 
     try:
-        project_generator.update_bot_files(bot_files)
+        project_generator.replace_all_bot_files(bot_files)
 
         # 1. Validating
         await push_job_status_event(job, JobStatus.validating)
@@ -226,8 +238,8 @@ async def run_update_files_job(
 
         # 2. Training
         await push_job_status_event(job, JobStatus.training)
-        app.ctx.agent = await train_and_load_agent(training_input)
-        input_channel.agent = app.ctx.agent
+        agent = await train_and_load_agent(training_input)
+        update_agent(agent, app)
         await push_job_status_event(job, JobStatus.train_success)
 
         await push_job_status_event(job, JobStatus.done)
@@ -238,7 +250,7 @@ async def run_update_files_job(
         if config.VALIDATION_FAIL_ON_WARNINGS:
             log_levels.append("warning")
         structlogger.debug(
-            "update_files_job.validation_error",
+            "replace_all_files_job.validation_error",
             job_id=job.id,
             error=str(exc),
             validation_logs=exc.validation_logs,
@@ -252,7 +264,7 @@ async def run_update_files_job(
 
     except TrainingError as exc:
         structlogger.debug(
-            "update_files_job.train_error",
+            "replace_all_files_job.train_error",
             job_id=job.id,
             error=str(exc),
         )
@@ -262,7 +274,7 @@ async def run_update_files_job(
     except Exception as exc:
         # Capture full traceback for anything truly unexpected
         structlogger.exception(
-            "update_files_job.unexpected_error",
+            "replace_all_files_job.unexpected_error",
             job_id=job.id,
             error=str(exc),
         )

rasa/builder/main.py

@@ -22,10 +22,9 @@ from rasa.builder.logging_utils import (
     log_request_start,
 )
 from rasa.builder.service import bp, setup_project_generator
-from rasa.core.agent import Agent, load_agent
-from rasa.core.available_endpoints import AvailableEndpoints
+from rasa.builder.training_service import try_load_existing_agent, update_agent
 from rasa.core.channels.studio_chat import StudioChatInput
-from rasa.model import get_latest_model
+from rasa.model_manager.warm_rasa_process import warmup
 from rasa.server import configure_cors
 from rasa.utils.common import configure_logging_and_warnings
 from rasa.utils.log_utils import configure_structlog
@@ -60,63 +59,6 @@ def setup_input_channel() -> StudioChatInput:
     return StudioChatInput.from_credentials(credentials=studio_chat_credentials)
 
 
-async def try_load_existing_agent(project_folder: str) -> Optional[Agent]:
-    """Try to load an existing agent from the project's models directory.
-
-    Args:
-        project_folder: Path to the project folder
-
-    Returns:
-        Loaded Agent instance if successful, None otherwise
-    """
-    models_dir = os.path.join(project_folder, "models")
-
-    if not os.path.exists(models_dir) or not os.path.isdir(models_dir):
-        structlogger.debug("No models directory found", models_dir=models_dir)
-        return None
-
-    try:
-        # Find the latest model in the models directory
-        latest_model_path = get_latest_model(models_dir)
-        if not latest_model_path:
-            structlogger.debug(
-                "No models found in models directory", models_dir=models_dir
-            )
-            return None
-
-        structlogger.info(
-            "Found existing model, attempting to load", model_path=latest_model_path
-        )
-
-        # Get available endpoints for agent loading
-        available_endpoints = AvailableEndpoints.get_instance()
-
-        # Load the agent
-        agent = await load_agent(
-            model_path=latest_model_path, endpoints=available_endpoints
-        )
-
-        if agent and agent.is_ready():
-            structlogger.info(
-                "Successfully loaded existing agent", model_path=latest_model_path
-            )
-            return agent
-        else:
-            structlogger.warning(
-                "Agent loaded but not ready", model_path=latest_model_path
-            )
-            return None
-
-    except Exception as e:
-        structlogger.warning(
-            "Failed to load existing agent",
-            models_dir=models_dir,
-            error=str(e),
-            exc_info=True,
-        )
-        return None
-
-
 def setup_middleware(app: Sanic) -> None:
     """Setup middleware for request/response processing."""
 
@@ -157,7 +99,6 @@ def create_app(project_folder: str) -> Sanic:
         use_authentication=app.config.USE_AUTHENTICATION,
         rasa_version=rasa.__version__,
     )
-    app.ctx.agent = None
 
     # Set up project generator and store in app context
     app.ctx.project_generator = setup_project_generator(project_folder)
@@ -165,6 +106,8 @@ def create_app(project_folder: str) -> Sanic:
     # Set up input channel and store in app context
     app.ctx.input_channel = setup_input_channel()
 
+    update_agent(None, app)
+
     # Register the blueprint
     app.blueprint(bp)
 
@@ -196,7 +139,7 @@ def create_app(project_folder: str) -> Sanic:
         try:
             existing_agent = await try_load_existing_agent(project_folder)
             if existing_agent:
-                app.ctx.agent = existing_agent
+                update_agent(existing_agent, app)
                 structlogger.info("Agent loaded on server startup")
             else:
                 structlogger.info(
@@ -208,6 +151,33 @@ def create_app(project_folder: str) -> Sanic:
     return app
 
 
+def _apply_llm_overrides_from_builder_env() -> None:
+    # Prefer a dedicated builder key, fall back to license if you proxy with it
+    if not config.HELLO_LLM_PROXY_BASE_URL:
+        return
+
+    structlogger.debug(
+        "builder.main.using_llm_proxy", base_url=config.HELLO_LLM_PROXY_BASE_URL
+    )
+
+    if not config.RASA_PRO_LICENSE:
+        structlogger.error(
+            "copilot.proxy_missing_license",
+            event_info=(
+                "HELLO_LLM_PROXY_BASE_URL is set but RASA_PRO_LICENSE is missing."
+            ),
+        )
+        return
+
+    if not os.getenv("OPENAI_API_BASE") and not os.getenv("OPENAI_API_KEY"):
+        base_url = config.HELLO_LLM_PROXY_BASE_URL.rstrip("/")
+        # needed for litellm client
+        os.environ["OPENAI_API_BASE"] = base_url
+        # needed for openai async client
+        os.environ["OPENAI_BASE_URL"] = base_url
+        os.environ["OPENAI_API_KEY"] = config.RASA_PRO_LICENSE
+
+
 def main(project_folder: Optional[str] = None) -> None:
     """Main entry point."""
     try:
@@ -218,6 +188,12 @@ def main(project_folder: Optional[str] = None) -> None:
         rasa.telemetry.initialize_telemetry()
         rasa.telemetry.initialize_error_reporting(private_mode=False)
 
+        _apply_llm_overrides_from_builder_env()
+
+        if config.HELLO_RASA_PROJECT_ID:
+            # ensures long import times for modules are ahead of time
+            warmup()
+
         # working directory needs to be the project folder, e.g.
         # for relative paths (./docs) in a projects config to work
         if not project_folder:

rasa/builder/models.py

@@ -6,7 +6,7 @@ from pathlib import Path
 from typing import Any, Dict, List, Literal, Optional
 
 import structlog
-from pydantic import BaseModel, Field, validator
+from pydantic import BaseModel, ConfigDict, Field, field_validator
 
 from rasa.cli.scaffold import ProjectTemplateName
 from rasa.shared.importers.importer import TrainingDataImporter
@@ -21,7 +21,8 @@ class PromptRequest(BaseModel):
         ..., min_length=1, max_length=10000, description="The skill description prompt"
     )
 
-    @validator("prompt")
+    @field_validator("prompt")
+    @classmethod
     def validate_prompt(cls, v: str) -> str:
         if not v.strip():
             raise ValueError("Prompt cannot be empty or whitespace only")
@@ -38,7 +39,8 @@ class TemplateRequest(BaseModel):
         ),
     )
 
-    @validator("template_name")
+    @field_validator("template_name")
+    @classmethod
     def validate_template_name(cls, v: Any) -> Any:
         if v not in ProjectTemplateName:
             raise ValueError(
@@ -54,10 +56,8 @@ class BotDataUpdateRequest(BaseModel):
     flows_yml: Optional[str] = Field(None, alias="flows.yml")
     config_yml: Optional[str] = Field(None, alias="config.yml")
 
-    class Config:
-        """Config for BotDataUpdateRequest."""
-
-        allow_population_by_field_name = True
+    # Allow using either field names or aliases when creating the model
+    model_config = ConfigDict(populate_by_name=True)
 
 
 class BotData(BaseModel):

rasa/builder/project_generator.py

@@ -3,30 +3,27 @@
 import json
 import os
 import shutil
-import tarfile
-import tempfile
 from pathlib import Path
 from textwrap import dedent
 from typing import Any, Dict, Generator, List, Optional
 
-import aiofiles
-import aiohttp
 import structlog
 
-import rasa.version
 from rasa.builder import config
 from rasa.builder.exceptions import ProjectGenerationError, ValidationError
 from rasa.builder.llm_service import get_skill_generation_messages, llm_service
 from rasa.builder.logging_utils import capture_exception_with_context
 from rasa.builder.models import BotFiles
 from rasa.builder.project_info import ProjectInfo, ensure_first_used, load_project_info
+from rasa.builder.template_cache import copy_cache_for_template_if_available
 from rasa.builder.training_service import TrainingInput
 from rasa.builder.validation_service import validate_project
 from rasa.cli.scaffold import ProjectTemplateName, create_initial_project
+from rasa.shared.constants import DEFAULT_MODELS_PATH
 from rasa.shared.core.flows import yaml_flows_io
 from rasa.shared.importers.importer import TrainingDataImporter
 from rasa.shared.utils.yaml import dump_obj_as_yaml_to_string
-from rasa.utils.io import subpath
+from rasa.utils.io import InvalidPathException, subpath
 
 structlogger = structlog.get_logger()
 
@@ -48,12 +45,25 @@ class ProjectGenerator:
         """Get the project info."""
         return load_project_info(self.project_folder)
 
+    def is_empty(self) -> bool:
+        """Check if the project folder is empty.
+
+        Excluding hidden paths.
+        """
+        return not any(
+            file.is_file()
+            for file in self.project_folder.iterdir()
+            if not file.name.startswith(".")
+        )
+
     async def init_from_template(self, template: ProjectTemplateName) -> None:
         """Create the initial project files."""
         self.cleanup()
         create_initial_project(self.project_folder.as_posix(), template)
-        await download_cache_for_template(template, self.project_folder.as_posix())
-        # needs to happen after caching, as we download .rasa and that would
+        # If a local cache for this template exists, copy it into the project.
+        # We no longer download here to avoid blocking project creation.
+        await copy_cache_for_template_if_available(template, self.project_folder)
+        # needs to happen after caching, as we download/copy .rasa and that would
         # overwrite the project info file in .rasa
         ensure_first_used(self.project_folder)
 
@@ -225,25 +235,9 @@ class ProjectGenerator:
         """
         bot_files: BotFiles = {}
 
-        for file in self.project_folder.glob("**/*"):
-            # Skip directories
-            if not file.is_file():
-                continue
-
+        for file in self.bot_file_paths():
             relative_path = file.relative_to(self.project_folder)
 
-            # Skip hidden files and directories (any path component starting with '.')
-            # as well as `__pycache__` folders
-            if any(part.startswith(".") for part in relative_path.parts):
-                continue
-
-            if "__pycache__" in relative_path.parts:
-                continue
-
-            # exclude the project_folder / models folder
-            if relative_path.parts[0] == "models":
-                continue
-
             # Exclude the docs directory if specified
             if exclude_docs_directory and relative_path.parts[0] == "docs":
                 continue
@@ -255,7 +249,6 @@ class ProjectGenerator:
                 ]
                 if file.suffix.lstrip(".").lower() not in allowed_file_extensions:
                     continue
-
             # Read file content and store with relative path as key
             try:
                 bot_files[relative_path.as_posix()] = file.read_text(encoding="utf-8")
@@ -266,9 +259,40 @@ class ProjectGenerator:
                     file_path=file.as_posix(),
                 )
                 bot_files[relative_path.as_posix()] = None
-
         return bot_files
 
+    def is_restricted_path(self, path: Path) -> bool:
+        """Check if the path is restricted.
+
+        These paths are excluded from deletion and editing by the user.
+        """
+        relative_path = path.relative_to(self.project_folder)
+
+        # Skip hidden files and directories (any path component starting with '.')
+        # as well as `__pycache__` folders
+        if any(part.startswith(".") for part in relative_path.parts):
+            return True
+
+        if "__pycache__" in relative_path.parts:
+            return True
+
+        # exclude the project_folder / models folder
+        if relative_path.parts[0] == DEFAULT_MODELS_PATH:
+            return True
+
+        return False
+
+    def bot_file_paths(
+        self,
+    ) -> Generator[Path, None, None]:
+        """Get the paths of all bot files."""
+        for file in self.project_folder.glob("**/*"):
+            # Skip directories
+            if not file.is_file() or self.is_restricted_path(file):
+                continue
+
+            yield file
+
     def _get_bot_data_for_llm(self) -> Dict[str, Any]:
         """Get the current bot data for the LLM."""
         file_importer = self._create_importer()
@@ -313,7 +337,7 @@ class ProjectGenerator:
     def update_bot_files(self, files: Dict[str, Optional[str]]) -> None:
         """Update bot files with new content by writing to disk."""
         for filename, content in files.items():
-            file_path = Path(subpath(self.project_folder, filename))
+            file_path = Path(subpath(str(self.project_folder), filename))
             # Disallow updates inside .rasa project metadata directory
             if any(
                 part.startswith(".")
@@ -324,6 +348,95 @@ class ProjectGenerator:
             file_path.parent.mkdir(parents=True, exist_ok=True)
             file_path.write_text(content, encoding="utf-8")
 
+    def ensure_all_files_are_writable(self, files: Dict[str, Optional[str]]) -> None:
+        """Ensure all files are writable."""
+        for filename, content in files.items():
+            file_path = Path(subpath(str(self.project_folder), filename))
+            if self.is_restricted_path(file_path):
+                raise InvalidPathException(
+                    f"This file or folder is restricted from editing: {file_path}"
+                )
+
+    def replace_all_bot_files(self, files: Dict[str, Optional[str]]) -> None:
+        """Replace all bot files with new content, deleting files not in the request.
+
+        Files/folders starting with .rasa/ or models/ are excluded from deletion.
+
+        Args:
+            files: Dictionary mapping file names to their content
+        """
+        self.ensure_all_files_are_writable(files)
+        # Collect all existing files - any files not in the new `files` dict will be
+        # deleted from this set
+        existing_files = set(path.as_posix() for path in self.bot_file_paths())
+
+        # Write all new files
+        for filename, content in files.items():
+            if content is None:
+                continue
+
+            file_path = Path(subpath(str(self.project_folder), filename))
+            file_path.parent.mkdir(parents=True, exist_ok=True)
+
+            try:
+                file_path.write_text(content, encoding="utf-8")
+            except Exception as e:
+                # Log write failure and avoid deleting an existing file by mistake
+                capture_exception_with_context(
+                    e,
+                    "project_generator.replace_all_bot_files.write_error",
+                    extra={"file_path": file_path},
+                )
+                if file_path.as_posix() in existing_files:
+                    # Keep the original file if it already existed
+                    existing_files.discard(file_path.as_posix())
+                continue
+
+            # Remove from deletion set since this file is in the new set of files
+            existing_files.discard(file_path.as_posix())
+
+        # Delete files that weren't in the request
+        for file_to_delete in existing_files:
+            file_path = Path(file_to_delete)
+            try:
+                file_path.unlink()
+            except Exception as e:
+                capture_exception_with_context(
+                    e,
+                    "project_generator.replace_all_bot_files.delete_error",
+                    extra={"file_path": file_path},
+                )
+
+        # Clean up empty directories (except excluded ones)
+        self._cleanup_empty_directories()
+
+    def _cleanup_empty_directories(self) -> None:
+        """Remove empty directories from the project folder.
+
+        Excludes hidden files and directories, and models/ from cleanup.
+        """
+        # Walk directories in reverse order (deepest first)
+        for dirpath, dirnames, filenames in os.walk(self.project_folder, topdown=False):
+            # Skip if this is the project root
+            if dirpath == str(self.project_folder):
+                continue
+
+            if self.is_restricted_path(Path(dirpath)):
+                continue
+
+            relative_path = Path(dirpath).relative_to(self.project_folder)
+
+            try:
+                # Only remove if directory is empty
+                if not os.listdir(dirpath):
+                    os.rmdir(dirpath)
+            except Exception as e:
+                capture_exception_with_context(
+                    e,
+                    "project_generator.cleanup_empty_directories.error",
+                    extra={"directory": relative_path.as_posix()},
+                )
+
     def cleanup(self) -> None:
         """Cleanup the project folder."""
         # remove all the files and folders in the project folder resulting
@@ -331,6 +444,8 @@ class ProjectGenerator:
         for filename in os.listdir(self.project_folder):
             file_path = os.path.join(self.project_folder, filename)
             try:
+                if filename == "lost+found":
+                    continue
                 if os.path.isfile(file_path) or os.path.islink(file_path):
                     os.unlink(file_path)
                 elif os.path.isdir(file_path):
@@ -341,122 +456,3 @@ class ProjectGenerator:
                     error=str(e),
                     file_path=file_path,
                 )
-
-
-CACHE_BUCKET_URL = "https://trained-templates.s3.us-east-1.amazonaws.com"
-
-
-def _safe_tar_members(
-    tar: tarfile.TarFile, destination_directory: Path
-) -> Generator[tarfile.TarInfo, None, None]:
-    """Yield safe members for extraction to prevent path traversal and links.
-
-    Args:
-        tar: Open tar file handle
-        destination_directory: Directory to which files will be extracted
-
-    Yields:
-        Members that are safe to extract within destination_directory
-    """
-    base_path = destination_directory.resolve()
-
-    for member in tar.getmembers():
-        name = member.name
-        # Skip empty names and absolute paths
-        if not name or name.startswith("/") or name.startswith("\\"):
-            continue
-
-        # Disallow symlinks and hardlinks
-        if member.issym() or member.islnk():
-            continue
-
-        # Compute the final path and ensure it's within base_path
-        target_path = (base_path / name).resolve()
-        try:
-            target_path.relative_to(base_path)
-        except ValueError:
-            # Member would escape the destination directory
-            continue
-
-        yield member
-
-
-async def download_cache_for_template(
-    template: ProjectTemplateName, project_folder: str
-) -> None:
-    # get a temp path for the cache file download
-    temporary_cache_file = tempfile.NamedTemporaryFile(suffix=".tar.gz", delete=False)
-
-    try:
-        url = f"{CACHE_BUCKET_URL}/{rasa.version.__version__}-{template.value}.tar.gz"
-        async with aiohttp.ClientSession() as session:
-            async with session.get(url) as response:
-                response.raise_for_status()
-                async with aiofiles.open(temporary_cache_file.name, "wb") as f:
-                    async for chunk in response.content.iter_chunked(1024 * 1024):
-                        await f.write(chunk)
-
-        # extract the cache to the project folder using safe member filtering
-        with tarfile.open(temporary_cache_file.name, "r:gz") as tar:
-            destination = Path(project_folder)
-            destination.mkdir(parents=True, exist_ok=True)
-            tar.extractall(
-                path=destination,
-                members=_safe_tar_members(tar, destination),
-            )
-
-        structlogger.info(
-            "project_generator.download_cache_for_template.success",
-            template=template,
-            event_info=(
-                f"Downloaded cache for template, extracted to {project_folder}."
-            ),
-        )
-    except aiohttp.ClientResponseError as e:
-        if e.status == 403:
-            structlogger.debug(
-                "project_generator.download_cache_for_template.no_cache_found",
-                template=template,
-                event_info=("No cache found for template, continuing without it."),
-            )
-        else:
-            structlogger.debug(
-                "project_generator.download_cache_for_template.response_error",
-                error=str(e),
-                status=e.status,
-                template=template,
-                event_info=(
-                    "Failed to download cache for template, continuing without it."
-                ),
-            )
-            capture_exception_with_context(
-                e,
-                "project_generator.download_cache_for_template.response_error",
-                tags={"template": template.value, "status": str(e.status)},
-            )
-    except Exception as exc:
-        structlogger.debug(
-            "project_generator.download_cache_for_template.unexpected_error",
-            error=str(exc),
-            template=template,
-            event_info=(
-                "Unexpected error when downloading cache for template, "
-                "continuing without it."
-            ),
-        )
-        capture_exception_with_context(
-            exc,
-            "project_generator.download_cache_for_template.unexpected_error",
-            tags={"template": template.value},
-        )
-    finally:
-        # Clean up the temporary file
-        try:
-            Path(temporary_cache_file.name).unlink(missing_ok=True)
-        except Exception as exc:
-            structlogger.debug(
-                "project_generator.download_cache_for_template.cleanup_error",
-                error=str(exc),
-                template=template,
-                event_info=("Failed to cleanup cache for template, ignoring."),
-            )