rasa-pro 3.14.0a15__py3-none-any.whl → 3.14.0a17__py3-none-any.whl

rasa/builder/main.py

@@ -22,9 +22,11 @@ from rasa.builder.logging_utils import (
     log_request_start,
 )
 from rasa.builder.service import bp, setup_project_generator
+from rasa.builder.template_cache import (
+    background_download_template_caches,
+)
 from rasa.builder.training_service import try_load_existing_agent, update_agent
 from rasa.core.channels.studio_chat import StudioChatInput
-from rasa.model_manager.warm_rasa_process import warmup
 from rasa.server import configure_cors
 from rasa.utils.common import configure_logging_and_warnings
 from rasa.utils.log_utils import configure_structlog
@@ -148,6 +150,16 @@ def create_app(project_folder: str) -> Sanic:
         except Exception as e:
             structlogger.warning("Failed to load agent on server startup", error=str(e))
 
+    if config.HELLO_RASA_PROJECT_ID and app.ctx.project_generator.is_empty():
+        app.register_listener(background_download_template_caches, "after_server_start")
+    else:
+        structlogger.debug(
+            "builder.main.background_cache_download.disabled",
+            event_info=(
+                "No hello rasa project id set; skipping background cache download"
+            ),
+        )
+
     return app
 
 
@@ -188,12 +200,9 @@ def main(project_folder: Optional[str] = None) -> None:
         rasa.telemetry.initialize_telemetry()
         rasa.telemetry.initialize_error_reporting(private_mode=False)
 
+        # TODO: don't do this when running locally
         _apply_llm_overrides_from_builder_env()
 
-        if config.HELLO_RASA_PROJECT_ID:
-            # ensures long import times for modules are ahead of time
-            warmup()
-
         # working directory needs to be the project folder, e.g.
         # for relative paths (./docs) in a projects config to work
         if not project_folder:

rasa/builder/models.py

@@ -6,7 +6,7 @@ from pathlib import Path
 from typing import Any, Dict, List, Literal, Optional
 
 import structlog
-from pydantic import BaseModel, Field, validator
+from pydantic import BaseModel, ConfigDict, Field, field_validator
 
 from rasa.cli.scaffold import ProjectTemplateName
 from rasa.shared.importers.importer import TrainingDataImporter
@@ -21,7 +21,8 @@ class PromptRequest(BaseModel):
         ..., min_length=1, max_length=10000, description="The skill description prompt"
     )
 
-    @validator("prompt")
+    @field_validator("prompt")
+    @classmethod
     def validate_prompt(cls, v: str) -> str:
         if not v.strip():
             raise ValueError("Prompt cannot be empty or whitespace only")
@@ -38,7 +39,8 @@ class TemplateRequest(BaseModel):
         ),
     )
 
-    @validator("template_name")
+    @field_validator("template_name")
+    @classmethod
     def validate_template_name(cls, v: Any) -> Any:
         if v not in ProjectTemplateName:
             raise ValueError(
@@ -54,10 +56,8 @@ class BotDataUpdateRequest(BaseModel):
     flows_yml: Optional[str] = Field(None, alias="flows.yml")
     config_yml: Optional[str] = Field(None, alias="config.yml")
 
-    class Config:
-        """Config for BotDataUpdateRequest."""
-
-        allow_population_by_field_name = True
+    # Allow using either field names or aliases when creating the model
+    model_config = ConfigDict(populate_by_name=True)
 
 
 class BotData(BaseModel):

rasa/builder/project_generator.py

@@ -5,23 +5,25 @@ import os
 import shutil
 from pathlib import Path
 from textwrap import dedent
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, Generator, List, Optional
 
 import structlog
 
 from rasa.builder import config
 from rasa.builder.exceptions import ProjectGenerationError, ValidationError
 from rasa.builder.llm_service import get_skill_generation_messages, llm_service
+from rasa.builder.logging_utils import capture_exception_with_context
 from rasa.builder.models import BotFiles
 from rasa.builder.project_info import ProjectInfo, ensure_first_used, load_project_info
 from rasa.builder.template_cache import copy_cache_for_template_if_available
 from rasa.builder.training_service import TrainingInput
 from rasa.builder.validation_service import validate_project
 from rasa.cli.scaffold import ProjectTemplateName, create_initial_project
+from rasa.shared.constants import DEFAULT_MODELS_PATH
 from rasa.shared.core.flows import yaml_flows_io
 from rasa.shared.importers.importer import TrainingDataImporter
 from rasa.shared.utils.yaml import dump_obj_as_yaml_to_string
-from rasa.utils.io import subpath
+from rasa.utils.io import InvalidPathException, subpath
 
 structlogger = structlog.get_logger()
 
@@ -60,7 +62,7 @@ class ProjectGenerator:
         create_initial_project(self.project_folder.as_posix(), template)
         # If a local cache for this template exists, copy it into the project.
         # We no longer download here to avoid blocking project creation.
-        await copy_cache_for_template_if_available(template, self.project_folder)
+        copy_cache_for_template_if_available(template, self.project_folder)
         # needs to happen after caching, as we download/copy .rasa and that would
         # overwrite the project info file in .rasa
         ensure_first_used(self.project_folder)
@@ -233,25 +235,9 @@ class ProjectGenerator:
         """
         bot_files: BotFiles = {}
 
-        for file in self.project_folder.glob("**/*"):
-            # Skip directories
-            if not file.is_file():
-                continue
-
+        for file in self.bot_file_paths():
             relative_path = file.relative_to(self.project_folder)
 
-            # Skip hidden files and directories (any path component starting with '.')
-            # as well as `__pycache__` folders
-            if any(part.startswith(".") for part in relative_path.parts):
-                continue
-
-            if "__pycache__" in relative_path.parts:
-                continue
-
-            # exclude the project_folder / models folder
-            if relative_path.parts[0] == "models":
-                continue
-
             # Exclude the docs directory if specified
             if exclude_docs_directory and relative_path.parts[0] == "docs":
                 continue
@@ -263,7 +249,6 @@ class ProjectGenerator:
                 ]
                 if file.suffix.lstrip(".").lower() not in allowed_file_extensions:
                     continue
-
             # Read file content and store with relative path as key
             try:
                 bot_files[relative_path.as_posix()] = file.read_text(encoding="utf-8")
@@ -274,9 +259,40 @@ class ProjectGenerator:
                     file_path=file.as_posix(),
                 )
                 bot_files[relative_path.as_posix()] = None
-
         return bot_files
 
+    def is_restricted_path(self, path: Path) -> bool:
+        """Check if the path is restricted.
+
+        These paths are excluded from deletion and editing by the user.
+        """
+        relative_path = path.relative_to(self.project_folder)
+
+        # Skip hidden files and directories (any path component starting with '.')
+        # as well as `__pycache__` folders
+        if any(part.startswith(".") for part in relative_path.parts):
+            return True
+
+        if "__pycache__" in relative_path.parts:
+            return True
+
+        # exclude the project_folder / models folder
+        if relative_path.parts[0] == DEFAULT_MODELS_PATH:
+            return True
+
+        return False
+
+    def bot_file_paths(
+        self,
+    ) -> Generator[Path, None, None]:
+        """Get the paths of all bot files."""
+        for file in self.project_folder.glob("**/*"):
+            # Skip directories
+            if not file.is_file() or self.is_restricted_path(file):
+                continue
+
+            yield file
+
     def _get_bot_data_for_llm(self) -> Dict[str, Any]:
         """Get the current bot data for the LLM."""
         file_importer = self._create_importer()
@@ -321,7 +337,7 @@ class ProjectGenerator:
     def update_bot_files(self, files: Dict[str, Optional[str]]) -> None:
         """Update bot files with new content by writing to disk."""
         for filename, content in files.items():
-            file_path = Path(subpath(self.project_folder, filename))
+            file_path = Path(subpath(str(self.project_folder), filename))
             # Disallow updates inside .rasa project metadata directory
             if any(
                 part.startswith(".")
@@ -332,6 +348,95 @@ class ProjectGenerator:
             file_path.parent.mkdir(parents=True, exist_ok=True)
             file_path.write_text(content, encoding="utf-8")
 
+    def ensure_all_files_are_writable(self, files: Dict[str, Optional[str]]) -> None:
+        """Ensure all files are writable."""
+        for filename, content in files.items():
+            file_path = Path(subpath(str(self.project_folder), filename))
+            if self.is_restricted_path(file_path):
+                raise InvalidPathException(
+                    f"This file or folder is restricted from editing: {file_path}"
+                )
+
+    def replace_all_bot_files(self, files: Dict[str, Optional[str]]) -> None:
+        """Replace all bot files with new content, deleting files not in the request.
+
+        Files/folders starting with .rasa/ or models/ are excluded from deletion.
+
+        Args:
+            files: Dictionary mapping file names to their content
+        """
+        self.ensure_all_files_are_writable(files)
+        # Collect all existing files - any files not in the new `files` dict will be
+        # deleted from this set
+        existing_files = set(path.as_posix() for path in self.bot_file_paths())
+
+        # Write all new files
+        for filename, content in files.items():
+            if content is None:
+                continue
+
+            file_path = Path(subpath(str(self.project_folder), filename))
+            file_path.parent.mkdir(parents=True, exist_ok=True)
+
+            try:
+                file_path.write_text(content, encoding="utf-8")
+            except Exception as e:
+                # Log write failure and avoid deleting an existing file by mistake
+                capture_exception_with_context(
+                    e,
+                    "project_generator.replace_all_bot_files.write_error",
+                    extra={"file_path": file_path},
+                )
+                if file_path.as_posix() in existing_files:
+                    # Keep the original file if it already existed
+                    existing_files.discard(file_path.as_posix())
+                continue
+
+            # Remove from deletion set since this file is in the new set of files
+            existing_files.discard(file_path.as_posix())
+
+        # Delete files that weren't in the request
+        for file_to_delete in existing_files:
+            file_path = Path(file_to_delete)
+            try:
+                file_path.unlink()
+            except Exception as e:
+                capture_exception_with_context(
+                    e,
+                    "project_generator.replace_all_bot_files.delete_error",
+                    extra={"file_path": file_path},
+                )
+
+        # Clean up empty directories (except excluded ones)
+        self._cleanup_empty_directories()
+
+    def _cleanup_empty_directories(self) -> None:
+        """Remove empty directories from the project folder.
+
+        Excludes hidden files and directories, and models/ from cleanup.
+        """
+        # Walk directories in reverse order (deepest first)
+        for dirpath, dirnames, filenames in os.walk(self.project_folder, topdown=False):
+            # Skip if this is the project root
+            if dirpath == str(self.project_folder):
+                continue
+
+            if self.is_restricted_path(Path(dirpath)):
+                continue
+
+            relative_path = Path(dirpath).relative_to(self.project_folder)
+
+            try:
+                # Only remove if directory is empty
+                if not os.listdir(dirpath):
+                    os.rmdir(dirpath)
+            except Exception as e:
+                capture_exception_with_context(
+                    e,
+                    "project_generator.cleanup_empty_directories.error",
+                    extra={"directory": relative_path.as_posix()},
+                )
+
     def cleanup(self) -> None:
         """Cleanup the project folder."""
         # remove all the files and folders in the project folder resulting

rasa/builder/service.py

@@ -54,8 +54,8 @@ from rasa.builder.guardrails.utils import (
 from rasa.builder.job_manager import job_manager
 from rasa.builder.jobs import (
     run_prompt_to_bot_job,
+    run_replace_all_files_job,
     run_template_to_bot_job,
-    run_update_files_job,
 )
 from rasa.builder.llm_service import llm_service
 from rasa.builder.logging_utils import (
@@ -512,26 +512,31 @@ async def get_bot_files(request: Request) -> HTTPResponse:
         )
 
 
-@bp.route("/files", methods=["PUT"])
-@openapi.summary("Update bot files")
+@bp.route("/files", methods=["POST"])
+@openapi.summary("Replace all bot files")
 @openapi.description(
-    "Updates the bot configuration files and retrains the model. "
-    "Returns immediately with a job ID. Connect to `/job-events/<job_id>` "
-    "for real-time SSE progress tracking."
+    "Replaces all bot configuration files with the provided files, deletes any "
+    "files not included in the request (excluding .rasa/ and models/ directories), "
+    "and retrains the model. Returns immediately with a job ID. Connect to "
+    "`/job-events/<job_id>` for real-time SSE progress tracking."
     "\n\n"
+    "**File Management:**\n"
+    "- All files in the request are written to the project folder\n"
+    "- Files not included in the request are deleted from the project\n"
+    "- Files/folders starting with `.rasa/` or `models/` are excluded from deletion\n\n"
     "**SSE Event Flow:** (available via /job-events/<job_id>)\n"
     "1. `received` - Request received by server\n"
     "2. `validating` - Validating bot configuration files\n"
     "3. `validation_success` - File validation completed successfully\n"
     "4. `training` - Training the bot model with updated files\n"
     "5. `train_success` - Model training completed\n"
-    "6. `done` - Bot files update completed\n\n"
+    "6. `done` - Bot files replacement completed\n\n"
     "**Error Events (can occur at any time):**\n"
     "- `validation_error` - Bot configuration files are invalid\n"
     "- `train_error` - Files updated but training failed\n"
     "- `error` - Unexpected error occurred\n\n"
     "**Usage:**\n"
-    "1. Send PUT request with Content-Type: application/json\n"
+    "1. Send POST request with Content-Type: application/json\n"
     "2. The response will be a JSON object `{job_id: ...}`\n"
     "3. Connect to `/job-events/<job_id>` for a server-sent event stream of progress."
 )
@@ -539,9 +544,11 @@ async def get_bot_files(request: Request) -> HTTPResponse:
 @openapi.body(
     {"application/json": {str: Optional[str]}},
     description=(
-        "A dictionary mapping file names to their updated content. "
-        "The file name should be the name of the file in the project folder. "
-        "Files that are not in the request will not be updated."
+        "A dictionary mapping file names to their complete content. "
+        "All files in the project will be replaced with these files. "
+        "Files not included in the request will be deleted from the project "
+        "(except for .rasa/ and models/ directories). "
+        "The file name should be the relative path from the project root."
     ),
     required=True,
     example={
@@ -558,6 +565,10 @@ async def get_bot_files(request: Request) -> HTTPResponse:
             "  - name: CountVectorsFeaturizer\n"
             "policies:\n  - name: MemoizationPolicy\n  - name: RulePolicy"
         ),
+        "data/nlu.yml": (
+            "version: '3.1'\n"
+            "nlu:\n- intent: greet\n  examples: |\n    - hello\n    - hi"
+        ),
     },
 )
 @openapi.response(
@@ -565,7 +576,7 @@ async def get_bot_files(request: Request) -> HTTPResponse:
     {"application/json": model_to_schema(JobCreateResponse)},
     description=(
         "Job created. Poll or subscribe to /job-events/<job_id> "
-        "for progress and SSE updates."
+        "for progress and SSE updates on file replacement and training."
     ),
 )
 @openapi.response(
@@ -607,8 +618,8 @@ async def get_bot_files(request: Request) -> HTTPResponse:
     schema=str,
 )
 @protected()
-async def update_bot_files(request: Request) -> HTTPResponse:
-    """Update bot files with server-sent events for progress tracking."""
+async def replace_all_bot_files(request: Request) -> HTTPResponse:
+    """Replace all bot files with server-sent events for progress tracking."""
     try:
         bot_files = request.json
     except Exception as exc:
@@ -621,17 +632,17 @@ async def update_bot_files(request: Request) -> HTTPResponse:
 
     try:
         job = job_manager.create_job()
-        request.app.add_task(run_update_files_job(request.app, job, bot_files))
+        request.app.add_task(run_replace_all_files_job(request.app, job, bot_files))
         return response.json(JobCreateResponse(job_id=job.id).model_dump(), status=200)
     except Exception as exc:
         capture_exception_with_context(
             exc,
-            "bot_builder_service.update_bot_files.unexpected_error",
-            tags={"endpoint": "/api/files", "method": "PUT"},
+            "bot_builder_service.replace_all_bot_files.unexpected_error",
+            tags={"endpoint": "/api/files", "method": "POST"},
         )
         return response.json(
             ApiErrorResponse(
-                error="Failed to update bot files", details={"error": str(exc)}
+                error="Failed to replace bot files", details={"error": str(exc)}
             ).model_dump(),
             status=HTTPStatus.INTERNAL_SERVER_ERROR,
         )
@@ -1122,11 +1133,9 @@ async def copilot(request: Request) -> None:
         #    copilot response handler
         start_timestamp = time.perf_counter()
         copilot_client = llm_service.instantiate_copilot()
-        (
-            original_stream,
-            used_documents,
-            system_prompt,
-        ) = await copilot_client.generate_response(context)
+        (original_stream, generation_context) = await copilot_client.generate_response(
+            context
+        )
 
         copilot_response_handler = llm_service.instantiate_handler(
             COPILOT_HANDLER_ROLLING_BUFFER_SIZE
@@ -1142,18 +1151,24 @@ async def copilot(request: Request) -> None:
             asyncio.to_thread(
                 telemetry.log_copilot_from_handler,
                 handler=copilot_response_handler,
-                used_documents=used_documents,
+                used_documents=generation_context.relevant_documents,
                 latency_ms=int((time.perf_counter() - start_timestamp) * 1000),
-                system_prompt=system_prompt,
+                system_message=generation_context.system_message,
+                chat_history=generation_context.chat_history,
+                last_user_message=(
+                    req.last_message.get_text_content()
+                    if (req.last_message and req.last_message.role == ROLE_USER)
+                    else None
+                ),
                 **copilot_client.usage_statistics.model_dump(),
             )
         )
 
         # 9. Once the stream is over, extract and send references
         #    if any documents were used
-        if used_documents:
+        if generation_context.relevant_documents:
             reference_section = copilot_response_handler.extract_references(
-                used_documents
+                generation_context.relevant_documents
             )
             await sse.send(reference_section.to_sse_event().format())