rasa-pro 3.14.1__py3-none-any.whl → 3.15.0a3__py3-none-any.whl

rasa/builder/jobs.py

@@ -1,9 +1,16 @@
-from typing import Any, Dict, Optional
+import tarfile
+from pathlib import Path
+from typing import Any, Dict, List, Optional
 
 import structlog
 from sanic import Sanic
 
 from rasa.builder import config
+from rasa.builder.constants import (
+    MAX_ARCHIVE_FILE_SIZE,
+    MAX_ARCHIVE_FILES,
+    MAX_ARCHIVE_TOTAL_SIZE,
+)
 from rasa.builder.copilot.constants import (
     PROMPT_TO_BOT_KEY,
 )
@@ -19,6 +26,7 @@ from rasa.builder.copilot.models import (
     ResponseCategory,
     TrainingErrorLog,
 )
+from rasa.builder.download import download_backup_from_url
 from rasa.builder.exceptions import (
     LLMGenerationError,
     ProjectGenerationError,
@@ -39,6 +47,11 @@ from rasa.builder.training_service import (
 )
 from rasa.builder.validation_service import validate_project
 from rasa.cli.scaffold import ProjectTemplateName
+from rasa.core.agent import load_agent
+from rasa.core.config.configuration import Configuration
+from rasa.exceptions import ModelNotFound
+from rasa.model import get_local_model
+from rasa.shared.constants import DEFAULT_ENDPOINTS_PATH
 
 structlogger = structlog.get_logger()
 
@@ -607,3 +620,197 @@ async def run_copilot_training_success_job(
         )
         await push_job_status_event(job, JobStatus.error, message=str(exc))
         job_manager.mark_done(job, error=str(exc))
+
+
+def _safe_tar_members(
+    tar: tarfile.TarFile, destination_directory: Path
+) -> List[tarfile.TarInfo]:
+    """Get safe members for extraction to prevent path traversal and resource attacks.
+
+    Args:
+        tar: Open tar file handle
+        destination_directory: Directory to which files will be extracted
+
+    Returns:
+        List of members that are safe to extract within destination_directory
+
+    Raises:
+        ProjectGenerationError: If archive violates security constraints
+    """
+    base_path = destination_directory.resolve()
+    safe_members = []
+    total_size = 0
+    file_count = 0
+
+    for member in tar.getmembers():
+        name = member.name
+
+        # Check file count limit
+        file_count += 1
+        if file_count > MAX_ARCHIVE_FILES:
+            raise ProjectGenerationError(
+                f"Archive contains too many files (>{MAX_ARCHIVE_FILES}).", attempts=1
+            )
+
+        # Skip empty names and absolute paths
+        if not name or name.startswith("/") or name.startswith("\\"):
+            continue
+
+        # Disallow symlinks and hardlinks
+        if member.issym() or member.islnk():
+            continue
+
+        # Check individual file size limit
+        if member.size > MAX_ARCHIVE_FILE_SIZE:
+            raise ProjectGenerationError(
+                f"Archive contains file '{name}' that is too large "
+                f"({member.size} bytes > {MAX_ARCHIVE_FILE_SIZE} bytes).",
+                attempts=1,
+            )
+
+        # Check total size limit
+        total_size += member.size
+        if total_size > MAX_ARCHIVE_TOTAL_SIZE:
+            raise ProjectGenerationError(
+                "Archive total size too large "
+                f"({total_size} bytes > {MAX_ARCHIVE_TOTAL_SIZE} bytes).",
+                attempts=1,
+            )
+
+        # Compute the final path and ensure it's within base_path
+        target_path = (base_path / name).resolve()
+        try:
+            target_path.relative_to(base_path)
+        except ValueError:
+            # Member would escape the destination directory
+            continue
+
+        safe_members.append(member)
+
+    return safe_members
+
+
+async def run_backup_to_bot_job(
+    app: "Sanic",
+    job: JobInfo,
+    presigned_url: str,
+) -> None:
+    """Run the backup-to-bot job in the background.
+
+    Args:
+        app: The Sanic application instance.
+        job: The job information instance.
+        presigned_url: Presigned URL to download tar.gz backup data.
+    """
+    project_generator: ProjectGenerator = app.ctx.project_generator
+    await push_job_status_event(job, JobStatus.received)
+
+    temp_file_path = None
+    try:
+        # 1) Download and extract backup
+        await push_job_status_event(job, JobStatus.generating)
+        temp_file_path = await download_backup_from_url(presigned_url)
+
+        # Clear existing project files, keeping .rasa and __pycache__
+        project_path = Path(project_generator.project_folder)
+        project_generator.cleanup(skip_files=[".rasa", "__pycache__"])
+
+        # Extract the backup archive
+        with tarfile.open(temp_file_path, "r:gz") as tar:
+            safe_members = _safe_tar_members(tar, project_path)
+            tar.extractall(path=project_path, members=safe_members)  # nosec B202:tarfile_unsafe_members
+
+        await push_job_status_event(job, JobStatus.generation_success)
+
+        # 2) Load existing model or train new one
+        models_dir = project_path / "models"
+        try:
+            latest_model = get_local_model(str(models_dir))
+        except ModelNotFound:
+            latest_model = None
+
+        if latest_model:
+            # Load existing model
+            structlogger.info(
+                "backup_to_bot_job.loading_existing_model",
+                job_id=job.id,
+                model_path=latest_model,
+            )
+            await push_job_status_event(job, JobStatus.training)
+            available_endpoints = Configuration.initialise_endpoints(
+                endpoints_path=project_path / DEFAULT_ENDPOINTS_PATH
+            ).endpoints
+            agent = await load_agent(
+                model_path=latest_model, endpoints=available_endpoints
+            )
+            update_agent(agent, app)
+            await push_job_status_event(job, JobStatus.train_success)
+        else:
+            # Train new model
+            await push_job_status_event(job, JobStatus.training)
+            training_input = project_generator.get_training_input()
+            agent = await train_and_load_agent(training_input)
+            update_agent(agent, app)
+            await push_job_status_event(job, JobStatus.train_success)
+
+        # 3) Complete successfully
+        bot_files = project_generator.get_bot_files()
+        structlogger.info(
+            "bot_builder_service.backup_to_bot.success",
+            files_restored=list(bot_files.keys()),
+            had_existing_model=bool(latest_model),
+        )
+        await push_job_status_event(job, JobStatus.done)
+        job_manager.mark_done(job)
+
+    except tarfile.ReadError as exc:
+        raise ProjectGenerationError(
+            f"Failed to extract backup archive: {exc}. "
+            f"Please ensure the backup file is a valid tar.gz archive.",
+            attempts=1,
+        )
+    except TrainingError as exc:
+        structlogger.debug(
+            "backup_to_bot_job.training_error", job_id=job.id, error=str(exc)
+        )
+        await push_job_status_event(job, JobStatus.train_error, message=str(exc))
+        job_manager.mark_done(job, error=str(exc))
+
+    except ValidationError as exc:
+        log_levels = ["error"]
+        if config.VALIDATION_FAIL_ON_WARNINGS:
+            log_levels.append("warning")
+
+        structlogger.debug(
+            "backup_to_bot_job.validation_error",
+            job_id=job.id,
+            error=str(exc),
+            all_validation_logs=exc.validation_logs,
+            included_log_levels=log_levels,
+        )
+        error_message = exc.get_error_message_with_logs(log_levels=log_levels)
+        await push_job_status_event(
+            job, JobStatus.validation_error, message=error_message
+        )
+        job_manager.mark_done(job, error=error_message)
+
+    except ProjectGenerationError as exc:
+        structlogger.debug(
+            "backup_to_bot_job.generation_error", job_id=job.id, error=str(exc)
+        )
+        await push_job_status_event(job, JobStatus.generation_error, message=str(exc))
+        job_manager.mark_done(job, error=str(exc))
+
+    except Exception as exc:
+        structlogger.exception(
+            "backup_to_bot_job.unexpected_error", job_id=job.id, error=str(exc)
+        )
+        await push_job_status_event(job, JobStatus.error, message=str(exc))
+        job_manager.mark_done(job, error=str(exc))
+    finally:
+        # Always clean up temp file
+        if temp_file_path:
+            try:
+                Path(temp_file_path).unlink(missing_ok=True)
+            except Exception:
+                pass

rasa/builder/logging_utils.py

@@ -1,6 +1,7 @@
 """Logging and Sentry utilities for the builder service."""
 
 import collections
+import contextvars
 import logging
 import threading
 import time
@@ -20,8 +21,10 @@ structlogger = structlog.get_logger()
 # Thread-safe deque for collecting recent logs
 _recent_logs: Deque[str] = collections.deque(maxlen=config.MAX_LOG_ENTRIES)
 _logs_lock = threading.RLock()
-# Thread-local storage for validation logs
-_validation_logs = threading.local()
+# Context variable for validation logs (async-safe)
+_validation_logs: contextvars.ContextVar[Optional[List[Dict[str, Any]]]] = (
+    contextvars.ContextVar("validation_logs", default=None)
+)
 
 
 def collecting_logs_processor(
@@ -42,11 +45,12 @@ def collecting_logs_processor(
 
 
 def collecting_validation_logs_processor(
-    logger: Any, method_name: str, event_dict: Dict[str, Any]
-) -> Dict[str, Any]:
-    """Structlog processor that captures validation logs in thread-local storage.
+    logger: Any, method_name: str, event_dict: MutableMapping[str, Any]
+) -> MutableMapping[str, Any]:
+    """Structlog processor that captures validation logs in context variable storage.
 
     It's designed to be used with the capture_validation_logs context manager.
+    Uses contextvars for async-safe log capture across async tasks.
 
     Args:
         logger: The structlog logger instance
@@ -57,41 +61,38 @@ def collecting_validation_logs_processor(
         The unmodified event_dict (this processor doesn't modify the log data)
     """
     # Only capture logs if we're in a validation context
-    # (logs list exists for this thread)
-    if hasattr(_validation_logs, "logs"):
+    # (logs list exists in the current context)
+    logs = _validation_logs.get()
+    if logs is not None:
         log_entry = {"log_level": method_name, **event_dict}
-        _validation_logs.logs.append(log_entry)
+        logs.append(log_entry)
 
     return event_dict
 
 
 @contextmanager
 def capture_validation_logs() -> Generator[List[Dict[str, Any]], Any, None]:
-    """Context manager to capture validation logs using thread-local storage.
+    """Context manager to capture validation logs using context variables.
 
-    This context manager temporarily reconfigures structlog to capture all logs
-    during validation and stores them in thread-local storage. It's thread-safe
-    and automatically cleans up after use.
+    This context manager stores logs in a context variable WITHOUT reconfiguring
+    structlog globally. The processor checks the context variable and captures
+    logs if present. This avoids race conditions with concurrent requests.
 
     Yields:
         A list of captured log entries, each containing the log level and all
         original log data from the event_dict.
     """
-    # Temporarily reconfigure structlog to add our capture processor
-    original_processors = structlog.get_config()["processors"]
-    new_processors = [collecting_validation_logs_processor] + original_processors
-    structlog.configure(processors=new_processors)
-
-    # Initialize thread-local logs storage
-    _validation_logs.logs = []
+    # Initialize context variable logs storage
+    # The processor is ALWAYS installed (see module init), it just checks
+    # this context var
+    logs: List[Dict[str, Any]] = []
+    token = _validation_logs.set(logs)
 
     try:
-        yield _validation_logs.logs
+        yield logs
     finally:
-        # Restore original configuration and clean up thread-local storage
-        structlog.configure(processors=original_processors)
-        if hasattr(_validation_logs, "logs"):
-            delattr(_validation_logs, "logs")
+        # Clean up context variable
+        _validation_logs.reset(token)
 
 
 def attach_request_id_processor(

rasa/builder/main.py

@@ -18,6 +18,7 @@ from rasa.builder import config
 from rasa.builder.logging_utils import (
     attach_request_id_processor,
     collecting_logs_processor,
+    collecting_validation_logs_processor,
     log_request_end,
     log_request_start,
 )
@@ -47,7 +48,11 @@ def setup_logging() -> None:
     configure_structlog(
         log_level,
         include_time=True,
-        additional_processors=[attach_request_id_processor, collecting_logs_processor],
+        additional_processors=[
+            attach_request_id_processor,
+            collecting_logs_processor,
+            collecting_validation_logs_processor,
+        ],
     )
 
 

rasa/builder/models.py

@@ -49,6 +49,29 @@ class TemplateRequest(BaseModel):
         return v
 
 
+class RestoreFromBackupRequest(BaseModel):
+    """Request model for backup-to-bot endpoint."""
+
+    presigned_url: str = Field(
+        ...,
+        min_length=1,
+        description="Presigned URL to download tar.gz backup file.",
+    )
+
+    @field_validator("presigned_url")
+    @classmethod
+    def validate_presigned_url(cls, v: str) -> str:
+        if not v.strip():
+            raise ValueError("Presigned URL cannot be empty or whitespace only")
+
+        # Basic URL validation
+        url = v.strip()
+        if not url.startswith(("http://", "https://")):
+            raise ValueError("Presigned URL must be a valid HTTP/HTTPS URL")
+
+        return url
+
+
 class BotDataUpdateRequest(BaseModel):
     """Request model for bot data updates."""
 

rasa/builder/project_generator.py

@@ -226,6 +226,7 @@ class ProjectGenerator:
         self,
         allowed_file_extensions: Optional[List[str]] = None,
         exclude_docs_directory: bool = False,
+        exclude_models_directory: bool = True,
     ) -> BotFiles:
         """Get the current bot files by reading from disk.
 
@@ -234,13 +235,14 @@ class ProjectGenerator:
                 If None, fetch all files. If provided, only fetch files with matching
                 extensions. Use `""` empty string to allow files with no extensions.
             exclude_docs_directory: Optional boolean indicating whether to exclude.
+            exclude_models_directory: Optional boolean indicating whether to exclude.
 
         Returns:
             Dictionary of file contents with relative paths as keys
         """
         bot_files: BotFiles = {}
 
-        for file in self.bot_file_paths():
+        for file in self.bot_file_paths(exclude_models_directory):
             relative_path = file.relative_to(self.project_folder)
 
             # Exclude the docs directory if specified
@@ -266,7 +268,9 @@ class ProjectGenerator:
                 bot_files[relative_path.as_posix()] = None
         return bot_files
 
-    def is_restricted_path(self, path: Path) -> bool:
+    def is_restricted_path(
+        self, path: Path, exclude_models_directory: bool = True
+    ) -> bool:
         """Check if the path is restricted.
 
         These paths are excluded from deletion and editing by the user.
@@ -281,19 +285,21 @@ class ProjectGenerator:
         if "__pycache__" in relative_path.parts:
             return True
 
-        # exclude the project_folder / models folder
-        if relative_path.parts[0] == DEFAULT_MODELS_PATH:
+        # exclude the project_folder / models folder if specified
+        if exclude_models_directory and relative_path.parts[0] == DEFAULT_MODELS_PATH:
             return True
 
         return False
 
     def bot_file_paths(
-        self,
+        self, exclude_models_directory: bool = True
     ) -> Generator[Path, None, None]:
         """Get the paths of all bot files."""
         for file in self.project_folder.glob("**/*"):
             # Skip directories
-            if not file.is_file() or self.is_restricted_path(file):
+            if not file.is_file() or self.is_restricted_path(
+                file, exclude_models_directory
+            ):
                 continue
 
             yield file
@@ -373,7 +379,10 @@ class ProjectGenerator:
         self.ensure_all_files_are_writable(files)
         # Collect all existing files - any files not in the new `files` dict will be
         # deleted from this set
-        existing_files = set(path.as_posix() for path in self.bot_file_paths())
+        existing_files = set(
+            path.as_posix()
+            for path in self.bot_file_paths(exclude_models_directory=True)
+        )
 
         # Write all new files
         for filename, content in files.items():
@@ -442,14 +451,24 @@ class ProjectGenerator:
                     extra={"directory": relative_path.as_posix()},
                 )
 
-    def cleanup(self) -> None:
-        """Cleanup the project folder."""
+    def cleanup(self, skip_files: Optional[List[str]] = None) -> None:
+        """Cleanup the project folder.
+
+        Args:
+            skip_files: List of file/directory names to skip during cleanup.
+        """
+        if skip_files is None:
+            skip_files = []
+
+        # Always include "lost+found" in skip files
+        skip_files = list(skip_files) + ["lost+found"]
+
         # remove all the files and folders in the project folder resulting
         # in an empty folder
         for filename in os.listdir(self.project_folder):
             file_path = os.path.join(self.project_folder, filename)
             try:
-                if filename == "lost+found":
+                if filename in skip_files:
                     continue
                 if os.path.isfile(file_path) or os.path.islink(file_path):
                     os.unlink(file_path)