rasa-pro 3.15.0a1__py3-none-any.whl → 3.15.0a3__py3-none-any.whl

rasa/builder/jobs.py

@@ -1,9 +1,16 @@
-from typing import Any, Dict, Optional
+import tarfile
+from pathlib import Path
+from typing import Any, Dict, List, Optional
 
 import structlog
 from sanic import Sanic
 
 from rasa.builder import config
+from rasa.builder.constants import (
+    MAX_ARCHIVE_FILE_SIZE,
+    MAX_ARCHIVE_FILES,
+    MAX_ARCHIVE_TOTAL_SIZE,
+)
 from rasa.builder.copilot.constants import (
     PROMPT_TO_BOT_KEY,
 )
@@ -19,6 +26,7 @@ from rasa.builder.copilot.models import (
     ResponseCategory,
     TrainingErrorLog,
 )
+from rasa.builder.download import download_backup_from_url
 from rasa.builder.exceptions import (
     LLMGenerationError,
     ProjectGenerationError,
@@ -39,6 +47,11 @@ from rasa.builder.training_service import (
 )
 from rasa.builder.validation_service import validate_project
 from rasa.cli.scaffold import ProjectTemplateName
+from rasa.core.agent import load_agent
+from rasa.core.config.configuration import Configuration
+from rasa.exceptions import ModelNotFound
+from rasa.model import get_local_model
+from rasa.shared.constants import DEFAULT_ENDPOINTS_PATH
 
 structlogger = structlog.get_logger()
 
@@ -607,3 +620,197 @@ async def run_copilot_training_success_job(
         )
         await push_job_status_event(job, JobStatus.error, message=str(exc))
         job_manager.mark_done(job, error=str(exc))
+
+
+def _safe_tar_members(
+    tar: tarfile.TarFile, destination_directory: Path
+) -> List[tarfile.TarInfo]:
+    """Get safe members for extraction to prevent path traversal and resource attacks.
+
+    Args:
+        tar: Open tar file handle
+        destination_directory: Directory to which files will be extracted
+
+    Returns:
+        List of members that are safe to extract within destination_directory
+
+    Raises:
+        ProjectGenerationError: If archive violates security constraints
+    """
+    base_path = destination_directory.resolve()
+    safe_members = []
+    total_size = 0
+    file_count = 0
+
+    for member in tar.getmembers():
+        name = member.name
+
+        # Check file count limit
+        file_count += 1
+        if file_count > MAX_ARCHIVE_FILES:
+            raise ProjectGenerationError(
+                f"Archive contains too many files (>{MAX_ARCHIVE_FILES}).", attempts=1
+            )
+
+        # Skip empty names and absolute paths
+        if not name or name.startswith("/") or name.startswith("\\"):
+            continue
+
+        # Disallow symlinks and hardlinks
+        if member.issym() or member.islnk():
+            continue
+
+        # Check individual file size limit
+        if member.size > MAX_ARCHIVE_FILE_SIZE:
+            raise ProjectGenerationError(
+                f"Archive contains file '{name}' that is too large "
+                f"({member.size} bytes > {MAX_ARCHIVE_FILE_SIZE} bytes).",
+                attempts=1,
+            )
+
+        # Check total size limit
+        total_size += member.size
+        if total_size > MAX_ARCHIVE_TOTAL_SIZE:
+            raise ProjectGenerationError(
+                "Archive total size too large "
+                f"({total_size} bytes > {MAX_ARCHIVE_TOTAL_SIZE} bytes).",
+                attempts=1,
+            )
+
+        # Compute the final path and ensure it's within base_path
+        target_path = (base_path / name).resolve()
+        try:
+            target_path.relative_to(base_path)
+        except ValueError:
+            # Member would escape the destination directory
+            continue
+
+        safe_members.append(member)
+
+    return safe_members
+
+
+async def run_backup_to_bot_job(
+    app: "Sanic",
+    job: JobInfo,
+    presigned_url: str,
+) -> None:
+    """Run the backup-to-bot job in the background.
+
+    Args:
+        app: The Sanic application instance.
+        job: The job information instance.
+        presigned_url: Presigned URL to download tar.gz backup data.
+    """
+    project_generator: ProjectGenerator = app.ctx.project_generator
+    await push_job_status_event(job, JobStatus.received)
+
+    temp_file_path = None
+    try:
+        # 1) Download and extract backup
+        await push_job_status_event(job, JobStatus.generating)
+        temp_file_path = await download_backup_from_url(presigned_url)
+
+        # Clear existing project files, keeping .rasa and __pycache__
+        project_path = Path(project_generator.project_folder)
+        project_generator.cleanup(skip_files=[".rasa", "__pycache__"])
+
+        # Extract the backup archive
+        with tarfile.open(temp_file_path, "r:gz") as tar:
+            safe_members = _safe_tar_members(tar, project_path)
+            tar.extractall(path=project_path, members=safe_members)  # nosec B202:tarfile_unsafe_members
+
+        await push_job_status_event(job, JobStatus.generation_success)
+
+        # 2) Load existing model or train new one
+        models_dir = project_path / "models"
+        try:
+            latest_model = get_local_model(str(models_dir))
+        except ModelNotFound:
+            latest_model = None
+
+        if latest_model:
+            # Load existing model
+            structlogger.info(
+                "backup_to_bot_job.loading_existing_model",
+                job_id=job.id,
+                model_path=latest_model,
+            )
+            await push_job_status_event(job, JobStatus.training)
+            available_endpoints = Configuration.initialise_endpoints(
+                endpoints_path=project_path / DEFAULT_ENDPOINTS_PATH
+            ).endpoints
+            agent = await load_agent(
+                model_path=latest_model, endpoints=available_endpoints
+            )
+            update_agent(agent, app)
+            await push_job_status_event(job, JobStatus.train_success)
+        else:
+            # Train new model
+            await push_job_status_event(job, JobStatus.training)
+            training_input = project_generator.get_training_input()
+            agent = await train_and_load_agent(training_input)
+            update_agent(agent, app)
+            await push_job_status_event(job, JobStatus.train_success)
+
+        # 3) Complete successfully
+        bot_files = project_generator.get_bot_files()
+        structlogger.info(
+            "bot_builder_service.backup_to_bot.success",
+            files_restored=list(bot_files.keys()),
+            had_existing_model=bool(latest_model),
+        )
+        await push_job_status_event(job, JobStatus.done)
+        job_manager.mark_done(job)
+
+    except tarfile.ReadError as exc:
+        raise ProjectGenerationError(
+            f"Failed to extract backup archive: {exc}. "
+            f"Please ensure the backup file is a valid tar.gz archive.",
+            attempts=1,
+        )
+    except TrainingError as exc:
+        structlogger.debug(
+            "backup_to_bot_job.training_error", job_id=job.id, error=str(exc)
+        )
+        await push_job_status_event(job, JobStatus.train_error, message=str(exc))
+        job_manager.mark_done(job, error=str(exc))
+
+    except ValidationError as exc:
+        log_levels = ["error"]
+        if config.VALIDATION_FAIL_ON_WARNINGS:
+            log_levels.append("warning")
+
+        structlogger.debug(
+            "backup_to_bot_job.validation_error",
+            job_id=job.id,
+            error=str(exc),
+            all_validation_logs=exc.validation_logs,
+            included_log_levels=log_levels,
+        )
+        error_message = exc.get_error_message_with_logs(log_levels=log_levels)
+        await push_job_status_event(
+            job, JobStatus.validation_error, message=error_message
+        )
+        job_manager.mark_done(job, error=error_message)
+
+    except ProjectGenerationError as exc:
+        structlogger.debug(
+            "backup_to_bot_job.generation_error", job_id=job.id, error=str(exc)
+        )
+        await push_job_status_event(job, JobStatus.generation_error, message=str(exc))
+        job_manager.mark_done(job, error=str(exc))
+
+    except Exception as exc:
+        structlogger.exception(
+            "backup_to_bot_job.unexpected_error", job_id=job.id, error=str(exc)
+        )
+        await push_job_status_event(job, JobStatus.error, message=str(exc))
+        job_manager.mark_done(job, error=str(exc))
+    finally:
+        # Always clean up temp file
+        if temp_file_path:
+            try:
+                Path(temp_file_path).unlink(missing_ok=True)
+            except Exception:
+                pass

rasa/builder/logging_utils.py

@@ -1,6 +1,7 @@
 """Logging and Sentry utilities for the builder service."""
 
 import collections
+import contextvars
 import logging
 import threading
 import time
@@ -20,8 +21,10 @@ structlogger = structlog.get_logger()
 # Thread-safe deque for collecting recent logs
 _recent_logs: Deque[str] = collections.deque(maxlen=config.MAX_LOG_ENTRIES)
 _logs_lock = threading.RLock()
-# Thread-local storage for validation logs
-_validation_logs = threading.local()
+# Context variable for validation logs (async-safe)
+_validation_logs: contextvars.ContextVar[Optional[List[Dict[str, Any]]]] = (
+    contextvars.ContextVar("validation_logs", default=None)
+)
 
 
 def collecting_logs_processor(
@@ -42,11 +45,12 @@ def collecting_logs_processor(
 
 
 def collecting_validation_logs_processor(
-    logger: Any, method_name: str, event_dict: Dict[str, Any]
-) -> Dict[str, Any]:
-    """Structlog processor that captures validation logs in thread-local storage.
+    logger: Any, method_name: str, event_dict: MutableMapping[str, Any]
+) -> MutableMapping[str, Any]:
+    """Structlog processor that captures validation logs in context variable storage.
 
     It's designed to be used with the capture_validation_logs context manager.
+    Uses contextvars for async-safe log capture across async tasks.
 
     Args:
         logger: The structlog logger instance
@@ -57,41 +61,38 @@ def collecting_validation_logs_processor(
         The unmodified event_dict (this processor doesn't modify the log data)
     """
     # Only capture logs if we're in a validation context
-    # (logs list exists for this thread)
-    if hasattr(_validation_logs, "logs"):
+    # (logs list exists in the current context)
+    logs = _validation_logs.get()
+    if logs is not None:
         log_entry = {"log_level": method_name, **event_dict}
-        _validation_logs.logs.append(log_entry)
+        logs.append(log_entry)
 
     return event_dict
 
 
 @contextmanager
 def capture_validation_logs() -> Generator[List[Dict[str, Any]], Any, None]:
-    """Context manager to capture validation logs using thread-local storage.
+    """Context manager to capture validation logs using context variables.
 
-    This context manager temporarily reconfigures structlog to capture all logs
-    during validation and stores them in thread-local storage. It's thread-safe
-    and automatically cleans up after use.
+    This context manager stores logs in a context variable WITHOUT reconfiguring
+    structlog globally. The processor checks the context variable and captures
+    logs if present. This avoids race conditions with concurrent requests.
 
     Yields:
         A list of captured log entries, each containing the log level and all
         original log data from the event_dict.
     """
-    # Temporarily reconfigure structlog to add our capture processor
-    original_processors = structlog.get_config()["processors"]
-    new_processors = [collecting_validation_logs_processor] + original_processors
-    structlog.configure(processors=new_processors)
-
-    # Initialize thread-local logs storage
-    _validation_logs.logs = []
+    # Initialize context variable logs storage
+    # The processor is ALWAYS installed (see module init), it just checks
+    # this context var
+    logs: List[Dict[str, Any]] = []
+    token = _validation_logs.set(logs)
 
     try:
-        yield _validation_logs.logs
+        yield logs
     finally:
-        # Restore original configuration and clean up thread-local storage
-        structlog.configure(processors=original_processors)
-        if hasattr(_validation_logs, "logs"):
-            delattr(_validation_logs, "logs")
+        # Clean up context variable
+        _validation_logs.reset(token)
 
 
 def attach_request_id_processor(

rasa/builder/main.py

@@ -18,6 +18,7 @@ from rasa.builder import config
 from rasa.builder.logging_utils import (
     attach_request_id_processor,
     collecting_logs_processor,
+    collecting_validation_logs_processor,
     log_request_end,
     log_request_start,
 )
@@ -47,7 +48,11 @@ def setup_logging() -> None:
     configure_structlog(
         log_level,
         include_time=True,
-        additional_processors=[attach_request_id_processor, collecting_logs_processor],
+        additional_processors=[
+            attach_request_id_processor,
+            collecting_logs_processor,
+            collecting_validation_logs_processor,
+        ],
     )
 
 

rasa/builder/models.py

@@ -49,6 +49,29 @@ class TemplateRequest(BaseModel):
         return v
 
 
+class RestoreFromBackupRequest(BaseModel):
+    """Request model for backup-to-bot endpoint."""
+
+    presigned_url: str = Field(
+        ...,
+        min_length=1,
+        description="Presigned URL to download tar.gz backup file.",
+    )
+
+    @field_validator("presigned_url")
+    @classmethod
+    def validate_presigned_url(cls, v: str) -> str:
+        if not v.strip():
+            raise ValueError("Presigned URL cannot be empty or whitespace only")
+
+        # Basic URL validation
+        url = v.strip()
+        if not url.startswith(("http://", "https://")):
+            raise ValueError("Presigned URL must be a valid HTTP/HTTPS URL")
+
+        return url
+
+
 class BotDataUpdateRequest(BaseModel):
     """Request model for bot data updates."""
 

rasa/builder/project_generator.py

@@ -226,6 +226,7 @@ class ProjectGenerator:
         self,
         allowed_file_extensions: Optional[List[str]] = None,
         exclude_docs_directory: bool = False,
+        exclude_models_directory: bool = True,
     ) -> BotFiles:
         """Get the current bot files by reading from disk.
 
@@ -234,13 +235,14 @@ class ProjectGenerator:
                 If None, fetch all files. If provided, only fetch files with matching
                 extensions. Use `""` empty string to allow files with no extensions.
             exclude_docs_directory: Optional boolean indicating whether to exclude.
+            exclude_models_directory: Optional boolean indicating whether to exclude.
 
         Returns:
             Dictionary of file contents with relative paths as keys
         """
         bot_files: BotFiles = {}
 
-        for file in self.bot_file_paths():
+        for file in self.bot_file_paths(exclude_models_directory):
             relative_path = file.relative_to(self.project_folder)
 
             # Exclude the docs directory if specified
@@ -266,7 +268,9 @@ class ProjectGenerator:
                 bot_files[relative_path.as_posix()] = None
         return bot_files
 
-    def is_restricted_path(self, path: Path) -> bool:
+    def is_restricted_path(
+        self, path: Path, exclude_models_directory: bool = True
+    ) -> bool:
         """Check if the path is restricted.
 
         These paths are excluded from deletion and editing by the user.
@@ -281,19 +285,21 @@ class ProjectGenerator:
         if "__pycache__" in relative_path.parts:
             return True
 
-        # exclude the project_folder / models folder
-        if relative_path.parts[0] == DEFAULT_MODELS_PATH:
+        # exclude the project_folder / models folder if specified
+        if exclude_models_directory and relative_path.parts[0] == DEFAULT_MODELS_PATH:
             return True
 
         return False
 
     def bot_file_paths(
-        self,
+        self, exclude_models_directory: bool = True
     ) -> Generator[Path, None, None]:
         """Get the paths of all bot files."""
         for file in self.project_folder.glob("**/*"):
             # Skip directories
-            if not file.is_file() or self.is_restricted_path(file):
+            if not file.is_file() or self.is_restricted_path(
+                file, exclude_models_directory
+            ):
                 continue
 
             yield file
@@ -373,7 +379,10 @@ class ProjectGenerator:
         self.ensure_all_files_are_writable(files)
         # Collect all existing files - any files not in the new `files` dict will be
         # deleted from this set
-        existing_files = set(path.as_posix() for path in self.bot_file_paths())
+        existing_files = set(
+            path.as_posix()
+            for path in self.bot_file_paths(exclude_models_directory=True)
+        )
 
         # Write all new files
         for filename, content in files.items():
@@ -442,14 +451,24 @@ class ProjectGenerator:
                     extra={"directory": relative_path.as_posix()},
                 )
 
-    def cleanup(self) -> None:
-        """Cleanup the project folder."""
+    def cleanup(self, skip_files: Optional[List[str]] = None) -> None:
+        """Cleanup the project folder.
+
+        Args:
+            skip_files: List of file/directory names to skip during cleanup.
+        """
+        if skip_files is None:
+            skip_files = []
+
+        # Always include "lost+found" in skip files
+        skip_files = list(skip_files) + ["lost+found"]
+
         # remove all the files and folders in the project folder resulting
         # in an empty folder
         for filename in os.listdir(self.project_folder):
             file_path = os.path.join(self.project_folder, filename)
             try:
-                if filename == "lost+found":
+                if filename in skip_files:
                     continue
                 if os.path.isfile(file_path) or os.path.islink(file_path):
                     os.unlink(file_path)

rasa/builder/service.py

@@ -51,6 +51,7 @@ from rasa.builder.guardrails.constants import (
 from rasa.builder.guardrails.store import guardrails_store
 from rasa.builder.job_manager import job_manager
 from rasa.builder.jobs import (
+    run_backup_to_bot_job,
     run_prompt_to_bot_job,
     run_replace_all_files_job,
     run_template_to_bot_job,
@@ -70,6 +71,7 @@ from rasa.builder.models import (
     JobStatus,
     JobStatusEvent,
     PromptRequest,
+    RestoreFromBackupRequest,
     ServerSentEvent,
     TemplateRequest,
 )
@@ -466,6 +468,95 @@ async def handle_template_to_bot(request: Request) -> HTTPResponse:
         )
 
 
+@bp.route("/backup-to-bot", methods=["POST"])
+@openapi.summary("Generate bot from backup archive")
+@openapi.description(
+    "Creates a complete conversational AI bot from a backup tar.gz archive. "
+    "Returns immediately with a job ID. Connect to `/job-events/<job_id>` to "
+    "receive server-sent events (SSE) for real-time progress tracking "
+    "throughout the bot restoration process.\n\n"
+    "**SSE Event Flow** (via `/job-events/<job_id>`):\n"
+    "1. `received` - Request received by server\n"
+    "2. `generating` - Extracting and restoring bot from backup\n"
+    "3. `generation_success` - Backup restoration completed successfully\n"
+    "4. `training` - Training the bot model (if no existing model found)\n"
+    "5. `train_success` - Model training completed (if training was needed)\n"
+    "6. `done` - Bot restoration completed\n\n"
+    "**Error Events:**\n"
+    "- `generation_error` - Failed to restore bot from backup\n"
+    "- `train_error` - Backup restored but training failed\n"
+    "- `validation_error` - Restored bot configuration is invalid\n"
+    "- `error` - Unexpected error occurred\n\n"
+    "**Usage:**\n"
+    "1. Send POST request with Content-Type: application/json\n"
+    "2. The response will be a JSON object `{job_id: ...}`\n"
+    "3. Connect to `/job-events/<job_id>` for a server-sent event stream of progress."
+)
+@openapi.tag("bot-generation")
+@openapi.body(
+    {"application/json": model_to_schema(RestoreFromBackupRequest)},
+    description="Backup request with presigned URL to tar.gz archive.",
+    required=True,
+    example={"presigned_url": "https://s3.amazonaws.com/bucket/path?signature=..."},
+)
+@openapi.response(
+    200,
+    {"application/json": model_to_schema(JobCreateResponse)},
+    description="Job created. Poll or subscribe to /job-events/<job_id> for progress.",
+)
+@openapi.response(
+    400,
+    {"application/json": model_to_schema(ApiErrorResponse)},
+    description="Validation error in request payload or invalid presigned URL",
+)
+@openapi.response(
+    500,
+    {"application/json": model_to_schema(ApiErrorResponse)},
+    description="Internal server error",
+)
+@openapi.parameter(
+    HEADER_USER_ID,
+    description=(
+        "Optional user id to associate requests (e.g., for telemetry/guardrails)."
+    ),
+    _in="header",
+    required=False,
+    schema=str,
+)
+async def handle_backup_to_bot(request: Request) -> HTTPResponse:
+    """Handle backup-to-bot restoration requests."""
+    try:
+        payload = RestoreFromBackupRequest(**request.json)
+    except Exception as exc:
+        return response.json(
+            ApiErrorResponse(
+                error="Invalid request", details={"error": str(exc)}
+            ).model_dump(),
+            status=400,
+        )
+
+    try:
+        # Allocate job and schedule background task
+        job = job_manager.create_job()
+        request.app.add_task(
+            run_backup_to_bot_job(request.app, job, payload.presigned_url)
+        )
+        return response.json(JobCreateResponse(job_id=job.id).model_dump(), status=200)
+    except Exception as exc:
+        capture_exception_with_context(
+            exc,
+            "bot_builder_service.backup_to_bot.unexpected_error",
+            tags={"endpoint": "/api/backup-to-bot"},
+        )
+        return response.json(
+            ApiErrorResponse(
+                error="Failed to create backup-to-bot job",
+                details={"error": str(exc)},
+            ).model_dump(),
+            status=HTTPStatus.INTERNAL_SERVER_ERROR,
+        )
+
+
 @bp.route("/files", methods=["GET"])
 @openapi.summary("Get bot files")
 @openapi.description(
@@ -801,16 +892,14 @@ async def get_bot_info(request: Request) -> HTTPResponse:
 @openapi.summary("Download bot project as tar.gz")
 @openapi.description(
     "Downloads the current bot project files as a compressed tar.gz archive. "
-    "Includes all configuration files and a .env file with RASA_PRO_LICENSE. "
-    "Requires valid JWT token in Authorization header."
+    "Includes all configuration files and a .env file with RASA_PRO_LICENSE."
 )
 @openapi.tag("bot-files")
 @openapi.parameter(
-    "Authorization",
-    description=("Bearer token for authentication. Always required for this endpoint."),
-    _in="header",
-    required=True,
-    schema=str,
+    "exclude_models_directory",
+    bool,
+    location="query",
+    description="Whether to exclude the models directory",
 )
 @openapi.parameter(
     HEADER_USER_ID,
@@ -833,32 +922,25 @@ async def get_bot_info(request: Request) -> HTTPResponse:
     {"application/gzip": bytes},
     description="Bot project downloaded successfully as tar.gz",
 )
-@openapi.response(
-    401,
-    {"application/json": model_to_schema(ApiErrorResponse)},
-    description=(
-        "Authentication failed - Authorization header missing or invalid. "
-        "Authentication is always required for this endpoint."
-    ),
-)
 @openapi.response(
     500,
     {"application/json": model_to_schema(ApiErrorResponse)},
     description="Internal server error",
 )
-@protected(always_required=True)
 async def download_bot_project(request: Request) -> HTTPResponse:
     """Download bot project as tar.gz archive."""
     try:
-        # Token verification is enforced by the
-        # protected(always_required=True) decorator.
+        # Get query parameters
+        exclude_models_directory = (
+            request.args.get("exclude_models_directory", "true").lower() == "true"
+        )
+        project_name = request.args.get("project_name", "bot-project")
 
         # Get bot files
         project_generator = get_project_generator(request)
-        bot_files = project_generator.get_bot_files()
-
-        # Get project name from query parameters, default to "bot-project"
-        project_name = request.args.get("project_name", "bot-project")
+        bot_files = project_generator.get_bot_files(
+            exclude_models_directory=exclude_models_directory
+        )
 
         # Create tar.gz archive
         tar_data = create_bot_project_archive(bot_files, project_name)