rasa-pro 3.13.1a19__py3-none-any.whl → 3.13.1a20__py3-none-any.whl

rasa/builder/service.py

@@ -1,7 +1,9 @@
-"""Main service for the prompt-to-bot functionality."""
-
+# mypy: disable-error-code=misc
+import asyncio
 import os
 import sys
+import time
+from http import HTTPStatus
 from typing import Any, Optional
 
 import structlog
@@ -9,30 +11,61 @@ from sanic import Blueprint, HTTPResponse, response
 from sanic.request import Request
 from sanic_openapi import openapi
 
-from rasa.builder.exceptions import (
-    LLMGenerationError,
-    ProjectGenerationError,
-    TrainingError,
-    ValidationError,
+from rasa.builder.auth import HEADER_USER_ID, extract_and_verify_auth0_token
+from rasa.builder.config import (
+    COPILOT_ASSISTANT_TRACKER_MAX_TURNS,
+    COPILOT_HANDLER_ROLLING_BUFFER_SIZE,
+    HELLO_RASA_PROJECT_ID,
+)
+from rasa.builder.copilot.constants import ROLE_USER
+from rasa.builder.copilot.exceptions import CopilotStreamError
+from rasa.builder.copilot.models import (
+    CopilotContext,
+    CopilotRequest,
+    GeneratedContent,
+    ReferenceEntry,
+    ReferenceSection,
+    ResponseCategory,
+    ResponseCompleteness,
+)
+from rasa.builder.copilot.telemetry import CopilotTelemetry
+from rasa.builder.download import create_bot_project_archive
+from rasa.builder.guardrails.utils import (
+    check_assistant_chat_for_policy_violations,
+    check_copilot_chat_for_policy_violations,
+)
+from rasa.builder.job_manager import job_manager
+from rasa.builder.jobs import (
+    run_prompt_to_bot_job,
+    run_template_to_bot_job,
+    run_update_files_job,
 )
 from rasa.builder.llm_service import llm_service
-from rasa.builder.logging_utils import get_recent_logs
+from rasa.builder.logging_utils import (
+    capture_exception_with_context,
+    get_recent_logs,
+)
 from rasa.builder.models import (
     ApiErrorResponse,
-    CopilotChatMessage,
-    CopilotContext,
-    CopilotRequest,
+    AssistantInfo,
+    BotData,
+    JobCreateResponse,
+    JobStatus,
+    JobStatusEvent,
     PromptRequest,
     ServerSentEvent,
     TemplateRequest,
 )
 from rasa.builder.project_generator import ProjectGenerator
-from rasa.builder.training_service import train_and_load_agent
-from rasa.builder.validation_service import validate_project
-from rasa.cli.scaffold import ProjectTemplateName
+from rasa.builder.shared.tracker_context import TrackerContext
+from rasa.core.agent import Agent
 from rasa.core.channels.studio_chat import StudioChatInput
+from rasa.core.exceptions import AgentNotReady
+from rasa.shared.core.flows.flows_list import FlowsList
+from rasa.shared.core.flows.yaml_flows_io import get_flows_as_json
 from rasa.shared.core.trackers import DialogueStateTracker
-from rasa.studio.upload import CALMUserData, extract_calm_import_parts_from_importer
+from rasa.shared.importers.utils import DOMAIN_KEYS
+from rasa.utils.json_utils import extract_values
 from rasa.utils.openapi import model_to_schema
 
 structlogger = structlog.get_logger()
@@ -73,34 +106,24 @@ def get_input_channel(request: Request) -> StudioChatInput:
     return request.app.ctx.input_channel
 
 
-def extract_calm_import_parts_from_project_generator(
-    project_generator: ProjectGenerator,
-) -> CALMUserData:
-    """Extract CALMUserData from a ProjectGenerator.
+async def extract_bot_data_from_agent(agent: Agent) -> BotData:
+    """Extract BotData from an Agent.
 
     Args:
-        project_generator: The project generator to extract data from
+        agent: The agent to extract data from
 
     Returns:
-        CALMUserData containing flows, domain, config, endpoints, and nlu data
+        BotData containing flows, domain, config, endpoints, and nlu data
     """
-    # Get the training data importer
-    importer = project_generator._create_importer()
-
-    # Extract endpoints (if exists)
-    endpoints_path = project_generator.project_folder / "endpoints.yml"
-    if endpoints_path.exists():
-        from rasa.shared.utils.yaml import read_yaml_file
-
-        endpoints = read_yaml_file(endpoints_path, expand_env_vars=False)
-    else:
-        endpoints = {}
-
-    # Use the shared function with the importer and project data paths
-    return extract_calm_import_parts_from_importer(
-        importer=importer,
-        config=None,  # Let the shared function get config from importer
-        endpoints=endpoints,
+    domain = agent.domain.as_dict() if agent.domain else {}
+    flows = (
+        await agent.processor.get_flows()
+        if agent.processor
+        else FlowsList(underlying_flows=[])
+    )
+    return BotData(
+        flows=get_flows_as_json(flows),
+        domain=extract_values(domain, DOMAIN_KEYS),
     )
 
 
@@ -115,26 +138,111 @@ async def health(request: Request) -> HTTPResponse:
     return response.json({"status": "ok", "service": "bot-builder"})
 
 
+@bp.route("/job-events/<job_id>", methods=["GET"])
+@openapi.summary("Stream job progress events")
+@openapi.description(
+    "Stream server-sent events (SSE) tracking real-time job progress.\n\n"
+    "**Connect with:** `Accept: text/event-stream`.\n\n"
+    "**SSE Event Example:**\n"
+    "```text\n"
+    "event: received\n"
+    'data: {"status": "received"}\n'
+    "```\n\n"
+)
+@openapi.tag("job-events")
+@openapi.parameter(
+    "job_id", str, location="path", description="The id of the job to stream events for"
+)
+@openapi.response(
+    200,
+    {"text/event-stream": str},
+    description="Server-sent events stream. See documentation for event types.",
+    example=(
+        "event: received\n"
+        'data: {"status": "received"}\n'
+        "\n"
+        "event: generating\n"
+        'data: {"status": "generating"}\n'
+    ),
+)
+@openapi.response(
+    404,
+    {"application/json": model_to_schema(ApiErrorResponse)},
+    description="Unknown job_id: No such job exists.",
+)
+async def job_events(request: Request, job_id: str) -> HTTPResponse:
+    try:
+        job = job_manager.get_job(job_id)
+        if job is None:
+            return response.json(
+                ApiErrorResponse(
+                    error="Job not found", details={"job_id": job_id}
+                ).model_dump(),
+                status=404,
+            )
+
+        stream = await request.respond(content_type="text/event-stream")
+
+        try:
+            async for evt in job.event_stream():
+                await stream.send(evt.format())
+        except Exception as exc:
+            # Handle exceptions within the SSE stream context
+            capture_exception_with_context(
+                exc,
+                "bot_builder_service.job_events.streaming_error",
+                extra={"job_id": job_id},
+                tags={"endpoint": "/api/job-events/<job_id>"},
+            )
+            # Send error event in SSE format instead of JSON response
+            error_event = JobStatusEvent.from_status(
+                status=JobStatus.error.value,
+                message=f"Failed to stream job events: {exc}",
+            ).format()
+            await stream.send(error_event)
+        finally:
+            await stream.eof()
+
+        return stream
+    except Exception as exc:
+        # This exception handler only applies before stream.respond() is called
+        capture_exception_with_context(
+            exc,
+            "bot_builder_service.job_events.unexpected_error",
+            extra={"job_id": job_id},
+            tags={"endpoint": "/api/job-events/<job_id>"},
+        )
+        return response.json(
+            ApiErrorResponse(
+                error="Failed to stream job events", details={"error": str(exc)}
+            ).model_dump(),
+            status=HTTPStatus.INTERNAL_SERVER_ERROR,
+        )
+
+
 @bp.route("/prompt-to-bot", methods=["POST"])
 @openapi.summary("Generate bot from natural language prompt")
 @openapi.description(
-    "Creates a complete conversational AI bot from a natural language prompt "
-    "using LLM generation. Returns server-sent events (SSE) for real-time "
-    "progress tracking through the entire bot creation process.\n\n"
-    "**SSE Event Flow:**\n"
+    "Creates a complete conversational AI bot from a natural language prompt. "
+    "Returns immediately with a job ID. Connect to `/job-events/<job_id>` to "
+    "receive server-sent events (SSE) for real-time progress tracking "
+    "throughout the bot creation process.\n\n"
+    "**SSE Event Flow** (via `/job-events/<job_id>`):\n"
     "1. `received` - Request received by server\n"
     "2. `generating` - Generating bot project files\n"
     "3. `generation_success` - Bot generation completed successfully\n"
     "4. `training` - Training the bot model\n"
     "5. `train_success` - Model training completed\n"
     "6. `done` - Bot creation completed\n\n"
-    "**Error Events (can occur at any time):**\n"
+    "**Error Events:**\n"
     "- `generation_error` - Failed to generate bot from prompt\n"
     "- `train_error` - Bot generated but training failed\n"
     "- `validation_error` - Generated bot configuration is invalid\n"
     "- `error` - Unexpected error occurred\n\n"
-    "**Usage:** Send POST request with Content-Type: application/json and "
-    "Accept: text/event-stream"
+    "**Usage:**\n"
+    "1. Send POST request with Content-Type: application/json\n"
+    "2. The response will be a JSON object `{job_id: ...}`\n"
+    "3. Connect to `/job-events/<job_id>` for a server-sent event stream of progress."
 )
 @openapi.tag("bot-generation")
 @openapi.body(
@@ -144,8 +252,8 @@ async def health(request: Request) -> HTTPResponse:
 )
 @openapi.response(
     200,
-    {"text/event-stream": str},
-    description="Server-sent events stream with real-time progress updates",
+    {"application/json": model_to_schema(JobCreateResponse)},
+    description="Job created. Poll or subscribe to /job-events/<job_id> for progress.",
 )
 @openapi.response(
     400,
@@ -157,148 +265,61 @@ async def health(request: Request) -> HTTPResponse:
     {"application/json": model_to_schema(ApiErrorResponse)},
     description="Internal server error",
 )
-async def handle_prompt_to_bot(request: Request) -> None:
+async def handle_prompt_to_bot(request: Request) -> HTTPResponse:
     """Handle prompt-to-bot generation requests."""
-    sse_response = await request.respond(content_type="text/event-stream")
-    project_generator = get_project_generator(request)
-    input_channel = get_input_channel(request)
-
     try:
-        # 1. Received
-        await _send_sse_event(
-            sse_response,
-            ServerSentEvent(event="received", data={"status": "received"}),
-        )
-
-        # Validate request
-        prompt_data = PromptRequest(**request.json)
-
-        # 2. Generating
-        await _send_sse_event(
-            sse_response,
-            ServerSentEvent(event="generating", data={"status": "generating"}),
-        )
-
-        try:
-            # Generate project with retries
-            bot_files = await project_generator.generate_project_with_retries(
-                prompt_data.prompt,
-                template=ProjectTemplateName.PLAIN,
-            )
-
-            await _send_sse_event(
-                sse_response,
-                ServerSentEvent(
-                    event="generation_success",
-                    data={"status": "generation_success"},
-                ),
-            )
-
-        except (ProjectGenerationError, LLMGenerationError) as e:
-            await _send_sse_event(
-                sse_response,
-                ServerSentEvent(
-                    event="generation_error",
-                    data={"status": "generation_error", "error": str(e)},
-                ),
-            )
-            await sse_response.eof()
-            return
-
-        # 3. Training
-        await _send_sse_event(
-            sse_response,
-            ServerSentEvent(event="training", data={"status": "training"}),
-        )
-
-        try:
-            # Train and load agent
-            importer = project_generator._create_importer()
-            request.app.ctx.agent = await train_and_load_agent(importer)
-
-            # Update input channel with new agent
-            input_channel.agent = request.app.ctx.agent
-
-            await _send_sse_event(
-                sse_response,
-                ServerSentEvent(
-                    event="train_success", data={"status": "train_success"}
-                ),
-            )
-
-        except TrainingError as e:
-            await _send_sse_event(
-                sse_response,
-                ServerSentEvent(
-                    event="train_error",
-                    data={"status": "train_error", "error": str(e)},
-                ),
-            )
-            await sse_response.eof()
-            return
-
-        # 4. Done
-        await _send_sse_event(
-            sse_response,
-            ServerSentEvent(
-                event="done",
-                data={
-                    "status": "done",
-                },
-            ),
-        )
-
-        structlogger.info(
-            "bot_builder_service.prompt_to_bot.success",
-            files_generated=list(bot_files.keys()),
-        )
-
-    except ValidationError as e:
-        structlogger.error(
-            "bot_builder_service.prompt_to_bot.validation_error", error=str(e)
-        )
-        await _send_sse_event(
-            sse_response,
-            ServerSentEvent(
-                event="validation_error",
-                data={"status": "validation_error", "error": str(e)},
-            ),
+        payload = PromptRequest(**request.json)
+    except Exception as exc:
+        return response.json(
+            ApiErrorResponse(
+                error="Invalid request", details={"error": str(exc)}
+            ).model_dump(),
+            status=400,
         )
 
-    except Exception as e:
-        structlogger.error(
-            "bot_builder_service.prompt_to_bot.unexpected_error", error=str(e)
+    try:
+        # Allocate job and schedule background task
+        job = job_manager.create_job()
+        request.app.add_task(run_prompt_to_bot_job(request.app, job, payload.prompt))
+        return response.json(JobCreateResponse(job_id=job.id).model_dump(), status=200)
+    except Exception as exc:
+        capture_exception_with_context(
+            exc,
+            "bot_builder_service.prompt_to_bot.unexpected_error",
+            tags={"endpoint": "/api/prompt-to-bot"},
         )
-        await _send_sse_event(
-            sse_response,
-            ServerSentEvent(event="error", data={"status": "error", "error": str(e)}),
+        return response.json(
+            ApiErrorResponse(
+                error="Failed to create prompt-to-bot job",
+                details={"error": str(exc)},
+            ).model_dump(),
+            status=HTTPStatus.INTERNAL_SERVER_ERROR,
         )
-    finally:
-        await sse_response.eof()
 
 
 @bp.route("/template-to-bot", methods=["POST"])
 @openapi.summary("Generate bot from predefined template")
 @openapi.description(
-    "Creates a complete conversational AI bot from a predefined template with "
-    "immediate setup. Returns server-sent events (SSE) for real-time progress "
-    "tracking through the entire bot creation process.\n\n"
-    "**SSE Event Flow:**\n"
+    "Creates a complete conversational AI bot from a predefined template. "
+    "Returns immediately with a job ID. Connect to `/job-events/<job_id>` to "
+    "receive server-sent events (SSE) for real-time progress tracking "
+    "throughout the bot creation process.\n\n"
+    "**SSE Event Flow** (via `/job-events/<job_id>`):\n"
     "1. `received` - Request received by server\n"
     "2. `generating` - Initializing bot from template\n"
     "3. `generation_success` - Template initialization completed successfully\n"
     "4. `training` - Training the bot model\n"
     "5. `train_success` - Model training completed\n"
     "6. `done` - Bot creation completed\n\n"
-    "**Error Events (can occur at any time):**\n"
+    "**Error Events:**\n"
     "- `generation_error` - Failed to initialize bot from template\n"
     "- `train_error` - Template loaded but training failed\n"
     "- `validation_error` - Template configuration is invalid\n"
     "- `error` - Unexpected error occurred\n\n"
-    "**Usage:** Send POST request with Content-Type: application/json and "
-    "Accept: text/event-stream\n"
-    "**Templates Available:** Check available templates through the API or "
-    "documentation"
+    "**Usage:**\n"
+    "1. Send POST request with Content-Type: application/json\n"
+    "2. The response will be a JSON object `{job_id: ...}`\n"
+    "3. Connect to `/job-events/<job_id>` for a server-sent event stream of progress."
 )
 @openapi.tag("bot-generation")
 @openapi.body(
@@ -308,12 +329,8 @@ async def handle_prompt_to_bot(request: Request) -> None:
 )
 @openapi.response(
     200,
-    {"text/event-stream": model_to_schema(ServerSentEvent)},
-    description="Server-sent events stream with real-time progress updates",
-    example=ServerSentEvent(
-        event="generation_success",
-        data={"status": "generation_success"},
-    ).model_dump(),
+    {"application/json": model_to_schema(JobCreateResponse)},
+    description="Job created. Poll or subscribe to /job-events/<job_id> for progress.",
 )
 @openapi.response(
     400,
@@ -325,124 +342,38 @@ async def handle_prompt_to_bot(request: Request) -> None:
     {"application/json": model_to_schema(ApiErrorResponse)},
     description="Internal server error",
 )
-async def handle_template_to_bot(request: Request) -> None:
-    """Handle template-to-bot generation requests."""
-    sse_response = await request.respond(content_type="text/event-stream")
-    project_generator = get_project_generator(request)
-    input_channel = get_input_channel(request)
-
+async def handle_template_to_bot(request: Request) -> HTTPResponse:
+    """Create a new template-to-bot job and return job_id immediately."""
     try:
-        # 1. Received
-        await _send_sse_event(
-            sse_response,
-            ServerSentEvent(event="received", data={"status": "received"}),
-        )
-
-        # Validate request
         template_data = TemplateRequest(**request.json)
-
-        # 2. Generating
-        await _send_sse_event(
-            sse_response,
-            ServerSentEvent(event="generating", data={"status": "generating"}),
-        )
-
-        try:
-            # Generate project with retries
-            project_generator.init_from_template(
-                template_data.template_name,
-            )
-            bot_files = project_generator.get_bot_files()
-
-            await _send_sse_event(
-                sse_response,
-                ServerSentEvent(
-                    event="generation_success",
-                    data={"status": "generation_success"},
-                ),
-            )
-
-        except ProjectGenerationError as e:
-            await _send_sse_event(
-                sse_response,
-                ServerSentEvent(
-                    event="generation_error",
-                    data={"status": "generation_error", "error": str(e)},
-                ),
-            )
-            await sse_response.eof()
-            return
-
-        # 3. Training
-        await _send_sse_event(
-            sse_response,
-            ServerSentEvent(event="training", data={"status": "training"}),
-        )
-
-        try:
-            # Train and load agent
-            importer = project_generator._create_importer()
-            request.app.ctx.agent = await train_and_load_agent(importer)
-
-            # Update input channel with new agent
-            input_channel.agent = request.app.ctx.agent
-
-            await _send_sse_event(
-                sse_response,
-                ServerSentEvent(
-                    event="train_success", data={"status": "train_success"}
-                ),
-            )
-
-        except TrainingError as e:
-            await _send_sse_event(
-                sse_response,
-                ServerSentEvent(
-                    event="train_error",
-                    data={"status": "train_error", "error": str(e)},
-                ),
-            )
-            await sse_response.eof()
-            return
-
-        # 4. Done
-        await _send_sse_event(
-            sse_response,
-            ServerSentEvent(
-                event="done",
-                data={
-                    "status": "done",
-                },
-            ),
-        )
-
-        structlogger.info(
-            "bot_builder_service.template_to_bot.success",
-            files_generated=list(bot_files.keys()),
+    except Exception as exc:
+        return response.json(
+            ApiErrorResponse(
+                error="Invalid request", details={"error": str(exc)}
+            ).model_dump(),
+            status=400,
         )
 
-    except ValidationError as e:
-        structlogger.error(
-            "bot_builder_service.template_to_bot.validation_error", error=str(e)
-        )
-        await _send_sse_event(
-            sse_response,
-            ServerSentEvent(
-                event="validation_error",
-                data={"status": "validation_error", "error": str(e)},
-            ),
+    try:
+        # allocate job and schedule background task
+        job = job_manager.create_job()
+        request.app.add_task(
+            run_template_to_bot_job(request.app, job, template_data.template_name)
         )
-
-    except Exception as e:
-        structlogger.error(
-            "bot_builder_service.template_to_bot.unexpected_error", error=str(e)
+        return response.json(JobCreateResponse(job_id=job.id).model_dump(), status=200)
+    except Exception as exc:
+        capture_exception_with_context(
+            exc,
+            "bot_builder_service.template_to_bot.unexpected_error",
+            tags={"endpoint": "/api/template-to-bot"},
         )
-        await _send_sse_event(
-            sse_response,
-            ServerSentEvent(event="error", data={"status": "error", "error": str(e)}),
+        return response.json(
+            ApiErrorResponse(
+                error="Failed to create template-to-bot job",
+                details={"error": str(exc)},
+            ).model_dump(),
+            status=HTTPStatus.INTERNAL_SERVER_ERROR,
         )
-    finally:
-        await sse_response.eof()
 
 
 @bp.route("/files", methods=["GET"])
@@ -461,18 +392,32 @@ async def handle_template_to_bot(request: Request) -> None:
 )
 async def get_bot_files(request: Request) -> HTTPResponse:
     """Get current bot files."""
-    project_generator = get_project_generator(request)
-    bot_files = project_generator.get_bot_files()
-    return response.json(bot_files)
+    try:
+        project_generator = get_project_generator(request)
+        bot_files = project_generator.get_bot_files()
+        return response.json(bot_files)
+    except Exception as exc:
+        capture_exception_with_context(
+            exc,
+            "bot_builder_service.get_bot_files.unexpected_error",
+            tags={"endpoint": "/api/files", "method": "GET"},
+        )
+        return response.json(
+            ApiErrorResponse(
+                error="Failed to retrieve bot files", details={"error": str(exc)}
+            ).model_dump(),
+            status=HTTPStatus.INTERNAL_SERVER_ERROR,
+        )
 
 
 @bp.route("/files", methods=["PUT"])
 @openapi.summary("Update bot files")
 @openapi.description(
     "Updates the bot configuration files and retrains the model. "
-    "Returns server-sent events (SSE) for real-time progress tracking "
-    "through the entire update process.\n\n"
-    "**SSE Event Flow:**\n"
+    "Returns immediately with a job ID. Connect to `/job-events/<job_id>` "
+    "for real-time SSE progress tracking."
+    "\n\n"
+    "**SSE Event Flow:** (available via /job-events/<job_id>)\n"
     "1. `received` - Request received by server\n"
     "2. `validating` - Validating bot configuration files\n"
     "3. `validation_success` - File validation completed successfully\n"
@@ -483,8 +428,10 @@ async def get_bot_files(request: Request) -> HTTPResponse:
     "- `validation_error` - Bot configuration files are invalid\n"
     "- `train_error` - Files updated but training failed\n"
     "- `error` - Unexpected error occurred\n\n"
-    "**Usage:** Send PUT request with Content-Type: application/json and "
-    "Accept: text/event-stream"
+    "**Usage:**\n"
+    "1. Send PUT request with Content-Type: application/json\n"
+    "2. The response will be a JSON object `{job_id: ...}`\n"
+    "3. Connect to `/job-events/<job_id>` for a server-sent event stream of progress."
 )
 @openapi.tag("bot-files")
 @openapi.body(
@@ -496,8 +443,11 @@ async def get_bot_files(request: Request) -> HTTPResponse:
 )
 @openapi.response(
     200,
-    {"text/event-stream": str},
-    description="Server-sent events stream with update progress",
+    {"application/json": model_to_schema(JobCreateResponse)},
+    description=(
+        "Job created. Poll or subscribe to /job-events/<job_id> "
+        "for progress and SSE updates."
+    ),
 )
 @openapi.response(
     400,
@@ -509,101 +459,34 @@ async def get_bot_files(request: Request) -> HTTPResponse:
     {"application/json": model_to_schema(ApiErrorResponse)},
     description="Internal server error",
 )
-async def update_bot_files(request: Request) -> None:
+async def update_bot_files(request: Request) -> HTTPResponse:
     """Update bot files with server-sent events for progress tracking."""
-    sse_response = await request.respond(content_type="text/event-stream")
-    project_generator = get_project_generator(request)
-    input_channel = get_input_channel(request)
-
     try:
-        # 1. Received
-        await _send_sse_event(
-            sse_response,
-            ServerSentEvent(event="received", data={"status": "received"}),
-        )
-
-        # Update bot files
         bot_files = request.json
-        project_generator.update_bot_files(bot_files)
-
-        # 2. Validating
-        await _send_sse_event(
-            sse_response,
-            ServerSentEvent(event="validating", data={"status": "validating"}),
-        )
-
-        try:
-            importer = project_generator._create_importer()
-            validation_error = await validate_project(importer)
-
-            if validation_error:
-                raise ValidationError(validation_error)
-
-            await _send_sse_event(
-                sse_response,
-                ServerSentEvent(
-                    event="validation_success",
-                    data={"status": "validation_success"},
-                ),
-            )
-
-        except ValidationError as e:
-            await _send_sse_event(
-                sse_response,
-                ServerSentEvent(
-                    event="validation_error",
-                    data={"status": "validation_error", "error": str(e)},
-                ),
-            )
-            await sse_response.eof()
-            return
-
-        # 3. Training
-        await _send_sse_event(
-            sse_response,
-            ServerSentEvent(event="training", data={"status": "training"}),
+    except Exception as exc:
+        return response.json(
+            ApiErrorResponse(
+                error="Invalid request", details={"error": str(exc)}
+            ).model_dump(),
+            status=400,
         )
 
-        try:
-            request.app.ctx.agent = await train_and_load_agent(importer)
-            input_channel.agent = request.app.ctx.agent
-
-            await _send_sse_event(
-                sse_response,
-                ServerSentEvent(
-                    event="train_success", data={"status": "train_success"}
-                ),
-            )
-
-        except TrainingError as e:
-            await _send_sse_event(
-                sse_response,
-                ServerSentEvent(
-                    event="train_error",
-                    data={"status": "train_error", "error": str(e)},
-                ),
-            )
-            await sse_response.eof()
-            return
-
-        # 4. Done
-        await _send_sse_event(
-            sse_response,
-            ServerSentEvent(
-                event="done",
-                data={
-                    "status": "done",
-                },
-            ),
+    try:
+        job = job_manager.create_job()
+        request.app.add_task(run_update_files_job(request.app, job, bot_files))
+        return response.json(JobCreateResponse(job_id=job.id).model_dump(), status=200)
+    except Exception as exc:
+        capture_exception_with_context(
+            exc,
+            "bot_builder_service.update_bot_files.unexpected_error",
+            tags={"endpoint": "/api/files", "method": "PUT"},
         )
-
-    except Exception as e:
-        await _send_sse_event(
-            sse_response,
-            ServerSentEvent(event="error", data={"status": "error", "error": str(e)}),
+        return response.json(
+            ApiErrorResponse(
+                error="Failed to update bot files", details={"error": str(exc)}
+            ).model_dump(),
+            status=HTTPStatus.INTERNAL_SERVER_ERROR,
         )
-    finally:
-        await sse_response.eof()
 
 
 @bp.route("/data", methods=["GET"])
@@ -615,9 +498,14 @@ async def update_bot_files(request: Request) -> None:
 @openapi.tag("bot-data")
 @openapi.response(
     200,
-    {"application/json": model_to_schema(CALMUserData)},
+    {"application/json": model_to_schema(BotData)},
     description="Bot data retrieved successfully",
 )
+@openapi.response(
+    409,
+    {"application/json": model_to_schema(ApiErrorResponse)},
+    description="Agent not ready",
+)
 @openapi.response(
     500,
     {"application/json": model_to_schema(ApiErrorResponse)},
@@ -626,17 +514,196 @@ async def update_bot_files(request: Request) -> None:
 async def get_bot_data(request: Request) -> HTTPResponse:
     """Get current bot data in CALM import format."""
     try:
-        project_generator = get_project_generator(request)
-        calm_parts = extract_calm_import_parts_from_project_generator(project_generator)
+        agent: Optional[Agent] = request.app.ctx.agent
+        if not agent:
+            raise AgentNotReady(
+                "Can't retrieve the data without an agent being loaded."
+            )
 
-        return response.json(calm_parts.model_dump())
-    except Exception as e:
-        structlogger.error("bot_builder_service.get_bot_data.error", error=str(e))
+        bot_data = await extract_bot_data_from_agent(agent)
+
+        return response.json(bot_data.model_dump())
+    except AgentNotReady as e:
+        return response.json(
+            ApiErrorResponse(
+                error="Agent not ready",
+                details={"error": str(e)},
+            ).model_dump(),
+            status=HTTPStatus.CONFLICT,
+        )
+    except Exception as exc:
+        capture_exception_with_context(
+            exc,
+            "bot_builder_service.get_bot_data.unexpected_error",
+            tags={"endpoint": "/api/data"},
+        )
         return response.json(
             ApiErrorResponse(
                 error="Failed to retrieve bot data",
+                details={"error": str(exc)},
+            ).model_dump(),
+            status=HTTPStatus.INTERNAL_SERVER_ERROR,
+        )
+
+
+@bp.route("/assistant", methods=["GET"])
+@openapi.summary("Get assistant info")
+@openapi.description(
+    "Returns basic information about the loaded assistant, including the assistant id "
+    "as configured in the model's metadata (from config.yml)."
+)
+@openapi.tag("assistant")
+@openapi.response(
+    200,
+    {"application/json": model_to_schema(AssistantInfo)},
+    description="Assistant info retrieved successfully",
+)
+@openapi.response(
+    409,
+    {"application/json": model_to_schema(ApiErrorResponse)},
+    description="Agent not ready",
+)
+@openapi.response(
+    500,
+    {"application/json": model_to_schema(ApiErrorResponse)},
+    description="Internal server error",
+)
+async def get_bot_info(request: Request) -> HTTPResponse:
+    """Return assistant info including assistant id from model metadata."""
+    try:
+        agent: Optional[Agent] = request.app.ctx.agent
+        if not agent:
+            raise AgentNotReady("Can't retrieve bot info without a loaded agent.")
+
+        assistant_id: Optional[str] = (
+            agent.processor.model_metadata.assistant_id
+            if agent.processor
+            and agent.processor.model_metadata
+            and hasattr(agent.processor.model_metadata, "assistant_id")
+            else None
+        )
+
+        return response.json(AssistantInfo(assistant_id=assistant_id).model_dump())
+    except AgentNotReady as e:
+        return response.json(
+            ApiErrorResponse(
+                error="Agent not ready",
                 details={"error": str(e)},
             ).model_dump(),
+            status=HTTPStatus.CONFLICT,
+        )
+    except Exception as exc:
+        capture_exception_with_context(
+            exc,
+            "bot_builder_service.get_bot_info.unexpected_error",
+            tags={"endpoint": "/api/assistant"},
+        )
+        return response.json(
+            ApiErrorResponse(
+                error="Failed to retrieve bot info",
+                details={"error": str(exc)},
+            ).model_dump(),
+            status=HTTPStatus.INTERNAL_SERVER_ERROR,
+        )
+
+
+@bp.route("/download", methods=["GET"])
+@openapi.summary("Download bot project as tar.gz")
+@openapi.description(
+    "Downloads the current bot project files as a compressed tar.gz archive. "
+    "Includes all configuration files and a .env file with RASA_PRO_LICENSE. "
+    "Requires valid JWT token in Authorization header."
+)
+@openapi.tag("bot-files")
+@openapi.parameter(
+    "Authorization",
+    description="Bearer token for authentication",
+    _in="header",
+    required=True,
+    schema=str,
+)
+@openapi.parameter(
+    "project_name",
+    description="Name of the project for the archive filename and pyproject.toml",
+    _in="query",
+    required=False,
+    schema=str,
+)
+@openapi.response(
+    200,
+    {"application/gzip": bytes},
+    description="Bot project downloaded successfully as tar.gz",
+)
+@openapi.response(
+    401,
+    {"application/json": model_to_schema(ApiErrorResponse)},
+    description="Authentication failed - invalid or missing token",
+)
+@openapi.response(
+    500,
+    {"application/json": model_to_schema(ApiErrorResponse)},
+    description="Internal server error",
+)
+async def download_bot_project(request: Request) -> HTTPResponse:
+    """Download bot project as tar.gz archive."""
+    try:
+        # Extract and verify token
+        auth_header = request.headers.get("Authorization", "")
+        verification_result = extract_and_verify_auth0_token(auth_header)
+
+        if verification_result.error_message is not None:
+            structlogger.error(
+                "bot_builder_service.download_bot_project.token_verification_failed",
+                error=verification_result.error_message,
+            )
+            return response.json(
+                ApiErrorResponse(
+                    error=verification_result.error_message,
+                    details={"expected": "Bearer <valid_token>"},
+                ).model_dump(),
+                status=401,
+            )
+
+        # Get bot files
+        project_generator = get_project_generator(request)
+        bot_files = project_generator.get_bot_files()
+
+        # Get project name from query parameters, default to "bot-project"
+        project_name = request.args.get("project_name", "bot-project")
+
+        # Create tar.gz archive
+        tar_data = create_bot_project_archive(bot_files, project_name)
+
+        structlogger.info(
+            "bot_builder_service.download_bot_project.success",
+            user_sub=verification_result.payload.get("sub")
+            if verification_result.payload
+            else None,
+            files_count=len(bot_files),
+            archive_size=len(tar_data),
+            payload=verification_result.payload,
+            project_name=project_name,
+        )
+
+        return response.raw(
+            tar_data,
+            content_type="application/gzip",
+            headers={
+                "Content-Disposition": f"attachment; filename={project_name}.tar.gz"
+            },
+        )
+
+    except Exception as exc:
+        capture_exception_with_context(
+            exc,
+            "bot_builder_service.download_bot_project.unexpected_error",
+            tags={"endpoint": "/api/download"},
+        )
+        return response.json(
+            ApiErrorResponse(
+                error="Failed to create bot project archive",
+                details={"error": str(exc)},
+            ).model_dump(),
             status=500,
         )
 
@@ -644,8 +711,10 @@ async def get_bot_data(request: Request) -> HTTPResponse:
 @bp.route("/copilot", methods=["POST"])
 @openapi.summary("AI copilot for bot building")
 @openapi.description(
-    "Provides LLM-powered copilot assistance for conversational bot development, "
-    "including debugging, suggestions, and explanations."
+    "Provides LLM-powered copilot assistance with streaming markdown responses. "
+    "Returns server-sent events (SSE) for real-time streaming of copilot responses.\n\n"
+    "The event's `event` field is the type of event. For this endpoint it will always "
+    "be: `copilot_response`.\n\n"
 )
 @openapi.tag("copilot")
 @openapi.body(
@@ -653,15 +722,74 @@ async def get_bot_data(request: Request) -> HTTPResponse:
     description=(
         "Copilot request containing: "
         "1. conversation history between user and copilot, "
-        "2. session ID for tracking conversation context with the bot being built, and"
-        "3. additional context."
+        "2. session ID for tracking conversation context with the bot being built."
     ),
     required=True,
 )
 @openapi.response(
     200,
-    {"application/json": model_to_schema(CopilotChatMessage)},
-    description="Copilot response with assistance and suggestions.",
+    {"text/event-stream": model_to_schema(ServerSentEvent)},
+    description=(
+        "Server-sent events stream with copilot responses and references. "
+        "The event's `event` field is the type "
+        "of event. For this endpoint it will always be: `copilot_response`. "
+        "The event's data field is a JSON dump. The following describes the event data "
+        "field:\n"
+        "- `response_category` can be one of the following:\n"
+        "  - `copilot` - Stream token generated by the copilot.\n"
+        "  - `out_of_scope_detection` - Response coming from the the out of scope detection.\n"  # noqa: E501
+        "  - `roleplay_detection` - Response coming from the the roleplay detection.\n"
+        "  - `reference` - Reference section.\n"
+        "- `completeness`: Whether this is a streaming token or complete response.\n"
+        "- `content`: The actual response content (for streaming tokens).\n"
+        "- `references`: (Only for `reference` response category) List of reference entries. Each reference entry is a dictionary with the following keys:\n"  # noqa: E501
+        "   - `index`: Reference index as an integer number.\n"
+        "   - `title`: Reference title as a string.\n"
+        "   - `url`: Reference URL as a string.\n\n"
+    ),
+    examples={
+        "Streaming token from copilot": {
+            "summary": "Streaming token from copilot",
+            "value": GeneratedContent(
+                content="<token generated by the copilot>",
+                response_category=ResponseCategory.COPILOT,
+                response_completeness=ResponseCompleteness.TOKEN,
+            )
+            .to_sse_event()
+            .model_dump(),
+        },
+        "Complete response from out of scope / roleplay detection / guardrail policy violation": {  # noqa: E501
+            "summary": "Complete response from copilot",
+            "value": GeneratedContent(
+                content="<response from the out of scope detection>",
+                response_category=ResponseCategory.OUT_OF_SCOPE_DETECTION,
+                response_completeness=ResponseCompleteness.COMPLETE,
+            )
+            .to_sse_event()
+            .model_dump(),
+        },
+        "Response with the references": {
+            "summary": "Reference section with all references",
+            "value": ReferenceSection(
+                references=[
+                    ReferenceEntry(
+                        index=1,
+                        title="Title of the reference",
+                        url="https://rasa.com/docs/...",
+                    ),
+                    ReferenceEntry(
+                        index=2,
+                        title="Title of another reference",
+                        url="https://rasa.com/docs/...",
+                    ),
+                ],
+                response_category=ResponseCategory.REFERENCE,
+                response_completeness=ResponseCompleteness.COMPLETE,
+            )
+            .to_sse_event()
+            .model_dump(),
+        },
+    },
 )
 @openapi.response(
     400,
@@ -678,71 +806,135 @@ async def get_bot_data(request: Request) -> HTTPResponse:
     {"application/json": model_to_schema(ApiErrorResponse)},
     description="Internal server error.",
 )
-async def copilot(request: Request) -> HTTPResponse:
-    """Handle copilot requests."""
+async def copilot(request: Request) -> None:
+    """Handle copilot requests with streaming markdown responses."""
+    sse = await request.respond(content_type="text/event-stream")
     project_generator = get_project_generator(request)
-    input_channel = get_input_channel(request)
 
     try:
-        # Validate request
-        copilot_request = CopilotRequest(**request.json)
+        # 1. Validate and unpack input
+        req = CopilotRequest(**request.json)
 
-        # Get current conversation context
-        current_tracker = await current_tracker_from_input_channel(
-            request.app, input_channel
+        telemetry = CopilotTelemetry(
+            project_id=HELLO_RASA_PROJECT_ID,
+            user_id=request.headers.get(HEADER_USER_ID),
         )
-        assistant_logs = get_recent_logs()
-        assistant_files = project_generator.get_bot_files()
+        structlogger.debug("builder.copilot.telemetry.request.init")
+
+        if req.last_message and req.last_message.role == ROLE_USER:
+            structlogger.debug("builder.copilot.telemetry.request.user_turn")
+            # Offload telemetry logging to a background task
+            request.app.add_task(
+                asyncio.to_thread(
+                    telemetry.log_user_turn, req.last_message.get_text_content()
+                )
+            )
 
-        # Create LLM builder context
+        # 2. Get the necessary context for the copilot
+        tracker = await current_tracker_from_input_channel(request.app, req.session_id)
+        tracker_context = TrackerContext.from_tracker(
+            tracker, max_turns=COPILOT_ASSISTANT_TRACKER_MAX_TURNS
+        )
+        tracker_context = await check_assistant_chat_for_policy_violations(
+            tracker_context=tracker_context,
+            hello_rasa_user_id=request.headers.get(HEADER_USER_ID),
+            hello_rasa_project_id=HELLO_RASA_PROJECT_ID,
+        )
         context = CopilotContext(
-            tracker=current_tracker,
-            assistant_logs=assistant_logs,
-            assistant_files=assistant_files,
-            copilot_chat_history=copilot_request.copilot_chat_history,
+            tracker_context=tracker_context,
+            assistant_logs=get_recent_logs(),
+            assistant_files=project_generator.get_bot_files(),
+            copilot_chat_history=req.copilot_chat_history,
         )
 
-        # Generate response
-        copilot_response = await llm_service.copilot.generate_response(context)
+        # 3. Run guardrail policy checks. If any policy violations are detected,
+        #    send a response and end the stream.
+        guardrail_response: Optional[
+            GeneratedContent
+        ] = await check_copilot_chat_for_policy_violations(
+            context=context,
+            hello_rasa_user_id=request.headers.get(HEADER_USER_ID),
+            hello_rasa_project_id=HELLO_RASA_PROJECT_ID,
+        )
+        if guardrail_response is not None:
+            await sse.send(guardrail_response.to_sse_event().format())
+            await sse.eof()
+            return
 
-        return response.json(copilot_response.model_dump())
+        # 4. Get the original response stream from copilot and handle it with the
+        #    copilot response handler
+        start_timestamp = time.perf_counter()
+        copilot_client = llm_service.instantiate_copilot()
+        (
+            original_stream,
+            used_documents,
+        ) = await copilot_client.generate_response(context)
+
+        copilot_response_handler = llm_service.instantiate_handler(
+            COPILOT_HANDLER_ROLLING_BUFFER_SIZE
+        )
+        intercepted_stream = copilot_response_handler.handle_response(original_stream)
+
+        # 5. Stream the intercepted response
+        async for token in intercepted_stream:
+            await sse.send(token.to_sse_event().format())
+
+        # Offload telemetry logging to a background task
+        request.app.add_task(
+            asyncio.to_thread(
+                telemetry.log_copilot_from_handler,
+                handler=copilot_response_handler,
+                used_documents=used_documents,
+                latency_ms=int((time.perf_counter() - start_timestamp) * 1000),
+                **copilot_client.usage_statistics.model_dump(),
+            )
+        )
 
-    except LLMGenerationError as e:
-        structlogger.error(
-            "bot_builder_service.copilot_helper.generation_error", error=str(e)
+        # 6. Once the stream is over, extract and send references
+        #    if any documents were used
+        if used_documents:
+            reference_section = copilot_response_handler.extract_references(
+                used_documents
+            )
+            await sse.send(reference_section.to_sse_event().format())
+
+    except CopilotStreamError as e:
+        capture_exception_with_context(
+            e,
+            "bot_builder_service.copilot.generation_error",
+            extra={"session_id": req.session_id},
+            tags={"endpoint": "/api/copilot"},
         )
-        return response.json(
-            ApiErrorResponse(
-                error="LLM helper generation failed", details={"llm_error": str(e)}
-            ).model_dump(),
-            status=502,
+        await sse.send(
+            ServerSentEvent(
+                event="error",
+                data={"error": str(e)},
+            ).format()
         )
 
-    except Exception as e:
-        structlogger.error(
-            "bot_builder_service.copilot_helper.unexpected_error", error=str(e)
+    except Exception as exc:
+        capture_exception_with_context(
+            exc,
+            "bot_builder_service.copilot.unexpected_error",
+            extra={"session_id": req.session_id if "req" in locals() else None},
+            tags={"endpoint": "/api/copilot"},
         )
-        return response.json(
-            ApiErrorResponse(
-                error="Unexpected error in LLM builder",
-                details=None,
-            ).model_dump(),
-            status=500,
+        await sse.send(
+            ServerSentEvent(
+                event="error",
+                data={"error": str(exc)},
+            ).format()
         )
 
+    finally:
+        await sse.eof()
+
 
 async def current_tracker_from_input_channel(
-    app: Any, input_channel: StudioChatInput
+    app: Any, session_id: str
 ) -> Optional[DialogueStateTracker]:
     """Generate chat bot context from current conversation."""
-    if app.ctx.agent and input_channel.latest_tracker_session_id:
-        return await app.ctx.agent.tracker_store.retrieve(
-            input_channel.latest_tracker_session_id
-        )
+    if app.ctx.agent and session_id:
+        return await app.ctx.agent.tracker_store.retrieve(session_id)
     else:
         return None
-
-
-async def _send_sse_event(sse_response: HTTPResponse, event: ServerSentEvent) -> None:
-    """Send a server-sent event."""
-    await sse_response.send(event.format())