rasa-pro 3.14.0.dev7__py3-none-any.whl → 3.14.0.dev9__py3-none-any.whl

rasa/builder/project_generator.py

@@ -0,0 +1,458 @@
+"""Service for generating Rasa projects from prompts."""
+
+import json
+import os
+import shutil
+from pathlib import Path
+from textwrap import dedent
+from typing import Any, Dict, Generator, List, Optional
+
+import structlog
+
+from rasa.builder import config
+from rasa.builder.exceptions import ProjectGenerationError, ValidationError
+from rasa.builder.llm_service import get_skill_generation_messages, llm_service
+from rasa.builder.logging_utils import capture_exception_with_context
+from rasa.builder.models import BotFiles
+from rasa.builder.project_info import ProjectInfo, ensure_first_used, load_project_info
+from rasa.builder.template_cache import copy_cache_for_template_if_available
+from rasa.builder.training_service import TrainingInput
+from rasa.builder.validation_service import validate_project
+from rasa.cli.scaffold import ProjectTemplateName, create_initial_project
+from rasa.shared.constants import DEFAULT_MODELS_PATH
+from rasa.shared.core.flows import yaml_flows_io
+from rasa.shared.importers.importer import TrainingDataImporter
+from rasa.shared.utils.yaml import dump_obj_as_yaml_to_string
+from rasa.utils.io import InvalidPathException, subpath
+
+structlogger = structlog.get_logger()
+
+
+class ProjectGenerator:
+    """Service for generating Rasa projects from skill descriptions."""
+
+    def __init__(self, project_folder: str) -> None:
+        """Initialize the project generator with a folder for file persistence.
+
+        Args:
+            project_folder: Path to the folder where project files will be stored
+        """
+        self.project_folder = Path(project_folder)
+        self.project_folder.mkdir(parents=True, exist_ok=True)
+
+    @property
+    def project_info(self) -> ProjectInfo:
+        """Get the project info."""
+        return load_project_info(self.project_folder)
+
+    def is_empty(self) -> bool:
+        """Check if the project folder is empty.
+
+        Excluding hidden paths.
+        """
+        return not any(
+            file.is_file()
+            for file in self.project_folder.iterdir()
+            if not file.name.startswith(".")
+        )
+
+    async def init_from_template(self, template: ProjectTemplateName) -> None:
+        """Create the initial project files."""
+        self.cleanup()
+        create_initial_project(self.project_folder.as_posix(), template)
+        # If a local cache for this template exists, copy it into the project.
+        # We no longer download here to avoid blocking project creation.
+        copy_cache_for_template_if_available(template, self.project_folder)
+        # needs to happen after caching, as we download/copy .rasa and that would
+        # overwrite the project info file in .rasa
+        ensure_first_used(self.project_folder)
+
+    async def generate_project_with_retries(
+        self,
+        skill_description: str,
+        template: ProjectTemplateName,
+        max_retries: Optional[int] = None,
+    ) -> Dict[str, Optional[str]]:
+        """Generate a Rasa project with retry logic for validation failures.
+
+        Args:
+            skill_description: Natural language description of the skill
+            rasa_config: Rasa configuration dictionary
+            template: Project template to use for the initial project
+            max_retries: Maximum number of retry attempts
+
+        Returns:
+            Dictionary of generated file contents (filename -> content)
+
+        Raises:
+            ProjectGenerationError: If generation fails after all retries
+        """
+        if max_retries is None:
+            max_retries = config.MAX_RETRIES
+
+        # if ever we do not use the template, we need to ensure first_used is set
+        # separately!
+        await self.init_from_template(template)
+
+        project_data = self._get_bot_data_for_llm()
+
+        initial_messages = get_skill_generation_messages(
+            skill_description, project_data
+        )
+
+        async def _generate_with_retry(
+            messages: List[Dict[str, Any]], attempts_left: int
+        ) -> Dict[str, Optional[str]]:
+            try:
+                # Generate project data using LLM
+                project_data = await llm_service.generate_rasa_project(messages)
+
+                # Update stored bot data
+                self._update_bot_files_from_llm_response(project_data)
+
+                bot_files = self.get_bot_files()
+                structlogger.info(
+                    "project_generator.generated_project",
+                    attempts_left=attempts_left,
+                    files=list(bot_files.keys()),
+                )
+
+                # Validate the generated project
+                await self._validate_generated_project()
+
+                structlogger.info(
+                    "project_generator.validation_success", attempts_left=attempts_left
+                )
+
+                return bot_files
+
+            except ValidationError as e:
+                structlogger.error(
+                    "project_generator.validation_error",
+                    error=str(e),
+                    attempts_left=attempts_left,
+                )
+
+                if attempts_left <= 0:
+                    raise ProjectGenerationError(
+                        f"Failed to generate valid Rasa project: {e}", max_retries
+                    )
+
+                # Create error feedback for next attempt
+                error_feedback_messages = messages + [
+                    {
+                        "role": "assistant",
+                        "content": json.dumps(project_data),
+                    },
+                    {
+                        "role": "user",
+                        "content": dedent(f"""
+                            Previous attempt failed validation with error: {e}
+
+                            Please fix the issues and generate a valid Rasa project.
+                            Pay special attention to:
+                            - Proper YAML syntax
+                            - Required fields in domain and flows
+                            - Consistent naming between flows and domain
+                            - Valid slot types and mappings
+                        """).strip(),
+                    },
+                ]
+
+                return await _generate_with_retry(
+                    error_feedback_messages, attempts_left - 1
+                )
+
+            except Exception as e:
+                structlogger.error(
+                    "project_generator.generation_error",
+                    error=str(e),
+                    attempts_left=attempts_left,
+                )
+
+                if attempts_left <= 0:
+                    raise ProjectGenerationError(
+                        f"Failed to generate Rasa project: {e}", max_retries
+                    )
+
+                # For non-validation errors, retry with original messages
+                return await _generate_with_retry(initial_messages, attempts_left - 1)
+
+        return await _generate_with_retry(initial_messages, max_retries)
+
+    async def _validate_generated_project(self) -> None:
+        """Validate the generated project using the validation service."""
+        importer = self._create_importer()
+        validation_error = await validate_project(importer)
+
+        if validation_error:
+            raise ValidationError(validation_error)
+
+    def _get_endpoints_file(self) -> Path:
+        """Get the endpoints file."""
+        return self.project_folder / "endpoints.yml"
+
+    def get_training_input(self) -> TrainingInput:
+        """Get the training input."""
+        return TrainingInput(
+            importer=self._create_importer(),
+            endpoints_file=self._get_endpoints_file(),
+        )
+
+    def _create_importer(self) -> TrainingDataImporter:
+        """Create a training data importer from the current bot files."""
+        try:
+            if (self.project_folder / "domain.yml").exists():
+                domain_path = self.project_folder / "domain.yml"
+            else:
+                domain_path = self.project_folder / "domain"
+
+            return TrainingDataImporter.load_from_config(
+                config_path=str(self.project_folder / "config.yml"),
+                domain_path=str(domain_path),
+                training_data_paths=[str(self.project_folder / "data")],
+                args={},
+            )
+
+        except Exception as e:
+            raise ValidationError(f"Failed to create importer: {e}")
+
+    def get_bot_files(
+        self,
+        allowed_file_extensions: Optional[List[str]] = None,
+        exclude_docs_directory: bool = False,
+    ) -> BotFiles:
+        """Get the current bot files by reading from disk.
+
+        Args:
+            allowed_file_extensions: Optional list of file extensions to include.
+                If None, fetch all files. If provided, only fetch files with matching
+                extensions. Use `""` empty string to allow files with no extensions.
+            exclude_docs_directory: Optional boolean indicating whether to exclude.
+
+        Returns:
+            Dictionary of file contents with relative paths as keys
+        """
+        bot_files: BotFiles = {}
+
+        for file in self.bot_file_paths():
+            relative_path = file.relative_to(self.project_folder)
+
+            # Exclude the docs directory if specified
+            if exclude_docs_directory and relative_path.parts[0] == "docs":
+                continue
+
+            # Exclude the files by file extensions if specified
+            if allowed_file_extensions is not None:
+                allowed_file_extensions = [
+                    ext.lower() for ext in allowed_file_extensions
+                ]
+                if file.suffix.lstrip(".").lower() not in allowed_file_extensions:
+                    continue
+            # Read file content and store with relative path as key
+            try:
+                bot_files[relative_path.as_posix()] = file.read_text(encoding="utf-8")
+            except Exception as e:
+                structlogger.debug(
+                    "project_generator.get_bot_files.error",
+                    error=str(e),
+                    file_path=file.as_posix(),
+                )
+                bot_files[relative_path.as_posix()] = None
+        return bot_files
+
+    def is_restricted_path(self, path: Path) -> bool:
+        """Check if the path is restricted.
+
+        These paths are excluded from deletion and editing by the user.
+        """
+        relative_path = path.relative_to(self.project_folder)
+
+        # Skip hidden files and directories (any path component starting with '.')
+        # as well as `__pycache__` folders
+        if any(part.startswith(".") for part in relative_path.parts):
+            return True
+
+        if "__pycache__" in relative_path.parts:
+            return True
+
+        # exclude the project_folder / models folder
+        if relative_path.parts[0] == DEFAULT_MODELS_PATH:
+            return True
+
+        return False
+
+    def bot_file_paths(
+        self,
+    ) -> Generator[Path, None, None]:
+        """Get the paths of all bot files."""
+        for file in self.project_folder.glob("**/*"):
+            # Skip directories
+            if not file.is_file() or self.is_restricted_path(file):
+                continue
+
+            yield file
+
+    def _get_bot_data_for_llm(self) -> Dict[str, Any]:
+        """Get the current bot data for the LLM."""
+        file_importer = self._create_importer()
+
+        # only include data created by the user (or the builder llm)
+        # avoid including to many defaults that are not customized
+        domain = file_importer.get_user_domain()
+        flows = file_importer.get_user_flows()
+
+        return {
+            "domain": domain.as_dict(should_clean_json=True),
+            "flows": yaml_flows_io.get_flows_as_json(flows, should_clean_json=True),
+        }
+
+    def _path_for_flow(self, flow_id: str) -> str:
+        """Get the path for a flow."""
+        if flow_id.startswith("pattern_"):
+            return f"data/patterns/{flow_id}.yml"
+        else:
+            return f"data/flows/{flow_id}.yml"
+
+    def _update_bot_files_from_llm_response(self, project_data: Dict[str, Any]) -> None:
+        """Update the bot files with generated data by writing to disk."""
+        files = {"domain.yml": dump_obj_as_yaml_to_string(project_data["domain"])}
+        # split up flows into one file per flow in the /flows folder
+        for flow_id, flow_data in project_data["flows"].get("flows", {}).items():
+            flow_file_path = self._path_for_flow(flow_id)
+            single_flow_file_data = {"flows": {flow_id: flow_data}}
+            files[flow_file_path] = dump_obj_as_yaml_to_string(single_flow_file_data)
+
+        # removes any other flows that the LLM didn't generate
+        self._cleanup_flows()
+        self.update_bot_files(files)
+
+    def _cleanup_flows(self) -> None:
+        """Cleanup the flows folder."""
+        flows_folder = self.project_folder / "data" / "flows"
+        if flows_folder.exists():
+            shutil.rmtree(flows_folder)
+        flows_folder.mkdir(parents=True, exist_ok=True)
+
+    def update_bot_files(self, files: Dict[str, Optional[str]]) -> None:
+        """Update bot files with new content by writing to disk."""
+        for filename, content in files.items():
+            file_path = Path(subpath(str(self.project_folder), filename))
+            # Disallow updates inside .rasa project metadata directory
+            if any(
+                part.startswith(".")
+                for part in file_path.relative_to(self.project_folder).parts
+            ):
+                # silently ignore hidden paths
+                continue
+            file_path.parent.mkdir(parents=True, exist_ok=True)
+            file_path.write_text(content, encoding="utf-8")
+
+    def ensure_all_files_are_writable(self, files: Dict[str, Optional[str]]) -> None:
+        """Ensure all files are writable."""
+        for filename, content in files.items():
+            file_path = Path(subpath(str(self.project_folder), filename))
+            if self.is_restricted_path(file_path):
+                raise InvalidPathException(
+                    f"This file or folder is restricted from editing: {file_path}"
+                )
+
+    def replace_all_bot_files(self, files: Dict[str, Optional[str]]) -> None:
+        """Replace all bot files with new content, deleting files not in the request.
+
+        Files/folders starting with .rasa/ or models/ are excluded from deletion.
+
+        Args:
+            files: Dictionary mapping file names to their content
+        """
+        self.ensure_all_files_are_writable(files)
+        # Collect all existing files - any files not in the new `files` dict will be
+        # deleted from this set
+        existing_files = set(path.as_posix() for path in self.bot_file_paths())
+
+        # Write all new files
+        for filename, content in files.items():
+            if content is None:
+                continue
+
+            file_path = Path(subpath(str(self.project_folder), filename))
+            file_path.parent.mkdir(parents=True, exist_ok=True)
+
+            try:
+                file_path.write_text(content, encoding="utf-8")
+            except Exception as e:
+                # Log write failure and avoid deleting an existing file by mistake
+                capture_exception_with_context(
+                    e,
+                    "project_generator.replace_all_bot_files.write_error",
+                    extra={"file_path": file_path},
+                )
+                if file_path.as_posix() in existing_files:
+                    # Keep the original file if it already existed
+                    existing_files.discard(file_path.as_posix())
+                continue
+
+            # Remove from deletion set since this file is in the new set of files
+            existing_files.discard(file_path.as_posix())
+
+        # Delete files that weren't in the request
+        for file_to_delete in existing_files:
+            file_path = Path(file_to_delete)
+            try:
+                file_path.unlink()
+            except Exception as e:
+                capture_exception_with_context(
+                    e,
+                    "project_generator.replace_all_bot_files.delete_error",
+                    extra={"file_path": file_path},
+                )
+
+        # Clean up empty directories (except excluded ones)
+        self._cleanup_empty_directories()
+
+    def _cleanup_empty_directories(self) -> None:
+        """Remove empty directories from the project folder.
+
+        Excludes hidden files and directories, and models/ from cleanup.
+        """
+        # Walk directories in reverse order (deepest first)
+        for dirpath, dirnames, filenames in os.walk(self.project_folder, topdown=False):
+            # Skip if this is the project root
+            if dirpath == str(self.project_folder):
+                continue
+
+            if self.is_restricted_path(Path(dirpath)):
+                continue
+
+            relative_path = Path(dirpath).relative_to(self.project_folder)
+
+            try:
+                # Only remove if directory is empty
+                if not os.listdir(dirpath):
+                    os.rmdir(dirpath)
+            except Exception as e:
+                capture_exception_with_context(
+                    e,
+                    "project_generator.cleanup_empty_directories.error",
+                    extra={"directory": relative_path.as_posix()},
+                )
+
+    def cleanup(self) -> None:
+        """Cleanup the project folder."""
+        # remove all the files and folders in the project folder resulting
+        # in an empty folder
+        for filename in os.listdir(self.project_folder):
+            file_path = os.path.join(self.project_folder, filename)
+            try:
+                if filename == "lost+found":
+                    continue
+                if os.path.isfile(file_path) or os.path.islink(file_path):
+                    os.unlink(file_path)
+                elif os.path.isdir(file_path):
+                    shutil.rmtree(file_path)
+            except Exception as e:
+                structlogger.error(
+                    "project_generator.cleanup_error",
+                    error=str(e),
+                    file_path=file_path,
+                )

rasa/builder/project_info.py

@@ -0,0 +1,72 @@
+from __future__ import annotations
+
+import json
+from dataclasses import asdict, dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Optional
+
+from rasa.constants import RASA_DIR_NAME
+
+
+@dataclass
+class ProjectInfo:
+    """Metadata persisted for the builder about the current project.
+
+    - first_used_utc_iso: RFC3339/ISO8601 string of when the project was first used.
+    """
+
+    first_used_utc_iso: Optional[str] = None
+
+    @classmethod
+    def from_json(cls, data: str) -> "ProjectInfo":
+        payload = json.loads(data) if data else {}
+        return cls(**payload)
+
+    def to_json(self) -> str:
+        return json.dumps(asdict(self))
+
+    def first_used_dt(self) -> Optional[datetime]:
+        if not self.first_used_utc_iso:
+            return None
+        value = self.first_used_utc_iso.strip()
+        if value.endswith("Z"):
+            value = value[:-1] + "+00:00"
+        try:
+            dt = datetime.fromisoformat(value)
+        except Exception:
+            return None
+        if dt.tzinfo is None:
+            dt = dt.replace(tzinfo=timezone.utc)
+        return dt.astimezone(timezone.utc)
+
+
+def _project_info_path(project_folder: Path) -> Path:
+    return project_folder / RASA_DIR_NAME / "project_info.json"
+
+
+def load_project_info(project_folder: Path) -> ProjectInfo:
+    path = _project_info_path(project_folder)
+    try:
+        if path.exists():
+            return ProjectInfo.from_json(path.read_text(encoding="utf-8"))
+    except Exception:
+        pass
+    return ProjectInfo()
+
+
+def save_project_info(project_folder: Path, info: ProjectInfo) -> None:
+    path = _project_info_path(project_folder)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(info.to_json(), encoding="utf-8")
+
+
+def ensure_first_used(project_folder: Path) -> datetime:
+    info = load_project_info(project_folder)
+    if existing := info.first_used_dt():
+        return existing
+
+    now = datetime.now(timezone.utc)
+    info.first_used_utc_iso = now.isoformat()
+    save_project_info(project_folder, info)
+    return now

rasa/builder/scrape_rasa_docs.py

@@ -0,0 +1,97 @@
+import json
+import os
+from pathlib import Path
+from urllib.parse import urljoin, urlparse
+
+import requests
+from bs4 import BeautifulSoup
+
+BASE_URL = "https://rasa.com"
+DOCS_ROOT = "https://rasa.com/docs"
+OUTPUT_DIR = "rasa_docs_md"
+MAX_PAGES = 100  # Optional limit for safety
+
+visited = set()
+to_visit = [DOCS_ROOT]
+
+os.makedirs(OUTPUT_DIR, exist_ok=True)
+
+
+def is_valid_doc_url(url: str) -> bool:
+    return url.startswith(DOCS_ROOT) and not any(
+        [url.endswith(".pdf"), "#" in url, "mailto:" in url]
+    )
+
+
+def slugify_url(url: str) -> str:
+    path = urlparse(url).path.strip("/").replace("/", "_")
+    return path if path else "index"
+
+
+def clean_text(html: str) -> str:
+    soup = BeautifulSoup(html, "html.parser")
+
+    # Remove navs, footers, and code tabs (customize if needed)
+    for tag in soup(["nav", "footer", "script", "style", "form", "button"]):
+        tag.decompose()
+
+    main = soup.find("main") or soup.body
+    if not main:
+        return ""
+
+    # Replace <code> with backticks
+    for code in main.find_all("code"):
+        code.string = f"`{code.get_text(strip=True)}`"
+
+    text = main.get_text(separator="\n", strip=True)
+    return text
+
+
+def save_as_markdown(text: str, url: str) -> str:
+    slug = slugify_url(url)
+    file_name = f"{slug}.md"
+    md_path = Path(OUTPUT_DIR) / file_name
+    with open(md_path, "w", encoding="utf-8") as f:
+        f.write(text)
+
+    print(f"✅ Saved: {md_path}")
+    return file_name
+
+
+pages_scraped = 0
+markdown_to_url = {}
+
+while to_visit and pages_scraped < MAX_PAGES:
+    url = to_visit.pop(0)
+    if url in visited:
+        continue
+
+    try:
+        print(f"Scraping: {url}")
+        response = requests.get(url)
+        response.raise_for_status()
+
+        html = response.text
+        text = clean_text(html)
+        if len(text) < 200:  # skip very short pages
+            print("⏭️ Skipped (too short)")
+            continue
+
+        file_name = save_as_markdown(text, url)
+        markdown_to_url[file_name] = url
+        pages_scraped += 1
+
+        soup = BeautifulSoup(html, "html.parser")
+        for link_tag in soup.find_all("a", href=True):
+            link = urljoin(url, link_tag["href"])
+            if is_valid_doc_url(link) and link not in visited:
+                to_visit.append(link)
+
+        visited.add(url)
+
+    except Exception as e:
+        print(f"⚠️ Failed to scrape {url}: {e}")
+
+
+with open("markdown_to_url.json", "w") as f:
+    json.dump(markdown_to_url, f, indent=2)