rasa-pro 3.13.0a1.dev5__py3-none-any.whl → 3.13.0a1.dev7__py3-none-any.whl

rasa/builder/README.md

@@ -0,0 +1,120 @@
+# Rasa Prompt-to-Bot Service
+
+A production-ready service that generates Rasa chatbots from natural language descriptions using LLMs.
+
+## Architecture
+
+The service follows functional programming principles with minimal use of classes:
+
+### Core Modules
+
+- **`config.py`** - Configuration management using module-level constants
+- **`exceptions.py`** - Custom exception hierarchy for error handling
+- **`models.py`** - Pydantic models for request/response validation
+- **`llm_service.py`** - LLM interactions (minimal class for state management)
+- **`validation_service.py`** - Project validation functions
+- **`training_service.py`** - Model training functions
+- **`project_generator.py`** - Project generation (class for bot files state)
+- **`service.py`** - Main orchestrating service (class for app state)
+- **`logging_utils.py`** - Thread-safe logging utilities
+- **`llm_context.py`** - Conversation context formatting
+- **`main.py`** - Application entry point
+
+### Utility Scripts
+
+- **`scrape_rasa_docs.py`** - Documentation scraping
+- **`create_openai_vector_store.py`** - Documentation indexing
+
+## Key Design Principles
+
+1. **Functional First**: Use functions for stateless operations
+2. **Minimal Classes**: Classes only when state management is needed
+3. **Configuration**: Environment variables with sensible defaults
+4. **Error Handling**: Structured exception hierarchy
+5. **Type Safety**: Full type annotations and Pydantic validation
+6. **Thread Safety**: Safe concurrent operations
+7. **Resource Management**: Proper cleanup and lifecycle handling
+
+## Usage
+
+### Running the Service
+
+```bash
+python rasa/builder/main.py
+```
+
+### Environment Configuration
+
+```bash
+# OpenAI Settings
+export OPENAI_MODEL="gpt-4.1-2025-04-14"
+export OPENAI_TEMPERATURE="0.7"
+export OPENAI_VECTOR_STORE_ID="vs_xxxxx"
+export OPENAI_TIMEOUT="30"
+
+# Server Settings
+export SERVER_HOST="0.0.0.0"
+export SERVER_PORT="5005"
+export MAX_RETRIES="5"
+export CORS_ORIGINS="http://localhost:3000,https://example.com"
+
+# Validation Settings
+export VALIDATION_FAIL_ON_WARNINGS="false"
+```
+
+### API Endpoints
+
+- `POST /api/prompt-to-bot` - Generate bot from description
+- `GET /api/bot-data` - Get current bot configuration
+- `PUT /api/bot-data` - Update bot configuration (SSE)
+- `POST /api/llm-builder` - LLM helper for bot development
+- `GET /` - Health check
+
+### Documentation Setup
+
+```bash
+# 1. Scrape Rasa documentation
+python rasa/builder/scrape_rasa_docs.py
+
+# 2. Create OpenAI vector store
+python rasa/builder/create_openai_vector_store.py
+```
+
+## Benefits of Functional Approach
+
+- **Simpler**: Easy to understand and reason about
+- **Testable**: Functions are easier to unit test
+- **Reusable**: Pure functions can be composed
+- **Maintainable**: Clear separation of concerns
+- **Performant**: No unnecessary object overhead
+- **Debuggable**: Clear call stacks and data flow
+
+## Error Handling
+
+The service uses a structured exception hierarchy:
+
+- `PromptToBotError` - Base exception
+- `ValidationError` - Project validation failures
+- `TrainingError` - Model training issues
+- `LLMGenerationError` - LLM API problems
+- `ProjectGenerationError` - Generation retry exhaustion
+- `AgentLoadError` - Agent loading failures
+
+## Logging
+
+Structured logging with context using `structlog`:
+
+```python
+structlogger.info("operation.success", key="value")
+structlogger.error("operation.failed", error=str(e))
+```
+
+## State Management
+
+Only classes that truly need state:
+
+1. **`LLMService`** - Caches schemas and manages OpenAI client
+2. **`ProjectGenerator`** - Maintains current bot files
+3. **`PromptToBotService`** - Manages Sanic app and agent state
+
+Everything else uses pure functions for maximum simplicity and testability.

rasa/builder/config.py

@@ -0,0 +1,69 @@
+"""Configuration module for the prompt-to-bot service."""
+
+import os
+from typing import Any, Dict
+
+import importlib_resources
+
+from rasa.constants import PACKAGE_NAME
+from rasa.shared.utils.yaml import read_yaml, read_yaml_file
+
+# OpenAI Configuration
+OPENAI_MODEL = os.getenv("OPENAI_MODEL", "gpt-4.1-2025-04-14")
+OPENAI_TEMPERATURE = float(os.getenv("OPENAI_TEMPERATURE", "0.7"))
+OPENAI_VECTOR_STORE_ID = os.getenv(
+    "OPENAI_VECTOR_STORE_ID", "vs_685123376e288191a005b6b144d3026f"
+)
+OPENAI_MAX_VECTOR_RESULTS = int(os.getenv("OPENAI_MAX_VECTOR_RESULTS", "10"))
+OPENAI_TIMEOUT = int(os.getenv("OPENAI_TIMEOUT", "30"))
+
+# Server Configuration
+SERVER_HOST = os.getenv("SERVER_HOST", "0.0.0.0")
+SERVER_PORT = int(os.getenv("SERVER_PORT", "5005"))
+MAX_RETRIES = int(os.getenv("MAX_RETRIES", "5"))
+MAX_LOG_ENTRIES = int(os.getenv("MAX_LOG_ENTRIES", "30"))
+
+# CORS Configuration
+_cors_origins_env = os.getenv("CORS_ORIGINS", "*")
+CORS_ORIGINS = _cors_origins_env.split(",") if _cors_origins_env != "*" else ["*"]
+
+# Validation Configuration
+VALIDATION_FAIL_ON_WARNINGS = (
+    os.getenv("VALIDATION_FAIL_ON_WARNINGS", "false").lower() == "true"
+)
+VALIDATION_MAX_HISTORY = None  # Could be configured if needed
+
+
+def get_default_config(assistant_id: str) -> Dict[str, Any]:
+    """Get default Rasa configuration."""
+    base_config = read_yaml_file(
+        str(
+            importlib_resources.files(PACKAGE_NAME).joinpath(
+                "cli/project_templates/default/config.yml"
+            )
+        )
+    )
+    base_config["assistant_id"] = assistant_id
+    return base_config
+
+
+def get_default_endpoints() -> Dict[str, Any]:
+    """Get default endpoints configuration."""
+    return read_yaml_file(
+        str(
+            importlib_resources.files(PACKAGE_NAME).joinpath(
+                "cli/project_templates/default/endpoints.yml"
+            )
+        )
+    )
+
+
+def get_default_credentials() -> Dict[str, Any]:
+    """Get default credentials configuration."""
+    default_credentials_yaml = """
+    studio_chat:
+      user_message_evt: "user_message"
+      bot_message_evt: "bot_message"
+      session_persistence: true
+    """
+    return read_yaml(default_credentials_yaml)

rasa/builder/create_openai_vector_store.py

@@ -1,69 +1,228 @@
+#!/usr/bin/env python3
+"""Script to create and populate OpenAI vector store with Rasa documentation."""
+
 import json
-from dataclasses import dataclass
+import sys
 from pathlib import Path
+from typing import Dict, List
 
 import openai
+import structlog
+
+structlogger = structlog.get_logger()
+
+# Configuration
+DOCS_DIR = "rasa_docs_md"
+FILE_IDS_FILE = "file_ids.json"
+MARKDOWN_TO_URL_FILE = "markdown_to_url.json"
+ASSISTANT_NAME = "Rasa Docs Assistant"
+
+
+def load_url_mapping() -> Dict[str, str]:
+    """Load the markdown filename to URL mapping."""
+    markdown_to_url_file = Path(MARKDOWN_TO_URL_FILE)
+
+    if not markdown_to_url_file.exists():
+        raise FileNotFoundError(
+            f"URL mapping file {markdown_to_url_file} not found. "
+            "Please run scrape_rasa_docs.py first."
+        )
+
+    with open(markdown_to_url_file, "r") as f:
+        return json.load(f)
+
+
+def get_markdown_files(docs_dir: str = DOCS_DIR) -> List[Path]:
+    """Get all markdown files in the docs directory."""
+    docs_path = Path(docs_dir)
+
+    if not docs_path.exists():
+        raise FileNotFoundError(f"Documentation directory {docs_dir} not found")
 
-DOCS_DIR = "rasa_docs_md"  # Folder with scraped Markdown files
-assistant_name = "Rasa Docs Assistant"
+    md_files = list(docs_path.glob("*.md"))
+    if not md_files:
+        raise FileNotFoundError(f"No markdown files found in {docs_dir}")
 
+    structlogger.info("vector_store.found_files", count=len(md_files))
+    return md_files
 
-@dataclass
-class FileWithAttributes:
-    file_id: str
-    file_name: str
-    attributes: dict
 
+def load_or_upload_files(client: openai.OpenAI) -> List[Dict[str, str]]:
+    """Load existing file IDs or upload files to OpenAI."""
+    file_ids_file = Path(FILE_IDS_FILE)
+
+    if file_ids_file.exists():
+        structlogger.info("vector_store.loading_existing_files")
+        with open(file_ids_file, "r") as f:
+            return json.load(f)
+
+    return upload_files(client)
+
+
+def upload_files(client: openai.OpenAI) -> List[Dict[str, str]]:
+    """Upload markdown files to OpenAI."""
+    structlogger.info("vector_store.uploading_files")
+
+    md_files = get_markdown_files()
+    uploaded_files = []
+
+    for md_file in md_files:
+        try:
+            with open(md_file, "rb") as f:
+                uploaded = client.files.create(file=f, purpose="assistants")
+
+            file_info = {"file_id": uploaded.id, "file_name": md_file.name}
+            uploaded_files.append(file_info)
+
+            structlogger.info(
+                "vector_store.file_uploaded",
+                file_name=md_file.name,
+                file_id=uploaded.id,
+            )
+
+        except Exception as e:
+            structlogger.error(
+                "vector_store.upload_failed", file_name=md_file.name, error=str(e)
+            )
+            raise
 
-def get_files_with_metadata(files) -> list[FileWithAttributes]:
-    with open("markdown_to_url.json", "r") as f:
-        markdown_to_url = json.load(f)
+    # Save file IDs for future use
+    with open(FILE_IDS_FILE, "w") as f:
+        json.dump(uploaded_files, f, indent=2)
+
+    structlogger.info("vector_store.upload_complete", count=len(uploaded_files))
+    return uploaded_files
+
+
+def prepare_files_with_metadata(files: List[Dict[str, str]]) -> List[Dict[str, str]]:
+    """Prepare files with URL metadata."""
+    url_mapping = load_url_mapping()
 
     files_with_metadata = []
-    for file in files:
+    for file_info in files:
+        file_name = file_info["file_name"]
+        url = url_mapping.get(file_name, "")
+
+        if not url:
+            structlogger.warning("vector_store.missing_url", file_name=file_name)
+
         files_with_metadata.append(
-            FileWithAttributes(
-                file_id=file["file_id"],
-                file_name=file["file_name"],
-                attributes={"url": markdown_to_url[file["file_name"]]},
-            )
+            {"file_id": file_info["file_id"], "file_name": file_name, "url": url}
         )
 
     return files_with_metadata
 
 
-# Step 1: Upload files
-print("📤 Uploading files to OpenAI...")
+def create_vector_store(
+    client: openai.OpenAI, files_with_metadata: List[Dict[str, str]]
+) -> str:
+    """Create vector store and add files."""
+    try:
+        # Create vector store
+        structlogger.info("vector_store.creating")
+        vector_store = client.vector_stores.create(name=ASSISTANT_NAME)
+
+        # Add files to vector store
+        for file_meta in files_with_metadata:
+            try:
+                client.vector_stores.files.create(
+                    vector_store_id=vector_store.id,
+                    file_id=file_meta["file_id"],
+                    attributes={"url": file_meta["url"]},
+                )
+
+                structlogger.info(
+                    "vector_store.file_added",
+                    file_name=file_meta["file_name"],
+                    url=file_meta["url"],
+                )
+
+            except Exception as e:
+                structlogger.error(
+                    "vector_store.file_add_failed",
+                    file_name=file_meta["file_name"],
+                    error=str(e),
+                )
+                # Continue with other files
+
+        structlogger.info(
+            "vector_store.created",
+            vector_store_id=vector_store.id,
+            files_count=len(files_with_metadata),
+        )
+
+        return vector_store.id
+
+    except Exception as e:
+        structlogger.error("vector_store.creation_failed", error=str(e))
+        raise
+
+
+def run_vector_store_creation() -> str:
+    """Run the complete vector store creation process."""
+    client = openai.OpenAI()
+
+    try:
+        # Load or upload files
+        files = load_or_upload_files(client)
+
+        # Prepare files with metadata
+        files_with_metadata = prepare_files_with_metadata(files)
+
+        # Create vector store
+        vector_store_id = create_vector_store(client, files_with_metadata)
+
+        print("\n🎉 Vector store created successfully!")
+        print(f"Vector store ID: {vector_store_id}")
+        print(f"Files processed: {len(files_with_metadata)}")
+
+        return vector_store_id
+
+    except Exception as e:
+        structlogger.error("vector_store.process_failed", error=str(e))
+        print(f"\n❌ Vector store creation failed: {e}")
+        raise
+
+
+def setup_logging():
+    """Setup basic logging."""
+    import logging.config
+
+    logging_config = {
+        "version": 1,
+        "disable_existing_loggers": False,
+        "formatters": {
+            "standard": {"format": "%(asctime)s [%(levelname)s] %(name)s: %(message)s"},
+        },
+        "handlers": {
+            "default": {
+                "level": "INFO",
+                "formatter": "standard",
+                "class": "logging.StreamHandler",
+            },
+        },
+        "loggers": {"": {"handlers": ["default"], "level": "INFO", "propagate": False}},
+    }
+
+    logging.config.dictConfig(logging_config)
 
-files = []
 
-# try to read the file ids from the json file
-if Path("file_ids.json").exists():
-    with open("file_ids.json", "r") as f:
-        files = json.load(f)
-else:
-    for md_file in Path(DOCS_DIR).glob("*.md"):
-        with open(md_file, "rb") as f:
-            uploaded = openai.files.create(file=f, purpose="assistants")
-            files.append({"file_id": uploaded.id, "file_name": md_file.name})
-            print(f"✅ Uploaded {md_file.name} → {uploaded.id}")
+def main():
+    """Main entry point for the script."""
+    setup_logging()
 
-    # persist the file ids in a json file
-    with open("file_ids.json", "w") as f:
-        json.dump(files, f, indent=2)
+    try:
+        run_vector_store_creation()
+        return 0
 
-# Step 2: Create Vector Store
-print("\n🤖 Creating vector store...")
+    except KeyboardInterrupt:
+        print("\n⏹️ Process interrupted by user")
+        return 1
 
-vector_store = openai.vector_stores.create(name="Rasa Docs")
+    except Exception as e:
+        print(f"\n💥 Unexpected error: {e}")
+        return 1
 
-for file in get_files_with_metadata(files):
-    openai.vector_stores.files.create(
-        vector_store_id=vector_store.id,
-        file_id=file.file_id,
-        attributes=file.attributes,
-    )
-    print(f"✅ Added {file.file_name} to vector store.")
 
-print("\n🎉 Vector store created successfully!")
-print(f"Vector store ID: {vector_store.id}")
+if __name__ == "__main__":
+    sys.exit(main())

rasa/builder/exceptions.py

@@ -0,0 +1,49 @@
+"""Custom exceptions for the prompt-to-bot service."""
+
+from typing import Any, Optional
+
+
+class PromptToBotError(Exception):
+    """Base exception for prompt-to-bot service."""
+
+    pass
+
+
+class ValidationError(PromptToBotError):
+    """Raised when Rasa project validation fails."""
+
+    def __init__(self, message: str, validation_logs: Optional[Any] = None):
+        super().__init__(message)
+        self.validation_logs = validation_logs
+
+
+class TrainingError(PromptToBotError):
+    """Raised when model training fails."""
+
+    pass
+
+
+class LLMGenerationError(PromptToBotError):
+    """Raised when LLM generation fails."""
+
+    pass
+
+
+class SchemaValidationError(PromptToBotError):
+    """Raised when schema validation fails."""
+
+    pass
+
+
+class AgentLoadError(PromptToBotError):
+    """Raised when agent loading fails."""
+
+    pass
+
+
+class ProjectGenerationError(PromptToBotError):
+    """Raised when project generation fails after retries."""
+
+    def __init__(self, message: str, attempts: int):
+        super().__init__(f"{message} (failed after {attempts} attempts)")
+        self.attempts = attempts