amd-gaia 0.14.1__py3-none-any.whl

gaia/rag/sdk.py

@@ -0,0 +1,2194 @@
+#!/usr/bin/env python3
+# Copyright(C) 2024-2025 Advanced Micro Devices, Inc. All rights reserved.
+# SPDX-License-Identifier: MIT
+
+"""
+GAIA RAG SDK - Simple PDF document retrieval and Q&A
+"""
+
+import hashlib
+import os
+import pickle
+import re
+import time
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+import numpy as np
+
+try:
+    from pypdf import PdfReader
+except ImportError:
+    try:
+        from PyPDF2 import PdfReader
+    except ImportError:
+        PdfReader = None
+
+try:
+    from sentence_transformers import SentenceTransformer
+except ImportError:
+    SentenceTransformer = None
+
+try:
+    import faiss
+except ImportError:
+    faiss = None
+
+from gaia.chat.sdk import ChatConfig, ChatSDK
+from gaia.logger import get_logger
+from gaia.security import PathValidator
+
+
+@dataclass
+class RAGConfig:
+    """Configuration for RAG SDK."""
+
+    model: str = "Qwen3-Coder-30B-A3B-Instruct-GGUF"
+    max_tokens: int = 1024
+    chunk_size: int = 500
+    chunk_overlap: int = 100  # Increased to 20% overlap for better context preservation
+    max_chunks: int = 5  # Increased to retrieve more context
+    embedding_model: str = (
+        "nomic-embed-text-v2-moe-GGUF"  # Lemonade GGUF embedding model
+    )
+    cache_dir: str = ".gaia"
+    show_stats: bool = False
+    use_local_llm: bool = True
+    base_url: str = "http://localhost:8000/api/v1"  # Lemonade server API URL
+    # Memory management settings
+    max_indexed_files: int = 100  # Maximum number of files to keep indexed
+    max_total_chunks: int = 10000  # Maximum total chunks across all files
+    enable_lru_eviction: bool = (
+        True  # Enable automatic eviction of least recently used documents
+    )
+    # File size limits (prevent OOM)
+    max_file_size_mb: int = 100  # Maximum file size in MB (default: 100MB)
+    warn_file_size_mb: int = 50  # Warn if file exceeds this size (default: 50MB)
+    # LLM-based chunking
+    use_llm_chunking: bool = (
+        False  # Enable LLM-based intelligent chunking (requires LLM client)
+    )
+    # VLM settings (enabled if available, errors out if model can't be loaded)
+    vlm_model: str = "Qwen2.5-VL-7B-Instruct-GGUF"
+    # Security settings
+    allowed_paths: Optional[List[str]] = None
+
+
+@dataclass
+class RAGResponse:
+    """Response from RAG operations with enhanced metadata."""
+
+    text: str
+    chunks: Optional[List[str]] = None
+    chunk_scores: Optional[List[float]] = None
+    stats: Optional[Dict[str, Any]] = None
+    # Enhanced metadata
+    source_files: Optional[List[str]] = None  # List of source files for each chunk
+    chunk_metadata: Optional[List[Dict[str, Any]]] = None  # Detailed metadata per chunk
+    query_metadata: Optional[Dict[str, Any]] = None  # Query-level metadata
+
+
+class RAGSDK:
+    """
+    Simple RAG SDK for PDF document Q&A following GAIA patterns.
+
+    Example usage:
+        ```python
+        from gaia.rag.sdk import RAGSDK, RAGConfig
+
+        # Initialize
+        config = RAGConfig(show_stats=True)
+        rag = RAGSDK(config)
+
+        # Index document
+        rag.index_document("document.pdf")
+
+        # Query
+        response = rag.query("What are the key features?")
+        print(response.text)
+        ```
+    """
+
+    def __init__(self, config: Optional[RAGConfig] = None):
+        """Initialize RAG SDK."""
+        self.config = config or RAGConfig()
+        self.log = get_logger(__name__)
+
+        # Check dependencies
+        self._check_dependencies()
+
+        # Initialize components
+        self.embedder = None
+        self.llm_client = None
+        self.use_lemonade_embeddings = False
+        self.index = None
+        self.chunks = []
+        self.indexed_files = set()
+
+        # Per-file indexing: maps file paths to their chunk indices
+        # This enables efficient per-file searches
+        self.file_to_chunk_indices = {}  # {file_path: [chunk_idx1, chunk_idx2, ...]}
+        self.chunk_to_file = {}  # {chunk_idx: file_path} for reverse lookup
+
+        # Per-file FAISS indices and embeddings (CACHED for performance)
+        self.file_indices = {}  # {file_path: faiss.Index}
+        self.file_embeddings = {}  # {file_path: numpy.array}
+
+        # Per-file metadata (for /dump command and stats)
+        self.file_metadata = (
+            {}
+        )  # {file_path: {'full_text': str, 'num_pages': int, 'vlm_pages': int, ...}}
+
+        # LRU tracking for memory management
+        self.file_access_times = {}  # {file_path: last_access_time}
+        self.file_index_times = {}  # {file_path: index_time}
+
+        # Create cache directory
+        os.makedirs(self.config.cache_dir, exist_ok=True)
+
+        # Initialize chat SDK for LLM responses
+        chat_config = ChatConfig(
+            model=self.config.model,
+            max_tokens=self.config.max_tokens,
+            show_stats=self.config.show_stats,
+            use_local_llm=self.config.use_local_llm,
+        )
+        self.chat = ChatSDK(chat_config)
+
+        # Initialize path validator
+        self.path_validator = PathValidator(self.config.allowed_paths)
+
+        self.log.debug("RAG SDK initialized")
+
+    def _check_dependencies(self):
+        """Check if required dependencies are available."""
+        missing = []
+        if PdfReader is None:
+            missing.append("pypdf (or PyPDF2)")
+        if SentenceTransformer is None:
+            missing.append("sentence-transformers")
+        if faiss is None:
+            missing.append("faiss-cpu")
+
+        if missing:
+            error_msg = (
+                f"\n❌ Error: Missing required RAG dependencies: {', '.join(missing)}\n\n"
+                f"Please install the RAG dependencies:\n"
+                f"  pip install -e .[rag]\n\n"
+                f"Or install packages directly:\n"
+                f"  pip install {' '.join(missing)}\n"
+            )
+            raise ImportError(error_msg)
+
+    def _safe_open(self, file_path: str, mode="rb"):
+        """
+        Safely open file with path validation and O_NOFOLLOW to prevent symlink attacks.
+
+        Args:
+            file_path: Path to file
+            mode: Open mode ('rb', 'r', 'w', 'wb', etc.)
+
+        Returns:
+            File handle
+
+        Raises:
+            PermissionError: If file is outside allowed paths or is a symlink
+            IOError: If file cannot be opened
+        """
+        # Security check: Validate path against allowed directories
+        if not self.path_validator.is_path_allowed(file_path):
+            raise PermissionError(f"Access denied: {file_path} is not in allowed paths")
+
+        import stat
+
+        # Determine flags based on mode
+        if "r" in mode and "+" not in mode:
+            flags = os.O_RDONLY
+        elif "w" in mode:
+            flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC
+        elif "a" in mode:
+            flags = os.O_WRONLY | os.O_CREAT | os.O_APPEND
+        else:
+            flags = os.O_RDONLY
+
+        # CRITICAL: Add O_NOFOLLOW to reject symlinks
+        # This prevents TOCTOU attacks where symlinks are swapped
+        if hasattr(os, "O_NOFOLLOW"):
+            flags |= os.O_NOFOLLOW
+
+        try:
+            # Open file descriptor with O_NOFOLLOW
+            fd = os.open(str(file_path), flags)
+        except OSError as e:
+            if e.errno == 40:  # ELOOP - too many symbolic links
+                raise PermissionError(f"Symlinks not allowed: {file_path}")
+            raise IOError(f"Cannot open file {file_path}: {e}")
+
+        # Verify it's a regular file (not directory or special file)
+        try:
+            file_stat = os.fstat(fd)
+            if not stat.S_ISREG(file_stat.st_mode):
+                os.close(fd)
+                raise PermissionError(f"Not a regular file: {file_path}")
+
+            # Convert to file object with appropriate mode
+            mode_str = "rb" if "b" in mode else "r"
+            if "w" in mode:
+                mode_str = "wb" if "b" in mode else "w"
+            elif "a" in mode:
+                mode_str = "ab" if "b" in mode else "a"
+
+            return os.fdopen(fd, mode_str)
+
+        except Exception as _e:
+            os.close(fd)
+            raise
+
+    def _get_cache_path(self, file_path: str) -> str:
+        """
+        Get cache file path for a document using content-based hashing.
+
+        Uses SHA-256 hash of actual file content for cache key.
+        This ensures proper cache invalidation even for:
+        - Same-size file edits
+        - Files modified within same second (low mtime resolution)
+        - Content changes that preserve size
+
+        Args:
+            file_path: Path to the document
+
+        Returns:
+            Path to cache file
+        """
+        path = Path(file_path).absolute()
+
+        try:
+            # Hash the actual file CONTENT for reliable cache invalidation
+            # This is more reliable than mtime + size
+            hasher = hashlib.sha256()
+
+            # Read file in chunks to handle large files efficiently
+            # Use _safe_open to prevent symlink attacks
+            with self._safe_open(path, "rb") as f:
+                while chunk := f.read(8192):
+                    hasher.update(chunk)
+
+            content_hash = hasher.hexdigest()
+
+            # Include path in hash to avoid collisions between identical files
+            path_hash = hashlib.sha256(str(path).encode()).hexdigest()[:16]
+            cache_key = f"{path_hash}_{content_hash[:32]}"
+
+            return os.path.join(self.config.cache_dir, f"{cache_key}.pkl")
+
+        except (OSError, IOError) as e:
+            # If file doesn't exist or can't be read, use path-based key
+            # This will fail later during indexing anyway
+            self.log.warning(f"Cannot read file for cache key: {e}")
+            file_hash = hashlib.sha256(str(path).encode()).hexdigest()
+            return os.path.join(self.config.cache_dir, f"{file_hash}_notfound.pkl")
+
+    def _load_embedder(self):
+        """Load embedding model via Lemonade server for hardware acceleration.
+
+        Forces a fresh load with --ubatch-size 2048 to prevent llama.cpp issues
+        after VLM processing. Must unload first since Lemonade skips reload
+        if model already loaded.
+        """
+        if self.embedder is None:
+            self.log.info(
+                f"Loading embedding model via Lemonade: {self.config.embedding_model}"
+            )
+
+            from gaia.llm.lemonade_client import LemonadeClient
+
+            if not hasattr(self, "llm_client") or self.llm_client is None:
+                self.llm_client = LemonadeClient()
+
+            # Force fresh load - must unload first
+            try:
+                self.llm_client.unload_model()
+            except Exception:
+                pass  # Ignore if nothing to unload
+
+            try:
+                self.llm_client.load_model(
+                    self.config.embedding_model,
+                    llamacpp_args="--ubatch-size 2048",
+                )
+                self.log.info("Loaded embedding model with ubatch-size=2048")
+            except Exception as e:
+                self.log.warning(f"Could not pre-load embedding model: {e}")
+
+            self.embedder = self.llm_client
+            self.use_lemonade_embeddings = True
+
+            self.log.info("Using Lemonade server for hardware-accelerated embeddings")
+
+    def _encode_texts(
+        self, texts: List[str], show_progress: bool = False
+    ) -> "np.ndarray":
+        """
+        Encode texts to embeddings using Lemonade server with batching and timing.
+
+        Args:
+            texts: List of text strings to encode
+            show_progress: Whether to show progress
+
+        Returns:
+            numpy array of embeddings with shape (num_texts, embedding_dim)
+        """
+
+        # Batch embedding requests to avoid timeouts
+        BATCH_SIZE = 25  # Smaller batches for reliability (25 chunks ~= 12KB text)
+        all_embeddings = []
+
+        total_batches = (len(texts) + BATCH_SIZE - 1) // BATCH_SIZE
+        total_start = time.time()
+
+        for batch_idx in range(0, len(texts), BATCH_SIZE):
+            batch_texts = texts[batch_idx : batch_idx + BATCH_SIZE]
+            batch_num = (batch_idx // BATCH_SIZE) + 1
+
+            batch_start = time.time()
+
+            if show_progress or self.config.show_stats:
+                self.log.info(
+                    f"   📦 Embedding batch {batch_num}/{total_batches} ({len(batch_texts)} chunks)..."
+                )
+
+            # Call Lemonade embeddings API for this batch with retry
+            max_retries = 2
+            for attempt in range(max_retries + 1):
+                try:
+                    # Use longer timeout for embedding batches (180s = 3 minutes per batch)
+                    response = self.embedder.embeddings(
+                        batch_texts, model=self.config.embedding_model, timeout=180
+                    )
+                    break  # Success, exit retry loop
+                except Exception as e:
+                    if attempt < max_retries:
+                        self.log.warning(
+                            f"   ⚠️  Batch {batch_num} attempt {attempt + 1} failed, retrying: {e}"
+                        )
+                        time.sleep(2)  # Wait before retry
+                    else:
+                        self.log.error(
+                            f"   ❌ Batch {batch_num} failed after {max_retries + 1} attempts"
+                        )
+                        raise
+
+            batch_duration = time.time() - batch_start
+
+            if show_progress or self.config.show_stats:
+                chunks_per_sec = (
+                    len(batch_texts) / batch_duration if batch_duration > 0 else 0
+                )
+                self.log.info(
+                    f"   ✅ Batch {batch_num} complete in {batch_duration:.2f}s ({chunks_per_sec:.1f} chunks/sec)"
+                )
+
+            # Extract embeddings from response
+            # Expected format: {"data": [{"embedding": [...]}, ...]}
+            for item in response.get("data", []):
+                embedding = item.get("embedding", [])
+                all_embeddings.append(embedding)
+
+        total_duration = time.time() - total_start
+        if len(texts) > BATCH_SIZE:
+            overall_rate = len(texts) / total_duration if total_duration > 0 else 0
+            self.log.info(
+                f"   🎯 Total embedding time: {total_duration:.2f}s ({overall_rate:.1f} chunks/sec, {total_batches} batches)"
+            )
+
+        # Convert to numpy array
+        return np.array(all_embeddings, dtype=np.float32)
+
+    def _get_file_type(self, file_path: str) -> str:
+        """Detect file type from extension."""
+        ext = Path(file_path).suffix.lower()
+        return ext if ext else ".unknown"
+
+    def _extract_text_from_pdf(self, pdf_path: str) -> tuple:
+        """
+        Extract text from PDF file with VLM for images (always enabled if available).
+
+        Returns:
+            (text, num_pages, metadata) tuple where metadata contains:
+            - num_pages: int
+            - vlm_pages: int (number of pages enhanced with VLM)
+            - total_images: int (total images processed)
+        """
+        import time as time_module  # pylint: disable=reimported
+
+        try:
+            extract_start = time_module.time()
+            reader = PdfReader(pdf_path)
+            total_pages = len(reader.pages)
+            self.log.info(f"📄 Extracting text from {total_pages} pages...")
+
+            # Initialize VLM client (auto-enabled if available)
+            vlm = None
+            vlm_available = False
+            try:
+                from gaia.llm.vlm_client import VLMClient
+                from gaia.rag.pdf_utils import (
+                    count_images_in_page,
+                    extract_images_from_page_pymupdf,
+                )
+
+                vlm = VLMClient(
+                    vlm_model=self.config.vlm_model, base_url=self.config.base_url
+                )
+                vlm_available = vlm.check_availability()
+
+                if vlm_available and self.config.show_stats:
+                    print("  🔍 VLM enabled: Will extract text from images")
+                elif not vlm_available and self.config.show_stats:
+                    print("  ⚠️  VLM not available - images will not be processed")
+                    print("  📥 To enable VLM image extraction:")
+                    print("     1. Open Lemonade Model Manager (http://localhost:8000)")
+                    print(f"     2. Download model: {self.config.vlm_model}")
+
+            except Exception as vlm_error:
+                if self.config.show_stats:
+                    print(f"  ⚠️  VLM initialization failed: {vlm_error}")
+                self.log.warning(f"VLM initialization failed: {vlm_error}")
+                vlm_available = False
+
+            if self.config.show_stats:
+                print(f"\n{'='*60}")
+                print("  📄 COMPUTE INTENSIVE: PDF Text Extraction")
+                print(f"  📊 Total pages: {total_pages}")
+                print(f"  ⏱️  Estimated time: {total_pages * 0.2:.1f} seconds")
+                if vlm_available:
+                    print("  🖼️  VLM: Enabled for image text extraction")
+                else:
+                    print("  🖼️  VLM: Disabled (text-only extraction)")
+                print(f"{'='*60}")
+
+            pages_data = []
+            vlm_pages_count = 0
+            total_images_processed = 0
+
+            for i, page in enumerate(reader.pages, 1):
+                page_start = time_module.time()
+
+                # Step 1: Extract text with pypdf
+                pypdf_text = page.extract_text()
+
+                # Step 2: Check for images
+                has_imgs = False
+                num_imgs = 0
+                if vlm_available:
+                    try:
+                        has_imgs, num_imgs = count_images_in_page(page)
+                    except Exception:  # pylint: disable=broad-except
+                        pass
+
+                # Step 3: Extract from images if present
+                image_texts = []
+                if has_imgs and vlm_available:
+                    try:
+                        images = extract_images_from_page_pymupdf(pdf_path, page_num=i)
+                        if images:
+                            image_texts = vlm.extract_from_page_images(
+                                images, page_num=i
+                            )
+                            if image_texts:
+                                vlm_pages_count += 1
+                                total_images_processed += len(image_texts)
+                    except Exception as img_error:
+                        self.log.warning(
+                            f"Image extraction failed on page {i}: {img_error}"
+                        )
+
+                # Step 4: Merge
+                merged_text = self._merge_page_texts(
+                    pypdf_text, image_texts, page_num=i
+                )
+
+                pages_data.append(
+                    {
+                        "page": i,
+                        "text": merged_text,
+                        "has_images": has_imgs,
+                        "num_images": num_imgs,
+                        "vlm_used": len(image_texts) > 0,
+                    }
+                )
+
+                page_duration = time_module.time() - page_start
+
+                if self.config.show_stats:
+                    # Update progress with timing info
+                    progress_pct = (i / total_pages) * 100
+                    avg_time_per_page = (time_module.time() - extract_start) / i
+                    eta = avg_time_per_page * (total_pages - i)
+                    vlm_indicator = " 🖼️" if len(image_texts) > 0 else ""
+                    print(
+                        f"  📄 Page {i}/{total_pages} ({progress_pct:.0f}%){vlm_indicator} | "
+                        f"⏱️  {page_duration:.2f}s | ETA: {eta:.1f}s" + " " * 10,
+                        end="\r",
+                        flush=True,
+                    )
+
+            # Cleanup VLM
+            if vlm_available and vlm:
+                try:
+                    vlm.cleanup()
+                except Exception:  # pylint: disable=broad-except
+                    pass
+
+            extract_duration = time_module.time() - extract_start
+
+            # Build full text
+            full_text = "\n\n".join(
+                [f"[Page {p['page']}]\n{p['text']}" for p in pages_data]
+            )
+
+            if self.config.show_stats:
+                print(
+                    f"\n  ✅ Extracted {len(full_text):,} characters from {total_pages} pages"
+                )
+                print(
+                    f"  ⏱️  Total extraction time: {extract_duration:.2f}s ({total_pages/extract_duration:.1f} pages/sec)"
+                )
+                print(f"  💾 Text size: {len(full_text) / 1024:.1f} KB")
+                if vlm_pages_count > 0:
+                    print(
+                        f"  🖼️  VLM enhanced: {vlm_pages_count} pages, {total_images_processed} images"
+                    )
+                print(f"{'='*60}\n")
+
+            self.log.info(
+                f"📝 Extracted {len(full_text):,} characters in {extract_duration:.2f}s (VLM: {vlm_pages_count} pages)"
+            )
+
+            # Build metadata
+            metadata = {
+                "num_pages": total_pages,
+                "vlm_pages": vlm_pages_count,
+                "total_images": total_images_processed,
+                "vlm_checked": True,  # Indicates this cache was created with VLM capability check
+                "vlm_available": vlm_available,  # Whether VLM was actually available
+            }
+
+            return full_text, total_pages, metadata
+        except Exception as e:
+            self.log.error(f"Error reading PDF {pdf_path}: {e}")
+            raise
+
+    def _merge_page_texts(
+        self, pypdf_text: str, image_texts: list, page_num: int
+    ) -> str:
+        """
+        Merge pypdf text + VLM image texts.
+
+        Args:
+            pypdf_text: Text extracted by pypdf
+            image_texts: List of dicts from VLM extraction (each has 'image_num' and 'text')
+            page_num: Page number for logging
+
+        Returns:
+            Merged text with image content clearly marked
+        """
+        parts = []
+
+        # Add pypdf text first (if any)
+        if pypdf_text.strip():
+            parts.append(pypdf_text.strip())
+
+        # Add VLM-extracted image content (if any)
+        if image_texts:
+            parts.append("\n\n---\n")
+            parts.append(f"[Page {page_num}]\n**Content Extracted from Images:**\n")
+
+            for img_data in image_texts:
+                parts.append(
+                    f"\n[Page {page_num}] ### 🖼️ IMAGE {img_data['image_num']}\n\n"
+                )
+
+                # Clean up the VLM text for better structure
+                image_text = img_data["text"].strip()
+
+                # Ensure proper line breaks for list items (general pattern)
+                # Look for patterns like "- text" or "* text" or "1. text"
+                image_text = re.sub(r"(?<!\n)([•\-\*]|\d+\.)\s+", r"\n\1 ", image_text)
+
+                # Add double newline after what looks like a heading
+                # (line ending with colon or short line followed by longer text)
+                lines = image_text.split("\n")
+                formatted_lines = []
+                for i, line in enumerate(lines):
+                    formatted_lines.append(line)
+                    # Add extra newline after lines that look like headers
+                    if line.strip().endswith(":") and i < len(lines) - 1:
+                        formatted_lines.append("")
+
+                image_text = "\n".join(formatted_lines)
+
+                parts.append(image_text)
+                parts.append("\n\n")
+
+        return "\n".join(parts)
+
+    def _llm_based_chunking(
+        self, text: str, chunk_size: int, overlap: int
+    ) -> List[str]:
+        """
+        Use LLM to intelligently identify chunk boundaries.
+
+        The LLM analyzes the text structure and suggests optimal split points
+        that preserve semantic meaning and context.
+        """
+        self.log.info("🤖 Using LLM for intelligent text chunking...")
+
+        chunks = []
+
+        # Process text in segments (to handle long documents)
+        # Approximate: 1 token ≈ 4 characters
+        segment_size = chunk_size * 4 * 3  # Process 3 chunks worth at a time
+        text_length = len(text)
+        position = 0
+
+        while position < text_length:
+            # Get a segment to process
+            segment_end = min(position + segment_size, text_length)
+            segment = text[position:segment_end]
+
+            # Ask LLM to identify good chunk boundaries
+            prompt = """You are a document chunking expert. Your task is to identify optimal points to split the following text into chunks.
+
+The text should be split into chunks of approximately {chunk_size} tokens (roughly {chunk_size * 4} characters each).
+
+IMPORTANT RULES:
+1. Keep semantic units together (complete thoughts, paragraphs, sections)
+2. Never split in the middle of sentences
+3. Preserve context - each chunk should be understandable on its own
+4. Keep related information together (e.g., a heading with its content)
+5. For lists, try to keep the list introduction with at least some items
+
+Text to chunk:
+---
+{segment[:2000]}  # Limit prompt size
+{"..." if len(segment) > 2000 else ""}
+---
+
+Please identify the CHARACTER POSITIONS where the text should be split.
+Return ONLY a JSON array of split positions, like: [245, 502, 847]
+These positions indicate where to split the text."""
+
+            try:
+                # Get LLM response
+                response_data = self.llm_client.completions(
+                    model=self.config.model,
+                    prompt=prompt,
+                    temperature=0.0,  # Low temperature for deterministic chunking
+                    max_tokens=500,
+                )
+                response = response_data["choices"][0]["text"]
+
+                # Parse the split positions
+                import json
+
+                split_positions = json.loads(response)
+
+                # Create chunks based on LLM-suggested positions
+                last_pos = 0
+                for split_pos in split_positions:
+                    if split_pos > last_pos and split_pos < len(segment):
+                        chunk = segment[last_pos:split_pos].strip()
+                        if chunk:
+                            chunks.append(chunk)
+                        last_pos = split_pos
+
+                # Add remaining text
+                if last_pos < len(segment):
+                    chunk = segment[last_pos:].strip()
+                    if chunk:
+                        chunks.append(chunk)
+
+            except Exception as e:
+                self.log.warning(f"LLM chunking failed for segment: {e}")
+                # Fall back to simple splitting for this segment
+                segment_chunks = self._fallback_chunk_segment(segment, chunk_size)
+                chunks.extend(segment_chunks)
+
+            # Move to next segment with overlap
+            position = segment_end - (overlap * 4)  # Convert overlap tokens to chars
+
+        return chunks
+
+    def _fallback_chunk_segment(self, text: str, chunk_size: int) -> List[str]:
+        """Simple fallback chunking for a text segment."""
+        chunks = []
+        words = text.split()
+        current_chunk = []
+        current_size = 0
+
+        for word in words:
+            word_size = len(word) // 4  # Rough token estimate
+            if current_size + word_size > chunk_size and current_chunk:
+                chunks.append(" ".join(current_chunk))
+                current_chunk = [word]
+                current_size = word_size
+            else:
+                current_chunk.append(word)
+                current_size += word_size
+
+        if current_chunk:
+            chunks.append(" ".join(current_chunk))
+
+        return chunks
+
+    def _extract_text_from_text_file(self, file_path: str) -> str:
+        """Extract text from text-based file (txt, md, etc.)."""
+        try:
+            encodings = ["utf-8", "utf-8-sig", "latin-1", "cp1252"]
+            text = None
+
+            for encoding in encodings:
+                try:
+                    # Use _safe_open with binary mode, then decode
+                    with self._safe_open(file_path, "rb") as f:
+                        text = f.read().decode(encoding)
+                    break
+                except (UnicodeDecodeError, AttributeError):
+                    continue
+
+            if text is None:
+                raise ValueError(
+                    f"Failed to decode file: {file_path}\n"
+                    f"Tried encodings: {', '.join(encodings)}\n"
+                    "Suggestions:\n"
+                    "  1. Convert the file to UTF-8 encoding\n"
+                    "  2. Check if the file is corrupted\n"
+                    "  3. Ensure the file is a text file (not binary)"
+                )
+
+            if self.config.show_stats:
+                print(f"  ✅ Loaded text file ({len(text):,} characters)")
+
+            self.log.info(f"📝 Extracted {len(text):,} characters from text file")
+            return text.strip()
+        except Exception as e:
+            self.log.error(f"Error reading text file {file_path}: {e}")
+            raise
+
+    def _extract_text_from_csv(self, csv_path: str) -> str:
+        """Extract text from CSV file."""
+        try:
+            import csv
+
+            text_parts = []
+            encodings = ["utf-8", "utf-8-sig", "latin-1", "cp1252"]
+
+            for encoding in encodings:
+                try:
+                    # Use _safe_open with binary mode, then decode for csv.reader
+                    from io import StringIO
+
+                    with self._safe_open(csv_path, "rb") as f:
+                        text = f.read().decode(encoding)
+                        reader = csv.reader(StringIO(text))
+                        rows = list(reader)
+
+                        if not rows:
+                            raise ValueError("CSV file is empty")
+
+                        # Include header as context
+                        if rows:
+                            header = rows[0]
+                            text_parts.append(f"Columns: {', '.join(header)}\n")
+
+                        # Convert rows to readable text
+                        for row in rows[1:]:
+                            # Create a readable row format
+                            row_text = []
+                            for i, cell in enumerate(row):
+                                if i < len(header):
+                                    row_text.append(f"{header[i]}: {cell}")
+                                else:
+                                    row_text.append(cell)
+                            text_parts.append(" | ".join(row_text))
+
+                        text = "\n".join(text_parts)
+
+                        if self.config.show_stats:
+                            print(
+                                f"  ✅ Loaded CSV file ({len(rows)} rows, {len(header)} columns)"
+                            )
+
+                        self.log.info(f"📊 Extracted {len(rows)} rows from CSV")
+                        return text
+                except UnicodeDecodeError:
+                    continue
+
+            raise ValueError(
+                f"Failed to decode CSV file: {csv_path}\n"
+                f"Tried encodings: {', '.join(encodings)}\n"
+                "Suggestions:\n"
+                "  1. Save the CSV file with UTF-8 encoding in Excel/LibreOffice\n"
+                "  2. Check if the file is a valid CSV (not corrupted)\n"
+                "  3. Try opening and re-saving in a text editor"
+            )
+        except Exception as e:
+            self.log.error(f"Error reading CSV {csv_path}: {e}")
+            raise
+
+    def _extract_text_from_json(self, json_path: str) -> str:
+        """Extract text from JSON file."""
+        try:
+            import json
+
+            # Use _safe_open to prevent symlink attacks
+            with self._safe_open(json_path, "rb") as f:
+                data = json.load(f)
+
+            # Convert JSON to readable text format
+            def json_to_text(obj, indent=0):
+                """Recursively convert JSON to readable text."""
+                lines = []
+                prefix = "  " * indent
+
+                if isinstance(obj, dict):
+                    for key, value in obj.items():
+                        if isinstance(value, (dict, list)):
+                            lines.append(f"{prefix}{key}:")
+                            lines.extend(json_to_text(value, indent + 1))
+                        else:
+                            lines.append(f"{prefix}{key}: {value}")
+                elif isinstance(obj, list):
+                    for i, item in enumerate(obj):
+                        if isinstance(item, (dict, list)):
+                            lines.append(f"{prefix}Item {i + 1}:")
+                            lines.extend(json_to_text(item, indent + 1))
+                        else:
+                            lines.append(f"{prefix}- {item}")
+                else:
+                    lines.append(f"{prefix}{obj}")
+
+                return lines
+
+            text = "\n".join(json_to_text(data))
+
+            if self.config.show_stats:
+                print(f"  ✅ Loaded JSON file ({len(text):,} characters)")
+
+            self.log.info(f"📝 Extracted {len(text):,} characters from JSON")
+            return text
+        except Exception as e:
+            self.log.error(f"Error reading JSON {json_path}: {e}")
+            raise
+
+    def _extract_text_from_file(self, file_path: str) -> tuple:
+        """
+        Extract text from file based on type.
+
+        Returns:
+            (text, metadata_dict) tuple where metadata_dict contains:
+            - num_pages: int (for PDFs) or None
+            - vlm_pages: int (for PDFs with VLM) or None
+            - total_images: int (for PDFs with VLM) or None
+        """
+        file_type = self._get_file_type(file_path)
+        metadata = {"num_pages": None, "vlm_pages": None, "total_images": None}
+
+        # PDF files
+        if file_type == ".pdf":
+            text, num_pages, pdf_metadata = self._extract_text_from_pdf(file_path)
+            metadata["num_pages"] = num_pages
+            metadata["vlm_pages"] = pdf_metadata.get("vlm_pages", 0)
+            metadata["total_images"] = pdf_metadata.get("total_images", 0)
+            return text, metadata
+
+        # Text-based files
+        elif file_type in [".txt", ".md", ".markdown", ".rst", ".log"]:
+            return self._extract_text_from_text_file(file_path), metadata
+
+        # CSV files
+        elif file_type == ".csv":
+            return self._extract_text_from_csv(file_path), metadata
+
+        # JSON files
+        elif file_type == ".json":
+            return self._extract_text_from_json(file_path), metadata
+
+        # Code files (treat as text for Q&A purposes)
+        elif file_type in [
+            # Backend languages
+            ".py",
+            ".pyw",  # Python
+            ".java",  # Java
+            ".cpp",
+            ".cc",
+            ".cxx",
+            ".hpp",
+            ".h",  # C++
+            ".c",  # C
+            ".cs",  # C#
+            ".go",  # Go
+            ".rs",  # Rust
+            ".rb",  # Ruby
+            ".php",  # PHP
+            ".swift",  # Swift
+            ".kt",
+            ".kts",  # Kotlin
+            ".scala",  # Scala
+            # Web - JavaScript/TypeScript
+            ".js",
+            ".jsx",  # JavaScript
+            ".ts",
+            ".tsx",  # TypeScript
+            ".mjs",
+            ".cjs",  # JavaScript modules
+            # Web - Frameworks
+            ".vue",  # Vue.js
+            ".svelte",  # Svelte
+            ".astro",  # Astro
+            # Web - Styling
+            ".css",  # CSS
+            ".scss",
+            ".sass",  # Sass
+            ".less",  # Less
+            ".styl",
+            ".stylus",  # Stylus
+            # Web - Markup
+            ".html",
+            ".htm",  # HTML
+            ".svg",  # SVG
+            ".jsx",
+            ".tsx",  # JSX/TSX (already listed but emphasizing)
+            # Scripting
+            ".sh",
+            ".bash",  # Shell
+            ".ps1",  # PowerShell
+            ".r",
+            ".R",  # R
+            # Database
+            ".sql",  # SQL
+            # Configuration
+            ".yaml",
+            ".yml",  # YAML
+            ".xml",  # XML
+            ".toml",  # TOML
+            ".ini",
+            ".cfg",
+            ".conf",  # Config files
+            ".env",  # Environment files
+            ".properties",  # Properties files
+            # Build & Package
+            ".gradle",  # Gradle
+            ".cmake",  # CMake
+            ".mk",
+            ".make",  # Makefiles
+            # Documentation
+            ".rst",  # ReStructuredText
+        ]:
+            self.log.info(f"Indexing code/web file: {file_type}")
+            return self._extract_text_from_text_file(file_path), metadata
+
+        # Unknown file type - try as text
+        else:
+            self.log.warning(
+                f"Unknown file type {file_type}, attempting to read as text"
+            )
+            return self._extract_text_from_text_file(file_path), metadata
+
+    def _split_text_into_chunks(self, text: str) -> List[str]:
+        """
+        Split text into semantic chunks using LLM intelligence when available.
+
+        Uses intelligent splitting that:
+        - Leverages LLM to identify natural semantic boundaries (if available)
+        - Falls back to structural heuristics if LLM is not available
+        - Respects natural document boundaries (paragraphs, sections)
+        - Keeps semantic units together
+        - Maintains context with overlap
+
+        This dramatically improves Q&A quality over naive word splitting.
+        """
+        self.log.info("📝 Splitting text into semantic chunks...")
+
+        chunks = []
+        chunk_size_tokens = self.config.chunk_size
+        overlap_tokens = self.config.chunk_overlap
+
+        # Try to use LLM for intelligent chunking if available
+        if self.config.use_llm_chunking:
+            # Ensure LLM client is initialized for chunking
+            if self.llm_client is None:
+                try:
+                    from gaia.llm.lemonade_client import LemonadeClient
+
+                    self.llm_client = LemonadeClient()
+                    self.log.info("✅ Initialized LLM client for intelligent chunking")
+                except Exception as e:
+                    self.log.warning(
+                        f"Failed to initialize LLM client for chunking: {e}"
+                    )
+
+            if self.llm_client is not None:
+                try:
+                    return self._llm_based_chunking(
+                        text, chunk_size_tokens, overlap_tokens
+                    )
+                except Exception as e:
+                    self.log.warning(
+                        f"LLM chunking failed, falling back to heuristic: {e}"
+                    )
+
+        # Fall back to heuristic-based chunking
+
+        # STEP 1: Identify and protect VLM content blocks as atomic units
+        # VLM content starts with "[Page X] ### 🖼️ IMAGE" and continues until next image or end
+        # We'll mark these sections to prevent splitting during paragraph processing
+        vlm_pattern = r"\[Page \d+\] ### 🖼️ IMAGE \d+.*?(?=\[Page \d+\] ### 🖼️ IMAGE|\[Page \d+\]\n(?!### 🖼️)|\Z)"
+
+        # Find all VLM image blocks and replace them with placeholders temporarily
+        vlm_blocks = []
+        protected_text = text
+        for i, match in enumerate(re.finditer(vlm_pattern, text, re.DOTALL)):
+            placeholder = f"<<<VLM_BLOCK_{i}>>>"
+            vlm_blocks.append(
+                {"placeholder": placeholder, "content": match.group(0).strip()}
+            )
+            protected_text = protected_text.replace(match.group(0), placeholder, 1)
+
+        # STEP 2: Identify natural document boundaries
+        # Look for markdown headers, section breaks, or significant whitespace
+        # Use protected_text which has VLM blocks replaced with placeholders
+        lines = protected_text.split("\n")
+        sections = []
+        current_section = []
+
+        for i, line in enumerate(lines):
+            # Detect section boundaries:
+            # 1. Markdown headers (# Header, ## Header, ### Header)
+            # 2. Lines that look like titles (short, possibly capitalized)
+            # 3. Horizontal rules (---, ===, ___)
+            # 4. Significant whitespace gaps
+
+            is_boundary = False
+
+            # Check for markdown headers
+            if line.strip().startswith("#"):
+                is_boundary = True
+            # Check for horizontal rules
+            elif re.match(r"^[\-=_]{3,}$", line.strip()):
+                is_boundary = True
+            # Check for lines that look like section titles (short, might be all caps)
+            elif line.strip() and len(line.strip()) < 100 and i > 0:
+                # If previous line was empty and next line exists and is not empty
+                prev_empty = i > 0 and not lines[i - 1].strip()
+                next_exists = i < len(lines) - 1
+                next_not_empty = next_exists and lines[i + 1].strip()
+
+                # Heuristic: likely a section header if surrounded by whitespace
+                if prev_empty and next_not_empty:
+                    # Additional check: does it look like a title?
+                    # (starts with capital, no ending punctuation, relatively short)
+                    if line.strip()[0].isupper() and not line.strip()[-1] in ".!?,;":
+                        is_boundary = True
+
+            if is_boundary and current_section:
+                # Save the current section
+                sections.append("\n".join(current_section))
+                current_section = [line]
+            else:
+                current_section.append(line)
+
+        # Don't forget the last section
+        if current_section:
+            sections.append("\n".join(current_section))
+
+        # If we didn't find many sections, try paragraph-based splitting
+        if len(sections) <= 3:
+            # Split by double newlines (paragraphs)
+            paragraphs = re.split(r"\n\s*\n", text)
+            # Filter out empty paragraphs
+            paragraphs = [p.strip() for p in paragraphs if p.strip()]
+        else:
+            paragraphs = sections
+
+        # STEP 3: Mark paragraphs that are VLM content (should not be split)
+        vlm_paragraphs = set()
+        for idx, para in enumerate(paragraphs):
+            # Check if this paragraph contains VLM markers
+            if "### 🖼️ IMAGE" in para or "**Content Extracted from Images:**" in para:
+                vlm_paragraphs.add(idx)
+                self.log.debug(
+                    f"Paragraph {idx} marked as VLM content (will keep atomic)"
+                )
+
+        current_chunk = []
+        current_size = 0
+
+        for idx, para in enumerate(paragraphs):
+            para = para.strip()
+            if not para:
+                continue
+
+            # Estimate tokens (rough: 1 token ≈ 4 characters)
+            para_tokens = len(para) // 4
+
+            # Check if this is VLM content - if so, keep it atomic
+            is_vlm_content = idx in vlm_paragraphs
+
+            # If single paragraph exceeds chunk size AND it's not VLM content, split by sentences
+            if para_tokens > chunk_size_tokens and not is_vlm_content:
+                # Split into sentences
+                sentences = self._split_into_sentences(para)
+
+                for sentence in sentences:
+                    sentence_tokens = len(sentence) // 4
+
+                    # If adding this sentence exceeds chunk size, save current chunk
+                    if (
+                        current_size + sentence_tokens > chunk_size_tokens
+                        and current_chunk
+                    ):
+                        chunks.append(" ".join(current_chunk))
+
+                        # Keep overlap (last few sentences)
+                        overlap_text = " ".join(current_chunk)
+                        overlap_actual = len(overlap_text) // 4
+                        if overlap_actual > overlap_tokens:
+                            # Trim to overlap size
+                            current_chunk = self._get_last_n_tokens(
+                                overlap_text, overlap_tokens
+                            ).split()
+                            current_size = overlap_tokens
+                        else:
+                            current_chunk = []
+                            current_size = 0
+
+                    current_chunk.append(sentence)
+                    current_size += sentence_tokens
+            else:
+                # Small paragraph - try to keep intact
+                # SPECIAL CASE: If this is VLM content, keep it atomic even if it exceeds chunk size
+                if is_vlm_content:
+                    if current_chunk:
+                        # Save current chunk before adding VLM content
+                        chunks.append(" ".join(current_chunk))
+                        current_chunk = []
+                        current_size = 0
+
+                    # Add VLM content as its own chunk (atomic, not split)
+                    chunks.append(para)
+                    self.log.debug(
+                        f"Added VLM content as atomic chunk ({para_tokens} tokens)"
+                    )
+
+                elif current_size + para_tokens > chunk_size_tokens and current_chunk:
+                    # Save current chunk
+                    chunks.append(" ".join(current_chunk))
+
+                    # Keep overlap
+                    overlap_text = " ".join(current_chunk)
+                    current_chunk = self._get_last_n_tokens(
+                        overlap_text, overlap_tokens
+                    ).split()
+                    current_size = len(" ".join(current_chunk)) // 4
+
+                    current_chunk.append(para)
+                    current_size += para_tokens
+                else:
+                    current_chunk.append(para)
+                    current_size += para_tokens
+
+        # Add final chunk
+        if current_chunk:
+            chunks.append(" ".join(current_chunk))
+
+        # STEP 4: Restore VLM blocks from placeholders
+        if vlm_blocks:
+            restored_chunks = []
+            for chunk in chunks:
+                restored_chunk = chunk
+                # Replace placeholders with actual VLM content
+                for vlm_block in vlm_blocks:
+                    if vlm_block["placeholder"] in restored_chunk:
+                        restored_chunk = restored_chunk.replace(
+                            vlm_block["placeholder"], vlm_block["content"]
+                        )
+                restored_chunks.append(restored_chunk)
+            chunks = restored_chunks
+
+        if self.config.show_stats:
+            avg_size = sum(len(c) for c in chunks) // len(chunks) if chunks else 0
+            print(f"  ✅ Created {len(chunks)} semantic chunks (avg {avg_size} chars)")
+
+        self.log.info(f"📦 Created {len(chunks)} semantic chunks")
+        return chunks
+
+    def _split_into_sentences(self, text: str) -> List[str]:
+        """
+        Split text into sentences using simple heuristics.
+
+        Better than word splitting, doesn't require NLTK dependency.
+        """
+        # Split on sentence endings followed by space and capital letter
+
+        # Handle common abbreviations that shouldn't split
+        text = text.replace("Dr.", "Dr<DOT>")
+        text = text.replace("Mr.", "Mr<DOT>")
+        text = text.replace("Mrs.", "Mrs<DOT>")
+        text = text.replace("Ms.", "Ms<DOT>")
+        text = text.replace("Prof.", "Prof<DOT>")
+        text = text.replace("Sr.", "Sr<DOT>")
+        text = text.replace("Jr.", "Jr<DOT>")
+        text = text.replace("vs.", "vs<DOT>")
+        text = text.replace("e.g.", "e<DOT>g<DOT>")
+        text = text.replace("i.e.", "i<DOT>e<DOT>")
+        text = text.replace("etc.", "etc<DOT>")
+
+        # Split on sentence boundaries
+        sentences = re.split(r"(?<=[.!?])\s+(?=[A-Z])", text)
+
+        # Restore abbreviations
+        sentences = [s.replace("<DOT>", ".") for s in sentences]
+
+        return [s.strip() for s in sentences if s.strip()]
+
+    def _get_last_n_tokens(self, text: str, n_tokens: int) -> str:
+        """Get approximately the last n tokens from text."""
+        # Rough estimate: 1 token ≈ 4 characters
+        target_chars = n_tokens * 4
+        if len(text) <= target_chars:
+            return text
+
+        # Try to break on word boundary
+        trimmed = text[-target_chars:]
+        first_space = trimmed.find(" ")
+        if first_space > 0:
+            return trimmed[first_space + 1 :]
+        return trimmed
+
+    def _create_vector_index(self, chunks: List[str]) -> tuple:
+        """Create FAISS vector index from chunks with progress reporting."""
+        import time as time_module  # pylint: disable=reimported
+
+        self._load_embedder()
+
+        # Generate embeddings with detailed progress
+        self.log.info(f"🔍 Generating embeddings for {len(chunks)} chunks...")
+
+        if self.config.show_stats:
+            print(f"\n{'='*60}")
+            print("  🧠 COMPUTE INTENSIVE: Generating vector embeddings")
+            print(f"  📊 Processing {len(chunks)} chunks")
+            print(f"  ⏱️  Estimated time: {len(chunks) * 0.05:.1f} seconds")
+            print(f"{'='*60}")
+
+        embed_start = time_module.time()
+        embeddings = self._encode_texts(chunks, show_progress=self.config.show_stats)
+        embed_duration = time_module.time() - embed_start
+
+        if self.config.show_stats:
+            print(
+                f"\n  ✅ Generated {embeddings.shape[0]} embeddings ({embeddings.shape[1]} dimensions)"
+            )
+            print(
+                f"  ⏱️  Embedding time: {embed_duration:.2f}s ({len(chunks)/embed_duration:.1f} chunks/sec)"
+            )
+
+        # Create FAISS index
+        self.log.info("🏗️  Building FAISS search index...")
+
+        if self.config.show_stats:
+            print("\n  🏗️  Building FAISS search index...")
+
+        index_start = time_module.time()
+        dimension = embeddings.shape[1]
+        index = faiss.IndexFlatL2(dimension)
+        # pylint: disable=no-value-for-parameter
+        index.add(embeddings.astype("float32"))
+        index_duration = time_module.time() - index_start
+
+        if self.config.show_stats:
+            print(
+                f"  ✅ Built search index for {index.ntotal} vectors in {index_duration:.2f}s"
+            )
+            print(
+                f"  💾 Memory: ~{(embeddings.nbytes / (1024**2)):.1f}MB for embeddings"
+            )
+            print(f"{'='*60}\n")
+
+        self.log.info(
+            f"📚 Index ready with {index.ntotal} vectors "
+            f"(embed: {embed_duration:.2f}s, index: {index_duration:.2f}s)"
+        )
+        return index, chunks
+
+    def remove_document(self, file_path: str) -> bool:
+        """
+        Remove a document from the index.
+
+        Args:
+            file_path: Path to document to remove
+
+        Returns:
+            True if removal succeeded, False otherwise
+        """
+        file_path = str(Path(file_path).absolute())
+
+        if file_path not in self.indexed_files:
+            self.log.warning(f"Document not indexed: {file_path}")
+            return False
+
+        try:
+            # Get chunk indices for this file
+            if file_path in self.file_to_chunk_indices:
+                chunk_indices_set = set(self.file_to_chunk_indices[file_path])
+
+                # OPTIMIZED: Rebuild all structures in one O(N) pass
+                # This is much faster than deleting in a loop (which is O(N²))
+                new_chunks = []
+                new_chunk_to_file = {}
+                new_file_to_chunk_indices = {}
+
+                # Single pass through all chunks - O(N)
+                for old_idx, chunk in enumerate(self.chunks):
+                    # Skip chunks from file being removed
+                    if old_idx in chunk_indices_set:
+                        continue
+
+                    new_idx = len(new_chunks)
+                    new_chunks.append(chunk)
+
+                    # Update chunk_to_file mapping
+                    if old_idx in self.chunk_to_file:
+                        file = self.chunk_to_file[old_idx]
+                        new_chunk_to_file[new_idx] = file
+
+                        # Update file_to_chunk_indices for this file
+                        if file not in new_file_to_chunk_indices:
+                            new_file_to_chunk_indices[file] = []
+                        new_file_to_chunk_indices[file].append(new_idx)
+
+                # Atomic replacement - all or nothing
+                self.chunks = new_chunks
+                self.chunk_to_file = new_chunk_to_file
+                self.file_to_chunk_indices = new_file_to_chunk_indices
+
+            # Remove from indexed files
+            self.indexed_files.discard(file_path)
+
+            # Clean up LRU tracking
+            if file_path in self.file_access_times:
+                del self.file_access_times[file_path]
+            if file_path in self.file_index_times:
+                del self.file_index_times[file_path]
+
+            # Clean up cached per-file indices and embeddings
+            if file_path in self.file_indices:
+                del self.file_indices[file_path]
+            if file_path in self.file_embeddings:
+                del self.file_embeddings[file_path]
+
+            # Clean up cached metadata
+            if file_path in self.file_metadata:
+                del self.file_metadata[file_path]
+
+            # Rebuild index if chunks remain
+            if self.chunks:
+                self.index, self.chunks = self._create_vector_index(self.chunks)
+                if self.config.show_stats:
+                    print(f"✅ Removed {Path(file_path).name} from index")
+                    print(
+                        f"📊 Remaining: {len(self.indexed_files)} documents, {len(self.chunks)} chunks"
+                    )
+            else:
+                self.index = None
+                if self.config.show_stats:
+                    print("✅ Removed last document from index")
+
+            self.log.info(f"Successfully removed document: {file_path}")
+            return True
+
+        except Exception as e:
+            self.log.error(f"Failed to remove document {file_path}: {e}")
+            return False
+
+    def reindex_document(self, file_path: str) -> Dict[str, Any]:
+        """
+        Reindex a document (remove old chunks and add new ones).
+
+        Args:
+            file_path: Path to document to reindex
+
+        Returns:
+            Dict with indexing results and statistics (same as index_document)
+        """
+        file_path = str(Path(file_path).absolute())
+
+        # Remove old version if it exists
+        if file_path in self.indexed_files:
+            self.log.info(f"Removing old version of {file_path}")
+            if not self.remove_document(file_path):
+                return {
+                    "success": False,
+                    "error": "Failed to remove old version",
+                    "file_name": Path(file_path).name,
+                }
+
+        # Index the new version
+        self.log.info(f"Indexing new version of {file_path}")
+        result = self.index_document(file_path)
+        if result.get("success"):
+            result["reindexed"] = True
+        return result
+
+    def _evict_lru_document(self) -> bool:
+        """
+        Evict the least recently used document to free memory.
+
+        Returns:
+            True if a document was evicted, False otherwise
+        """
+        if not self.config.enable_lru_eviction or not self.file_access_times:
+            return False
+
+        # Find LRU file (oldest access time)
+        lru_file = min(self.file_access_times, key=self.file_access_times.get)
+
+        if self.config.show_stats:
+            print(
+                f"📦 Memory limit reached, evicting LRU document: {Path(lru_file).name}"
+            )
+
+        # Remove the LRU document
+        return self.remove_document(lru_file)
+
+    def _check_memory_limits(self) -> None:
+        """
+        Check memory limits and evict documents if necessary.
+        """
+        # Check total chunks limit
+        while (
+            self.config.max_total_chunks > 0
+            and len(self.chunks) > self.config.max_total_chunks
+            and len(self.indexed_files) > 1
+        ):  # Keep at least one file
+            if not self._evict_lru_document():
+                break
+
+        # Check indexed files limit
+        while (
+            self.config.max_indexed_files > 0
+            and len(self.indexed_files) > self.config.max_indexed_files
+        ):
+            if not self._evict_lru_document():
+                break
+
+    def index_document(self, file_path: str) -> Dict[str, Any]:
+        """
+        Index a document for retrieval.
+
+        Supports:
+        - Documents: PDF, TXT, MD, CSV, JSON
+        - Backend Code: Python, Java, C/C++, Go, Rust, Ruby, PHP, Swift, Kotlin, Scala
+        - Web Code: JavaScript/TypeScript, HTML, CSS/SCSS/SASS/LESS, Vue, Svelte, Astro
+        - Config: YAML, XML, TOML, INI, ENV, Properties
+        - Build: Gradle, CMake, Makefiles
+        - Database: SQL
+
+        Args:
+            file_path: Path to document or code file
+
+        Returns:
+            Dict with indexing results and statistics:
+            {
+                "success": bool,
+                "file_name": str,
+                "file_type": str,
+                "file_size_mb": float,
+                "num_pages": int (for PDFs),
+                "num_chunks": int,
+                "total_indexed_files": int,
+                "total_chunks": int,
+                "error": str (if failed)
+            }
+
+        Raises:
+            ValueError: If file_path is empty or file doesn't exist
+        """
+        # Validate input
+        if not file_path or not file_path.strip():
+            raise ValueError("File path cannot be empty")
+
+        # Initialize stats dict
+        stats = {
+            "success": False,
+            "file_name": Path(file_path).name if file_path else "",
+            "file_type": "",
+            "file_size_mb": 0.0,
+            "num_pages": None,
+            "vlm_pages": None,
+            "total_images": None,
+            "num_chunks": 0,
+            "total_indexed_files": len(self.indexed_files),
+            "total_chunks": len(self.chunks),
+        }
+
+        # Check if file exists before processing
+        if not os.path.exists(file_path):
+            self.log.error(f"File not found: {file_path}")
+            if self.config.show_stats:
+                print(f"❌ File not found: {file_path}")
+                print("   Please check the file path and try again")
+            stats["error"] = f"File not found: {file_path}"
+            return stats
+
+        # Check if file is empty (early validation to save time)
+        file_size = os.path.getsize(file_path)
+        file_size_mb = file_size / (1024 * 1024)
+        stats["file_size_mb"] = round(file_size_mb, 2)
+
+        if file_size == 0:
+            self.log.error(f"File is empty: {file_path}")
+            if self.config.show_stats:
+                print(f"❌ File is empty: {file_path}")
+                print("   The file has no content to index")
+            stats["error"] = "File is empty"
+            return stats
+
+        # Enforce maximum file size limit (prevent OOM)
+        if file_size_mb > self.config.max_file_size_mb:
+            error_msg = (
+                f"File too large: {Path(file_path).name} ({file_size_mb:.1f}MB)\n"
+                f"Maximum allowed: {self.config.max_file_size_mb}MB\n"
+                "Suggestions:\n"
+                "  1. Split the file into smaller documents\n"
+                "  2. Increase max_file_size_mb in RAGConfig\n"
+                "  3. Use a more powerful system with more RAM"
+            )
+            self.log.error(error_msg)
+            if self.config.show_stats:
+                print(f"❌ {error_msg}")
+            stats["error"] = (
+                f"File too large ({file_size_mb:.1f}MB > {self.config.max_file_size_mb}MB)"
+            )
+            return stats
+
+        # Warn if file is large
+        if file_size_mb > self.config.warn_file_size_mb:
+            if self.config.show_stats:
+                print(f"⚠️  Large file detected ({file_size_mb:.1f}MB)")
+                print("   This may take 30-60 seconds to process...")
+            self.log.warning(f"Processing large file: {file_size_mb:.1f}MB")
+
+        # Convert to absolute path only after validation
+        file_path = str(Path(file_path).absolute())
+
+        # Get file type for logging
+        file_type = self._get_file_type(file_path)
+        stats["file_type"] = file_type
+        stats["file_name"] = Path(file_path).name
+
+        # Check if already indexed
+        if file_path in self.indexed_files:
+            if self.config.show_stats:
+                print(f"📋 Document already indexed: {Path(file_path).name}")
+            self.log.info(f"Document already indexed: {file_path}")
+            stats["success"] = True
+            stats["already_indexed"] = True
+            stats["total_indexed_files"] = len(self.indexed_files)
+            stats["total_chunks"] = len(self.chunks)
+            return stats
+
+        # Check cache - the cache key is based on file content hash
+        cache_path = self._get_cache_path(file_path)
+
+        # Also check for cached Markdown file with hash-based name
+        # Extract the cache key from the pickle cache path to find matching MD file
+        cache_filename = Path(cache_path).stem  # Remove .pkl extension
+        md_cache_path = os.path.join(
+            self.config.cache_dir, f"{cache_filename}_extracted.md"
+        )
+
+        if os.path.exists(cache_path):
+            if self.config.show_stats:
+                print(f"💾 Loading from cache: {Path(file_path).name}")
+            self.log.info(f"📦 Loading cached index for: {file_path}")
+            try:
+                with open(cache_path, "rb") as f:
+                    cached_data = pickle.load(f)
+                    cached_chunks = cached_data["chunks"]
+                    cached_full_text = cached_data.get(
+                        "full_text", ""
+                    )  # May not exist in old caches
+                    cached_metadata = cached_data.get(
+                        "metadata", {}
+                    )  # May not exist in old caches
+
+                    # Check if cache might be missing VLM content
+                    # If metadata doesn't have VLM info, it's an old cache
+                    if not cached_metadata.get("vlm_checked", False):
+                        if self.config.show_stats:
+                            print(
+                                "  ⚠️  Cache might be missing image text (pre-VLM cache)"
+                            )
+                            print(
+                                "     💡 Use /clear-cache to force re-extraction with VLM"
+                            )
+
+                    # Verify Markdown cache exists alongside pickle cache
+                    if os.path.exists(md_cache_path):
+                        self.log.info(
+                            f"  ✅ Markdown cache also available: {md_cache_path}"
+                        )
+
+                    if self.config.show_stats:
+                        vlm_info = ""
+                        if cached_metadata.get("vlm_pages", 0) > 0:
+                            vlm_info = f" (VLM: {cached_metadata['vlm_pages']} pages)"
+                        print(
+                            f"  ✅ Loaded {len(cached_chunks)} cached chunks{vlm_info}"
+                        )
+
+                    # Track chunk indices for this file
+                    start_idx = len(self.chunks)
+                    file_chunk_indices = []
+
+                    if self.index is None:
+                        # First document - use cached index directly
+                        self.chunks = cached_chunks
+                        # Track indices for all chunks (0 to len-1)
+                        for i in range(len(cached_chunks)):
+                            file_chunk_indices.append(i)
+                            self.chunk_to_file[i] = file_path
+                        self.index, self.chunks = self._create_vector_index(self.chunks)
+                    else:
+                        # Merge with existing chunks and recreate index
+                        old_count = len(self.chunks)
+                        self.chunks.extend(cached_chunks)
+                        # Track indices for new chunks (start_idx to start_idx+len-1)
+                        for i in range(len(cached_chunks)):
+                            chunk_idx = start_idx + i
+                            file_chunk_indices.append(chunk_idx)
+                            self.chunk_to_file[chunk_idx] = file_path
+                        if self.config.show_stats:
+                            print(
+                                f"  🔄 Rebuilding index ({old_count} + {len(cached_chunks)} = {len(self.chunks)} chunks)"
+                            )
+                        self.index, self.chunks = self._create_vector_index(self.chunks)
+
+                    # Store file-to-chunk mapping
+                    self.file_to_chunk_indices[file_path] = file_chunk_indices
+
+                    # Restore metadata in memory
+                    if cached_full_text or cached_metadata:
+                        self.file_metadata[file_path] = {
+                            "full_text": cached_full_text,
+                            **cached_metadata,
+                        }
+
+                    self.indexed_files.add(file_path)
+                    if self.config.show_stats:
+                        print("  ✅ Successfully loaded from cache")
+
+                    # Update stats for cache load
+                    stats["success"] = True
+                    stats["num_chunks"] = len(cached_chunks)
+                    stats["num_pages"] = cached_metadata.get("num_pages")
+                    stats["vlm_pages"] = cached_metadata.get("vlm_pages")
+                    stats["total_images"] = cached_metadata.get("total_images")
+                    stats["total_indexed_files"] = len(self.indexed_files)
+                    stats["total_chunks"] = len(self.chunks)
+                    stats["from_cache"] = True
+                    return stats
+            except Exception as e:
+                self.log.warning(f"Cache load failed: {e}, reindexing")
+                if self.config.show_stats:
+                    print("  ⚠️  Cache loading failed, will reindex from scratch")
+
+        # Extract and process document
+        if self.config.show_stats:
+            print(f"🚀 Starting to index: {Path(file_path).name} ({file_type})")
+        self.log.info(f"📄 Indexing document: {file_path} ({file_type})")
+
+        try:
+            # Extract text based on file type
+            text, file_metadata = self._extract_text_from_file(file_path)
+
+            # Store metadata in stats if available (for PDFs)
+            if file_metadata.get("num_pages"):
+                stats["num_pages"] = file_metadata["num_pages"]
+            if file_metadata.get("vlm_pages"):
+                stats["vlm_pages"] = file_metadata["vlm_pages"]
+            if file_metadata.get("total_images"):
+                stats["total_images"] = file_metadata["total_images"]
+
+            if not text.strip():
+                error_msg = (
+                    f"No text content found in {file_type} file: {Path(file_path).name}\n"
+                    "Possible reasons:\n"
+                    "  1. The file contains only images or non-text content\n"
+                    "  2. The file is password-protected (PDFs)\n"
+                    "  3. The file uses an unsupported format\n"
+                    "  4. The text extraction failed\n"
+                    "Try opening the file manually to verify it contains readable text"
+                )
+                stats["error"] = "No text content found"
+                raise ValueError(error_msg)
+
+            # Split into chunks
+            new_chunks = self._split_text_into_chunks(text)
+
+            # Track which chunks belong to this file
+            file_chunk_indices = []
+            start_idx = len(self.chunks)
+
+            # Add to existing chunks or create new
+            if self.chunks:
+                old_count = len(self.chunks)
+                self.chunks.extend(new_chunks)
+
+                # Track the indices of chunks for this file
+                for i in range(start_idx, start_idx + len(new_chunks)):
+                    file_chunk_indices.append(i)
+                    self.chunk_to_file[i] = file_path
+
+                if self.config.show_stats:
+                    print(
+                        f"🔄 Rebuilding search index ({old_count} + {len(new_chunks)} = {len(self.chunks)} total chunks)"
+                    )
+                self.index, self.chunks = self._create_vector_index(self.chunks)
+            else:
+                # First file being indexed
+                for i in range(len(new_chunks)):
+                    file_chunk_indices.append(i)
+                    self.chunk_to_file[i] = file_path
+
+                if self.config.show_stats:
+                    print("🏗️  Building initial search index...")
+                self.index, self.chunks = self._create_vector_index(new_chunks)
+
+            # Store the file-to-chunks mapping
+            self.file_to_chunk_indices[file_path] = file_chunk_indices
+
+            # Build and cache per-file FAISS index for fast file-specific searches
+            if self.config.show_stats:
+                print("🔍 Building per-file search index...")
+
+            self._load_embedder()
+            # Generate embeddings for this file's chunks only
+            file_embeddings = self._encode_texts(new_chunks, show_progress=False)
+
+            # Create FAISS index for this file
+            dimension = file_embeddings.shape[1]
+            file_index = faiss.IndexFlatL2(dimension)
+            # pylint: disable=no-value-for-parameter
+            file_index.add(file_embeddings.astype("float32"))
+
+            # Cache the index and embeddings for this file
+            self.file_indices[file_path] = file_index
+            self.file_embeddings[file_path] = file_embeddings
+
+            if self.config.show_stats:
+                print(f"✅ Cached per-file index with {len(new_chunks)} chunks")
+
+            # Cache the results for this specific document
+            if self.config.show_stats:
+                print("💾 Caching processed chunks...")
+            cache_data = {
+                "chunks": new_chunks,  # Cache only new chunks for this document
+                "full_text": text,  # Cache full extracted text (for /dump)
+                "metadata": file_metadata,  # Cache metadata (num_pages, vlm_pages, etc.)
+            }
+            with open(cache_path, "wb") as f:
+                pickle.dump(cache_data, f)
+
+            # Store metadata in memory for fast access
+            self.file_metadata[file_path] = {
+                "full_text": text,
+                **file_metadata,  # num_pages, vlm_pages, total_images
+            }
+
+            # Auto-save markdown version to cache directory for easy access
+            self._save_extracted_markdown(file_path, text, file_metadata)
+
+            self.indexed_files.add(file_path)
+
+            # Track index time for LRU
+            current_time = time.time()
+            self.file_index_times[file_path] = current_time
+            self.file_access_times[file_path] = current_time
+
+            # Check memory limits and evict if necessary
+            self._check_memory_limits()
+
+            if self.config.show_stats:
+                print(f"✅ Successfully indexed {Path(file_path).name}")
+                print(
+                    f"📊 Total: {len(self.indexed_files)} documents, {len(self.chunks)} chunks"
+                )
+                if self.config.enable_lru_eviction:
+                    print(
+                        f"📈 Memory usage: {len(self.chunks)}/{self.config.max_total_chunks} chunks, "
+                        f"{len(self.indexed_files)}/{self.config.max_indexed_files} files"
+                    )
+
+            self.log.info(f"✅ Successfully indexed {file_path}")
+
+            # Update final stats
+            stats["success"] = True
+            stats["num_chunks"] = len(new_chunks)
+            stats["total_indexed_files"] = len(self.indexed_files)
+            stats["total_chunks"] = len(self.chunks)
+            return stats
+
+        except Exception as e:
+            if self.config.show_stats:
+                print(f"❌ Failed to index {Path(file_path).name}: {e}")
+            self.log.error(f"Failed to index {file_path}: {e}")
+            stats["error"] = str(e)
+            return stats
+
+    def _retrieve_chunks_from_file(self, query: str, file_path: str) -> tuple:
+        """
+        Retrieve relevant chunks from a specific file using cached per-file index.
+
+        This is much faster than the global search because:
+        1. Uses pre-computed embeddings (no re-encoding)
+        2. Searches smaller, file-specific FAISS index
+        3. No need to rebuild index on each query
+        """
+        if self.index is None or not self.chunks:
+            raise ValueError("No documents indexed. Call index_document() first.")
+
+        if file_path not in self.file_to_chunk_indices:
+            raise ValueError(f"File not indexed: {file_path}")
+
+        # Update access time for LRU tracking
+        self.file_access_times[file_path] = time.time()
+
+        # Get chunk indices for this file
+        file_chunk_indices = self.file_to_chunk_indices[file_path]
+        if not file_chunk_indices:
+            return [], []
+
+        # Get chunks for this file
+        file_chunks = [self.chunks[i] for i in file_chunk_indices]
+
+        # Use CACHED per-file index (this is the performance fix!)
+        if file_path not in self.file_indices:
+            # Index not cached - build it now (shouldn't happen normally)
+            self.log.warning(f"Per-file index not cached for {file_path}, rebuilding")
+            self._load_embedder()
+            file_embeddings = self._encode_texts(file_chunks, show_progress=False)
+            dimension = file_embeddings.shape[1]
+            file_index = faiss.IndexFlatL2(dimension)
+            # pylint: disable=no-value-for-parameter
+            file_index.add(file_embeddings.astype("float32"))
+            self.file_indices[file_path] = file_index
+            self.file_embeddings[file_path] = file_embeddings
+        else:
+            # Use cached index - FAST!
+            file_index = self.file_indices[file_path]
+
+        # Encode query only once
+        self._load_embedder()
+        query_embedding = self._encode_texts([query], show_progress=False)
+
+        # Search in cached file-specific index
+        k = min(self.config.max_chunks, len(file_chunks))
+        # pylint: disable=no-value-for-parameter
+        distances, indices = file_index.search(query_embedding.astype("float32"), k)
+
+        # Get matching chunks and scores
+        retrieved_chunks = []
+        scores = []
+        for idx, dist in zip(indices[0], distances[0]):
+            if idx < len(file_chunks):  # Safety check
+                retrieved_chunks.append(file_chunks[idx])
+                # Convert distance to similarity score
+                score = 1.0 / (1.0 + float(dist))
+                scores.append(score)
+
+        if self.config.show_stats:
+            print(
+                f"  ✅ Found {len(retrieved_chunks)} relevant chunks from {Path(file_path).name} (using cached index)"
+            )
+
+        return retrieved_chunks, scores
+
+    def _retrieve_chunks_with_metadata(self, query: str) -> tuple:
+        """
+        Retrieve chunks with metadata about their source files.
+
+        Returns:
+            (chunks, scores, metadata) tuple
+        """
+        chunks, scores = self._retrieve_chunks(query)
+
+        # Build metadata for each chunk
+        metadata = []
+        for i, (chunk, score) in enumerate(zip(chunks, scores)):
+            # Find which file this chunk came from
+            chunk_idx = self.chunks.index(chunk) if chunk in self.chunks else -1
+            source_file = self.chunk_to_file.get(chunk_idx, "unknown")
+
+            metadata.append(
+                {
+                    "chunk_index": i + 1,
+                    "source_file": (
+                        Path(source_file).name
+                        if source_file != "unknown"
+                        else "unknown"
+                    ),
+                    "source_path": source_file,
+                    "relevance_score": float(score),
+                    "chunk_length": len(chunk),
+                    "estimated_tokens": len(chunk) // 4,  # Rough token estimate
+                }
+            )
+
+        return chunks, scores, metadata
+
+    def _retrieve_chunks(self, query: str) -> tuple:
+        """Retrieve relevant chunks for query."""
+        if self.index is None or not self.chunks:
+            raise ValueError("No documents indexed. Call index_document() first.")
+
+        self._load_embedder()
+
+        # Generate query embedding
+        if self.config.show_stats:
+            print(f"🔍 Searching through {len(self.chunks)} chunks...")
+        self.log.debug(f"Encoding query: {query[:50]}...")
+        query_embedding = self._encode_texts([query], show_progress=False)
+
+        # Search for similar chunks
+        k = min(self.config.max_chunks, len(self.chunks))
+        if self.config.show_stats:
+            print(f"  🎯 Finding {k} most relevant chunks...")
+        # pylint: disable=no-value-for-parameter
+        distances, indices = self.index.search(query_embedding.astype("float32"), k)
+
+        # Get chunks and scores
+        retrieved_chunks = [self.chunks[i] for i in indices[0]]
+        # Convert distances to similarity scores (lower distance = higher similarity)
+        scores = [1.0 / (1.0 + dist) for dist in distances[0]]
+
+        if self.config.show_stats:
+            print(
+                f"  ✅ Retrieved {len(retrieved_chunks)} chunks (avg relevance: {sum(scores)/len(scores):.3f})"
+            )
+
+        self.log.debug(
+            f"Retrieved {len(retrieved_chunks)} chunks with scores: {[f'{s:.3f}' for s in scores]}"
+        )
+        return retrieved_chunks, scores
+
+    def query(self, question: str, include_metadata: bool = True) -> RAGResponse:
+        """
+        Query the indexed documents with enhanced metadata tracking.
+
+        Args:
+            question: Question to ask about the documents
+            include_metadata: Whether to include detailed metadata in response
+
+        Returns:
+            RAGResponse with answer, retrieved chunks, and metadata
+        """
+        if self.index is None:
+            raise ValueError("No documents indexed. Call index_document() first.")
+
+        # Retrieve relevant chunks with metadata
+        if include_metadata:
+            chunks, scores, chunk_metadata = self._retrieve_chunks_with_metadata(
+                question
+            )
+        else:
+            chunks, scores = self._retrieve_chunks(question)
+            chunk_metadata = None
+
+        # Build context from retrieved chunks
+        context = "\n\n".join(
+            [f"Context {i+1}:\n{chunk}" for i, chunk in enumerate(chunks)]
+        )
+
+        # Create prompt
+        prompt = f"""Based on the following context, please answer the question. If the answer is not in the context, say so.
+
+Context:
+{context}
+
+Question: {question}
+
+Answer:"""
+
+        # Get LLM response
+        response = self.chat.send(prompt)
+
+        # Build query metadata
+        query_metadata = None
+        if include_metadata and chunk_metadata:
+            # Get unique source files
+            source_files = list(
+                set(
+                    m["source_file"]
+                    for m in chunk_metadata
+                    if m["source_file"] != "unknown"
+                )
+            )
+
+            query_metadata = {
+                "question": question,
+                "num_chunks_retrieved": len(chunks),
+                "source_files": source_files,
+                "total_indexed_files": len(self.indexed_files),
+                "total_indexed_chunks": len(self.chunks),
+                "average_relevance_score": float(np.mean(scores)) if scores else 0.0,
+                "max_relevance_score": float(max(scores)) if scores else 0.0,
+                "min_relevance_score": float(min(scores)) if scores else 0.0,
+            }
+
+            # Collect source files list
+            source_files_list = [m["source_file"] for m in chunk_metadata]
+        else:
+            source_files_list = None
+
+        return RAGResponse(
+            text=response.text,
+            chunks=chunks,
+            chunk_scores=scores,
+            stats=response.stats,
+            source_files=source_files_list,
+            chunk_metadata=chunk_metadata,
+            query_metadata=query_metadata,
+        )
+
+    def _save_extracted_markdown(
+        self, file_path: str, text: str, metadata: Dict[str, Any]
+    ):
+        """
+        Save extracted text as markdown file in cache directory.
+
+        This creates a human-readable markdown version of the extracted text
+        that can be used for /dump commands and debugging without re-extraction.
+        Uses hash-based naming to match the pickle cache for consistency.
+
+        Args:
+            file_path: Path to original document
+            text: Extracted text content
+            metadata: File metadata (num_pages, vlm_pages, etc.)
+        """
+        try:
+            from datetime import datetime
+
+            # Calculate file hash for consistency with pickle cache
+            path = Path(file_path).absolute()
+            hasher = hashlib.sha256()
+            with self._safe_open(path, "rb") as f:
+                while chunk := f.read(8192):
+                    hasher.update(chunk)
+            content_hash = hasher.hexdigest()
+
+            # Use hash-based filename similar to pickle cache
+            path_hash = hashlib.sha256(str(path).encode()).hexdigest()[:16]
+            cache_key = f"{path_hash}_{content_hash[:32]}"
+            md_filename = f"{cache_key}_extracted.md"
+            md_path = os.path.join(self.config.cache_dir, md_filename)
+
+            # Create markdown content with metadata header
+            vlm_status = (
+                "✅ Enabled"
+                if metadata.get("vlm_available", False)
+                else "❌ Not Available"
+            )
+            markdown_content = f"""# Extracted Text from {Path(file_path).name}
+
+## Metadata
+**Source File:** {file_path}
+**File Hash (SHA-256):** {content_hash[:32]}
+**Extraction Date:** {datetime.now().isoformat()}
+**Pages:** {metadata.get('num_pages', 'N/A')}
+**VLM Status:** {vlm_status}
+**VLM Pages (with images):** {metadata.get('vlm_pages', 0)}
+**Total Images Processed:** {metadata.get('total_images', 0)}
+
+---
+
+## Extracted Content
+{text}
+"""
+
+            # Write markdown file
+            with open(md_path, "w", encoding="utf-8") as f:
+                f.write(markdown_content)
+
+            self.log.debug(f"Saved extracted markdown to {md_path}")
+
+        except Exception as e:
+            # Don't fail indexing if markdown save fails
+            self.log.warning(
+                f"Failed to save markdown cache for {Path(file_path).name}: {e}"
+            )
+
+    def clear_cache(self):
+        """Clear the RAG cache."""
+        import shutil
+
+        if os.path.exists(self.config.cache_dir):
+            shutil.rmtree(self.config.cache_dir)
+            os.makedirs(self.config.cache_dir, exist_ok=True)
+        self.log.info("Cache cleared")
+
+    def get_status(self) -> Dict[str, Any]:
+        """Get RAG system status."""
+        return {
+            "indexed_files": len(self.indexed_files),
+            "total_chunks": len(self.chunks) if self.chunks else 0,
+            "cache_dir": self.config.cache_dir,
+            "embedding_model": self.config.embedding_model,
+            "config": {
+                "chunk_size": self.config.chunk_size,
+                "chunk_overlap": self.config.chunk_overlap,
+                "max_chunks": self.config.max_chunks,
+            },
+        }
+
+
+def quick_rag(pdf_path: str, question: str, **kwargs) -> str:
+    """
+    Convenience function for quick RAG query.
+
+    Args:
+        pdf_path: Path to PDF file
+        question: Question to ask
+        **kwargs: Additional config parameters
+
+    Returns:
+        Answer text
+
+    Raises:
+        ValueError: If pdf_path is empty, question is empty, or file doesn't exist
+    """
+    # Validate inputs
+    if not pdf_path or not pdf_path.strip():
+        raise ValueError("PDF path cannot be empty")
+
+    if not question or not question.strip():
+        raise ValueError("Question cannot be empty")
+
+    # Check if file exists
+    if not os.path.exists(pdf_path):
+        raise ValueError(f"PDF file not found: {pdf_path}")
+
+    config = RAGConfig(**kwargs)
+    rag = RAGSDK(config)
+
+    result = rag.index_document(pdf_path)
+    if not result.get("success"):
+        error = result.get("error", "Unknown error")
+        raise ValueError(f"Failed to index document: {pdf_path}. Error: {error}")
+
+    response = rag.query(question)
+    return response.text