realtimex-deeptutor 0.5.0.post1__py3-none-any.whl → 0.5.0.post3__py3-none-any.whl

src/knowledge/add_documents.py

@@ -4,6 +4,12 @@
 Incrementally add documents to existing knowledge base.
 Improved version with Hash-based duplicate checking, robust error handling,
 and architectural improvements for data integrity and vision support.
+
+Supports multiple RAG providers with lazy loading:
+- llamaindex: Pure vector retrieval (load_index + insert + persist)
+- lightrag: Knowledge graph (LightRAG.ainsert, text-only)
+- raganything: Multimodal with MinerU parser
+- raganything_docling: Multimodal with Docling parser
 """
 
 import argparse
@@ -17,64 +23,18 @@ from pathlib import Path
 import shutil
 import sys
 import tempfile
-from typing import TYPE_CHECKING, Any, Dict, List
+from typing import Any, Dict, List
 
 from dotenv import load_dotenv
 
-# Attempt imports for dynamic dependencies
-try:
-    from lightrag.llm.openai import openai_complete_if_cache
-    from lightrag.utils import EmbeddingFunc
-except ImportError:
-    # These will be caught during runtime if needed
-    openai_complete_if_cache = None
-    EmbeddingFunc = None
-
-# Type hinting support for dynamic imports
-if TYPE_CHECKING:
-    try:
-        from raganything import RAGAnything
-        from raganything import RAGAnythingConfig as RAGAnythingConfigType
-    except ImportError:
-        RAGAnything = Any
-        RAGAnythingConfigType = Any
-else:
-    RAGAnything = None
-    RAGAnythingConfigType = None
-
-# Placeholder for runtime classes
-raganything_cls = None
-RAGAnythingConfig = None
-
-
-def load_dynamic_imports(project_root: Path):
-    """Handle the path injections and dynamic imports safely."""
-    global raganything_cls, RAGAnythingConfig
-
-    sys.path.insert(0, str(project_root))
-    raganything_path = project_root.parent / "raganything" / "RAG-Anything"
-    if raganything_path.exists():
-        sys.path.insert(0, str(raganything_path))
-
-    try:
-        from raganything import RAGAnything as RA
-        from raganything import RAGAnythingConfig as RAC
-
-        raganything_cls = RA
-        RAGAnythingConfig = RAC
-    except ImportError:
-        pass
-
-
 from src.knowledge.extract_numbered_items import process_content_list
 from src.logging import LightRAGLogContext, get_logger
-from src.services.embedding import (
-    get_embedding_client,
-    get_embedding_config,
-    reset_embedding_client,
-)
 from src.services.llm import get_llm_config
 
+# Load LLM config early to ensure OPENAI_API_KEY env var is set before LightRAG imports
+# This is critical because LightRAG reads os.environ["OPENAI_API_KEY"] directly
+from src.services.llm.config import get_llm_config as _early_config_load  # noqa: F401
+
 logger = get_logger("KnowledgeInit")
 
 # Default base directory for knowledge bases
@@ -82,7 +42,14 @@ DEFAULT_BASE_DIR = "./data/knowledge_bases"
 
 
 class DocumentAdder:
-    """Add documents to existing knowledge base with Hash-validation"""
+    """Add documents to existing knowledge base with Hash-validation.
+
+    Supports multiple RAG providers with lazy loading to avoid unnecessary imports:
+    - llamaindex: Only imports llama_index modules
+    - lightrag: Only imports lightrag modules (no raganything)
+    - raganything: Imports raganything with MinerU parser
+    - raganything_docling: Imports raganything with Docling parser
+    """
 
     def __init__(
         self,
@@ -106,15 +73,61 @@ class DocumentAdder:
         self.content_list_dir = self.kb_dir / "content_list"
         self.metadata_file = self.kb_dir / "metadata.json"
 
-        if not self.rag_storage_dir.exists():
-            raise ValueError(f"Knowledge base not initialized: {kb_name}")
+        # For llamaindex, check llamaindex_storage instead of rag_storage
+        provider = self._get_provider_from_metadata()
+        if provider == "llamaindex":
+            llamaindex_storage = self.kb_dir / "llamaindex_storage"
+            if not llamaindex_storage.exists():
+                raise ValueError(f"Knowledge base not initialized (llamaindex): {kb_name}")
+        else:
+            if not self.rag_storage_dir.exists():
+                raise ValueError(f"Knowledge base not initialized: {kb_name}")
 
         self.api_key = api_key
         self.base_url = base_url
         self.progress_tracker = progress_tracker
-        self.rag_provider = rag_provider
+
+        # IMPORTANT: rag_provider parameter is IGNORED for incremental add
+        # We always use the provider from KB metadata to ensure consistency
+        # This prevents mixing different index formats in the same KB
+        self._resolved_provider = provider
+        if rag_provider and rag_provider != provider:
+            logger.warning(
+                f"Requested provider '{rag_provider}' ignored. "
+                f"Using KB's existing provider '{provider}' for consistency."
+            )
+        logger.info(f"Incremental add will use provider: {provider} (from KB metadata)")
         self._ensure_working_directories()
 
+    def _get_provider_from_metadata(self) -> str:
+        """
+        Get the RAG provider from KB metadata.
+
+        This is the ONLY source of truth for incremental adds - we must use
+        the same provider that was used during initialization to ensure
+        data consistency and correct storage format.
+
+        Returns:
+            Provider name (llamaindex, lightrag, raganything, raganything_docling)
+        """
+        if self.metadata_file.exists():
+            try:
+                with open(self.metadata_file, "r", encoding="utf-8") as f:
+                    metadata = json.load(f)
+                    provider = metadata.get("rag_provider")
+                    if provider:
+                        return provider
+            except Exception as e:
+                logger.warning(f"Failed to read provider from metadata: {e}")
+
+        # Fallback: detect from storage structure
+        llamaindex_storage = self.kb_dir / "llamaindex_storage"
+        if llamaindex_storage.exists():
+            return "llamaindex"
+
+        # Default to raganything for backward compatibility
+        return "raganything"
+
     def _ensure_working_directories(self):
         for directory in [self.raw_dir, self.images_dir, self.content_list_dir]:
             directory.mkdir(parents=True, exist_ok=True)
@@ -202,118 +215,271 @@ class DocumentAdder:
         """
         Async phase: Ingests files into the RAG system.
 
-        Uses FileTypeRouter to classify files and route them appropriately:
-        - PDF/DOCX/images -> MinerU parser (full document analysis)
-        - Text/Markdown -> Direct read + LightRAG insert (fast)
+        Uses lazy loading to only import dependencies for the actual provider:
+        - llamaindex: Only imports llama_index
+        - lightrag: Only imports lightrag (no raganything)
+        - raganything: Imports raganything with MinerU
+        - raganything_docling: Imports raganything with Docling
         """
         if not new_files:
             return None
 
-        if raganything_cls is None:
-            raise ImportError("RAGAnything module not found.")
+        provider = self._resolved_provider
+        logger.info(f"Processing {len(new_files)} files with provider: {provider}")
+
+        # Dispatch to provider-specific implementation
+        if provider == "llamaindex":
+            return await self._process_llamaindex(new_files)
+        elif provider == "lightrag":
+            return await self._process_lightrag(new_files)
+        elif provider == "raganything":
+            return await self._process_raganything(new_files, parser="mineru")
+        elif provider == "raganything_docling":
+            return await self._process_raganything(new_files, parser="docling")
+        else:
+            raise ValueError(f"Unknown RAG provider: {provider}")
+
+    async def _process_llamaindex(self, new_files: List[Path]) -> List[Path]:
+        """
+        Incremental add for LlamaIndex pipeline.
+        Lazy imports llama_index only when needed.
+        """
+        logger.info("Using LlamaIndex incremental add...")
+
+        # Lazy import llama_index
+        try:
+            from src.services.rag.pipelines.llamaindex import LlamaIndexPipeline
+        except ImportError as e:
+            raise ImportError(
+                f"LlamaIndex dependencies not installed. "
+                f"Install with: pip install llama-index llama-index-core. Error: {e}"
+            ) from e
+
+        # Pre-import progress stage if needed
+        ProgressStage: Any = None
+        if self.progress_tracker:
+            from src.knowledge.progress_tracker import ProgressStage
+
+        pipeline = LlamaIndexPipeline(kb_base_dir=str(self.base_dir))
+        file_paths = [str(f) for f in new_files]
+
+        # Use the new add_documents method for incremental add
+        processed_files = []
+        total_files = len(file_paths)
+
+        for idx, file_path in enumerate(file_paths, 1):
+            doc_file = Path(file_path)
+            try:
+                if self.progress_tracker and ProgressStage:
+                    self.progress_tracker.update(
+                        ProgressStage.PROCESSING_FILE,
+                        f"Indexing (LlamaIndex) {doc_file.name}",
+                        current=idx,
+                        total=total_files,
+                    )
+
+                # Use add_documents for incremental add
+                success = await pipeline.add_documents(self.kb_name, [file_path])
+                if success:
+                    processed_files.append(doc_file)
+                    self._record_successful_hash(doc_file)
+                    logger.info(f"  ✓ Processed (LlamaIndex): {doc_file.name}")
+                else:
+                    logger.error(f"  ✗ Failed to index: {doc_file.name}")
+            except Exception as e:
+                logger.exception(f"  ✗ Failed {doc_file.name}: {e}")
+
+        return processed_files
+
+    async def _process_lightrag(self, new_files: List[Path]) -> List[Path]:
+        """
+        Incremental add for LightRAG pipeline (text-only).
+        Lazy imports lightrag only when needed - does NOT require raganything.
+        """
+        logger.info("Using LightRAG incremental add (text-only)...")
 
+        # Lazy import lightrag
+        try:
+            from lightrag import LightRAG
+            from lightrag.utils import EmbeddingFunc
+        except ImportError as e:
+            raise ImportError(
+                f"LightRAG dependencies not installed. "
+                f"Install with: pip install lightrag. Error: {e}"
+            ) from e
+
+        from src.services.embedding import (
+            get_embedding_client,
+            get_embedding_config,
+            reset_embedding_client,
+        )
+        from src.services.llm import get_llm_client
         from src.services.rag.components.routing import FileTypeRouter
 
-        # Pre-import progress stage if needed to avoid overhead in loop
+        # Pre-import progress stage if needed
         ProgressStage: Any = None
         if self.progress_tracker:
             from src.knowledge.progress_tracker import ProgressStage
 
-        self.llm_cfg = get_llm_config()
-        model = self.llm_cfg.model
-        api_key = self.api_key or self.llm_cfg.api_key
-        base_url = self.base_url or self.llm_cfg.base_url
-
-        # LLM Function Wrapper
-        def llm_model_func(prompt, system_prompt=None, history_messages=None, **kwargs):
-            if history_messages is None:
-                history_messages = []
-            return openai_complete_if_cache(
-                model,
-                prompt,
-                system_prompt=system_prompt,
-                history_messages=history_messages,
-                api_key=api_key,
-                base_url=base_url,
-                **kwargs,
+        # Setup LLM and embedding
+        llm_client = get_llm_client()
+        self.llm_cfg = llm_client.config
+        llm_model_func = llm_client.get_model_func()
+
+        reset_embedding_client()
+        embedding_cfg = get_embedding_config()
+        embedding_client = get_embedding_client()
+
+        async def unified_embed_func(texts):
+            return await embedding_client.embed(texts)
+
+        embedding_func = EmbeddingFunc(
+            embedding_dim=embedding_cfg.dim,
+            max_token_size=embedding_cfg.max_tokens,
+            func=unified_embed_func,
+        )
+
+        # Create LightRAG instance (text-only, no raganything)
+        with LightRAGLogContext(scene="knowledge_incremental"):
+            rag = LightRAG(
+                working_dir=str(self.rag_storage_dir),
+                llm_model_func=llm_model_func,
+                embedding_func=embedding_func,
             )
+            await rag.initialize_storages()
 
-        # Vision Function Wrapper - Robust history handling
-        def vision_model_func(
-            prompt,
-            system_prompt=None,
-            history_messages=None,
-            image_data=None,
-            messages=None,
-            **kwargs,
-        ):
-            if history_messages is None:
-                history_messages = []
-            # If pre-formatted messages are provided, sanitize them
-            if messages:
-                safe_messages = self._filter_valid_messages(messages)
-                return openai_complete_if_cache(
-                    model,
-                    prompt="",
-                    messages=safe_messages,
-                    api_key=api_key,
-                    base_url=base_url,
-                    **kwargs,
-                )
+            from lightrag.kg.shared_storage import initialize_pipeline_status
 
-            # --- Construct Message History ---
-            current_messages = []
-
-            # 1. Add System Prompt (if provided)
-            if system_prompt:
-                current_messages.append({"role": "system", "content": system_prompt})
-
-            # 2. Add History (Filtering out conflicting system prompts)
-            if history_messages:
-                # Filter out system messages from history to avoid duplicates/conflicts with the new system_prompt
-                filtered_history = [
-                    msg
-                    for msg in history_messages
-                    if isinstance(msg, dict) and msg.get("role") != "system"
-                ]
-                current_messages.extend(filtered_history)
-
-            # 3. Construct New User Message
-            user_content: List[Dict[str, Any]] = [{"type": "text", "text": prompt}]
-            if image_data:
-                user_content.append(
-                    {
-                        "type": "image_url",
-                        "image_url": {"url": f"data:image/jpeg;base64,{image_data}"},
-                    }
-                )
+            await initialize_pipeline_status()
 
-            # 4. Merge Logic: Avoid back-to-back user messages
-            if current_messages and current_messages[-1].get("role") == "user":
-                last_msg = current_messages[-1]
-                # If last content is string, convert to list format first
-                if isinstance(last_msg["content"], str):
-                    last_msg["content"] = [{"type": "text", "text": last_msg["content"]}]
+        # Classify files
+        file_paths_str = [str(f) for f in new_files]
+        classification = FileTypeRouter.classify_files(file_paths_str)
 
-                # Append new content blocks
-                if isinstance(last_msg["content"], list):
-                    last_msg["content"].extend(user_content)
+        logger.info(
+            f"File classification: {len(classification.needs_mineru)} need parsing, "
+            f"{len(classification.text_files)} text files, "
+            f"{len(classification.unsupported)} unsupported"
+        )
+
+        processed_files = []
+        total_files = len(classification.needs_mineru) + len(classification.text_files)
+        idx = 0
+
+        # For LightRAG (text-only), use basic PDF text extraction for PDFs
+        for doc_file_str in classification.needs_mineru:
+            doc_file = Path(doc_file_str)
+            idx += 1
+            try:
+                if self.progress_tracker and ProgressStage:
+                    self.progress_tracker.update(
+                        ProgressStage.PROCESSING_FILE,
+                        f"Extracting text from {doc_file.name}",
+                        current=idx,
+                        total=total_files,
+                    )
+
+                if not doc_file.exists():
+                    logger.error(f"  ✗ Failed: File missing {doc_file.name}")
+                    continue
+
+                # Basic text extraction
+                content = await self._extract_text_basic(doc_file)
+                if content.strip():
+                    await rag.ainsert(content)
+                    processed_files.append(doc_file)
+                    self._record_successful_hash(doc_file)
+                    logger.info(f"  ✓ Processed (LightRAG): {doc_file.name}")
                 else:
-                    # Fallback if structure is unexpected, just append new message
-                    current_messages.append({"role": "user", "content": user_content})
-            else:
-                current_messages.append({"role": "user", "content": user_content})
-
-            return openai_complete_if_cache(
-                model,
-                prompt="",
-                messages=current_messages,
-                api_key=api_key,
-                base_url=base_url,
-                **kwargs,
-            )
+                    logger.warning(f"  ⚠ No text extracted: {doc_file.name}")
+            except Exception as e:
+                logger.exception(f"  ✗ Failed {doc_file.name}: {e}")
+
+        # Process text files directly
+        for doc_file_str in classification.text_files:
+            doc_file = Path(doc_file_str)
+            idx += 1
+            try:
+                if self.progress_tracker and ProgressStage:
+                    self.progress_tracker.update(
+                        ProgressStage.PROCESSING_FILE,
+                        f"Ingesting (text) {doc_file.name}",
+                        current=idx,
+                        total=total_files,
+                    )
+
+                if not doc_file.exists():
+                    logger.error(f"  ✗ Failed: File missing {doc_file.name}")
+                    continue
+
+                content = await FileTypeRouter.read_text_file(str(doc_file))
+                if content.strip():
+                    await rag.ainsert(content)
+                    processed_files.append(doc_file)
+                    self._record_successful_hash(doc_file)
+                    logger.info(f"  ✓ Processed (text): {doc_file.name}")
+                else:
+                    logger.warning(f"  ⚠ Skipped empty file: {doc_file.name}")
+            except Exception as e:
+                logger.exception(f"  ✗ Failed {doc_file.name}: {e}")
+
+        for doc_file_str in classification.unsupported:
+            logger.warning(f"  ⚠ Skipped unsupported file: {Path(doc_file_str).name}")
+
+        return processed_files
+
+    async def _process_raganything(
+        self, new_files: List[Path], parser: str = "mineru"
+    ) -> List[Path]:
+        """
+        Incremental add for RAGAnything pipeline (multimodal).
+        Lazy imports raganything only when needed.
+
+        Args:
+            parser: "mineru" for RAGAnything, "docling" for RAGAnything Docling
+        """
+        parser_name = "MinerU" if parser == "mineru" else "Docling"
+        logger.info(f"Using RAGAnything incremental add with {parser_name} parser...")
+
+        # Lazy import raganything
+        try:
+            # Add RAG-Anything to path if needed
+            project_root = Path(__file__).resolve().parent.parent.parent
+            raganything_path = project_root.parent / "raganything" / "RAG-Anything"
+            if raganything_path.exists() and str(raganything_path) not in sys.path:
+                sys.path.insert(0, str(raganything_path))
+
+            from lightrag.utils import EmbeddingFunc
+            from raganything import RAGAnything, RAGAnythingConfig
+        except ImportError as e:
+            raise ImportError(
+                f"RAGAnything dependencies not installed. "
+                f"Please install raganything package. Error: {e}"
+            ) from e
+
+        from src.services.embedding import (
+            get_embedding_client,
+            get_embedding_config,
+            reset_embedding_client,
+        )
+        from src.services.llm import get_llm_client
+        from src.services.rag.components.routing import FileTypeRouter
+        from src.services.rag.utils.image_migration import (
+            cleanup_parser_output_dirs,
+            migrate_images_and_update_paths,
+        )
+
+        # Pre-import progress stage if needed
+        ProgressStage: Any = None
+        if self.progress_tracker:
+            from src.knowledge.progress_tracker import ProgressStage
+
+        # Setup LLM and embedding
+        llm_client = get_llm_client()
+        self.llm_cfg = llm_client.config
+        llm_model_func = llm_client.get_model_func()
+        vision_model_func = llm_client.get_vision_model_func()
 
-        # Embedding Setup
         reset_embedding_client()
         embedding_cfg = get_embedding_config()
         embedding_client = get_embedding_client()
@@ -327,16 +493,17 @@ class DocumentAdder:
             func=unified_embed_func,
         )
 
+        # Configure RAGAnything with the appropriate parser
         config = RAGAnythingConfig(
             working_dir=str(self.rag_storage_dir),
-            parser="mineru",
+            parser=parser,
             enable_image_processing=True,
             enable_table_processing=True,
             enable_equation_processing=True,
         )
 
-        with LightRAGLogContext(scene="knowledge_init"):
-            rag = raganything_cls(
+        with LightRAGLogContext(scene="knowledge_incremental"):
+            rag = RAGAnything(
                 config=config,
                 llm_model_func=llm_model_func,
                 vision_model_func=vision_model_func,
@@ -345,12 +512,12 @@ class DocumentAdder:
             if hasattr(rag, "_ensure_lightrag_initialized"):
                 await rag._ensure_lightrag_initialized()
 
-        # Classify files by type
+        # Classify files
         file_paths_str = [str(f) for f in new_files]
         classification = FileTypeRouter.classify_files(file_paths_str)
 
         logger.info(
-            f"File classification: {len(classification.needs_mineru)} need MinerU, "
+            f"File classification: {len(classification.needs_mineru)} need {parser_name}, "
             f"{len(classification.text_files)} text files, "
             f"{len(classification.unsupported)} unsupported"
         )
@@ -358,8 +525,9 @@ class DocumentAdder:
         processed_files = []
         total_files = len(classification.needs_mineru) + len(classification.text_files)
         idx = 0
+        total_images_migrated = 0
 
-        # Process files requiring MinerU (PDF, DOCX, images)
+        # Process files requiring parser (PDF, DOCX, images)
         for doc_file_str in classification.needs_mineru:
             doc_file = Path(doc_file_str)
             idx += 1
@@ -367,32 +535,53 @@ class DocumentAdder:
                 if self.progress_tracker and ProgressStage:
                     self.progress_tracker.update(
                         ProgressStage.PROCESSING_FILE,
-                        f"Ingesting (MinerU) {doc_file.name}",
+                        f"Ingesting ({parser_name}) {doc_file.name}",
                         current=idx,
                         total=total_files,
                     )
 
-                # Verify file still exists in raw/ (it should, as we staged it)
                 if not doc_file.exists():
-                    logger.error(f"  ✗ Failed: Staged file missing {doc_file.name}")
+                    logger.error(f"  ✗ Failed: File missing {doc_file.name}")
                     continue
 
-                await asyncio.wait_for(
-                    rag.process_document_complete(
-                        file_path=str(doc_file),
-                        output_dir=str(self.content_list_dir),
-                        parse_method="auto",
-                    ),
-                    timeout=600.0,
+                # Step 1: Parse document
+                logger.info(f"  Step 1/3: Parsing {doc_file.name}...")
+                content_list, doc_id = await rag.parse_document(
+                    file_path=str(doc_file),
+                    output_dir=str(self.content_list_dir),
+                    parse_method="auto",
                 )
+
+                # Step 2: Migrate images
+                logger.info("  Step 2/3: Migrating images...")
+                updated_content_list, num_migrated = await migrate_images_and_update_paths(
+                    content_list=content_list,
+                    source_base_dir=self.content_list_dir,
+                    target_images_dir=self.images_dir,
+                    batch_size=50,
+                )
+                total_images_migrated += num_migrated
+
+                # Save content_list
+                content_list_file = self.content_list_dir / f"{doc_file.stem}.json"
+                with open(content_list_file, "w", encoding="utf-8") as f:
+                    json.dump(updated_content_list, f, ensure_ascii=False, indent=2)
+
+                # Step 3: Insert into RAG
+                logger.info("  Step 3/3: Inserting into knowledge graph...")
+                await rag.insert_content_list(
+                    content_list=updated_content_list,
+                    file_path=str(doc_file),
+                    doc_id=doc_id,
+                )
+
                 processed_files.append(doc_file)
-                # Store hash on success - "Canonizing" the file
                 self._record_successful_hash(doc_file)
-                logger.info(f"  ✓ Processed (MinerU): {doc_file.name}")
+                logger.info(f"  ✓ Processed ({parser_name}): {doc_file.name}")
             except Exception as e:
                 logger.exception(f"  ✗ Failed {doc_file.name}: {e}")
 
-        # Process text files directly (fast path - no MinerU)
+        # Process text files directly
         for doc_file_str in classification.text_files:
             doc_file = Path(doc_file_str)
             idx += 1
@@ -405,15 +594,12 @@ class DocumentAdder:
                         total=total_files,
                     )
 
-                # Verify file still exists
                 if not doc_file.exists():
-                    logger.error(f"  ✗ Failed: Staged file missing {doc_file.name}")
+                    logger.error(f"  ✗ Failed: File missing {doc_file.name}")
                     continue
 
-                # Read text file directly
                 content = await FileTypeRouter.read_text_file(str(doc_file))
                 if content.strip():
-                    # Insert directly into LightRAG, bypassing MinerU
                     await rag.lightrag.ainsert(content)
                     processed_files.append(doc_file)
                     self._record_successful_hash(doc_file)
@@ -423,13 +609,50 @@ class DocumentAdder:
             except Exception as e:
                 logger.exception(f"  ✗ Failed {doc_file.name}: {e}")
 
-        # Log unsupported files
         for doc_file_str in classification.unsupported:
             logger.warning(f"  ⚠ Skipped unsupported file: {Path(doc_file_str).name}")
 
+        # Cleanup parser output directories
+        if total_images_migrated > 0:
+            logger.info("Cleaning up temporary parser output directories...")
+            await cleanup_parser_output_dirs(self.content_list_dir)
+
         await self.fix_structure()
         return processed_files
 
+    async def _extract_text_basic(self, file_path: Path) -> str:
+        """Basic text extraction for LightRAG (text-only pipeline)."""
+        suffix = file_path.suffix.lower()
+
+        if suffix == ".pdf":
+            try:
+                import fitz  # PyMuPDF
+
+                doc = fitz.open(file_path)
+                texts = []
+                for page in doc:
+                    texts.append(page.get_text())
+                doc.close()
+                return "\n\n".join(texts)
+            except ImportError:
+                logger.warning("PyMuPDF not installed. Cannot extract PDF text.")
+                return ""
+            except Exception as e:
+                logger.error(f"Failed to extract PDF text: {e}")
+                return ""
+        else:
+            # Try to read as text
+            try:
+                with open(file_path, "r", encoding="utf-8") as f:
+                    return f.read()
+            except Exception:
+                try:
+                    with open(file_path, "r", encoding="latin-1") as f:
+                        return f.read()
+                except Exception as e:
+                    logger.error(f"Failed to read file as text: {e}")
+                    return ""
+
     def _record_successful_hash(self, file_path: Path):
         """Update metadata with the hash of a successfully processed file."""
         file_hash = self._get_file_hash(file_path)
@@ -464,36 +687,50 @@ class DocumentAdder:
         ]
 
     async def fix_structure(self):
-        """Robustly moves nested outputs and cleans up."""
-        logger.info("Organizing storage structure...")
+        """
+        Clean up parser output directories after image migration.
+
+        NOTE: Image migration and path updates are now handled by the RAG pipeline
+        (raganything.py / raganything_docling.py) BEFORE RAG insertion. This ensures
+        RAG stores the correct canonical image paths (kb/images/) from the start.
+
+        This method now only cleans up empty temporary parser output directories.
+        """
+        logger.info("Checking for leftover parser output directories...")
 
-        # 1. Identify moves
-        moves = []
-        for doc_dir in self.content_list_dir.glob("*"):
+        # Support both 'auto' (MinerU) and 'docling' parser output directories
+        parser_subdirs = ["auto", "docling"]
+        cleaned_count = 0
+
+        for doc_dir in list(self.content_list_dir.glob("*")):
             if not doc_dir.is_dir():
                 continue
 
-            # Content List
-            json_src = next(doc_dir.glob("auto/*_content_list.json"), None)
-            if json_src:
-                moves.append((json_src, self.content_list_dir / f"{doc_dir.name}.json"))
+            for parser_subdir in parser_subdirs:
+                subdir = doc_dir / parser_subdir
+                if subdir.exists():
+                    try:
+                        # Check if directory is empty or only contains empty subdirs
+                        has_content = any(
+                            f.is_file() or (f.is_dir() and any(f.iterdir()))
+                            for f in subdir.iterdir()
+                        )
 
-            # Images
-            for img in doc_dir.glob("auto/images/*"):
-                moves.append((img, self.images_dir / img.name))
+                        if not has_content:
+                            await self._run_in_executor(shutil.rmtree, subdir)
+                            cleaned_count += 1
+                    except Exception as e:
+                        logger.debug(f"Could not clean up {subdir}: {e}")
 
-        # 2. Execute moves
-        for src, dest in moves:
-            if src.exists():
-                await self._run_in_executor(shutil.copy2, src, dest)
+            # Remove doc_dir if it's now empty
+            try:
+                if doc_dir.exists() and not any(doc_dir.iterdir()):
+                    doc_dir.rmdir()
+            except Exception:
+                pass
 
-        # 3. Safe Cleanup: Only delete directories we actually processed
-        for doc_dir in self.content_list_dir.glob("*"):
-            if doc_dir.is_dir():
-                # Safety check: only delete if it looks like a parser output (has 'auto' subdir)
-                # This prevents wiping manual user folders in content_list_dir
-                if (doc_dir / "auto").exists():
-                    await self._run_in_executor(shutil.rmtree, doc_dir, ignore_errors=True)
+        if cleaned_count > 0:
+            logger.info(f"Cleaned up {cleaned_count} empty parser directories")
 
     def extract_numbered_items_for_new_docs(self, processed_files, batch_size=20):
         if not processed_files:
@@ -519,6 +756,11 @@ class DocumentAdder:
                 )
 
     def update_metadata(self, added_count: int):
+        """Update metadata after incremental add.
+
+        Note: We do NOT update rag_provider here - incremental adds must use
+        the same provider as the original initialization for consistency.
+        """
         if not self.metadata_file.exists():
             return
         try:
@@ -527,18 +769,9 @@ class DocumentAdder:
 
             metadata["last_updated"] = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
 
-            # Update RAG provider if specified
-            if self.rag_provider:
-                metadata["rag_provider"] = self.rag_provider
-
-                # Also save to centralized config file
-                try:
-                    from src.services.config import get_kb_config_service
-
-                    kb_config_service = get_kb_config_service()
-                    kb_config_service.set_rag_provider(self.kb_name, self.rag_provider)
-                except Exception as config_err:
-                    logger.warning(f"Failed to save to centralized config: {config_err}")
+            # Record the provider used (should match what's already in metadata)
+            if "rag_provider" not in metadata and self._resolved_provider:
+                metadata["rag_provider"] = self._resolved_provider
 
             history = metadata.get("update_history", [])
             history.append(
@@ -546,6 +779,7 @@ class DocumentAdder:
                     "timestamp": metadata["last_updated"],
                     "action": "incremental_add",
                     "count": added_count,
+                    "provider": self._resolved_provider,
                 }
             )
             metadata["update_history"] = history
@@ -557,7 +791,17 @@ class DocumentAdder:
 
 
 async def main():
-    parser = argparse.ArgumentParser(description="Incrementally add documents to RAG KB")
+    parser = argparse.ArgumentParser(
+        description="Incrementally add documents to RAG KB",
+        epilog="""
+Example usage:
+  # Add documents to existing KB (uses provider from KB metadata)
+  python -m src.knowledge.add_documents my_kb --docs doc1.pdf doc2.txt
+
+  # Add all documents from a directory
+  python -m src.knowledge.add_documents my_kb --docs-dir ./documents/
+        """,
+    )
     parser.add_argument("kb_name", help="KB Name")
     parser.add_argument("--docs", nargs="+", help="Files")
     parser.add_argument("--docs-dir", help="Directory")
@@ -568,10 +812,6 @@ async def main():
 
     args = parser.parse_args()
 
-    # Initialize dynamic paths
-    project_root = Path(__file__).parent.parent.parent
-    load_dynamic_imports(project_root)
-
     load_dotenv()
 
     doc_files = []