realtimex-deeptutor 0.5.0.post1__py3-none-any.whl → 0.5.0.post3__py3-none-any.whl

src/services/rag/components/retrievers/lightrag.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 """
 LightRAG Retriever
 ==================
@@ -52,57 +53,17 @@ class LightRAGRetriever(BaseComponent):
 
         try:
             from lightrag import LightRAG
-            from openai import AsyncOpenAI
 
             from src.services.embedding import get_embedding_client
             from src.services.llm import get_llm_client
 
+            # Use unified LLM client from src/services/llm
             llm_client = get_llm_client()
             embed_client = get_embedding_client()
 
-            # Create AsyncOpenAI client directly
-            openai_client = AsyncOpenAI(
-                api_key=llm_client.config.api_key,
-                base_url=llm_client.config.base_url,
-            )
-
-            # LLM function using services (ASYNC - LightRAG expects async functions)
-            async def llm_model_func(prompt, system_prompt=None, history_messages=None, **kwargs):
-                """Custom async LLM function that bypasses LightRAG's openai_complete_if_cache."""
-                if history_messages is None:
-                    history_messages = []
-
-                # Build messages
-                messages = []
-                if system_prompt:
-                    messages.append({"role": "system", "content": system_prompt})
-                messages.extend(history_messages)
-                messages.append({"role": "user", "content": prompt})
-
-                # Whitelist only valid OpenAI parameters
-                valid_params = {
-                    "temperature",
-                    "top_p",
-                    "n",
-                    "stream",
-                    "stop",
-                    "max_tokens",
-                    "presence_penalty",
-                    "frequency_penalty",
-                    "logit_bias",
-                    "user",
-                    "seed",
-                }
-                clean_kwargs = {k: v for k, v in kwargs.items() if k in valid_params}
-
-                # Call OpenAI API directly (async)
-                response = await openai_client.chat.completions.create(
-                    model=llm_client.config.model,
-                    messages=messages,
-                    **clean_kwargs,
-                )
-
-                return response.choices[0].message.content
+            # Get model function from unified LLM client
+            # This handles all provider differences and env var setup for LightRAG
+            llm_model_func = llm_client.get_model_func()
 
             # Create pure LightRAG instance (no multimodal)
             rag = LightRAG(

src/services/rag/components/routing.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 """
 File Type Router
 ================
@@ -284,3 +285,50 @@ class FileTypeRouter:
         """
         doc_type = cls.get_document_type(file_path)
         return doc_type in (DocumentType.TEXT, DocumentType.MARKDOWN)
+
+    @classmethod
+    def get_extensions_for_provider(cls, provider: str) -> set[str]:
+        """
+        Get supported file extensions for a specific RAG provider.
+
+        Args:
+            provider: RAG provider name (llamaindex, lightrag, raganything, raganything_docling)
+
+        Returns:
+            Set of supported file extensions (with leading dot, e.g., {".pdf", ".txt"})
+        """
+        # Base text extensions supported by all providers
+        text_extensions = cls.TEXT_EXTENSIONS.copy()
+
+        if provider == "llamaindex":
+            # LlamaIndex: PDF + all text files (reads any text file directly)
+            return cls.MINERU_EXTENSIONS | text_extensions
+
+        elif provider == "lightrag":
+            # LightRAG: PDF + all text files (uses FileTypeRouter)
+            return cls.MINERU_EXTENSIONS | text_extensions
+
+        elif provider in ("raganything", "raganything_docling"):
+            # RAGAnything: PDF + Word + Images + all text files (full multimodal via MinerU)
+            return (
+                cls.MINERU_EXTENSIONS | cls.DOCX_EXTENSIONS | cls.IMAGE_EXTENSIONS | text_extensions
+            )
+
+        else:
+            # Default: same as llamaindex (most conservative)
+            logger.warning(f"Unknown provider '{provider}', using default extensions")
+            return cls.MINERU_EXTENSIONS | text_extensions
+
+    @classmethod
+    def get_glob_patterns_for_provider(cls, provider: str) -> list[str]:
+        """
+        Get glob patterns for file searching based on RAG provider.
+
+        Args:
+            provider: RAG provider name (llamaindex, lightrag, raganything, raganything_docling)
+
+        Returns:
+            List of glob patterns (e.g., ["*.pdf", "*.txt", "*.md"])
+        """
+        extensions = cls.get_extensions_for_provider(provider)
+        return [f"*{ext}" for ext in sorted(extensions)]

src/services/rag/factory.py

@@ -1,25 +1,77 @@
+# -*- coding: utf-8 -*-
 """
 Pipeline Factory
 ================
 
 Factory for creating and managing RAG pipelines.
 
-LightRAG is the default pipeline (always available).
-RAGAnything and LlamaIndex are optional (require extra dependencies).
+Note: Pipeline imports are lazy to avoid importing heavy dependencies (lightrag, llama_index, etc.)
+at module load time. This allows the core services to be imported without RAG dependencies.
 """
 
 import logging
 from typing import Callable, Dict, List, Optional
-
-from .pipelines import lightrag
+import warnings
 
 logger = logging.getLogger(__name__)
 
-# Pipeline registry - start with always-available pipelines
-_PIPELINES: Dict[str, Callable] = {
-    "lightrag": lightrag.LightRAGPipeline,  # Knowledge graph: PDFParser, fast text-only (default)
-    "realtimex": lightrag.LightRAGPipeline,  # Alias: RealTimeX (uses LightRAG with RealTimeX AI config)
-}
+# Pipeline registry - populated lazily
+_PIPELINES: Dict[str, Callable] = {}
+_PIPELINES_INITIALIZED = False
+
+
+def _init_pipelines():
+    """Lazily initialize pipeline registry.
+
+    Important:
+    - Do NOT import optional heavy dependencies (e.g. llama_index) here.
+    - Pipelines must be imported inside their factory callables, so users can
+      use other providers without installing every optional dependency.
+    """
+    global _PIPELINES, _PIPELINES_INITIALIZED
+    if _PIPELINES_INITIALIZED:
+        return
+
+    def _build_raganything(**kwargs):
+        from .pipelines.raganything import RAGAnythingPipeline
+
+        return RAGAnythingPipeline(**kwargs)
+
+    def _build_raganything_docling(**kwargs):
+        from .pipelines.raganything_docling import RAGAnythingDoclingPipeline
+
+        return RAGAnythingDoclingPipeline(**kwargs)
+
+    def _build_lightrag(kb_base_dir: Optional[str] = None, **kwargs):
+        # LightRAGPipeline is a factory function returning a composed RAGPipeline
+        from .pipelines.lightrag import LightRAGPipeline
+
+        return LightRAGPipeline(kb_base_dir=kb_base_dir)
+
+    def _build_realtimex(kb_base_dir: Optional[str] = None, **kwargs):
+        # RealTimeX is an alias for LightRAG with RealTimeX branding
+        from .pipelines.lightrag import LightRAGPipeline
+
+        return LightRAGPipeline(kb_base_dir=kb_base_dir)
+
+    def _build_llamaindex(**kwargs):
+        # LlamaIndexPipeline depends on optional `llama_index` package.
+        # Import it only when explicitly requested.
+        from .pipelines.llamaindex import LlamaIndexPipeline
+
+        return LlamaIndexPipeline(**kwargs)
+
+    _PIPELINES.update(
+        {
+            "raganything": _build_raganything,  # Full multimodal: MinerU parser, deep analysis (slow, thorough)
+            "raganything_docling": _build_raganything_docling,  # Docling parser: Office/HTML friendly, easier setup
+            "lightrag": _build_lightrag,  # Knowledge graph: PDFParser, fast text-only (medium speed)
+            "realtimex": _build_realtimex,  # RealTimeX AI powered knowledge retrieval (recommended, uses LightRAG)
+            "llamaindex": _build_llamaindex,  # Vector-only: Simple chunking, fast (fastest)
+        }
+    )
+    _PIPELINES_INITIALIZED = True
+
 
 # Pipeline metadata for list_pipelines()
 _PIPELINE_INFO: Dict[str, Dict[str, str]] = {
@@ -37,14 +89,16 @@ _PIPELINE_INFO: Dict[str, Dict[str, str]] = {
     },
 }
 
+
 # Try to register optional pipelines
 def _register_optional_pipelines():
     """Register pipelines that have optional dependencies."""
     global _PIPELINES, _PIPELINE_INFO
-    
+
     # Try RAGAnything (requires raganything package)
     try:
         from .pipelines.raganything import RAGAnythingPipeline
+
         _PIPELINES["raganything"] = RAGAnythingPipeline
         _PIPELINE_INFO["raganything"] = {
             "id": "raganything",
@@ -61,10 +115,11 @@ def _register_optional_pipelines():
             "available": False,
         }
         logger.debug(f"RAGAnything not available: {e}")
-    
+
     # Try LlamaIndex (requires llama-index package)
     try:
         from .pipelines import llamaindex
+
         _PIPELINES["llamaindex"] = llamaindex.LlamaIndexPipeline
         _PIPELINE_INFO["llamaindex"] = {
             "id": "llamaindex",
@@ -87,13 +142,13 @@ def _register_optional_pipelines():
 _register_optional_pipelines()
 
 
-def get_pipeline(name: str = "lightrag", kb_base_dir: Optional[str] = None, **kwargs):
+def get_pipeline(name: str = "realtimex", kb_base_dir: Optional[str] = None, **kwargs):
     """
     Get a pre-configured pipeline by name.
 
     Args:
-        name: Pipeline name (lightrag, raganything, llamaindex)
-              Default is 'lightrag' (always available).
+        name: Pipeline name (raganything, raganything_docling, lightrag, realtimex, llamaindex)
+              Default is 'realtimex' (recommended, always available).
         kb_base_dir: Base directory for knowledge bases (passed to all pipelines)
         **kwargs: Additional arguments passed to pipeline constructor
 
@@ -103,6 +158,7 @@ def get_pipeline(name: str = "lightrag", kb_base_dir: Optional[str] = None, **kw
     Raises:
         ValueError: If pipeline name is not found or not available
     """
+    _init_pipelines()
     if name not in _PIPELINES:
         available = list(_PIPELINES.keys())
         # Check if it's a known but unavailable pipeline
@@ -116,20 +172,22 @@ def get_pipeline(name: str = "lightrag", kb_base_dir: Optional[str] = None, **kw
 
     factory = _PIPELINES[name]
 
-    # Handle different pipeline types:
-    # - lightrag, realtimex, academic: functions that return RAGPipeline
-    # - llamaindex, raganything: classes that need instantiation
-    if name in ("lightrag", "realtimex", "academic"):
-        # LightRAGPipeline and AcademicPipeline are factory functions
-        return factory(kb_base_dir=kb_base_dir)
-    elif name in ("llamaindex", "raganything"):
-        # LlamaIndexPipeline and RAGAnythingPipeline are classes
+    try:
+        # Handle different pipeline types:
+        # - lightrag, realtimex: callable that accepts kb_base_dir and returns a composed RAGPipeline
+        # - llamaindex, raganything, raganything_docling: callables that instantiate class-based pipelines
+        if name in ("lightrag", "realtimex"):
+            return factory(kb_base_dir=kb_base_dir, **kwargs)
+
         if kb_base_dir:
             kwargs["kb_base_dir"] = kb_base_dir
         return factory(**kwargs)
-    else:
-        # Default: try calling with kb_base_dir
-        return factory(kb_base_dir=kb_base_dir)
+    except ImportError as e:
+        # Common case: user didn't install optional RAG backend deps (e.g. llama_index).
+        raise ValueError(
+            f"Pipeline '{name}' is not available because an optional dependency is missing: {e}. "
+            f"Please install the required dependency for '{name}', or switch provider to 'realtimex'/'lightrag'."
+        ) from e
 
 
 def list_pipelines(include_unavailable: bool = False) -> List[Dict[str, str]]:
@@ -142,21 +200,33 @@ def list_pipelines(include_unavailable: bool = False) -> List[Dict[str, str]]:
     Returns:
         List of pipeline info dictionaries
     """
-    result = []
-    # Order: realtimex first (recommended), then others
-    order = ["realtimex", "lightrag", "raganything", "llamaindex"]
-    
-    for pipeline_id in order:
-        if pipeline_id in _PIPELINE_INFO:
-            info = _PIPELINE_INFO[pipeline_id]
-            if include_unavailable or info.get("available", False):
-                result.append({
-                    "id": info["id"],
-                    "name": info["name"],
-                    "description": info["description"],
-                })
-    
-    return result
+    return [
+        {
+            "id": "realtimex",
+            "name": "RealTimeX",
+            "description": "RealTimeX AI powered knowledge retrieval (recommended).",
+        },
+        {
+            "id": "lightrag",
+            "name": "LightRAG",
+            "description": "Lightweight knowledge graph retrieval, fast processing of text documents.",
+        },
+        {
+            "id": "raganything",
+            "name": "RAG-Anything (MinerU)",
+            "description": "Multimodal document processing with MinerU parser. Best for academic PDFs with complex equations and formulas.",
+        },
+        {
+            "id": "raganything_docling",
+            "name": "RAG-Anything (Docling)",
+            "description": "Multimodal document processing with Docling parser. Better for Office documents (.docx, .pptx) and HTML. Easier to install.",
+        },
+        {
+            "id": "llamaindex",
+            "name": "LlamaIndex",
+            "description": "Pure vector retrieval, fastest processing speed.",
+        },
+    ]
 
 
 def register_pipeline(name: str, factory: Callable):
@@ -167,6 +237,7 @@ def register_pipeline(name: str, factory: Callable):
         name: Pipeline name
         factory: Factory function or class that creates the pipeline
     """
+    _init_pipelines()
     _PIPELINES[name] = factory
 
 
@@ -180,6 +251,7 @@ def has_pipeline(name: str) -> bool:
     Returns:
         True if pipeline exists
     """
+    _init_pipelines()
     return name in _PIPELINES
 
 
@@ -190,8 +262,6 @@ def get_plugin(name: str) -> Dict[str, Callable]:
 
     Get a plugin by name (maps to pipeline API).
     """
-    import warnings
-
     warnings.warn(
         "get_plugin() is deprecated, use get_pipeline() instead",
         DeprecationWarning,
@@ -210,8 +280,6 @@ def list_plugins() -> List[Dict[str, str]]:
     """
     DEPRECATED: Use list_pipelines() instead.
     """
-    import warnings
-
     warnings.warn(
         "list_plugins() is deprecated, use list_pipelines() instead",
         DeprecationWarning,
@@ -224,8 +292,6 @@ def has_plugin(name: str) -> bool:
     """
     DEPRECATED: Use has_pipeline() instead.
     """
-    import warnings
-
     warnings.warn(
         "has_plugin() is deprecated, use has_pipeline() instead",
         DeprecationWarning,

src/services/rag/pipeline.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 """
 RAG Pipeline
 ============

src/services/rag/pipelines/__init__.py

@@ -1,32 +1,41 @@
+# -*- coding: utf-8 -*-
 """
 Pre-configured Pipelines
 ========================
 
 Ready-to-use RAG pipelines for common use cases.
-
-LightRAG and Academic pipelines are always available.
-LlamaIndex and RAGAnything require optional dependencies.
 """
 
-# Always available pipelines
-from .academic import AcademicPipeline
-from .lightrag import LightRAGPipeline
+from typing import Any
 
 __all__ = [
+    "RAGAnythingPipeline",
+    "RAGAnythingDoclingPipeline",
     "LightRAGPipeline",
-    "AcademicPipeline",
 ]
 
-# Optional pipelines - import only if dependencies are available
-try:
-    from .llamaindex import LlamaIndexPipeline
-    __all__.append("LlamaIndexPipeline")
-except ImportError:
-    LlamaIndexPipeline = None  # type: ignore
+# NOTE:
+# - Do NOT import heavy/optional backends at module import time.
+# - Users may want `llamaindex` without `raganything`, or vice versa.
+# - Accessing an attribute triggers a targeted import via __getattr__.
+
+
+def __getattr__(name: str) -> Any:
+    if name == "LightRAGPipeline":
+        from .lightrag import LightRAGPipeline
+
+        return LightRAGPipeline
+    if name == "RAGAnythingPipeline":
+        from .raganything import RAGAnythingPipeline
+
+        return RAGAnythingPipeline
+    if name == "RAGAnythingDoclingPipeline":
+        from .raganything_docling import RAGAnythingDoclingPipeline
 
-try:
-    from .raganything import RAGAnythingPipeline
-    __all__.append("RAGAnythingPipeline")
-except ImportError:
-    RAGAnythingPipeline = None  # type: ignore
+        return RAGAnythingDoclingPipeline
+    if name == "LlamaIndexPipeline":
+        # Optional dependency: llama_index
+        from .llamaindex import LlamaIndexPipeline
 
+        return LlamaIndexPipeline
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

src/services/rag/pipelines/lightrag.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 """
 LightRAG Pipeline
 =================

src/services/rag/pipelines/llamaindex.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 """
 LlamaIndex Pipeline
 ===================
@@ -291,6 +292,104 @@ class LlamaIndexPipeline:
                 "provider": "llamaindex",
             }
 
+    async def add_documents(self, kb_name: str, file_paths: List[str], **kwargs) -> bool:
+        """
+        Incrementally add documents to an existing LlamaIndex KB.
+
+        If the storage directory exists, loads the existing index and inserts
+        new documents. Otherwise, creates a new index.
+
+        Args:
+            kb_name: Knowledge base name
+            file_paths: List of file paths to add
+            **kwargs: Additional arguments
+
+        Returns:
+            True if successful
+        """
+        self.logger.info(f"Adding {len(file_paths)} documents to KB '{kb_name}' using LlamaIndex")
+
+        kb_dir = Path(self.kb_base_dir) / kb_name
+        storage_dir = kb_dir / "llamaindex_storage"
+
+        try:
+            # Parse new documents
+            documents = []
+            for file_path in file_paths:
+                file_path = Path(file_path)
+                self.logger.info(f"Parsing: {file_path.name}")
+
+                # Extract text based on file type
+                if file_path.suffix.lower() == ".pdf":
+                    text = self._extract_pdf_text(file_path)
+                else:
+                    try:
+                        with open(file_path, "r", encoding="utf-8") as f:
+                            text = f.read()
+                    except UnicodeDecodeError:
+                        with open(file_path, "r", encoding="latin-1") as f:
+                            text = f.read()
+
+                if text.strip():
+                    doc = Document(
+                        text=text,
+                        metadata={
+                            "file_name": file_path.name,
+                            "file_path": str(file_path),
+                        },
+                    )
+                    documents.append(doc)
+                    self.logger.info(f"Loaded: {file_path.name} ({len(text)} chars)")
+                else:
+                    self.logger.warning(f"Skipped empty document: {file_path.name}")
+
+            if not documents:
+                self.logger.warning("No valid documents to add")
+                return False
+
+            loop = asyncio.get_event_loop()
+
+            if storage_dir.exists():
+                # Load existing index and insert new documents
+                self.logger.info(f"Loading existing index from {storage_dir}...")
+
+                def load_and_insert():
+                    storage_context = StorageContext.from_defaults(persist_dir=str(storage_dir))
+                    index = load_index_from_storage(storage_context)
+
+                    # Insert new documents
+                    for doc in documents:
+                        index.insert(doc)
+
+                    # Persist updated index
+                    index.storage_context.persist(persist_dir=str(storage_dir))
+                    return len(documents)
+
+                num_added = await loop.run_in_executor(None, load_and_insert)
+                self.logger.info(f"Added {num_added} documents to existing index")
+            else:
+                # Create new index (first time)
+                self.logger.info(f"Creating new index with {len(documents)} documents...")
+                storage_dir.mkdir(parents=True, exist_ok=True)
+
+                def create_index():
+                    index = VectorStoreIndex.from_documents(documents, show_progress=True)
+                    index.storage_context.persist(persist_dir=str(storage_dir))
+                    return len(documents)
+
+                num_added = await loop.run_in_executor(None, create_index)
+                self.logger.info(f"Created new index with {num_added} documents")
+
+            self.logger.info(f"Successfully added documents to KB '{kb_name}'")
+            return True
+
+        except Exception as e:
+            self.logger.error(f"Failed to add documents: {e}")
+            import traceback
+
+            self.logger.error(traceback.format_exc())
+            return False
+
     async def delete(self, kb_name: str) -> bool:
         """
         Delete knowledge base.