realtimex-deeptutor 0.5.0.post1__py3-none-any.whl

src/services/rag/components/routing.py

@@ -0,0 +1,286 @@
+"""
+File Type Router
+================
+
+Centralized file type classification and routing for RAG pipelines.
+Determines the appropriate processing method for each document type.
+"""
+
+from dataclasses import dataclass
+from enum import Enum
+from pathlib import Path
+from typing import List
+
+from src.logging import get_logger
+
+logger = get_logger("FileTypeRouter")
+
+
+class DocumentType(Enum):
+    """Document type classification"""
+
+    PDF = "pdf"  # Requires MinerU complex parsing
+    TEXT = "text"  # Plain text, direct read
+    MARKDOWN = "markdown"  # Structured text
+    DOCX = "docx"  # Word documents
+    IMAGE = "image"  # Images (may need OCR)
+    UNKNOWN = "unknown"  # Unsupported
+
+
+@dataclass
+class FileClassification:
+    """
+    Result of file classification.
+
+    Attributes:
+        needs_mineru: Files requiring MinerU parsing (PDF, etc.)
+        text_files: Files that can be read directly as text
+        unsupported: Files with unsupported formats
+    """
+
+    needs_mineru: List[str]
+    text_files: List[str]
+    unsupported: List[str]
+
+
+class FileTypeRouter:
+    """
+    File type router for RAG pipelines.
+
+    Classifies files before processing to route them to appropriate handlers:
+    - PDF files -> MinerU parser (complex document parsing)
+    - Text files -> Direct read (fast, simple)
+    - Unsupported -> Skip with warning
+
+    Usage:
+        router = FileTypeRouter()
+        classification = router.classify_files(file_paths)
+
+        # Process PDF files with MinerU
+        for pdf in classification.needs_mineru:
+            await rag.process_document_complete(pdf, ...)
+
+        # Process text files directly
+        for txt in classification.text_files:
+            content = await FileTypeRouter.read_text_file(txt)
+            await rag.lightrag.ainsert(content)
+    """
+
+    # Extensions requiring MinerU parsing (complex document formats)
+    MINERU_EXTENSIONS = {".pdf"}
+
+    # Extensions for direct text reading
+    TEXT_EXTENSIONS = {
+        # Plain text
+        ".txt",
+        ".text",
+        ".log",
+        # Markup languages
+        ".md",
+        ".markdown",
+        ".rst",
+        ".asciidoc",
+        # Data formats
+        ".json",
+        ".yaml",
+        ".yml",
+        ".toml",
+        ".csv",
+        ".tsv",
+        # LaTeX
+        ".tex",
+        ".latex",
+        ".bib",
+        # Code files
+        ".py",
+        ".js",
+        ".ts",
+        ".jsx",
+        ".tsx",
+        ".java",
+        ".c",
+        ".cpp",
+        ".h",
+        ".hpp",
+        ".go",
+        ".rs",
+        ".rb",
+        ".php",
+        ".swift",
+        ".kt",
+        ".scala",
+        ".r",
+        ".sql",
+        ".sh",
+        ".bash",
+        ".zsh",
+        ".ps1",
+        # Web
+        ".html",
+        ".htm",
+        ".xml",
+        ".css",
+        ".scss",
+        ".sass",
+        ".less",
+        # Config
+        ".ini",
+        ".cfg",
+        ".conf",
+        ".env",
+        ".properties",
+    }
+
+    # Word document extensions (special handling)
+    DOCX_EXTENSIONS = {".docx", ".doc"}
+
+    # Image extensions (may need OCR)
+    IMAGE_EXTENSIONS = {".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp", ".tiff", ".tif"}
+
+    @classmethod
+    def get_document_type(cls, file_path: str) -> DocumentType:
+        """
+        Classify a single file by its type.
+
+        Args:
+            file_path: Path to the file
+
+        Returns:
+            DocumentType enum value
+        """
+        ext = Path(file_path).suffix.lower()
+
+        if ext in cls.MINERU_EXTENSIONS:
+            return DocumentType.PDF
+        elif ext in cls.TEXT_EXTENSIONS:
+            return DocumentType.TEXT
+        elif ext in cls.DOCX_EXTENSIONS:
+            return DocumentType.DOCX
+        elif ext in cls.IMAGE_EXTENSIONS:
+            return DocumentType.IMAGE
+        else:
+            # Try to detect if it's a text file by content
+            if cls._is_text_file(file_path):
+                return DocumentType.TEXT
+            return DocumentType.UNKNOWN
+
+    @classmethod
+    def _is_text_file(cls, file_path: str, sample_size: int = 8192) -> bool:
+        """
+        Detect if a file is text-based by examining its content.
+
+        Args:
+            file_path: Path to the file
+            sample_size: Number of bytes to sample
+
+        Returns:
+            True if file appears to be text
+        """
+        try:
+            with open(file_path, "rb") as f:
+                chunk = f.read(sample_size)
+
+            # Check for null bytes (binary file indicator)
+            if b"\x00" in chunk:
+                return False
+
+            # Try to decode as UTF-8
+            chunk.decode("utf-8")
+            return True
+        except (UnicodeDecodeError, IOError, OSError):
+            return False
+
+    @classmethod
+    def classify_files(cls, file_paths: List[str]) -> FileClassification:
+        """
+        Classify a list of files by processing method.
+
+        Args:
+            file_paths: List of file paths to classify
+
+        Returns:
+            FileClassification with files grouped by processing method
+        """
+        needs_mineru = []
+        text_files = []
+        unsupported = []
+
+        for path in file_paths:
+            doc_type = cls.get_document_type(path)
+
+            if doc_type == DocumentType.PDF:
+                needs_mineru.append(path)
+            elif doc_type in (DocumentType.TEXT, DocumentType.MARKDOWN):
+                text_files.append(path)
+            elif doc_type == DocumentType.DOCX:
+                # DOCX files need special handling
+                # For now, route to MinerU which can handle them
+                needs_mineru.append(path)
+            elif doc_type == DocumentType.IMAGE:
+                # Images might need OCR - route to MinerU if multimodal is enabled
+                needs_mineru.append(path)
+            else:
+                unsupported.append(path)
+
+        logger.debug(
+            f"Classified {len(file_paths)} files: "
+            f"{len(needs_mineru)} MinerU, {len(text_files)} text, {len(unsupported)} unsupported"
+        )
+
+        return FileClassification(
+            needs_mineru=needs_mineru,
+            text_files=text_files,
+            unsupported=unsupported,
+        )
+
+    @classmethod
+    async def read_text_file(cls, file_path: str) -> str:
+        """
+        Read a text file with automatic encoding detection.
+
+        Args:
+            file_path: Path to the text file
+
+        Returns:
+            File content as string
+        """
+        encodings = ["utf-8", "utf-8-sig", "gbk", "gb2312", "gb18030", "latin-1", "cp1252"]
+
+        for encoding in encodings:
+            try:
+                with open(file_path, "r", encoding=encoding) as f:
+                    return f.read()
+            except UnicodeDecodeError:
+                continue
+
+        # Last resort: read with error replacement
+        with open(file_path, "rb") as f:
+            return f.read().decode("utf-8", errors="replace")
+
+    @classmethod
+    def needs_mineru(cls, file_path: str) -> bool:
+        """
+        Quick check if a single file needs MinerU parsing.
+
+        Args:
+            file_path: Path to the file
+
+        Returns:
+            True if file requires MinerU
+        """
+        doc_type = cls.get_document_type(file_path)
+        return doc_type in (DocumentType.PDF, DocumentType.DOCX, DocumentType.IMAGE)
+
+    @classmethod
+    def is_text_readable(cls, file_path: str) -> bool:
+        """
+        Check if a file can be read directly as text.
+
+        Args:
+            file_path: Path to the file
+
+        Returns:
+            True if file can be read as text
+        """
+        doc_type = cls.get_document_type(file_path)
+        return doc_type in (DocumentType.TEXT, DocumentType.MARKDOWN)

src/services/rag/factory.py

@@ -0,0 +1,234 @@
+"""
+Pipeline Factory
+================
+
+Factory for creating and managing RAG pipelines.
+
+LightRAG is the default pipeline (always available).
+RAGAnything and LlamaIndex are optional (require extra dependencies).
+"""
+
+import logging
+from typing import Callable, Dict, List, Optional
+
+from .pipelines import lightrag
+
+logger = logging.getLogger(__name__)
+
+# Pipeline registry - start with always-available pipelines
+_PIPELINES: Dict[str, Callable] = {
+    "lightrag": lightrag.LightRAGPipeline,  # Knowledge graph: PDFParser, fast text-only (default)
+    "realtimex": lightrag.LightRAGPipeline,  # Alias: RealTimeX (uses LightRAG with RealTimeX AI config)
+}
+
+# Pipeline metadata for list_pipelines()
+_PIPELINE_INFO: Dict[str, Dict[str, str]] = {
+    "realtimex": {
+        "id": "realtimex",
+        "name": "RealTimeX",
+        "description": "RealTimeX AI powered knowledge retrieval (recommended).",
+        "available": True,
+    },
+    "lightrag": {
+        "id": "lightrag",
+        "name": "LightRAG",
+        "description": "Lightweight knowledge graph retrieval, fast processing of text documents.",
+        "available": True,
+    },
+}
+
+# Try to register optional pipelines
+def _register_optional_pipelines():
+    """Register pipelines that have optional dependencies."""
+    global _PIPELINES, _PIPELINE_INFO
+    
+    # Try RAGAnything (requires raganything package)
+    try:
+        from .pipelines.raganything import RAGAnythingPipeline
+        _PIPELINES["raganything"] = RAGAnythingPipeline
+        _PIPELINE_INFO["raganything"] = {
+            "id": "raganything",
+            "name": "RAG-Anything",
+            "description": "Multimodal document processing with chart and formula extraction.",
+            "available": True,
+        }
+        logger.debug("RAGAnything pipeline registered")
+    except ImportError as e:
+        _PIPELINE_INFO["raganything"] = {
+            "id": "raganything",
+            "name": "RAG-Anything",
+            "description": "Multimodal document processing (requires: pip install realtimex-deeptutor[raganything])",
+            "available": False,
+        }
+        logger.debug(f"RAGAnything not available: {e}")
+    
+    # Try LlamaIndex (requires llama-index package)
+    try:
+        from .pipelines import llamaindex
+        _PIPELINES["llamaindex"] = llamaindex.LlamaIndexPipeline
+        _PIPELINE_INFO["llamaindex"] = {
+            "id": "llamaindex",
+            "name": "LlamaIndex",
+            "description": "Pure vector retrieval, fastest processing speed.",
+            "available": True,
+        }
+        logger.debug("LlamaIndex pipeline registered")
+    except ImportError as e:
+        _PIPELINE_INFO["llamaindex"] = {
+            "id": "llamaindex",
+            "name": "LlamaIndex",
+            "description": "Vector retrieval (requires: pip install realtimex-deeptutor[llamaindex])",
+            "available": False,
+        }
+        logger.debug(f"LlamaIndex not available: {e}")
+
+
+# Register optional pipelines at module load
+_register_optional_pipelines()
+
+
+def get_pipeline(name: str = "lightrag", kb_base_dir: Optional[str] = None, **kwargs):
+    """
+    Get a pre-configured pipeline by name.
+
+    Args:
+        name: Pipeline name (lightrag, raganything, llamaindex)
+              Default is 'lightrag' (always available).
+        kb_base_dir: Base directory for knowledge bases (passed to all pipelines)
+        **kwargs: Additional arguments passed to pipeline constructor
+
+    Returns:
+        Pipeline instance
+
+    Raises:
+        ValueError: If pipeline name is not found or not available
+    """
+    if name not in _PIPELINES:
+        available = list(_PIPELINES.keys())
+        # Check if it's a known but unavailable pipeline
+        if name in _PIPELINE_INFO:
+            info = _PIPELINE_INFO[name]
+            raise ValueError(
+                f"Pipeline '{name}' is not available. {info['description']}. "
+                f"Available pipelines: {available}"
+            )
+        raise ValueError(f"Unknown pipeline: {name}. Available: {available}")
+
+    factory = _PIPELINES[name]
+
+    # Handle different pipeline types:
+    # - lightrag, realtimex, academic: functions that return RAGPipeline
+    # - llamaindex, raganything: classes that need instantiation
+    if name in ("lightrag", "realtimex", "academic"):
+        # LightRAGPipeline and AcademicPipeline are factory functions
+        return factory(kb_base_dir=kb_base_dir)
+    elif name in ("llamaindex", "raganything"):
+        # LlamaIndexPipeline and RAGAnythingPipeline are classes
+        if kb_base_dir:
+            kwargs["kb_base_dir"] = kb_base_dir
+        return factory(**kwargs)
+    else:
+        # Default: try calling with kb_base_dir
+        return factory(kb_base_dir=kb_base_dir)
+
+
+def list_pipelines(include_unavailable: bool = False) -> List[Dict[str, str]]:
+    """
+    List available pipelines.
+
+    Args:
+        include_unavailable: If True, also include pipelines that aren't installed
+
+    Returns:
+        List of pipeline info dictionaries
+    """
+    result = []
+    # Order: realtimex first (recommended), then others
+    order = ["realtimex", "lightrag", "raganything", "llamaindex"]
+    
+    for pipeline_id in order:
+        if pipeline_id in _PIPELINE_INFO:
+            info = _PIPELINE_INFO[pipeline_id]
+            if include_unavailable or info.get("available", False):
+                result.append({
+                    "id": info["id"],
+                    "name": info["name"],
+                    "description": info["description"],
+                })
+    
+    return result
+
+
+def register_pipeline(name: str, factory: Callable):
+    """
+    Register a custom pipeline.
+
+    Args:
+        name: Pipeline name
+        factory: Factory function or class that creates the pipeline
+    """
+    _PIPELINES[name] = factory
+
+
+def has_pipeline(name: str) -> bool:
+    """
+    Check if a pipeline exists.
+
+    Args:
+        name: Pipeline name
+
+    Returns:
+        True if pipeline exists
+    """
+    return name in _PIPELINES
+
+
+# Backward compatibility with old plugin API
+def get_plugin(name: str) -> Dict[str, Callable]:
+    """
+    DEPRECATED: Use get_pipeline() instead.
+
+    Get a plugin by name (maps to pipeline API).
+    """
+    import warnings
+
+    warnings.warn(
+        "get_plugin() is deprecated, use get_pipeline() instead",
+        DeprecationWarning,
+        stacklevel=2,
+    )
+
+    pipeline = get_pipeline(name)
+    return {
+        "initialize": pipeline.initialize,
+        "search": pipeline.search,
+        "delete": getattr(pipeline, "delete", lambda kb: True),
+    }
+
+
+def list_plugins() -> List[Dict[str, str]]:
+    """
+    DEPRECATED: Use list_pipelines() instead.
+    """
+    import warnings
+
+    warnings.warn(
+        "list_plugins() is deprecated, use list_pipelines() instead",
+        DeprecationWarning,
+        stacklevel=2,
+    )
+    return list_pipelines()
+
+
+def has_plugin(name: str) -> bool:
+    """
+    DEPRECATED: Use has_pipeline() instead.
+    """
+    import warnings
+
+    warnings.warn(
+        "has_plugin() is deprecated, use has_pipeline() instead",
+        DeprecationWarning,
+        stacklevel=2,
+    )
+    return has_pipeline(name)