realtimex-deeptutor 0.5.0.post1__py3-none-any.whl

src/services/rag/components/indexers/graph.py

@@ -0,0 +1,172 @@
+"""
+Graph Indexer
+=============
+
+Knowledge graph indexer using LightRAG.
+"""
+
+from pathlib import Path
+import sys
+from typing import Dict, List, Optional
+
+from ...types import Document
+from ..base import BaseComponent
+
+
+class GraphIndexer(BaseComponent):
+    """
+    Knowledge graph indexer using LightRAG.
+
+    Builds a knowledge graph from documents for graph-based retrieval.
+    """
+
+    name = "graph_indexer"
+    _instances: Dict[str, any] = {}  # Cache RAG instances
+
+    def __init__(self, kb_base_dir: Optional[str] = None):
+        """
+        Initialize graph indexer.
+
+        Args:
+            kb_base_dir: Base directory for knowledge bases
+        """
+        super().__init__()
+        self.kb_base_dir = kb_base_dir or str(
+            Path(__file__).resolve().parent.parent.parent.parent.parent.parent
+            / "data"
+            / "knowledge_bases"
+        )
+
+    def _get_rag_instance(self, kb_name: str):
+        """Get or create a RAGAnything instance."""
+        working_dir = str(Path(self.kb_base_dir) / kb_name / "rag_storage")
+
+        if working_dir in self._instances:
+            return self._instances[working_dir]
+
+        # Add RAG-Anything path
+        project_root = Path(__file__).resolve().parent.parent.parent.parent.parent.parent
+        raganything_path = project_root.parent / "raganything" / "RAG-Anything"
+        if raganything_path.exists() and str(raganything_path) not in sys.path:
+            sys.path.insert(0, str(raganything_path))
+
+        try:
+            from openai import AsyncOpenAI
+            from raganything import RAGAnything, RAGAnythingConfig
+
+            from src.services.embedding import get_embedding_client
+            from src.services.llm import get_llm_client
+
+            llm_client = get_llm_client()
+            embed_client = get_embedding_client()
+
+            # Create AsyncOpenAI client directly
+            openai_client = AsyncOpenAI(
+                api_key=llm_client.config.api_key,
+                base_url=llm_client.config.base_url,
+            )
+
+            # LLM function using services (ASYNC - LightRAG expects async functions)
+            async def llm_model_func(prompt, system_prompt=None, history_messages=None, **kwargs):
+                """Custom async LLM function that bypasses LightRAG's openai_complete_if_cache."""
+                if history_messages is None:
+                    history_messages = []
+
+                # Build messages
+                messages = []
+                if system_prompt:
+                    messages.append({"role": "system", "content": system_prompt})
+                messages.extend(history_messages)
+                messages.append({"role": "user", "content": prompt})
+
+                # Whitelist only valid OpenAI parameters
+                valid_params = {
+                    "temperature",
+                    "top_p",
+                    "n",
+                    "stream",
+                    "stop",
+                    "max_tokens",
+                    "presence_penalty",
+                    "frequency_penalty",
+                    "logit_bias",
+                    "user",
+                    "seed",
+                }
+                clean_kwargs = {k: v for k, v in kwargs.items() if k in valid_params}
+
+                # Call OpenAI API directly (async)
+                response = await openai_client.chat.completions.create(
+                    model=llm_client.config.model,
+                    messages=messages,
+                    **clean_kwargs,
+                )
+
+                return response.choices[0].message.content
+
+            config = RAGAnythingConfig(
+                working_dir=working_dir,
+                enable_image_processing=True,
+                enable_table_processing=True,
+                enable_equation_processing=True,
+            )
+
+            rag = RAGAnything(
+                config=config,
+                llm_model_func=llm_model_func,
+                embedding_func=embed_client.get_embedding_func(),
+            )
+
+            self._instances[working_dir] = rag
+            return rag
+
+        except ImportError as e:
+            self.logger.error(f"Failed to import RAG-Anything: {e}")
+            raise
+
+    async def process(self, kb_name: str, documents: List[Document], **kwargs) -> bool:
+        """
+        Build knowledge graph from documents.
+
+        Args:
+            kb_name: Knowledge base name
+            documents: List of documents to index
+            **kwargs: Additional arguments
+
+        Returns:
+            True if successful
+        """
+        self.logger.info(f"Building knowledge graph for {kb_name}...")
+
+        from src.logging.adapters import LightRAGLogContext
+
+        # Use log forwarding context
+        with LightRAGLogContext(scene="indexer"):
+            rag = self._get_rag_instance(kb_name)
+            await rag._ensure_lightrag_initialized()
+
+            for doc in documents:
+                if doc.content:
+                    # Write content to temporary file
+                    import os
+                    import tempfile
+
+                    tmp_path = None
+                    try:
+                        with tempfile.NamedTemporaryFile(
+                            mode="w", encoding="utf-8", suffix=".txt", delete=False
+                        ) as tmp_file:
+                            tmp_file.write(doc.content)
+                            tmp_path = tmp_file.name
+
+                        # Use RAGAnything API
+                        working_dir = str(Path(self.kb_base_dir) / kb_name / "rag_storage")
+                        output_dir = os.path.join(working_dir, "output")
+                        os.makedirs(output_dir, exist_ok=True)
+                        await rag.process_document_complete(tmp_path, output_dir)
+                    finally:
+                        if tmp_path and os.path.exists(tmp_path):
+                            os.unlink(tmp_path)
+
+        self.logger.info("Knowledge graph built successfully")
+        return True

src/services/rag/components/indexers/lightrag.py

@@ -0,0 +1,156 @@
+"""
+LightRAG Indexer
+================
+
+Pure LightRAG indexer (text-only, no multimodal processing).
+"""
+
+from pathlib import Path
+import sys
+from typing import Dict, List, Optional
+
+from ...types import Document
+from ..base import BaseComponent
+
+
+class LightRAGIndexer(BaseComponent):
+    """
+    Pure LightRAG knowledge graph indexer (text-only).
+
+    Uses LightRAG library directly without multimodal processing.
+    Faster than RAGAnything for text-only documents.
+    """
+
+    name = "lightrag_indexer"
+    _instances: Dict[str, any] = {}  # Cache LightRAG instances
+
+    def __init__(self, kb_base_dir: Optional[str] = None):
+        """
+        Initialize LightRAG indexer.
+
+        Args:
+            kb_base_dir: Base directory for knowledge bases
+        """
+        super().__init__()
+        self.kb_base_dir = kb_base_dir or str(
+            Path(__file__).resolve().parent.parent.parent.parent.parent.parent
+            / "data"
+            / "knowledge_bases"
+        )
+
+    def _get_lightrag_instance(self, kb_name: str):
+        """Get or create a LightRAG instance (text-only)."""
+        working_dir = str(Path(self.kb_base_dir) / kb_name / "rag_storage")
+
+        if working_dir in self._instances:
+            return self._instances[working_dir]
+
+        # Add LightRAG path
+        project_root = Path(__file__).resolve().parent.parent.parent.parent.parent.parent
+        raganything_path = project_root.parent / "raganything" / "RAG-Anything"
+        if raganything_path.exists() and str(raganything_path) not in sys.path:
+            sys.path.insert(0, str(raganything_path))
+
+        try:
+            from lightrag import LightRAG
+            from openai import AsyncOpenAI
+
+            from src.services.embedding import get_embedding_client
+            from src.services.llm import get_llm_client
+
+            llm_client = get_llm_client()
+            embed_client = get_embedding_client()
+
+            # Create AsyncOpenAI client directly
+            openai_client = AsyncOpenAI(
+                api_key=llm_client.config.api_key,
+                base_url=llm_client.config.base_url,
+            )
+
+            # LLM function using services (ASYNC - LightRAG expects async functions)
+            async def llm_model_func(prompt, system_prompt=None, history_messages=None, **kwargs):
+                """Custom async LLM function that bypasses LightRAG's openai_complete_if_cache."""
+                if history_messages is None:
+                    history_messages = []
+
+                # Build messages
+                messages = []
+                if system_prompt:
+                    messages.append({"role": "system", "content": system_prompt})
+                messages.extend(history_messages)
+                messages.append({"role": "user", "content": prompt})
+
+                # Whitelist only valid OpenAI parameters
+                valid_params = {
+                    "temperature",
+                    "top_p",
+                    "n",
+                    "stream",
+                    "stop",
+                    "max_tokens",
+                    "presence_penalty",
+                    "frequency_penalty",
+                    "logit_bias",
+                    "user",
+                    "seed",
+                }
+                clean_kwargs = {k: v for k, v in kwargs.items() if k in valid_params}
+
+                # Call OpenAI API directly (async)
+                response = await openai_client.chat.completions.create(
+                    model=llm_client.config.model,
+                    messages=messages,
+                    **clean_kwargs,
+                )
+
+                return response.choices[0].message.content
+
+            # Create pure LightRAG instance (no multimodal)
+            rag = LightRAG(
+                working_dir=working_dir,
+                llm_model_func=llm_model_func,
+                embedding_func=embed_client.get_embedding_func(),  # Use proper EmbeddingFunc object
+            )
+
+            self._instances[working_dir] = rag
+            return rag
+
+        except ImportError as e:
+            self.logger.error(f"Failed to import LightRAG: {e}")
+            raise
+
+    async def process(self, kb_name: str, documents: List[Document], **kwargs) -> bool:
+        """
+        Build knowledge graph from documents (text-only).
+
+        Args:
+            kb_name: Knowledge base name
+            documents: List of documents to index
+            **kwargs: Additional arguments
+
+        Returns:
+            True if successful
+        """
+        self.logger.info(f"Building knowledge graph for {kb_name} (text-only)...")
+
+        from src.logging.adapters import LightRAGLogContext
+
+        # Use log forwarding context
+        with LightRAGLogContext(scene="LightRAG-Indexer"):
+            rag = self._get_lightrag_instance(kb_name)
+
+            # Initialize storages (required for LightRAG)
+            await rag.initialize_storages()
+
+            # Initialize pipeline status (required for document processing)
+            from lightrag.kg.shared_storage import initialize_pipeline_status
+
+            await initialize_pipeline_status()
+
+            for doc in documents:
+                if doc.content:
+                    # Use direct LightRAG insert (text-only, fast)
+                    await rag.ainsert(doc.content)
+
+        self.logger.info("Knowledge graph built successfully (text-only)")
+        return True

src/services/rag/components/indexers/vector.py

@@ -0,0 +1,146 @@
+"""
+Vector Indexer
+==============
+
+Vector-based indexer using dense embeddings with FAISS.
+Provides fast similarity search for RAG retrieval.
+"""
+
+import json
+from pathlib import Path
+import pickle
+from typing import List, Optional
+
+import numpy as np
+
+from ...types import Document
+from ..base import BaseComponent
+
+
+class VectorIndexer(BaseComponent):
+    """
+    Vector indexer using FAISS for fast similarity search.
+
+    Creates and stores vector embeddings for efficient retrieval.
+    Falls back to simple vector storage if FAISS is not available.
+    """
+
+    name = "vector_indexer"
+
+    def __init__(self, kb_base_dir: Optional[str] = None):
+        """
+        Initialize vector indexer.
+
+        Args:
+            kb_base_dir: Base directory for knowledge bases
+        """
+        super().__init__()
+        self.kb_base_dir = kb_base_dir or str(
+            Path(__file__).resolve().parent.parent.parent.parent.parent.parent
+            / "data"
+            / "knowledge_bases"
+        )
+
+        # Try to import FAISS, fallback to simple storage if not available
+        self.use_faiss = False
+        try:
+            import faiss
+
+            self.faiss = faiss
+            self.use_faiss = True
+            self.logger.info("Using FAISS for vector indexing")
+        except ImportError:
+            self.logger.warning("FAISS not available, using simple vector storage")
+
+    async def process(self, kb_name: str, documents: List[Document], **kwargs) -> bool:
+        """
+        Index documents using vector embeddings.
+
+        Creates FAISS index for fast similarity search or falls back to
+        simple JSON storage if FAISS is unavailable.
+
+        Args:
+            kb_name: Knowledge base name
+            documents: List of documents to index
+            **kwargs: Additional arguments
+
+        Returns:
+            True if successful
+        """
+        self.logger.info(f"Indexing {len(documents)} documents into vector store for {kb_name}")
+
+        # Collect all chunks with embeddings
+        all_chunks = []
+        for doc in documents:
+            for chunk in doc.chunks:
+                # Check if embedding exists (handles numpy arrays and lists)
+                if chunk.embedding is not None and len(chunk.embedding) > 0:
+                    all_chunks.append(chunk)
+
+        if not all_chunks:
+            self.logger.warning("No chunks with embeddings to index")
+            return False
+
+        self.logger.info(f"Indexing {len(all_chunks)} chunks")
+
+        # Create vector store directory
+        kb_dir = Path(self.kb_base_dir) / kb_name / "vector_store"
+        kb_dir.mkdir(parents=True, exist_ok=True)
+
+        # Convert embeddings to numpy array
+        embeddings = np.array(
+            [
+                chunk.embedding if isinstance(chunk.embedding, list) else chunk.embedding.tolist()
+                for chunk in all_chunks
+            ],
+            dtype=np.float32,
+        )
+
+        # Store metadata separately
+        metadata = []
+        for i, chunk in enumerate(all_chunks):
+            metadata.append(
+                {
+                    "id": i,
+                    "content": chunk.content,
+                    "type": chunk.chunk_type,
+                    "metadata": chunk.metadata,
+                }
+            )
+
+        # Save metadata
+        with open(kb_dir / "metadata.json", "w", encoding="utf-8") as f:
+            json.dump(metadata, f, ensure_ascii=False, indent=2)
+
+        if self.use_faiss:
+            # Create FAISS index for inner product (cosine similarity with normalized vectors)
+            dimension = embeddings.shape[1]
+            index = self.faiss.IndexFlatIP(dimension)  # Inner product for cosine similarity
+
+            # Normalize vectors for cosine similarity (inner product of normalized vectors = cosine similarity)
+            self.faiss.normalize_L2(embeddings)
+
+            # Add vectors to index
+            index.add(embeddings)
+
+            # Save FAISS index
+            self.faiss.write_index(index, str(kb_dir / "index.faiss"))
+            self.logger.info(f"FAISS index saved with {index.ntotal} vectors")
+        else:
+            # Simple storage: save embeddings as pickle
+            with open(kb_dir / "embeddings.pkl", "wb") as f:
+                pickle.dump(embeddings, f)
+            self.logger.info(f"Embeddings saved for {len(all_chunks)} chunks")
+
+        # Save index info
+        info = {
+            "num_chunks": len(all_chunks),
+            "num_documents": len(documents),
+            "embedding_dim": embeddings.shape[1],
+            "use_faiss": self.use_faiss,
+        }
+        with open(kb_dir / "info.json", "w", encoding="utf-8") as f:
+            json.dump(info, f, indent=2)
+
+        self.logger.info(f"Vector index saved to {kb_dir}")
+        return True

src/services/rag/components/parsers/__init__.py

@@ -0,0 +1,18 @@
+"""
+Document Parsers
+================
+
+Parsers for extracting content from various document formats.
+"""
+
+from .base import BaseParser
+from .markdown import MarkdownParser
+from .pdf import PDFParser
+from .text import TextParser
+
+__all__ = [
+    "BaseParser",
+    "PDFParser",
+    "MarkdownParser",
+    "TextParser",
+]

src/services/rag/components/parsers/base.py

@@ -0,0 +1,35 @@
+"""
+Base Parser
+===========
+
+Base class for document parsers.
+"""
+
+from pathlib import Path
+from typing import Union
+
+from ...types import Document
+from ..base import BaseComponent
+
+
+class BaseParser(BaseComponent):
+    """
+    Base class for document parsers.
+
+    Parsers convert raw files into Document objects.
+    """
+
+    name = "base_parser"
+
+    async def process(self, file_path: Union[str, Path], **kwargs) -> Document:
+        """
+        Parse a file into a Document.
+
+        Args:
+            file_path: Path to the file to parse
+            **kwargs: Additional arguments
+
+        Returns:
+            Parsed Document
+        """
+        raise NotImplementedError("Subclasses must implement process()")

src/services/rag/components/parsers/markdown.py

@@ -0,0 +1,52 @@
+"""
+Markdown Parser
+===============
+
+Parser for Markdown documents.
+"""
+
+from pathlib import Path
+from typing import Union
+
+from ...types import Document
+from ..base import BaseComponent
+
+
+class MarkdownParser(BaseComponent):
+    """
+    Markdown parser.
+
+    Parses Markdown files into Document objects.
+    """
+
+    name = "markdown_parser"
+
+    async def process(self, file_path: Union[str, Path], **kwargs) -> Document:
+        """
+        Parse a Markdown file into a Document.
+
+        Args:
+            file_path: Path to the Markdown file
+            **kwargs: Additional arguments
+
+        Returns:
+            Parsed Document
+        """
+        file_path = Path(file_path)
+
+        if not file_path.exists():
+            raise FileNotFoundError(f"Markdown file not found: {file_path}")
+
+        self.logger.info(f"Parsing Markdown: {file_path.name}")
+
+        with open(file_path, "r", encoding="utf-8") as f:
+            content = f.read()
+
+        return Document(
+            content=content,
+            file_path=str(file_path),
+            metadata={
+                "filename": file_path.name,
+                "parser": self.name,
+            },
+        )

src/services/rag/components/parsers/pdf.py

@@ -0,0 +1,115 @@
+"""
+PDF Parser
+==========
+
+Parser for PDF documents using MinerU/RAG-Anything.
+"""
+
+import json
+from pathlib import Path
+from typing import Optional, Union
+
+from ...types import Document
+from ..base import BaseComponent
+
+
+class PDFParser(BaseComponent):
+    """
+    PDF parser using MinerU for extraction.
+
+    Can use RAG-Anything's MinerU integration or standalone MinerU.
+    """
+
+    name = "pdf_parser"
+
+    def __init__(self, use_mineru: bool = True, output_dir: Optional[str] = None):
+        """
+        Initialize PDF parser.
+
+        Args:
+            use_mineru: Whether to use MinerU for parsing
+            output_dir: Directory to store parsed output
+        """
+        super().__init__()
+        self.use_mineru = use_mineru
+        self.output_dir = output_dir
+
+    async def process(self, file_path: Union[str, Path], **kwargs) -> Document:
+        """
+        Parse a PDF file into a Document.
+
+        Args:
+            file_path: Path to the PDF file
+            **kwargs: Additional arguments
+
+        Returns:
+            Parsed Document with content and content_items
+        """
+        file_path = Path(file_path)
+
+        if not file_path.exists():
+            raise FileNotFoundError(f"PDF file not found: {file_path}")
+
+        self.logger.info(f"Parsing PDF: {file_path.name}")
+
+        # Check for existing parsed content
+        output_dir = Path(kwargs.get("output_dir", self.output_dir or file_path.parent))
+        content_list_file = output_dir / f"{file_path.stem}.json"
+
+        content_items = []
+        content = ""
+
+        if content_list_file.exists():
+            # Load existing parsed content
+            self.logger.info(f"Loading existing parsed content from {content_list_file}")
+            with open(content_list_file, "r", encoding="utf-8") as f:
+                content_items = json.load(f)
+
+            # Extract text content
+            content = self._extract_text_from_content_items(content_items)
+        else:
+            # Parse PDF (placeholder - actual MinerU parsing would happen here)
+            self.logger.warning(
+                "No pre-parsed content found. Use RAGAnythingPipeline for full PDF parsing."
+            )
+            # Basic text extraction fallback
+            content = await self._basic_pdf_extract(file_path)
+
+        return Document(
+            content=content,
+            file_path=str(file_path),
+            content_items=content_items,
+            metadata={
+                "filename": file_path.name,
+                "parser": self.name,
+            },
+        )
+
+    def _extract_text_from_content_items(self, content_items: list) -> str:
+        """Extract plain text from MinerU content items."""
+        texts = []
+        for item in content_items:
+            if isinstance(item, dict):
+                if "text" in item:
+                    texts.append(item["text"])
+                elif "content" in item:
+                    texts.append(item["content"])
+        return "\n\n".join(texts)
+
+    async def _basic_pdf_extract(self, file_path: Path) -> str:
+        """Basic PDF text extraction fallback."""
+        try:
+            import fitz  # PyMuPDF
+
+            doc = fitz.open(file_path)
+            texts = []
+            for page in doc:
+                texts.append(page.get_text())
+            doc.close()
+            return "\n\n".join(texts)
+        except ImportError:
+            self.logger.warning("PyMuPDF not installed. Cannot extract PDF text.")
+            return ""
+        except Exception as e:
+            self.logger.error(f"Failed to extract PDF text: {e}")
+            return ""