realtimex-deeptutor 0.5.0.post1__py3-none-any.whl

src/services/rag/components/parsers/text.py

@@ -0,0 +1,86 @@
+"""
+Text Parser
+===========
+
+Parser for plain text documents (.txt files).
+"""
+
+from pathlib import Path
+from typing import Union
+
+from ...types import Document
+from ..base import BaseComponent
+
+
+class TextParser(BaseComponent):
+    """
+    Plain text parser.
+
+    Parses text files (.txt) into Document objects.
+    Also handles common text-based formats.
+    """
+
+    name = "text_parser"
+
+    # Supported extensions
+    SUPPORTED_EXTENSIONS = {".txt", ".text", ".log", ".csv", ".tsv"}
+
+    async def process(self, file_path: Union[str, Path], **kwargs) -> Document:
+        """
+        Parse a text file into a Document.
+
+        Args:
+            file_path: Path to the text file
+            **kwargs: Additional arguments
+
+        Returns:
+            Parsed Document
+        """
+        file_path = Path(file_path)
+
+        if not file_path.exists():
+            raise FileNotFoundError(f"Text file not found: {file_path}")
+
+        self.logger.info(f"Parsing text file: {file_path.name}")
+
+        # Try different encodings
+        content = None
+        encodings = ["utf-8", "utf-8-sig", "gbk", "gb2312", "latin-1"]
+
+        for encoding in encodings:
+            try:
+                with open(file_path, "r", encoding=encoding) as f:
+                    content = f.read()
+                break
+            except UnicodeDecodeError:
+                continue
+
+        if content is None:
+            # Last resort: read as binary and decode with error handling
+            with open(file_path, "rb") as f:
+                content = f.read().decode("utf-8", errors="replace")
+
+        return Document(
+            content=content,
+            file_path=str(file_path),
+            metadata={
+                "filename": file_path.name,
+                "parser": self.name,
+                "extension": file_path.suffix.lower(),
+                "size_bytes": file_path.stat().st_size,
+            },
+        )
+
+    @classmethod
+    def can_parse(cls, file_path: Union[str, Path]) -> bool:
+        """
+        Check if this parser can handle the given file.
+
+        Args:
+            file_path: Path to check
+
+        Returns:
+            True if file can be parsed
+        """
+        suffix = Path(file_path).suffix.lower()
+        return suffix in cls.SUPPORTED_EXTENSIONS

src/services/rag/components/retrievers/__init__.py

@@ -0,0 +1,18 @@
+"""
+Document Retrievers
+===================
+
+Retrievers for searching indexed documents.
+"""
+
+from .base import BaseRetriever
+from .dense import DenseRetriever
+from .hybrid import HybridRetriever
+from .lightrag import LightRAGRetriever
+
+__all__ = [
+    "BaseRetriever",
+    "DenseRetriever",
+    "HybridRetriever",
+    "LightRAGRetriever",
+]

src/services/rag/components/retrievers/base.py

@@ -0,0 +1,34 @@
+"""
+Base Retriever
+==============
+
+Base class for document retrievers.
+"""
+
+from typing import Any, Dict
+
+from ..base import BaseComponent
+
+
+class BaseRetriever(BaseComponent):
+    """
+    Base class for document retrievers.
+
+    Retrievers search indexed documents and return relevant results.
+    """
+
+    name = "base_retriever"
+
+    async def process(self, query: str, kb_name: str, **kwargs) -> Dict[str, Any]:
+        """
+        Search for documents matching a query.
+
+        Args:
+            query: Search query
+            kb_name: Knowledge base name
+            **kwargs: Additional arguments
+
+        Returns:
+            Search results dictionary
+        """
+        raise NotImplementedError("Subclasses must implement process()")

src/services/rag/components/retrievers/dense.py

@@ -0,0 +1,200 @@
+"""
+Dense Retriever
+===============
+
+Dense vector-based retriever using FAISS or cosine similarity.
+"""
+
+import json
+from pathlib import Path
+import pickle
+from typing import Any, Dict, Optional
+
+import numpy as np
+
+from ..base import BaseComponent
+
+
+class DenseRetriever(BaseComponent):
+    """
+    Dense vector retriever.
+
+    Uses FAISS for fast similarity search or falls back to
+    cosine similarity if FAISS is unavailable.
+    """
+
+    name = "dense_retriever"
+
+    def __init__(self, kb_base_dir: Optional[str] = None, top_k: int = 5):
+        """
+        Initialize dense retriever.
+
+        Args:
+            kb_base_dir: Base directory for knowledge bases
+            top_k: Number of results to return
+        """
+        super().__init__()
+        self.kb_base_dir = kb_base_dir or str(
+            Path(__file__).resolve().parent.parent.parent.parent.parent.parent
+            / "data"
+            / "knowledge_bases"
+        )
+        self.top_k = top_k
+
+        # Try to import FAISS
+        self.use_faiss = False
+        try:
+            import faiss
+
+            self.faiss = faiss
+            self.use_faiss = True
+        except ImportError:
+            self.logger.warning("FAISS not available, using simple cosine similarity")
+
+    async def process(self, query: str, kb_name: str, **kwargs) -> Dict[str, Any]:
+        """
+        Search using dense embeddings with FAISS or cosine similarity.
+
+        Args:
+            query: Search query
+            kb_name: Knowledge base name
+            **kwargs: Additional arguments (mode, top_k, etc.)
+
+        Returns:
+            Search results dictionary with answer and sources
+        """
+        top_k = kwargs.get("top_k", self.top_k)
+        self.logger.info(f"Dense search in {kb_name}: {query[:50]}... (top_k={top_k})")
+
+        from src.services.embedding import get_embedding_client
+
+        # Get query embedding
+        client = get_embedding_client()
+        query_embedding = np.array((await client.embed([query]))[0], dtype=np.float32)
+
+        # Load index
+        kb_dir = Path(self.kb_base_dir) / kb_name / "vector_store"
+        metadata_file = kb_dir / "metadata.json"
+        info_file = kb_dir / "info.json"
+
+        if not metadata_file.exists():
+            self.logger.warning(f"No vector index found at {kb_dir}")
+            return {
+                "query": query,
+                "answer": "No documents indexed. Please upload documents first.",
+                "content": "",
+                "mode": "dense",
+                "provider": "llamaindex",
+                "results": [],
+            }
+
+        # Load metadata and info (info.json is optional)
+        with open(metadata_file, "r", encoding="utf-8") as f:
+            metadata = json.load(f)
+
+        if info_file.exists():
+            with open(info_file, "r", encoding="utf-8") as f:
+                info = json.load(f)
+        else:
+            info = {"use_faiss": False}
+
+        use_faiss = info.get("use_faiss", False)
+
+        if use_faiss and self.use_faiss:
+            # Use FAISS for fast search
+            index_file = kb_dir / "index.faiss"
+            if not index_file.exists():
+                self.logger.error(f"FAISS index file not found: {index_file}")
+                return self._empty_response(query)
+
+            # Load FAISS index
+            index = self.faiss.read_index(str(index_file))
+
+            # Normalize query vector for cosine similarity without modifying original
+            norm = np.linalg.norm(query_embedding)
+            if norm > 0:
+                query_vec = (query_embedding / norm).reshape(1, -1)
+            else:
+                query_vec = query_embedding.reshape(1, -1)
+
+            # Search
+            distances, indices = index.search(query_vec, min(top_k, len(metadata)))
+
+            # Build results
+            results = []
+            for dist, idx in zip(distances[0], indices[0]):
+                if idx < len(metadata):  # Valid index
+                    score = 1.0 / (1.0 + dist)  # Convert distance to similarity score
+                    results.append((score, metadata[idx]))
+        else:
+            # Fallback: Load embeddings and use cosine similarity
+            embeddings_file = kb_dir / "embeddings.pkl"
+            if not embeddings_file.exists():
+                self.logger.error(f"Embeddings file not found: {embeddings_file}")
+                return self._empty_response(query)
+
+            with open(embeddings_file, "rb") as f:
+                embeddings = pickle.load(f)
+
+            # Normalize for cosine similarity (avoid division by zero)
+            query_norm = np.linalg.norm(query_embedding)
+            if query_norm > 0:
+                query_vec = query_embedding / query_norm
+            else:
+                query_vec = query_embedding  # Keep as is if zero norm
+
+            norms = np.linalg.norm(embeddings, axis=1, keepdims=True)
+            # Replace zero norms with 1 to avoid division by zero
+            norms = np.where(norms == 0, 1, norms)
+            doc_vecs = embeddings / norms
+
+            # Compute similarities
+            similarities = np.dot(doc_vecs, query_vec)
+
+            # Get top-k results
+            top_indices = np.argsort(similarities)[::-1][:top_k]
+
+            results = []
+            for idx in top_indices:
+                score = float(similarities[idx])
+                results.append((score, metadata[idx]))
+
+        # Build response content
+        # Format chunks cleanly for LLM context (without score annotations)
+        content_parts = []
+        sources = []
+        for score, item in results:
+            content = item.get("content", "").strip()
+            if content:  # Only include non-empty chunks
+                # Add chunk without score prefix for clean LLM input
+                content_parts.append(content)
+                sources.append(
+                    {
+                        "content": content,
+                        "score": score,
+                        "metadata": item.get("metadata", {}),
+                    }
+                )
+
+        # Join chunks with clear separation
+        content = "\n\n".join(content_parts)
+
+        return {
+            "query": query,
+            "answer": content,  # Return clean context for LLM to use
+            "content": content,
+            "mode": "dense",
+            "provider": "llamaindex",
+            "results": sources,
+        }
+
+    def _empty_response(self, query: str) -> Dict[str, Any]:
+        """Return empty response when no results found."""
+        return {
+            "query": query,
+            "answer": "No relevant documents found.",
+            "content": "",
+            "mode": "dense",
+            "provider": "llamaindex",
+            "results": [],
+        }

src/services/rag/components/retrievers/hybrid.py

@@ -0,0 +1,164 @@
+"""
+Hybrid Retriever
+================
+
+Hybrid retriever combining multiple retrieval strategies.
+"""
+
+from pathlib import Path
+import sys
+from typing import Any, Dict, Optional
+
+from ..base import BaseComponent
+
+
+class HybridRetriever(BaseComponent):
+    """
+    Hybrid retriever combining graph and vector retrieval.
+
+    Uses LightRAG's hybrid mode for retrieval.
+    """
+
+    name = "hybrid_retriever"
+    _instances: Dict[str, Any] = {}
+
+    def __init__(self, kb_base_dir: Optional[str] = None):
+        """
+        Initialize hybrid retriever.
+
+        Args:
+            kb_base_dir: Base directory for knowledge bases
+        """
+        super().__init__()
+        self.kb_base_dir = kb_base_dir or str(
+            Path(__file__).resolve().parent.parent.parent.parent.parent.parent
+            / "data"
+            / "knowledge_bases"
+        )
+
+    def _get_rag_instance(self, kb_name: str):
+        """Get or create a RAGAnything instance."""
+        working_dir = str(Path(self.kb_base_dir) / kb_name / "rag_storage")
+
+        if working_dir in self._instances:
+            return self._instances[working_dir]
+
+        # Add RAG-Anything path
+        project_root = Path(__file__).resolve().parent.parent.parent.parent.parent.parent
+        raganything_path = project_root.parent / "raganything" / "RAG-Anything"
+        if raganything_path.exists() and str(raganything_path) not in sys.path:
+            sys.path.insert(0, str(raganything_path))
+
+        try:
+            from openai import AsyncOpenAI
+            from raganything import RAGAnything, RAGAnythingConfig
+
+            from src.services.embedding import get_embedding_client
+            from src.services.llm import get_llm_client
+
+            llm_client = get_llm_client()
+            embed_client = get_embedding_client()
+
+            # Create AsyncOpenAI client directly
+            openai_client = AsyncOpenAI(
+                api_key=llm_client.config.api_key,
+                base_url=llm_client.config.base_url,
+            )
+
+            # LLM function using services (ASYNC - LightRAG expects async functions)
+            async def llm_model_func(prompt, system_prompt=None, history_messages=None, **kwargs):
+                """Custom async LLM function that bypasses LightRAG's openai_complete_if_cache."""
+                if history_messages is None:
+                    history_messages = []
+
+                # Build messages
+                messages = []
+                if system_prompt:
+                    messages.append({"role": "system", "content": system_prompt})
+                messages.extend(history_messages)
+                messages.append({"role": "user", "content": prompt})
+
+                # Whitelist only valid OpenAI parameters
+                valid_params = {
+                    "temperature",
+                    "top_p",
+                    "n",
+                    "stream",
+                    "stop",
+                    "max_tokens",
+                    "presence_penalty",
+                    "frequency_penalty",
+                    "logit_bias",
+                    "user",
+                    "seed",
+                }
+                clean_kwargs = {k: v for k, v in kwargs.items() if k in valid_params}
+
+                # Call OpenAI API directly (async)
+                response = await openai_client.chat.completions.create(
+                    model=llm_client.config.model,
+                    messages=messages,
+                    **clean_kwargs,
+                )
+
+                return response.choices[0].message.content
+
+            config = RAGAnythingConfig(
+                working_dir=working_dir,
+                enable_image_processing=True,
+                enable_table_processing=True,
+                enable_equation_processing=True,
+            )
+
+            rag = RAGAnything(
+                config=config,
+                llm_model_func=llm_model_func,
+                embedding_func=embed_client.get_embedding_func(),
+            )
+
+            self._instances[working_dir] = rag
+            return rag
+
+        except ImportError as e:
+            self.logger.error(f"Failed to import RAG-Anything: {e}")
+            raise
+
+    async def process(
+        self,
+        query: str,
+        kb_name: str,
+        mode: str = "hybrid",
+        only_need_context: bool = False,
+        **kwargs,
+    ) -> Dict[str, Any]:
+        """
+        Search using hybrid retrieval.
+
+        Args:
+            query: Search query
+            kb_name: Knowledge base name
+            mode: Search mode (hybrid, local, global, naive)
+            only_need_context: Whether to only return context without answer
+            **kwargs: Additional arguments
+
+        Returns:
+            Search results dictionary
+        """
+        self.logger.info(f"Hybrid search ({mode}) in {kb_name}: {query[:50]}...")
+
+        from src.logging.adapters import LightRAGLogContext
+
+        with LightRAGLogContext(scene="rag_search"):
+            rag = self._get_rag_instance(kb_name)
+            await rag._ensure_lightrag_initialized()
+
+            answer = await rag.aquery(query, mode=mode, only_need_context=only_need_context)
+            answer_str = answer if isinstance(answer, str) else str(answer)
+
+            return {
+                "query": query,
+                "answer": answer_str,
+                "content": answer_str,
+                "mode": mode,
+                "provider": "hybrid",
+            }

src/services/rag/components/retrievers/lightrag.py

@@ -0,0 +1,169 @@
+"""
+LightRAG Retriever
+==================
+
+Pure LightRAG retriever (text-only, no multimodal).
+"""
+
+from pathlib import Path
+import sys
+from typing import Any, ClassVar, Dict, Optional
+
+from ..base import BaseComponent
+
+
+class LightRAGRetriever(BaseComponent):
+    """
+    Pure LightRAG retriever using LightRAG.query() directly.
+
+    Uses LightRAG's native retrieval modes (naive, local, global, hybrid).
+    Text-only, no multimodal processing.
+    """
+
+    name = "lightrag_retriever"
+    _instances: ClassVar[Dict[str, Any]] = {}
+
+    def __init__(self, kb_base_dir: Optional[str] = None):
+        """
+        Initialize LightRAG retriever.
+
+        Args:
+            kb_base_dir: Base directory for knowledge bases
+        """
+        super().__init__()
+        self.kb_base_dir = kb_base_dir or str(
+            Path(__file__).resolve().parent.parent.parent.parent.parent.parent
+            / "data"
+            / "knowledge_bases"
+        )
+
+    def _get_lightrag_instance(self, kb_name: str):
+        """Get or create a pure LightRAG instance (text-only)."""
+        working_dir = str(Path(self.kb_base_dir) / kb_name / "rag_storage")
+
+        if working_dir in self._instances:
+            return self._instances[working_dir]
+
+        # Add LightRAG path
+        project_root = Path(__file__).resolve().parent.parent.parent.parent.parent.parent
+        raganything_path = project_root.parent / "raganything" / "RAG-Anything"
+        if raganything_path.exists() and str(raganything_path) not in sys.path:
+            sys.path.insert(0, str(raganything_path))
+
+        try:
+            from lightrag import LightRAG
+            from openai import AsyncOpenAI
+
+            from src.services.embedding import get_embedding_client
+            from src.services.llm import get_llm_client
+
+            llm_client = get_llm_client()
+            embed_client = get_embedding_client()
+
+            # Create AsyncOpenAI client directly
+            openai_client = AsyncOpenAI(
+                api_key=llm_client.config.api_key,
+                base_url=llm_client.config.base_url,
+            )
+
+            # LLM function using services (ASYNC - LightRAG expects async functions)
+            async def llm_model_func(prompt, system_prompt=None, history_messages=None, **kwargs):
+                """Custom async LLM function that bypasses LightRAG's openai_complete_if_cache."""
+                if history_messages is None:
+                    history_messages = []
+
+                # Build messages
+                messages = []
+                if system_prompt:
+                    messages.append({"role": "system", "content": system_prompt})
+                messages.extend(history_messages)
+                messages.append({"role": "user", "content": prompt})
+
+                # Whitelist only valid OpenAI parameters
+                valid_params = {
+                    "temperature",
+                    "top_p",
+                    "n",
+                    "stream",
+                    "stop",
+                    "max_tokens",
+                    "presence_penalty",
+                    "frequency_penalty",
+                    "logit_bias",
+                    "user",
+                    "seed",
+                }
+                clean_kwargs = {k: v for k, v in kwargs.items() if k in valid_params}
+
+                # Call OpenAI API directly (async)
+                response = await openai_client.chat.completions.create(
+                    model=llm_client.config.model,
+                    messages=messages,
+                    **clean_kwargs,
+                )
+
+                return response.choices[0].message.content
+
+            # Create pure LightRAG instance (no multimodal)
+            rag = LightRAG(
+                working_dir=working_dir,
+                llm_model_func=llm_model_func,
+                embedding_func=embed_client.get_embedding_func(),  # Use proper EmbeddingFunc object
+            )
+
+            self._instances[working_dir] = rag
+            return rag
+
+        except ImportError as e:
+            self.logger.error(f"Failed to import LightRAG: {e}")
+            raise
+
+    async def process(
+        self,
+        query: str,
+        kb_name: str,
+        mode: str = "hybrid",
+        only_need_context: bool = False,
+        **kwargs,
+    ) -> Dict[str, Any]:
+        """
+        Search using pure LightRAG retrieval (text-only).
+
+        Args:
+            query: Search query
+            kb_name: Knowledge base name
+            mode: Search mode (hybrid, local, global, naive)
+            only_need_context: Whether to only return context without answer
+            **kwargs: Additional arguments
+
+        Returns:
+            Search results dictionary
+        """
+        self.logger.info(f"LightRAG search ({mode}) in {kb_name}: {query[:50]}...")
+
+        from src.logging.adapters import LightRAGLogContext
+
+        with LightRAGLogContext(scene="LightRAG-Search"):
+            rag = self._get_lightrag_instance(kb_name)
+
+            # Initialize storages if not already initialized
+            await rag.initialize_storages()
+            from lightrag.kg.shared_storage import initialize_pipeline_status
+
+            await initialize_pipeline_status()
+
+            # Import QueryParam for proper query parameter passing
+            from lightrag import QueryParam
+
+            # Use LightRAG's native query method with QueryParam object
+            query_param = QueryParam(mode=mode, only_need_context=only_need_context)
+            answer = await rag.aquery(query, param=query_param)
+            answer_str = answer if isinstance(answer, str) else str(answer)
+
+            return {
+                "query": query,
+                "answer": answer_str,
+                "content": answer_str,
+                "mode": mode,
+                "provider": "lightrag",
+            }