realtimex-deeptutor 0.5.0.post1__py3-none-any.whl

src/services/rag/components/__init__.py

@@ -0,0 +1,29 @@
+"""
+RAG Components
+==============
+
+Modular components for building RAG pipelines.
+
+Components follow a simple protocol:
+- Each component has a `name` attribute
+- Each component has an async `process()` method
+"""
+
+# Import component modules for convenience
+from . import chunkers, embedders, indexers, parsers, retrievers
+from .base import BaseComponent, Component
+from .routing import DocumentType, FileClassification, FileTypeRouter
+
+__all__ = [
+    "Component",
+    "BaseComponent",
+    "parsers",
+    "chunkers",
+    "embedders",
+    "indexers",
+    "retrievers",
+    # File type routing
+    "FileTypeRouter",
+    "FileClassification",
+    "DocumentType",
+]

src/services/rag/components/base.py

@@ -0,0 +1,59 @@
+"""
+Base Component
+==============
+
+Base classes and protocols for RAG components.
+"""
+
+from typing import Any, Protocol, runtime_checkable
+
+
+@runtime_checkable
+class Component(Protocol):
+    """
+    Base protocol for all RAG components.
+
+    All components must implement:
+    - name: str - Component identifier
+    - process(data, **kwargs) -> Any - Process input data
+    """
+
+    name: str
+
+    async def process(self, data: Any, **kwargs) -> Any:
+        """
+        Process input data.
+
+        Args:
+            data: Input data to process
+            **kwargs: Additional arguments
+
+        Returns:
+            Processed output
+        """
+        ...
+
+
+class BaseComponent:
+    """
+    Base class with common functionality for components.
+
+    Provides:
+    - Logger initialization
+    - Default name from class name
+    """
+
+    name: str = "base"
+
+    def __init__(self):
+        from src.logging import get_logger
+
+        self.logger = get_logger(self.__class__.__name__)
+
+    async def process(self, data: Any, **kwargs) -> Any:
+        """
+        Process input data.
+
+        Override this method in subclasses.
+        """
+        raise NotImplementedError("Subclasses must implement process()")

src/services/rag/components/chunkers/__init__.py

@@ -0,0 +1,18 @@
+"""
+Document Chunkers
+=================
+
+Chunkers for splitting documents into smaller pieces.
+"""
+
+from .base import BaseChunker
+from .fixed import FixedSizeChunker
+from .numbered_item import NumberedItemExtractor
+from .semantic import SemanticChunker
+
+__all__ = [
+    "BaseChunker",
+    "SemanticChunker",
+    "FixedSizeChunker",
+    "NumberedItemExtractor",
+]

src/services/rag/components/chunkers/base.py

@@ -0,0 +1,34 @@
+"""
+Base Chunker
+============
+
+Base class for document chunkers.
+"""
+
+from typing import List
+
+from ...types import Chunk, Document
+from ..base import BaseComponent
+
+
+class BaseChunker(BaseComponent):
+    """
+    Base class for document chunkers.
+
+    Chunkers split documents into smaller chunks for processing.
+    """
+
+    name = "base_chunker"
+
+    async def process(self, doc: Document, **kwargs) -> List[Chunk]:
+        """
+        Chunk a document.
+
+        Args:
+            doc: Document to chunk
+            **kwargs: Additional arguments
+
+        Returns:
+            List of Chunks
+        """
+        raise NotImplementedError("Subclasses must implement process()")

src/services/rag/components/chunkers/fixed.py

@@ -0,0 +1,71 @@
+"""
+Fixed Size Chunker
+==================
+
+Chunker that splits documents into fixed-size pieces.
+"""
+
+from typing import List
+
+from ...types import Chunk, Document
+from ..base import BaseComponent
+
+
+class FixedSizeChunker(BaseComponent):
+    """
+    Fixed-size chunker.
+
+    Splits documents into chunks of a fixed size with optional overlap.
+    """
+
+    name = "fixed_size_chunker"
+
+    def __init__(self, chunk_size: int = 1000, chunk_overlap: int = 100):
+        """
+        Initialize fixed-size chunker.
+
+        Args:
+            chunk_size: Size of each chunk in characters
+            chunk_overlap: Overlap between chunks
+        """
+        super().__init__()
+        self.chunk_size = chunk_size
+        self.chunk_overlap = chunk_overlap
+
+    async def process(self, doc: Document, **kwargs) -> List[Chunk]:
+        """
+        Chunk a document into fixed-size pieces.
+
+        Args:
+            doc: Document to chunk
+            **kwargs: Additional arguments
+
+        Returns:
+            List of fixed-size Chunks
+        """
+        self.logger.info(f"Chunking document: {doc.file_path or 'inline'}")
+
+        text = doc.content
+        if not text:
+            return []
+
+        chunks = []
+        step = self.chunk_size - self.chunk_overlap
+
+        for i in range(0, len(text), step):
+            chunk_text = text[i : i + self.chunk_size].strip()
+            if chunk_text:
+                chunks.append(
+                    Chunk(
+                        content=chunk_text,
+                        chunk_type="text",
+                        metadata={
+                            "start_pos": i,
+                            "end_pos": min(i + self.chunk_size, len(text)),
+                            "source": doc.file_path,
+                        },
+                    )
+                )
+
+        self.logger.info(f"Created {len(chunks)} chunks")
+        return chunks

src/services/rag/components/chunkers/numbered_item.py

@@ -0,0 +1,94 @@
+"""
+Numbered Item Extractor
+=======================
+
+Extracts numbered items (definitions, theorems, equations) from documents.
+"""
+
+from typing import List
+
+from ...types import Chunk, Document
+from ..base import BaseComponent
+
+
+class NumberedItemExtractor(BaseComponent):
+    """
+    Extract numbered items (definitions, theorems, equations) from documents.
+
+    Uses LLM to identify and extract structured academic content like
+    definitions, theorems, lemmas, propositions, equations, etc.
+    """
+
+    name = "numbered_item_extractor"
+
+    def __init__(self, batch_size: int = 20, max_concurrent: int = 5):
+        """
+        Initialize numbered item extractor.
+
+        Args:
+            batch_size: Number of content items to process per batch
+            max_concurrent: Maximum concurrent LLM calls
+        """
+        super().__init__()
+        self.batch_size = batch_size
+        self.max_concurrent = max_concurrent
+
+    async def process(self, doc: Document, **kwargs) -> List[Chunk]:
+        """
+        Extract numbered items from a document.
+
+        Args:
+            doc: Document to extract from (must have content_items)
+            **kwargs: Additional arguments
+
+        Returns:
+            List of Chunks representing numbered items
+        """
+        if not doc.content_items:
+            self.logger.warning("No content_items in document, skipping extraction")
+            return []
+
+        self.logger.info(f"Extracting numbered items from {len(doc.content_items)} content items")
+
+        try:
+            from src.knowledge.extract_numbered_items import (
+                extract_numbered_items_with_llm_async,
+            )
+            from src.services.llm import get_llm_client
+
+            llm_client = get_llm_client()
+
+            # Use existing extraction logic
+            items = await extract_numbered_items_with_llm_async(
+                doc.content_items,
+                api_key=llm_client.config.api_key,
+                base_url=llm_client.config.base_url,
+                batch_size=self.batch_size,
+                max_concurrent=self.max_concurrent,
+            )
+
+            # Convert to Chunks
+            chunks = []
+            for identifier, item_data in items.items():
+                chunks.append(
+                    Chunk(
+                        content=item_data["text"],
+                        chunk_type=item_data["type"],  # Definition, Theorem, Equation...
+                        metadata={
+                            "identifier": identifier,
+                            "page": item_data.get("page", 0),
+                            "img_paths": item_data.get("img_paths", []),
+                            "source": doc.file_path,
+                        },
+                    )
+                )
+
+            self.logger.info(f"Extracted {len(chunks)} numbered items")
+            return chunks
+
+        except ImportError as e:
+            self.logger.warning(f"Could not import extraction module: {e}")
+            return []
+        except Exception as e:
+            self.logger.error(f"Failed to extract numbered items: {e}")
+            return []

src/services/rag/components/chunkers/semantic.py

@@ -0,0 +1,97 @@
+"""
+Semantic Chunker
+================
+
+Chunker that splits documents based on semantic boundaries.
+"""
+
+from typing import List
+
+from ...types import Chunk, Document
+from ..base import BaseComponent
+
+
+class SemanticChunker(BaseComponent):
+    """
+    Semantic chunker.
+
+    Splits documents based on semantic boundaries like paragraphs,
+    sections, or natural breakpoints.
+    """
+
+    name = "semantic_chunker"
+
+    def __init__(
+        self,
+        chunk_size: int = 1000,
+        chunk_overlap: int = 200,
+        separators: List[str] = None,
+    ):
+        """
+        Initialize semantic chunker.
+
+        Args:
+            chunk_size: Target chunk size in characters
+            chunk_overlap: Overlap between chunks
+            separators: List of separators to split on
+        """
+        super().__init__()
+        self.chunk_size = chunk_size
+        self.chunk_overlap = chunk_overlap
+        self.separators = separators or ["\n\n", "\n", ". ", " "]
+
+    async def process(self, doc: Document, **kwargs) -> List[Chunk]:
+        """
+        Chunk a document semantically.
+
+        Args:
+            doc: Document to chunk
+            **kwargs: Additional arguments
+
+        Returns:
+            List of semantic Chunks
+        """
+        self.logger.info(f"Chunking document: {doc.file_path or 'inline'}")
+
+        text = doc.content
+        if not text:
+            return []
+
+        chunks = []
+        current_pos = 0
+
+        while current_pos < len(text):
+            # Find chunk end
+            end_pos = min(current_pos + self.chunk_size, len(text))
+
+            # Try to find a natural break point
+            if end_pos < len(text):
+                for sep in self.separators:
+                    # Look for separator in the last portion of the chunk
+                    search_start = max(current_pos + self.chunk_size - 200, current_pos)
+                    sep_pos = text.rfind(sep, search_start, end_pos)
+                    if sep_pos > current_pos:
+                        end_pos = sep_pos + len(sep)
+                        break
+
+            chunk_text = text[current_pos:end_pos].strip()
+            if chunk_text:
+                chunks.append(
+                    Chunk(
+                        content=chunk_text,
+                        chunk_type="text",
+                        metadata={
+                            "start_pos": current_pos,
+                            "end_pos": end_pos,
+                            "source": doc.file_path,
+                        },
+                    )
+                )
+
+            # Move to next position with overlap
+            current_pos = end_pos - self.chunk_overlap
+            if current_pos >= len(text) - self.chunk_overlap:
+                break
+
+        self.logger.info(f"Created {len(chunks)} chunks")
+        return chunks

src/services/rag/components/embedders/__init__.py

@@ -0,0 +1,14 @@
+"""
+Document Embedders
+==================
+
+Embedders for generating vector representations of text.
+"""
+
+from .base import BaseEmbedder
+from .openai import OpenAIEmbedder
+
+__all__ = [
+    "BaseEmbedder",
+    "OpenAIEmbedder",
+]

src/services/rag/components/embedders/base.py

@@ -0,0 +1,32 @@
+"""
+Base Embedder
+=============
+
+Base class for document embedders.
+"""
+
+from ...types import Document
+from ..base import BaseComponent
+
+
+class BaseEmbedder(BaseComponent):
+    """
+    Base class for document embedders.
+
+    Embedders generate vector representations for document chunks.
+    """
+
+    name = "base_embedder"
+
+    async def process(self, doc: Document, **kwargs) -> Document:
+        """
+        Embed a document's chunks.
+
+        Args:
+            doc: Document with chunks to embed
+            **kwargs: Additional arguments
+
+        Returns:
+            Document with embedded chunks
+        """
+        raise NotImplementedError("Subclasses must implement process()")

src/services/rag/components/embedders/openai.py

@@ -0,0 +1,63 @@
+"""
+OpenAI Embedder
+===============
+
+Embedder using OpenAI-compatible embedding API.
+"""
+
+from ...types import Document
+from ..base import BaseComponent
+
+
+class OpenAIEmbedder(BaseComponent):
+    """
+    OpenAI-compatible embedder.
+
+    Uses the embedding service to generate vectors for document chunks.
+    """
+
+    name = "openai_embedder"
+
+    def __init__(self, batch_size: int = 100):
+        """
+        Initialize OpenAI embedder.
+
+        Args:
+            batch_size: Number of texts to embed per API call
+        """
+        super().__init__()
+        self.batch_size = batch_size
+
+    async def process(self, doc: Document, **kwargs) -> Document:
+        """
+        Embed a document's chunks.
+
+        Args:
+            doc: Document with chunks to embed
+            **kwargs: Additional arguments
+
+        Returns:
+            Document with embedded chunks
+        """
+        if not doc.chunks:
+            self.logger.warning("No chunks to embed")
+            return doc
+
+        self.logger.info(f"Embedding {len(doc.chunks)} chunks")
+
+        from src.services.embedding import get_embedding_client
+
+        client = get_embedding_client()
+
+        # Batch embed
+        for i in range(0, len(doc.chunks), self.batch_size):
+            batch = doc.chunks[i : i + self.batch_size]
+            texts = [chunk.content for chunk in batch]
+
+            embeddings = await client.embed(texts)
+
+            for chunk, embedding in zip(batch, embeddings):
+                chunk.embedding = embedding
+
+        self.logger.info("Embedding complete")
+        return doc

src/services/rag/components/indexers/__init__.py

@@ -0,0 +1,18 @@
+"""
+Document Indexers
+=================
+
+Indexers for building searchable indexes from documents.
+"""
+
+from .base import BaseIndexer
+from .graph import GraphIndexer
+from .lightrag import LightRAGIndexer
+from .vector import VectorIndexer
+
+__all__ = [
+    "BaseIndexer",
+    "VectorIndexer",
+    "GraphIndexer",
+    "LightRAGIndexer",
+]

src/services/rag/components/indexers/base.py

@@ -0,0 +1,35 @@
+"""
+Base Indexer
+============
+
+Base class for document indexers.
+"""
+
+from typing import List
+
+from ...types import Document
+from ..base import BaseComponent
+
+
+class BaseIndexer(BaseComponent):
+    """
+    Base class for document indexers.
+
+    Indexers build searchable indexes from documents.
+    """
+
+    name = "base_indexer"
+
+    async def process(self, kb_name: str, documents: List[Document], **kwargs) -> bool:
+        """
+        Index documents into a knowledge base.
+
+        Args:
+            kb_name: Knowledge base name
+            documents: List of documents to index
+            **kwargs: Additional arguments
+
+        Returns:
+            True if successful
+        """
+        raise NotImplementedError("Subclasses must implement process()")