ragup 0.1.0__py3-none-any.whl

ragup/__init__.py

@@ -0,0 +1,7 @@
+from .core.document import Document
+
+__version__ = "1.0.0"
+
+__all__ = [
+    "Document",
+]

ragup/api/__init__.py

@@ -0,0 +1,5 @@
+from .server import RagUpServer
+
+__all__ = [
+    "RagUpServer",
+]

ragup/api/routes.py

@@ -0,0 +1,7 @@
+"""
+FastAPI routes.
+"""
+
+from fastapi import APIRouter
+
+router = APIRouter()

ragup/api/schemas.py

@@ -0,0 +1,15 @@
+"""
+Pydantic request models.
+"""
+
+from pydantic import BaseModel
+
+
+class SearchRequest(BaseModel):
+    query: str
+    top_k: int = 5
+
+
+class AskRequest(BaseModel):
+    question: str
+    top_k: int = 5

ragup/api/server.py

@@ -0,0 +1,81 @@
+"""
+FastAPI server for RagUp.
+"""
+
+from __future__ import annotations
+
+import uvicorn
+
+from fastapi import FastAPI
+
+from ..constants import (
+    DEFAULT_HOST,
+    DEFAULT_PORT,
+)
+from .schemas import (
+    AskRequest,
+    SearchRequest,
+)
+
+
+class RagUpServer:
+
+    def __init__(
+        self,
+        document,
+        api_key: str | None = None,
+    ):
+
+        self.document = document
+        self.api_key = api_key
+
+        self.app = FastAPI(
+            title="RagUp API",
+            version="1.0.0",
+        )
+
+        self._register_routes()
+
+    def _register_routes(self):
+
+        @self.app.get("/health")
+        def health():
+
+            return {
+                "status": "healthy"
+            }
+
+        @self.app.post("/search")
+        def search(request: SearchRequest):
+
+            return self.document.search(
+                query=request.query,
+                top_k=request.top_k,
+            )
+
+        if self.api_key:
+
+            @self.app.post("/ask")
+            def ask(request: AskRequest):
+
+                answer = self.document.ask(
+                    question=request.question,
+                    api_key=self.api_key,
+                    top_k=request.top_k,
+                )
+
+                return {
+                    "answer": answer
+                }
+
+    def run(
+        self,
+        host: str = DEFAULT_HOST,
+        port: int = DEFAULT_PORT,
+    ):
+
+        uvicorn.run(
+            self.app,
+            host=host,
+            port=port,
+        )

ragup/chunking/__init__.py

@@ -0,0 +1,9 @@
+
+
+from .base import BaseChunker
+from .recursive import RecursiveChunker
+
+__all__ = [
+    "BaseChunker",
+    "RecursiveChunker",
+]

ragup/chunking/base.py

@@ -0,0 +1,36 @@
+"""
+Base interface for all text chunkers.
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+
+
+class BaseChunker(ABC):
+    """
+    Base class for all chunking strategies.
+    """
+
+    def __init__(
+        self,
+        chunk_size: int = 500,
+        overlap: int = 50,
+    ):
+        self.chunk_size = chunk_size
+        self.overlap = overlap
+
+    @abstractmethod
+    def split(self, text: str) -> list[str]:
+        """
+        Split text into chunks.
+
+        Parameters
+        ----------
+        text : str
+
+        Returns
+        -------
+        list[str]
+        """
+        raise NotImplementedError

ragup/chunking/recursive.py

@@ -0,0 +1,59 @@
+"""
+Recursive text chunker.
+"""
+
+from __future__ import annotations
+
+from .base import BaseChunker
+
+
+class RecursiveChunker(BaseChunker):
+    """
+    Splits text into overlapping chunks.
+
+    Chunk sizes are based on words rather than characters,
+    producing more natural chunks.
+    """
+
+    def split(self, text: str) -> list[str]:
+        """
+        Split text into overlapping chunks.
+
+        Parameters
+        ----------
+        text : str
+
+        Returns
+        -------
+        list[str]
+        """
+
+        if not text.strip():
+            return []
+
+        words = text.split()
+
+        if len(words) <= self.chunk_size:
+            return [" ".join(words)]
+
+        chunks = []
+
+        step = self.chunk_size - self.overlap
+
+        if step <= 0:
+            raise ValueError(
+                "overlap must be smaller than chunk_size"
+            )
+
+        for start in range(0, len(words), step):
+
+            end = start + self.chunk_size
+
+            chunk = words[start:end]
+
+            if not chunk:
+                continue
+
+            chunks.append(" ".join(chunk))
+
+        return chunks

ragup/config.py

@@ -0,0 +1,48 @@
+"""
+Global configuration object for RagUp.
+"""
+
+from dataclasses import dataclass
+from typing import Optional
+
+from .constants import (
+    DEFAULT_CHUNK_OVERLAP,
+    DEFAULT_CHUNK_SIZE,
+    DEFAULT_EMBEDDING_MODEL,
+    DEFAULT_TOP_K,
+    DEFAULT_SCORE_THRESHOLD,
+    DEFAULT_PROVIDER,
+    DEFAULT_MODEL,
+    DEFAULT_TEMPERATURE,
+)
+
+
+@dataclass(slots=True)
+class RagUpConfig:
+    """
+    Central configuration for the RagUp pipeline.
+    """
+
+    # Chunking
+    chunk_size: int = DEFAULT_CHUNK_SIZE
+    overlap: int = DEFAULT_CHUNK_OVERLAP
+
+    # Embeddings
+    embedding_model: str = DEFAULT_EMBEDDING_MODEL
+
+    # Search
+    top_k: int = DEFAULT_TOP_K
+    score_threshold: float = DEFAULT_SCORE_THRESHOLD
+
+    # LLM
+    provider: str = DEFAULT_PROVIDER
+    model: Optional[str] = DEFAULT_MODEL
+    api_key: Optional[str] = None
+    temperature: float = DEFAULT_TEMPERATURE
+
+    # Indexing
+    cache: bool = True
+    rebuild: bool = False
+
+    # Logging
+    verbose: bool = False

ragup/constants.py

@@ -0,0 +1,73 @@
+"""
+Global constants used throughout RagUp.
+"""
+
+from pathlib import Path
+
+# ==========================================
+# Package
+# ==========================================
+
+PACKAGE_NAME = "ragup"
+PACKAGE_VERSION = "1.0.0"
+
+# ==========================================
+# Supported Files
+# ==========================================
+
+SUPPORTED_EXTENSIONS = {
+    ".pdf",
+    ".txt",
+    ".json",
+}
+
+# ==========================================
+# Chunking Defaults
+# ==========================================
+
+DEFAULT_CHUNK_SIZE = 500
+DEFAULT_CHUNK_OVERLAP = 50
+
+# ==========================================
+# Embedding Defaults
+# ==========================================
+
+DEFAULT_EMBEDDING_MODEL = "all-MiniLM-L6-v2"
+
+# ==========================================
+# Vector Database
+# ==========================================
+
+DEFAULT_RAGUP_DIR = Path(".ragup")
+
+DEFAULT_VECTOR_DB_DIR = DEFAULT_RAGUP_DIR / "db"
+
+DEFAULT_METADATA_FILE = DEFAULT_RAGUP_DIR / "metadata.json"
+
+DEFAULT_COLLECTION_NAME = "ragup_collection"
+
+# ==========================================
+# Search Defaults
+# ==========================================
+
+DEFAULT_TOP_K = 5
+
+DEFAULT_SCORE_THRESHOLD = 0.0
+
+# ==========================================
+# FastAPI Defaults
+# ==========================================
+
+DEFAULT_HOST = "127.0.0.1"
+
+DEFAULT_PORT = 8085
+
+# ==========================================
+# LLM Defaults
+# ==========================================
+
+DEFAULT_TEMPERATURE = 0.0
+
+DEFAULT_PROVIDER = "gemini"
+
+DEFAULT_MODEL = None

ragup/core/document.py

@@ -0,0 +1,282 @@
+"""
+Core Document class for RagUp.
+"""
+
+from __future__ import annotations
+import json
+from pathlib import Path
+from ..api import RagUpServer
+from ..chunking import RecursiveChunker
+from ..config import RagUpConfig
+from ..embeddings import Embedder
+from ..exceptions import FileNotIndexedError
+from ..loaders import get_loader
+from ..retrieval import Retriever, format_search_results
+from ..vectordb import VectorDBManager
+from ..llm import generate_answer
+from ..constants import (
+    DEFAULT_METADATA_FILE,
+)
+class Document:
+    """
+    Main entry point for RagUp.
+    """
+
+    def __init__(
+        self,
+        file_path: str | Path,
+    ):
+
+        self.file_path = Path(file_path)
+
+        self.config = RagUpConfig()
+
+        self.embedder = Embedder(
+            self.config.embedding_model
+        )
+
+        self.chunker = RecursiveChunker(
+            chunk_size=self.config.chunk_size,
+            overlap=self.config.overlap,
+        )
+
+        self.vectordb = VectorDBManager()
+
+        self.retriever = Retriever(
+            self.embedder,
+            self.vectordb,
+        )
+
+        self._indexed = False
+
+    def _load_document(self) -> str:
+        """
+        Load the document.
+        """
+
+        loader = get_loader(self.file_path)
+
+        return loader.load()
+
+    def _chunk_document(
+        self,
+        text: str,
+    ) -> list[str]:
+        """
+        Split the document into chunks.
+        """
+
+        return self.chunker.split(text)
+
+    def _embed_chunks(
+        self,
+        chunks: list[str],
+    ) -> list[list[float]]:
+        """
+        Generate embeddings for chunks.
+        """
+
+        return self.embedder.embed_documents(
+            chunks
+        )
+
+    def _build_metadata(
+        self,
+        chunks: list[str],
+    ) -> tuple[list[str], list[dict]]:
+            """
+            Build chunk ids and metadata.
+            """
+
+            stem = self.file_path.stem
+
+            ids = []
+            metadatas = []
+
+            for index in range(len(chunks)):
+
+                ids.append(
+                    f"{stem}_{index + 1:04d}"
+                )
+
+                metadatas.append(
+                    {
+                        "source": self.file_path.name,
+                        "chunk": index + 1,
+                    }
+                )
+
+            return ids, metadatas
+    
+    def _metadata(self) -> dict:
+
+        return {
+            "source": str(self.file_path.resolve()),
+            "last_modified": self.file_path.stat().st_mtime,
+            "chunk_size": self.chunker.chunk_size,
+            "overlap": self.chunker.overlap,
+            "embedding_model": self.config.embedding_model,
+        }
+
+
+    def _save_metadata(self) -> None:
+
+        DEFAULT_METADATA_FILE.parent.mkdir(
+            exist_ok=True
+        )
+
+        with open(
+            DEFAULT_METADATA_FILE,
+            "w",
+            encoding="utf-8",
+        ) as f:
+
+            json.dump(
+                self._metadata(),
+                f,
+                indent=4,
+            )
+
+
+    def _is_cached(self) -> bool:
+
+        if not DEFAULT_METADATA_FILE.exists():
+            return False
+
+        with open(
+            DEFAULT_METADATA_FILE,
+            "r",
+            encoding="utf-8",
+        ) as f:
+
+            metadata = json.load(f)
+
+        return metadata == self._metadata()
+    
+    def _ensure_index(self) -> None:
+        """
+        Ensure that an index exists for this document.
+        """
+
+        if self._indexed:
+            return
+
+        if self._is_cached():
+            self._indexed = True
+            return
+
+        raise FileNotIndexedError(
+            "No RagUp index found for this document. Run doc.ragup() first."
+        )
+
+    def _clear_index(self) -> None:
+            """
+            Remove the existing index before rebuilding.
+            """
+
+            self.vectordb.reset()
+
+    def ragup(
+    self,
+    chunk_size: int | None = None,
+    overlap: int | None = None,
+) -> None:
+        """
+        Build or load the RagUp index.
+        """
+
+        if chunk_size is not None:
+            self.chunker.chunk_size = chunk_size
+
+        if overlap is not None:
+            self.chunker.overlap = overlap
+
+        if self._is_cached():
+            print("✅ Using cached index.")
+            self._indexed = True
+
+            return
+        print("🔄 Building new index...")
+        self._clear_index()
+
+        text = self._load_document()
+
+        chunks = self._chunk_document(text)
+
+        embeddings = self._embed_chunks(chunks)
+
+        ids, metadatas = self._build_metadata(chunks)
+
+        self.vectordb.add(
+            ids=ids,
+            documents=chunks,
+            embeddings=embeddings,
+            metadatas=metadatas,
+        )
+
+        self._save_metadata()
+
+        self._indexed = True
+   
+
+    def search(
+        self,
+        query: str,
+        top_k: int = 5,
+    ) -> list[dict]:
+        """
+        Perform semantic search on the indexed document.
+        """
+
+        self._ensure_index()
+
+        results = self.retriever.retrieve(
+            query=query,
+            top_k=top_k,
+        )
+
+        return format_search_results(results)
+    def ask(
+        self,
+        question: str,
+        api_key: str,
+        top_k: int = 5,
+    ) -> str:
+        """
+        Answer a question using the indexed document.
+        """
+
+        self._ensure_index()
+
+        context = self.search(
+            query=question,
+            top_k=top_k,
+        )
+
+        return generate_answer(
+            question=question,
+            context=context,
+            api_key=api_key,
+        )
+    
+    def serve(
+    self,
+    api_key: str | None = None,
+    host: str = "127.0.0.1",
+    port: int = 8085,
+) -> None:
+        """
+        Launch the RagUp FastAPI server.
+        """
+
+        self._ensure_index()
+
+        server = RagUpServer(
+            document=self,
+            api_key=api_key,
+        )
+
+        server.run(
+            host=host,
+            port=port,
+        )

ragup/embeddings/__init__.py

@@ -0,0 +1,9 @@
+"""
+Embedding module.
+"""
+
+from .embedder import Embedder
+
+__all__ = [
+    "Embedder",
+]

ragup/embeddings/embedder.py

@@ -0,0 +1,61 @@
+"""
+Sentence Transformer embedder.
+"""
+
+from __future__ import annotations
+
+from sentence_transformers import SentenceTransformer
+
+from .models import (
+    DEFAULT_EMBEDDING_MODEL,
+    get_model_name,
+)
+
+
+class Embedder:
+    """
+    Wrapper around SentenceTransformer.
+    """
+
+    def __init__(
+        self,
+        model_name: str = DEFAULT_EMBEDDING_MODEL,
+    ):
+
+        self.model_name = get_model_name(model_name)
+
+        self.model = SentenceTransformer(
+            self.model_name
+        )
+
+    def embed_documents(
+        self,
+        documents: list[str],
+    ) -> list[list[float]]:
+        """
+        Generate embeddings for document chunks.
+        """
+
+        embeddings = self.model.encode(
+            documents,
+            convert_to_numpy=True,
+            normalize_embeddings=True,
+        )
+
+        return embeddings.tolist()
+
+    def embed_query(
+        self,
+        query: str,
+    ) -> list[float]:
+        """
+        Generate an embedding for a query.
+        """
+
+        embedding = self.model.encode(
+            query,
+            convert_to_numpy=True,
+            normalize_embeddings=True,
+        )
+
+        return embedding.tolist()