ragbox-core 1.0.0__tar.gz

ragbox_core-1.0.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 RAGBox Contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

ragbox_core-1.0.0/PKG-INFO

@@ -0,0 +1,121 @@
+Metadata-Version: 2.4
+Name: ragbox-core
+Version: 1.0.0
+Summary: RAG-in-a-Box: Zero-Configuration Self-Building Agentic RAG System
+License-File: LICENSE
+Author: Staff+ Engineer
+Author-email: staff@ragbox.ai
+Requires-Python: >=3.11,<4.0
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Requires-Dist: aiofiles (>=23.0,<24.0)
+Requires-Dist: anthropic
+Requires-Dist: camelot-py
+Requires-Dist: chromadb (>=0.4,<0.5)
+Requires-Dist: groq
+Requires-Dist: igraph
+Requires-Dist: leidenalg
+Requires-Dist: llama-cpp-python
+Requires-Dist: loguru (>=0.7,<0.8)
+Requires-Dist: neo4j (>=5.0,<6.0)
+Requires-Dist: networkx (>=3.0,<4.0)
+Requires-Dist: numpy (>=1.24,<2.0)
+Requires-Dist: openai
+Requires-Dist: paddleocr
+Requires-Dist: pdfplumber
+Requires-Dist: pydantic (>=2.0,<3.0)
+Requires-Dist: pypdf
+Requires-Dist: python-dotenv
+Requires-Dist: python-louvain
+Requires-Dist: python-magic
+Requires-Dist: python-pptx
+Requires-Dist: rank-bm25
+Requires-Dist: rich (>=13.0,<14.0)
+Requires-Dist: scikit-learn (>=1.3,<2.0)
+Requires-Dist: scipy (>=1.11,<2.0)
+Requires-Dist: sentence-transformers
+Requires-Dist: tenacity
+Requires-Dist: tiktoken
+Requires-Dist: tree-sitter
+Requires-Dist: tree-sitter-python
+Requires-Dist: typer (>=0.9,<0.10)
+Requires-Dist: watchdog (>=3.0,<4.0)
+Description-Content-Type: text/markdown
+
+# RAGBox
+
+**RAG-in-a-Box: Zero-Configuration Self-Building Agentic RAG System**
+
+RAGBox is a production-ready, auto-configuring, async-first RAG engine that combines Vector Search, Agentic Orchestration, and Graph Retrieval natively.
+
+## Installation
+
+```bash
+pip install ragbox
+```
+
+> **Note on Dependencies:** Advanced document processing features like OCR and complex PDF parsing require system-level dependencies. Depending on your OS, you may need to install standard C++ build tools or Tesseract for `paddleocr` and `pdfplumber` to function optimally.
+
+## Quick Start (3-Line API)
+
+```python
+from ragbox import RAGBox
+
+# Automatically ingests, builds graphs, configures vector db, and chunks
+rag = RAGBox("./company-docs")
+
+# Intelligent routing via query classification
+answer = rag.query("What's our vacation policy?")
+print(answer)
+```
+
+## CLI Interface
+
+RAGBox provides a dead-simple CLI for running locally without writing code:
+
+```bash
+# Point to your documents. RAGBox will self-build the index and graph.
+ragbox init ./company-docs
+
+# Query the active index
+ragbox query "What's our vacation policy?" -d ./company-docs
+```
+
+## Architecture
+
+```mermaid
+graph TD
+    A[Local Documents] --> B{Document Processor Auto-Router}
+    B --> C[AST / OCR / PDF Parsing]
+    C --> D[Chunking Engine]
+    D --> E[(Vector Store)]
+    C --> F[(Knowledge Graph)]
+    
+    Q[User Query] --> G[Agentic Orchestrator]
+    G --> H[Retrieval Fusion Engine]
+    E --> H
+    F --> H
+    H --> G
+    G --> I[Final Answer]
+```
+
+## Risk Surface Analysis
+
+*   **Temporal Edges (T=0 vs T=Scale):** At T=0, `ragbox init` is blocking to guarantee index availability. At T=scale, the background daemon handles delta updates (via watchdog) to prevent index staleness and thundering herds.
+*   **Adversarial Edges:** Subject to standard prompt injection if queries are exposed raw to external users. The Orchestrator currently assumes trusted inputs.
+*   **Resource Edges:** High concurrency read/write spikes memory due to dual maintenance of the local Vector DB and the Knowledge Graph.
+
+## Features
+
+* **Self-Healing Infrastructure:** Watchdog auto-detects changes and updates vector stores & knowledge graphs incrementally, preventing index staleness or storms.
+* **Auto-Document Intelligence:** Automatically detects PDF, Text, Images, and Code to use AST, OCR (`paddleocr`), or structural layouts (`pdfplumber`).
+* **Cost Estimator:** See the expected USD cost of indexing *before* it runs.
+* **Auto-Knowledge-Graph (GraphRAG):** Extracts entities and communities automatically using the Leiden algorithm for structured reasoning.
+* **Retrieval Fusion & Reranking:** Merges Dense Vectors and Graph Search using Reciprocal Rank Fusion, then reranks the massive candidate pool using a highly accurate `ms-marco` Cross-Encoder.
+* **Late Chunking:** Contextual sequence embeddings! Vectors are calculated over the full document bounds before being pooled into chunks, preserving global semantic context within local tokens.
+* **Agentic Orchestrator & Intelligent Routing:** Automatically routes incoming queries into 6 distinct pipelines: Vector, Keyword, Graph, Multi-Query, Time-Based, and Agentic.
+* **Multi-Query Expansion:** Broad intent queries are dynamically expanded into multiple variations by the LLM, retrieving and fusing results across all variations for unparalleled recall.
+

ragbox_core-1.0.0/README.md

@@ -0,0 +1,73 @@
+# RAGBox
+
+**RAG-in-a-Box: Zero-Configuration Self-Building Agentic RAG System**
+
+RAGBox is a production-ready, auto-configuring, async-first RAG engine that combines Vector Search, Agentic Orchestration, and Graph Retrieval natively.
+
+## Installation
+
+```bash
+pip install ragbox
+```
+
+> **Note on Dependencies:** Advanced document processing features like OCR and complex PDF parsing require system-level dependencies. Depending on your OS, you may need to install standard C++ build tools or Tesseract for `paddleocr` and `pdfplumber` to function optimally.
+
+## Quick Start (3-Line API)
+
+```python
+from ragbox import RAGBox
+
+# Automatically ingests, builds graphs, configures vector db, and chunks
+rag = RAGBox("./company-docs")
+
+# Intelligent routing via query classification
+answer = rag.query("What's our vacation policy?")
+print(answer)
+```
+
+## CLI Interface
+
+RAGBox provides a dead-simple CLI for running locally without writing code:
+
+```bash
+# Point to your documents. RAGBox will self-build the index and graph.
+ragbox init ./company-docs
+
+# Query the active index
+ragbox query "What's our vacation policy?" -d ./company-docs
+```
+
+## Architecture
+
+```mermaid
+graph TD
+    A[Local Documents] --> B{Document Processor Auto-Router}
+    B --> C[AST / OCR / PDF Parsing]
+    C --> D[Chunking Engine]
+    D --> E[(Vector Store)]
+    C --> F[(Knowledge Graph)]
+    
+    Q[User Query] --> G[Agentic Orchestrator]
+    G --> H[Retrieval Fusion Engine]
+    E --> H
+    F --> H
+    H --> G
+    G --> I[Final Answer]
+```
+
+## Risk Surface Analysis
+
+*   **Temporal Edges (T=0 vs T=Scale):** At T=0, `ragbox init` is blocking to guarantee index availability. At T=scale, the background daemon handles delta updates (via watchdog) to prevent index staleness and thundering herds.
+*   **Adversarial Edges:** Subject to standard prompt injection if queries are exposed raw to external users. The Orchestrator currently assumes trusted inputs.
+*   **Resource Edges:** High concurrency read/write spikes memory due to dual maintenance of the local Vector DB and the Knowledge Graph.
+
+## Features
+
+* **Self-Healing Infrastructure:** Watchdog auto-detects changes and updates vector stores & knowledge graphs incrementally, preventing index staleness or storms.
+* **Auto-Document Intelligence:** Automatically detects PDF, Text, Images, and Code to use AST, OCR (`paddleocr`), or structural layouts (`pdfplumber`).
+* **Cost Estimator:** See the expected USD cost of indexing *before* it runs.
+* **Auto-Knowledge-Graph (GraphRAG):** Extracts entities and communities automatically using the Leiden algorithm for structured reasoning.
+* **Retrieval Fusion & Reranking:** Merges Dense Vectors and Graph Search using Reciprocal Rank Fusion, then reranks the massive candidate pool using a highly accurate `ms-marco` Cross-Encoder.
+* **Late Chunking:** Contextual sequence embeddings! Vectors are calculated over the full document bounds before being pooled into chunks, preserving global semantic context within local tokens.
+* **Agentic Orchestrator & Intelligent Routing:** Automatically routes incoming queries into 6 distinct pipelines: Vector, Keyword, Graph, Multi-Query, Time-Based, and Agentic.
+* **Multi-Query Expansion:** Broad intent queries are dynamically expanded into multiple variations by the LLM, retrieving and fusing results across all variations for unparalleled recall.

ragbox_core-1.0.0/pyproject.toml

@@ -0,0 +1,56 @@
+[tool.poetry]
+name = "ragbox-core"
+version = "1.0.0"
+description = "RAG-in-a-Box: Zero-Configuration Self-Building Agentic RAG System"
+authors = ["Staff+ Engineer <staff@ragbox.ai>"]
+readme = "README.md"
+packages = [{include = "ragbox"}]
+exclude = ["tests/", "docs/", ".env", "*.db", "chroma_db/", "knowledge_graph/", "logs/"]
+[tool.poetry.scripts]
+ragbox = "ragbox.cli:app"
+
+[tool.poetry.dependencies]
+python = "^3.11"
+pydantic = "^2.0"
+aiofiles = "^23.0"
+watchdog = "^3.0"
+chromadb = "^0.4"
+neo4j = "^5.0"
+networkx = "^3.0"
+scikit-learn = "^1.3"
+pypdf = "*"
+pdfplumber = "*"
+camelot-py = "*"
+tree-sitter = "*"
+tree-sitter-python = "*"
+paddleocr = "*"
+sentence-transformers = "*"
+rank-bm25 = "*"
+numpy = "^1.24"
+scipy = "^1.11"
+loguru = "^0.7"
+typer = "^0.9"
+rich = "^13.0"
+llama-cpp-python = "*"
+openai = "*"
+anthropic = "*"
+groq = "*"
+python-dotenv = "*"
+python-pptx = "*"
+tenacity = "*"
+tiktoken = "*"
+python-magic = "*"
+leidenalg = "*"
+python-louvain = "*"
+igraph = "*"
+
+[tool.poetry.group.dev.dependencies]
+pytest = "^7.4.0"
+pytest-asyncio = "^0.21.1"
+mypy = "^1.5.0"
+black = "^23.7.0"
+ruff = "^0.0.285"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

ragbox_core-1.0.0/ragbox/__init__.py

@@ -0,0 +1,175 @@
+"""
+RAGBox: Zero-Configuration Self-Building Agentic RAG System.
+
+Provides the public RAGBox class for one-line integrations.
+"""
+import asyncio
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+from loguru import logger
+
+from ragbox.config.defaults import Settings
+from ragbox.core.agentic_orchestrator import AgenticOrchestrator
+from ragbox.core.chunking_engine import ChunkingEngine
+from ragbox.core.document_processor import DocumentProcessorRouter
+from ragbox.core.knowledge_graph import OptimizedKnowledgeGraph
+from ragbox.core.retrieval_fusion import RetrievalFusionEngine
+from ragbox.core.self_healing import SelfHealer
+from ragbox.models.queries import Answer
+from ragbox.utils.embeddings import EmbeddingAutoDetector
+from ragbox.utils.llm_clients import LLMAutoDetector
+from ragbox.utils.vector_stores import VectorStoreAutoDetector
+
+
+class RAGBox:
+    """
+    RAG-in-a-Box: Zero-Configuration Self-Building Agentic RAG System.
+    """
+
+    def __init__(self, document_dir: str | Path, config: Optional[Settings] = None) -> None:
+        """
+        Initialize a complete RAG system pointing to a directory.
+        
+        Args:
+            document_dir: Path to the directory containing documents.
+            config: Optional configuration overrides.
+        """
+        self.document_dir = Path(document_dir)
+        self.config = config or Settings()
+        
+        # Initialize Core Layers Auto-magically
+        self.llm_client = LLMAutoDetector.detect(self.config)
+        self.embedding_provider = EmbeddingAutoDetector.detect(self.config)
+        self.vector_store = VectorStoreAutoDetector.detect(self.config, self.embedding_provider)
+        
+        self.document_processor = DocumentProcessorRouter()
+        self.chunking_engine = ChunkingEngine(self.llm_client, self.embedding_provider)
+        self.knowledge_graph = OptimizedKnowledgeGraph(llm_client=self.llm_client)
+        
+        self.retriever = RetrievalFusionEngine(
+            vector_store=self.vector_store,
+            knowledge_graph=self.knowledge_graph,
+            embedding_provider=self.embedding_provider,
+            llm_client=self.llm_client
+        )
+        self.orchestrator = AgenticOrchestrator(
+            retriever=self.retriever,
+            llm_client=self.llm_client,
+            document_processor=self.document_processor,
+            knowledge_graph=self.knowledge_graph
+        )
+        self.self_healer = SelfHealer(
+            document_dir=self.document_dir,
+            document_processor=self.document_processor,
+            chunking_engine=self.chunking_engine,
+            vector_store=self.vector_store,
+            knowledge_graph=self.knowledge_graph
+        )
+        
+        # Start initial build and self-healing daemon
+        logger.info(f"Initializing RAGBox for {self.document_dir}")
+        self._ensure_built()
+        
+    def _ensure_built(self) -> None:
+        """Synchronously trigger the build process to guarantee readiness."""
+        try:
+            loop = asyncio.get_running_loop()
+        except RuntimeError:
+            loop = asyncio.new_event_loop()
+            asyncio.set_event_loop(loop)
+            
+        if loop.is_running():
+            asyncio.create_task(self.self_healer.initial_build())
+        else:
+            loop.run_until_complete(self.self_healer.initial_build())
+        
+        # Start watchdog
+        self.self_healer.start_watchdog()
+
+    def query(self, question: str) -> str:
+        """
+        Query the RAGBox system synchronously.
+        
+        Args:
+            question: The user's query.
+            
+        Returns:
+            The direct answer string.
+        """
+        try:
+            loop = asyncio.get_running_loop()
+            if loop.is_running():
+                raise RuntimeError("Cannot call synchronous `query` inside an active event loop. Use `aquery`.")
+        except RuntimeError:
+            loop = asyncio.new_event_loop()
+            asyncio.set_event_loop(loop)
+            
+        answer: Answer = loop.run_until_complete(self.orchestrator.execute(question))
+        return answer.content
+        
+    async def aquery(self, question: str) -> str:
+        """
+        Async evaluation of a query.
+        
+        Args:
+            question: The user's query.
+            
+        Returns:
+            The direct answer string.
+        """
+        answer: Answer = await self.orchestrator.execute(question)
+        return answer.content
+
+    def estimate_cost(self, query: Optional[str] = None) -> str:
+        """
+        Estimate the cost of a query or indexing the corpus upfront.
+
+        Args:
+            query: The query to run. If None, estimates the cost of indexing the entire corpus.
+
+        Returns:
+            A string detailing the estimated cost.
+        """
+        from ragbox.utils.cost_tracker import CostEstimator
+        
+        # We assume the default model for estimating cost
+        estimator = CostEstimator(self.llm_client.model_name if hasattr(self.llm_client, "model_name") else "gpt-4o")
+        
+        if query is None:
+            # Estimate cost of indexing the corpus based on file size
+            total_bytes = 0
+            for ext in self.document_processor.processors.keys():
+                for p in self.document_dir.rglob(f"*{ext}"):
+                    if p.is_file():
+                        total_bytes += p.stat().st_size
+            
+            # Wild guess: 1 byte ~ 0.25 tokens
+            estimated_tokens = int(total_bytes * 0.25)
+            # Embedding cost
+            embed_cost = (estimated_tokens / 1_000_000) * 0.13 # large embedding assumption
+            return f"Corpus Indexing Estimate: ~{estimated_tokens} tokens. Embedding Cost: ${embed_cost:.4f}"
+            
+        else:
+            # Simulate retrieval and estimation for query
+            try:
+                loop = asyncio.get_running_loop()
+                if loop.is_running():
+                    raise RuntimeError("Cannot call synchronous `estimate_cost` inside loop.")
+            except RuntimeError:
+                loop = asyncio.new_event_loop()
+                asyncio.set_event_loop(loop)
+            
+            # Retrieve mock sources
+            sources = loop.run_until_complete(self.retriever.retrieve(query))
+            context_text = "\n".join([s.text for s in sources])
+            
+            estimate = estimator.estimate_generation(
+                prompt=f"{context_text}\n{query}",
+                approx_output_tokens=500
+            )
+            return estimate.__str__()
+
+
+
+__all__ = ["RAGBox"]

ragbox_core-1.0.0/ragbox/cli.py

@@ -0,0 +1,106 @@
+import typer
+import asyncio
+from pathlib import Path
+from rich.console import Console
+from rich.panel import Panel
+from rich.markdown import Markdown
+
+from ragbox import RAGBox
+from ragbox.config.defaults import Settings
+
+app = typer.Typer(
+    name="ragbox",
+    help="RAGBox: Zero-Configuration Self-Building Agentic RAG System",
+    add_completion=False,
+)
+
+console = Console()
+
+@app.command()
+def init(
+    document_dir: Path = typer.Argument(
+        ...,
+        exists=True,
+        file_okay=False,
+        dir_okay=True,
+        help="Directory containing local documents to index.",
+    ),
+    config_file: Path = typer.Option(
+        None,
+        "--config",
+        "-c",
+        help="Path to an optional JSON/YAML configuration file.",
+    )
+):
+    """
+    Initialize a RAGBox instance in the specified directory.
+    
+    This command blocks until the initial index and knowledge graph are built.
+    Afterward, it starts the background daemon to watch for changes.
+    """
+    console.print(Panel(f"Initializing RAGBox at [cyan]{document_dir}[/cyan]", border_style="green"))
+    
+    # In a real implementation this might parse config_file into Settings
+    settings = Settings()
+    
+    try:
+        # Pre-flight cost estimation
+        rg_pre = RAGBox(document_dir, config=settings)
+        cost_estimate = rg_pre.estimate_cost(query=None)
+        console.print(Panel(f"[bold yellow]{cost_estimate}[/bold yellow]", title="Cost Estimate"))
+        
+        # RAGBox __init__ calls _ensure_built which builds synchronously implicitly via asyncio
+        with console.status("[bold green]Building Vector Store and Knowledge Graph...", spinner="dots"):
+            rg = rg_pre # It's already built by the init above
+        console.print("[bold green]✓ Indexing complete.[/bold green] RAGBox is ready for queries.")
+        console.print("The self-healing daemon is now active in the background for this session.")
+        
+        # Keep process alive to allow watchdog to run, or exit if init is purely 
+        # meant to pre-compute. Because RAGBox design spins up daemon threads, 
+        # we'll keep it alive here if we want continuous syncing.
+        console.print("[yellow]Press Ctrl+C to stop the daemon.[/yellow]")
+        try:
+            while True:
+                # Wait forever
+                asyncio.run(asyncio.sleep(3600))
+        except KeyboardInterrupt:
+            console.print("Daemon stopped.")
+        
+    except Exception as e:
+        console.print(f"[bold red]Error initializing RAGBox: {e}[/bold red]")
+        raise typer.Exit(code=1)
+
+@app.command()
+def query(
+    question: str = typer.Argument(..., help="The question to ask the RAG system."),
+    document_dir: Path = typer.Option(
+        ".",
+        "--dir",
+        "-d",
+        help="Directory where RAGBox is initialized.",
+        exists=True,
+        dir_okay=True,
+    )
+):
+    """
+    Query an existing RAGBox instance.
+    """
+    console.print(f"[bold blue]Querying RAGBox in {document_dir}...[/bold blue]\n")
+    try:
+        rg = RAGBox(document_dir)
+        
+        # Estimation
+        cost_estimate = rg.estimate_cost(query=question)
+        console.print(f"[yellow]Estimation:[/yellow] {cost_estimate}")
+        
+        with console.status("[bold green]Agentic Orchestrator thinking...", spinner="dots"):
+            answer = rg.query(question)
+            
+        console.print(Panel(Markdown(answer), title="Answer", border_style="blue"))
+        
+    except Exception as e:
+        console.print(f"[bold red]Error querying RAGBox: {e}[/bold red]")
+        raise typer.Exit(code=1)
+
+if __name__ == "__main__":
+    app()

ragbox_core-1.0.0/ragbox/config/__init__.py

@@ -0,0 +1,3 @@
+"""Config module initialization."""
+from .defaults import Settings
+__all__ = ["Settings"]

ragbox_core-1.0.0/ragbox/config/defaults.py

@@ -0,0 +1,28 @@
+"""
+Zero-Configuration Settings via Pydantic using dotenv.
+"""
+from typing import Optional
+from pydantic_settings import BaseSettings, SettingsConfigDict
+from pydantic import Field
+
+class Settings(BaseSettings):
+    """
+    RAGBox Configuration.
+    Prioritizes Env Vars, falls back to .env if present, otherwise defaults.
+    """
+    model_config = SettingsConfigDict(env_file='.env', env_file_encoding='utf-8', extra='ignore')
+
+    # API Keys
+    openai_api_key: Optional[str] = Field(default=None, description="OpenAI API Key")
+    anthropic_api_key: Optional[str] = Field(default=None, description="Anthropic API Key")
+    groq_api_key: Optional[str] = Field(default=None, description="Groq API Key")
+    pinecone_api_key: Optional[str] = Field(default=None, description="Pinecone API Key")
+
+    # Local paths
+    local_model_path: str = Field(default="models/llama-3.1-8b-instruct.gguf", description="Path for local LLM")
+    chroma_db_dir: str = Field(default="./chroma_db", description="Path for local vector DB")
+    
+    # Behaviors
+    log_level: str = "INFO"
+    max_agent_steps: int = 5
+    graph_community_level: int = 1

ragbox_core-1.0.0/ragbox/core/__init__.py

@@ -0,0 +1,18 @@
+"""
+Core RAGBox mechanisms.
+"""
+from .agentic_orchestrator import AgenticOrchestrator
+from .chunking_engine import ChunkingEngine
+from .document_processor import DocumentProcessorRouter
+from .knowledge_graph import OptimizedKnowledgeGraph
+from .retrieval_fusion import RetrievalFusionEngine
+from .self_healing import SelfHealer
+
+__all__ = [
+    "AgenticOrchestrator",
+    "ChunkingEngine",
+    "DocumentProcessorRouter",
+    "OptimizedKnowledgeGraph",
+    "RetrievalFusionEngine",
+    "SelfHealer",
+]