ragbandit-core 0.1.1__tar.gz

ragbandit_core-0.1.1/LICENSE.md

@@ -0,0 +1,9 @@
+MIT License
+
+Copyright (c) 2025 Martim Chaves
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

ragbandit_core-0.1.1/PKG-INFO

@@ -0,0 +1,145 @@
+Metadata-Version: 2.4
+Name: ragbandit-core
+Version: 0.1.1
+Summary: Core utilities for document processing, RAG configuration, querying, and evaluation.
+Author-email: Martim Chaves <martim@ragbandit.com>
+License: MIT
+Project-URL: Homepage, https://github.com/MartimChaves/ragbandit-core
+Project-URL: Documentation, https://github.com/MartimChaves/ragbandit-core#readme
+Project-URL: Source, https://github.com/MartimChaves/ragbandit-core
+Project-URL: Issues, https://github.com/MartimChaves/ragbandit-core/issues
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE.md
+Requires-Dist: pydantic>=2.11.7
+Requires-Dist: llama-index>=0.12.52
+Requires-Dist: mistralai>=1.7.0
+Requires-Dist: ragas>=0.3.0
+Requires-Dist: cryptography>=44.0.2
+Dynamic: license-file
+
+# ragbandit-core
+
+Core utilities for:
+
+* Document ingestion & processing (OCR, chunking, embedding)
+* Building and running Retrieval-Augmented Generation (RAG) pipelines
+* Evaluating answers with automated metrics
+
+## Quick start
+
+```bash
+pip install ragbandit-core
+```
+
+```python
+from ragbandit.documents import (
+    DocumentPipeline,
+    ReferencesProcessor,
+    FootnoteProcessor,
+    MistralOCRDocument,
+    MistralEmbedder,
+    SemanticChunker
+)
+import os
+import logging
+from dotenv import load_dotenv
+load_dotenv()
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s [%(levelname)s] %(name)s: %(message)s"
+)
+
+MISTRAL_API_KEY = os.getenv("MISTRAL_API_KEY")
+
+file_path = "./data/raw/[document_name].pdf"
+
+doc_pipeline = DocumentPipeline(
+    chunker=SemanticChunker(min_chunk_size=500, api_key=MISTRAL_API_KEY),
+    embedder=MistralEmbedder(model="mistral-embed", api_key=MISTRAL_API_KEY),  # noqa
+    ocr_processor=MistralOCRDocument(api_key=MISTRAL_API_KEY),
+    processors=[
+        ReferencesProcessor(api_key=MISTRAL_API_KEY),
+        FootnoteProcessor(api_key=MISTRAL_API_KEY),
+    ],
+)
+
+extended_response = doc_pipeline.process(file_path)
+
+```
+
+### Running Steps Manually
+
+For more control, you can run each pipeline step independently:
+
+```python
+from ragbandit.documents import (
+    DocumentPipeline,
+    ReferencesProcessor,
+    MistralOCRDocument,
+    MistralEmbedder,
+    SemanticChunker
+)
+import os
+from dotenv import load_dotenv
+load_dotenv()
+
+MISTRAL_API_KEY = os.getenv("MISTRAL_API_KEY")
+file_path = "./data/raw/[document_name].pdf"
+
+# Create pipeline with only the components you need
+pipeline = DocumentPipeline(
+    ocr_processor=MistralOCRDocument(api_key=MISTRAL_API_KEY),
+    processors=[ReferencesProcessor(api_key=MISTRAL_API_KEY)],
+    chunker=SemanticChunker(min_chunk_size=500, api_key=MISTRAL_API_KEY),
+    embedder=MistralEmbedder(model="mistral-embed", api_key=MISTRAL_API_KEY),
+)
+
+# Step 1: Run OCR
+ocr_result = pipeline.run_ocr(file_path)
+
+# Step 2: Run processors (optional)
+processing_results = pipeline.run_processors(ocr_result)
+final_doc = processing_results[-1]  # Get the last processor's output
+
+# Step 3: Chunk the document
+chunk_result = pipeline.run_chunker(final_doc)
+
+# Step 4: Embed chunks
+embedding_result = pipeline.run_embedder(chunk_result)
+```
+
+You can also create separate pipelines for different steps:
+
+```python
+# OCR-only pipeline
+ocr_pipeline = DocumentPipeline(
+    ocr_processor=MistralOCRDocument(api_key=MISTRAL_API_KEY)
+)
+ocr_result = ocr_pipeline.run_ocr(file_path)
+
+# Later, chunk with a different pipeline
+chunk_pipeline = DocumentPipeline(
+    chunker=SemanticChunker(min_chunk_size=500, api_key=MISTRAL_API_KEY)
+)
+chunks = chunk_pipeline.run_chunker(ocr_result)
+```
+
+## Package layout
+
+```
+ragbandit-core/
+├── src/ragbandit/
+│   ├── documents/   # document ingestion, OCR, chunking, 
+└── tests/
+```
+
+## License
+
+MIT

ragbandit_core-0.1.1/README.md

@@ -0,0 +1,121 @@
+# ragbandit-core
+
+Core utilities for:
+
+* Document ingestion & processing (OCR, chunking, embedding)
+* Building and running Retrieval-Augmented Generation (RAG) pipelines
+* Evaluating answers with automated metrics
+
+## Quick start
+
+```bash
+pip install ragbandit-core
+```
+
+```python
+from ragbandit.documents import (
+    DocumentPipeline,
+    ReferencesProcessor,
+    FootnoteProcessor,
+    MistralOCRDocument,
+    MistralEmbedder,
+    SemanticChunker
+)
+import os
+import logging
+from dotenv import load_dotenv
+load_dotenv()
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s [%(levelname)s] %(name)s: %(message)s"
+)
+
+MISTRAL_API_KEY = os.getenv("MISTRAL_API_KEY")
+
+file_path = "./data/raw/[document_name].pdf"
+
+doc_pipeline = DocumentPipeline(
+    chunker=SemanticChunker(min_chunk_size=500, api_key=MISTRAL_API_KEY),
+    embedder=MistralEmbedder(model="mistral-embed", api_key=MISTRAL_API_KEY),  # noqa
+    ocr_processor=MistralOCRDocument(api_key=MISTRAL_API_KEY),
+    processors=[
+        ReferencesProcessor(api_key=MISTRAL_API_KEY),
+        FootnoteProcessor(api_key=MISTRAL_API_KEY),
+    ],
+)
+
+extended_response = doc_pipeline.process(file_path)
+
+```
+
+### Running Steps Manually
+
+For more control, you can run each pipeline step independently:
+
+```python
+from ragbandit.documents import (
+    DocumentPipeline,
+    ReferencesProcessor,
+    MistralOCRDocument,
+    MistralEmbedder,
+    SemanticChunker
+)
+import os
+from dotenv import load_dotenv
+load_dotenv()
+
+MISTRAL_API_KEY = os.getenv("MISTRAL_API_KEY")
+file_path = "./data/raw/[document_name].pdf"
+
+# Create pipeline with only the components you need
+pipeline = DocumentPipeline(
+    ocr_processor=MistralOCRDocument(api_key=MISTRAL_API_KEY),
+    processors=[ReferencesProcessor(api_key=MISTRAL_API_KEY)],
+    chunker=SemanticChunker(min_chunk_size=500, api_key=MISTRAL_API_KEY),
+    embedder=MistralEmbedder(model="mistral-embed", api_key=MISTRAL_API_KEY),
+)
+
+# Step 1: Run OCR
+ocr_result = pipeline.run_ocr(file_path)
+
+# Step 2: Run processors (optional)
+processing_results = pipeline.run_processors(ocr_result)
+final_doc = processing_results[-1]  # Get the last processor's output
+
+# Step 3: Chunk the document
+chunk_result = pipeline.run_chunker(final_doc)
+
+# Step 4: Embed chunks
+embedding_result = pipeline.run_embedder(chunk_result)
+```
+
+You can also create separate pipelines for different steps:
+
+```python
+# OCR-only pipeline
+ocr_pipeline = DocumentPipeline(
+    ocr_processor=MistralOCRDocument(api_key=MISTRAL_API_KEY)
+)
+ocr_result = ocr_pipeline.run_ocr(file_path)
+
+# Later, chunk with a different pipeline
+chunk_pipeline = DocumentPipeline(
+    chunker=SemanticChunker(min_chunk_size=500, api_key=MISTRAL_API_KEY)
+)
+chunks = chunk_pipeline.run_chunker(ocr_result)
+```
+
+## Package layout
+
+```
+ragbandit-core/
+├── src/ragbandit/
+│   ├── documents/   # document ingestion, OCR, chunking, 
+└── tests/
+```
+
+## License
+
+MIT

ragbandit_core-0.1.1/pyproject.toml

@@ -0,0 +1,51 @@
+[build-system]
+requires = ["setuptools>=68", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "ragbandit-core"
+version = "0.1.1"
+description = "Core utilities for document processing, RAG configuration, querying, and evaluation."
+authors = [
+    { name="Martim Chaves", email="martim@ragbandit.com" }
+]
+readme = "README.md"
+requires-python = ">=3.10"
+license = {text = "MIT"}
+classifiers = [
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+]
+# Core runtime dependencies
+dependencies = [
+    "pydantic>=2.11.7",
+    "llama-index>=0.12.52",
+    "mistralai>=1.7.0",
+    "ragas>=0.3.0",
+    "cryptography>=44.0.2",
+]
+
+[tool.uv]
+dev-dependencies = [
+    "pytest<8.0.0,>=7.4.3",
+    "mypy<2.0.0,>=1.8.0",
+    "ruff<1.0.0,>=0.2.2",
+    "pre-commit<4.0.0,>=3.6.2",
+    "types-passlib<2.0.0.0,>=1.7.7.20240106",
+    "coverage<8.0.0,>=7.4.3",
+    "black>=25.1.0"
+]
+
+[tool.setuptools]
+package-dir = {"" = "src"}
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[project.urls]
+Homepage = "https://github.com/MartimChaves/ragbandit-core"
+Documentation = "https://github.com/MartimChaves/ragbandit-core#readme"
+Source = "https://github.com/MartimChaves/ragbandit-core"
+Issues = "https://github.com/MartimChaves/ragbandit-core/issues"

ragbandit_core-0.1.1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

ragbandit_core-0.1.1/src/ragbandit/__init__.py

@@ -0,0 +1,26 @@
+"""ragbandit core package.
+
+This package contains sub-modules for document processing,
+RAG pipeline configuration/execution, and evaluation utilities.
+Only lightweight interfaces and shared utilities are defined here;
+heavy logic resides in sub-packages.
+"""
+
+from importlib import metadata as _metadata
+
+__version__: str
+try:
+    __version__ = _metadata.version("ragbandit-core")
+except _metadata.PackageNotFoundError:  # pragma: no cover
+    __version__ = "0.0.0+dev"
+
+# Re-export public interfaces so that users can simply:
+#   from ragbandit import DocumentProcessor, RAGConfig, RAGPipeline, evaluate
+
+# from ragbandit.documents import DocumentPipeline
+
+
+__all__ = [
+    "__version__",
+    # "DocumentPipeline",
+]

ragbandit_core-0.1.1/src/ragbandit/config/__init__.py

@@ -0,0 +1,3 @@
+"""
+Configuration module for ragbandit package.
+"""

ragbandit_core-0.1.1/src/ragbandit/config/llms.py

@@ -0,0 +1,34 @@
+"""
+LLM configuration settings for ragbandit.
+
+This module defines default settings and constants for LLM interactions.
+"""
+
+# Default model settings
+DEFAULT_MODEL = "mistral-small-latest"
+DEFAULT_TEMPERATURE = 0.0
+
+# Retry settings
+DEFAULT_MAX_RETRIES = 3
+DEFAULT_RETRY_DELAY = 1.0  # seconds
+DEFAULT_BACKOFF_FACTOR = 2.0  # exponential backoff factor
+DEFAULT_TIMEOUT = 30.0  # seconds
+
+# Token limits
+MAX_PROMPT_TOKENS = {
+    "mistral-small-latest": 8000,
+    "mistral-medium-latest": 32000,
+    "mistral-large-latest": 32000,
+    "gpt-3.5-turbo": 4096,
+    "gpt-4": 8192,
+    "gpt-4-turbo": 128000,
+}
+
+# System prompts
+DEFAULT_SYSTEM_PROMPT = """You are a helpful AI assistant."""
+
+# Response formats
+JSON_FORMAT_INSTRUCTION = """
+Your response must be valid JSON that matches the following schema:
+{schema}
+"""

ragbandit_core-0.1.1/src/ragbandit/config/pricing.py

@@ -0,0 +1,38 @@
+"""
+Pricing configuration for LLM API calls.
+
+This module contains pricing constants for various
+LLM models and embedding models.
+"""
+
+# Token cost rates per 1M tokens (in USD)
+# Based on Mistral AI pricing as of July 2025
+MODEL_COSTS = {
+    # Format: "model_name": (input_cost_per_1M, output_cost_per_1M)
+    "mistral-small-latest": (2.00, 6.00),
+    "mistral-medium-latest": (6.00, 18.00),
+    "mistral-large-latest": (12.00, 36.00),
+    # Add other models as needed
+}
+
+# Embedding model costs per 1M tokens
+EMBEDDING_COSTS = {
+    # Format: "model_name": cost_per_1M_tokens
+    "mistral-embed": 0.10,
+    "text-embedding-3-small": 0.02,
+    "text-embedding-3-large": 0.13,
+    # Add other embedding models as needed
+}
+
+# OCR model costs per page (in EUR)
+OCR_MODEL_COSTS = {
+    # Format: "model_name": cost_per_page
+    "mistral-ocr-latest": 0.001,  # 1 EUR per 1000 pages
+    # Add other OCR models as needed
+}
+
+# Default OCR model to use if the specified model is not in OCR_MODEL_COSTS
+DEFAULT_OCR_MODEL = "mistral-ocr-latest"
+
+# Default model to use if the specified model is not in MODEL_COSTS
+DEFAULT_MODEL = "mistral-small-latest"

ragbandit_core-0.1.1/src/ragbandit/documents/__init__.py

@@ -0,0 +1,66 @@
+"""
+Document processing module for handling, analyzing, and transforming documents.
+
+This package provides tools for OCR, chunking,
+embedding, and processing documents.
+"""
+
+# Import key components from subdirectories
+from ragbandit.documents.document_pipeline import DocumentPipeline
+
+# Import from chunkers
+from ragbandit.documents.chunkers import (
+    BaseChunker,
+    FixedSizeChunker,
+    SemanticChunker,
+    SemanticBreak
+)
+
+# Import from processors
+from ragbandit.documents.processors import (
+    BaseProcessor,
+    FootnoteProcessor,
+    ReferencesProcessor
+)
+
+# Import from embedders
+from ragbandit.documents.embedders import (
+    BaseEmbedder,
+    MistralEmbedder
+)
+
+# Import from OCR
+from ragbandit.documents.ocr import (
+    BaseOCR,
+    MistralOCRDocument
+)
+
+# Import from utils
+from ragbandit.documents.utils import SecureFileHandler
+
+__all__ = [
+    # Main pipeline
+    "DocumentPipeline",
+
+    # Chunkers
+    "BaseChunker",
+    "FixedSizeChunker",
+    "SemanticChunker",
+    "SemanticBreak",
+
+    # Processors
+    "BaseProcessor",
+    "FootnoteProcessor",
+    "ReferencesProcessor",
+
+    # Embedders
+    "BaseEmbedder",
+    "MistralEmbedder",
+
+    # OCR
+    "BaseOCR",
+    "MistralOCRDocument",
+
+    # Utils
+    "SecureFileHandler"
+]

ragbandit_core-0.1.1/src/ragbandit/documents/chunkers/__init__.py

@@ -0,0 +1,18 @@
+"""
+Chunker implementations for document processing.
+
+This module provides various chunking strategies for documents.
+"""
+
+from ragbandit.documents.chunkers.base_chunker import BaseChunker
+from ragbandit.documents.chunkers.fixed_size_chunker import FixedSizeChunker
+from ragbandit.documents.chunkers.semantic_chunker import (
+    SemanticChunker, SemanticBreak
+)
+
+__all__ = [
+    "BaseChunker",
+    "FixedSizeChunker",
+    "SemanticChunker",
+    "SemanticBreak"
+]