ragmint 0.2.3__tar.gz → 0.3.1__tar.gz

{ragmint-0.2.3 → ragmint-0.3.1}/PKG-INFO

@@ -1,29 +1,45 @@
 Metadata-Version: 2.4
 Name: ragmint
-Version: 0.2.3
+Version: 0.3.1
 Summary: A modular framework for evaluating and optimizing RAG pipelines.
 Author-email: Andre Oliveira <oandreoliveira@outlook.com>
 License: Apache License 2.0
 Project-URL: Homepage, https://github.com/andyolivers/ragmint
 Project-URL: Documentation, https://andyolivers.com
 Project-URL: Issues, https://github.com/andyolivers/ragmint/issues
-Keywords: RAG,LLM,retrieval,optimization,AI,evaluation
+Keywords: RAG,LLM,retrieval,optimization,AI,evaluation,chunking,autotuning
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: numpy>=1.23
+Requires-Dist: numpy<2.0.0
 Requires-Dist: pandas>=2.0
 Requires-Dist: scikit-learn>=1.3
-Requires-Dist: openai>=1.0
-Requires-Dist: tqdm
-Requires-Dist: pyyaml
-Requires-Dist: chromadb>=0.4
+Requires-Dist: sentence-transformers>=2.2.2
+Requires-Dist: chromadb>=0.3.1
 Requires-Dist: faiss-cpu; sys_platform != "darwin"
+Requires-Dist: faiss-cpu==1.7.4; sys_platform == "darwin"
+Requires-Dist: rank-bm25>=0.2.2
 Requires-Dist: optuna>=3.0
-Requires-Dist: pytest
+Requires-Dist: tqdm
 Requires-Dist: colorama
+Requires-Dist: pyyaml
+Requires-Dist: python-dotenv
+Requires-Dist: openai>=1.0.0
 Requires-Dist: google-generativeai>=0.8.0
+Requires-Dist: anthropic>=0.25.0
 Requires-Dist: supabase>=2.4.0
+Requires-Dist: pytest
+Requires-Dist: langchain>=0.2.5
+Requires-Dist: langchain-community>=0.2.5
+Requires-Dist: langchain-text-splitters>=0.2.1
+Provides-Extra: dev
+Requires-Dist: black; extra == "dev"
+Requires-Dist: flake8; extra == "dev"
+Requires-Dist: isort; extra == "dev"
+Requires-Dist: pytest-cov; extra == "dev"
+Provides-Extra: docs
+Requires-Dist: mkdocs; extra == "docs"
+Requires-Dist: mkdocs-material; extra == "docs"
 Dynamic: license-file
 
 # Ragmint
@@ -38,7 +54,7 @@ Dynamic: license-file
 
 **Ragmint** (Retrieval-Augmented Generation Model Inspection & Tuning) is a modular, developer-friendly Python library for **evaluating, optimizing, and tuning RAG (Retrieval-Augmented Generation) pipelines**.
 
-It provides a complete toolkit for **retriever selection**, **embedding model tuning**, and **automated RAG evaluation** with support for **Optuna-based Bayesian optimization**, **Auto-RAG tuning**, and **explainability** through Gemini or Claude.
+It provides a complete toolkit for **retriever selection**, **embedding model tuning**, **automated RAG evaluation**, and **config-driven prebuilding** of pipelines with support for **Optuna-based Bayesian optimization**, **Auto-RAG tuning**, **chunking**, and **explainability** through Gemini or Claude.
 
 ---
 
@@ -49,10 +65,13 @@ It provides a complete toolkit for **retriever selection**, **embedding model tu
 - 🧠 **Explainability Layer** — interprets RAG performance via Gemini or Claude APIs  
 - 🏆 **Leaderboard Tracking** — stores and ranks experiment runs via JSON or external DB  
 - 🔍 **Built-in RAG evaluation metrics** — faithfulness, recall, BLEU, ROUGE, latency  
-- ⚙️ **Retrievers** — FAISS, Chroma, ElasticSearch  
-- 🧩 **Embeddings** — OpenAI, HuggingFace  
+- ⚙️ **Retrievers** — FAISS, Chroma, scikit-learn  
+- 🧩 **Embeddings** — Hugging Face  
 - 💾 **Caching, experiment tracking, and reproducibility** out of the box  
 - 🧰 **Clean modular structure** for easy integration in research and production setups  
+- 📦 **Chunking system** — automatic or configurable chunk_size and overlap for documents  
+- 🏗️ **Langchain Prebuilder** — prepares pipelines, applies chunking, embeddings, and vector store creation automatically  
+- ⚙️ **Config Adapter (LangchainConfigAdapter)** — normalizes configuration, fills defaults, validates retrievers
 
 ---
 
@@ -81,6 +100,8 @@ Example `configs/default.yaml`:
 ```yaml
 retriever: faiss
 embedding_model: text-embedding-3-small
+chunk_size: 500
+overlap: 100
 reranker:
   mode: mmr
   lambda_param: 0.5
@@ -94,15 +115,94 @@ optimization:
 ### 3️⃣ Manual Pipeline Usage
 
 ```python
-from ragmint.core.pipeline import RAGPipeline
+from ragmint.prebuilder import PreBuilder
+from ragmint.tuner import RAGMint
+
+# Prebuild pipeline (chunking, embeddings, vector store)
+prebuilder = PreBuilder(
+    docs_path="data/docs/",
+    config_path="configs/default.yaml"
+)
+pipeline = prebuilder.build_pipeline()
+
+# Initialize RAGMint with prebuilt components
+rag = RAGMint(pipeline=pipeline)
+
+# Run optimization
+best, results = rag.optimize(validation_set=None, metric="faithfulness", trials=3)
+print("Best configuration:", best)
 
-pipeline = RAGPipeline({
-    "embedding_model": "text-embedding-3-small",
-    "retriever": "faiss",
-})
+```
+---
+# 🧩 Embeddings and Retrievers
+
+**Ragmint** supports a flexible set of embeddings and retrievers, allowing you to adapt easily to various **RAG architectures**.
+
+---
+## 🧩 Chunking System
+
+* **Automatically splits documents** into chunks with `chunk_size` and `overlap` parameters.
+* **Supports default values** if not provided in configuration.
+* **Optimized** for downstream **retrieval and embeddings**.
+* **Enables adaptive chunking strategies** in future releases.
+
+---
+## 🧩 Langchain Config Adapter
+
+* **Ensures consistent configuration** across pipeline components.
+* **Normalizes retriever and embedding names** (e.g., `faiss`, `sentence-transformers/...`).
+* **Adds default chunk parameters** when missing.
+* **Validates retriever backends** and **raises clear errors** for unsupported options.
+
+---
+## 🧩 Langchain Prebuilder
 
-result = pipeline.run("What is retrieval-augmented generation?")
-print(result)
+**Automates pipeline preparation:**
+1. Reads documents
+2. Applies chunking
+3. Creates embeddings
+4. Initializes retriever / vector store
+5. Returns ready-to-use pipeline** for RAGMint or custom usage.
+
+---
+
+## 🔤 Available Embeddings (Hugging Face)
+
+You can select from the following models:
+
+* `sentence-transformers/all-MiniLM-L6-v2` — **lightweight**, general-purpose
+* `sentence-transformers/all-mpnet-base-v2` — **higher accuracy**, slower
+* `BAAI/bge-base-en-v1.5` — **multilingual**, dense embeddings
+* `intfloat/multilingual-e5-base` — ideal for **multilingual corpora**
+
+
+
+### Configuration Example
+
+Use the following format in your config file to specify the embedding model:
+
+```yaml
+embedding_model: sentence-transformers/all-MiniLM-L6-v2
+```
+---
+
+## 🔍 Available Retrievers
+
+**Ragmint** integrates multiple **retrieval backends** to suit different needs:
+
+| Retriever | Description |
+| :--- | :--- |
+| **FAISS** | Fast vector similarity search; efficient for dense embeddings |
+| **Chroma** | Persistent vector DB; works well for incremental updates |
+| **scikit-learn (NearestNeighbors)** | Lightweight, zero-dependency local retriever |
+
+
+### Configuration Example
+
+To specify the retriever in your configuration file, use the following format:
+
+```yaml
+retriever: faiss
 ```
 
 ---
@@ -174,8 +274,7 @@ lb.show_top(3)
 
 ## 🧠 Explainability with Gemini / Claude
 
-Compare two RAG configurations and receive natural language insights
-on **why** one performs better.
+Compare two RAG configurations and receive **natural language insights** on why one performs better.
 
 ```python
 from ragmint.explainer import explain_results
@@ -189,7 +288,7 @@ print(explanation)
 
 > Set your API keys in a `.env` file or via environment variables:
 > ```
-> export GOOGLE_API_KEY="your_gemini_key"
+> export GEMINI_API_KEY="your_gemini_key"
 > export ANTHROPIC_API_KEY="your_claude_key"
 > ```
 
@@ -203,8 +302,12 @@ ragmint/
 │   ├── pipeline.py
 │   ├── retriever.py
 │   ├── reranker.py
-│   ├── embedding.py
+│   ├── embeddings.py
+│   ├── chunking.py
 │   └── evaluation.py
+├── integration/
+│   ├── config_adapter.py
+│   └── langchain_prebuilder.py
 ├── autotuner.py
 ├── explainer.py
 ├── leaderboard.py
@@ -240,16 +343,42 @@ Your `pyproject.toml` includes all required dependencies:
 name = "ragmint"
 version = "0.1.0"
 dependencies = [
-    "numpy",
-    "optuna",
-    "scikit-learn",
-    "faiss-cpu",
-    "chromadb",
-    "pytest",
-    "openai",
-    "tqdm",
-    "google-generativeai",
-    "google-genai",
+  # Core ML + Embeddings
+  "numpy<2.0.0",
+  "pandas>=2.0",
+  "scikit-learn>=1.3",
+  "sentence-transformers>=2.2.2",
+
+  # Retrieval backends
+  "chromadb>=0.4",
+  "faiss-cpu; sys_platform != 'darwin'",       # For Linux/Windows
+  "faiss-cpu==1.7.4; sys_platform == 'darwin'", # Optional fix for macOS MPS
+  "rank-bm25>=0.2.2",                          # For BM25 retriever
+
+  # Optimization & evaluation
+  "optuna>=3.0",
+  "tqdm",
+  "colorama",
+
+  # RAG evaluation and data utils
+  "pyyaml",
+  "python-dotenv",
+
+  # Explainability and LLM APIs
+  "openai>=1.0.0",
+  "google-generativeai>=0.8.0",
+  "anthropic>=0.25.0",
+
+  # Integration / storage
+  "supabase>=2.4.0",
+
+  # Testing
+  "pytest",
+
+  # LangChain integration layer
+  "langchain>=0.2.5",
+  "langchain-community>=0.2.5",
+  "langchain-text-splitters>=0.2.1"
 ]
 ```
 

ragmint-0.2.3/src/ragmint.egg-info/PKG-INFO → ragmint-0.3.1/README.md

@@ -1,31 +1,3 @@
-Metadata-Version: 2.4
-Name: ragmint
-Version: 0.2.3
-Summary: A modular framework for evaluating and optimizing RAG pipelines.
-Author-email: Andre Oliveira <oandreoliveira@outlook.com>
-License: Apache License 2.0
-Project-URL: Homepage, https://github.com/andyolivers/ragmint
-Project-URL: Documentation, https://andyolivers.com
-Project-URL: Issues, https://github.com/andyolivers/ragmint/issues
-Keywords: RAG,LLM,retrieval,optimization,AI,evaluation
-Requires-Python: >=3.9
-Description-Content-Type: text/markdown
-License-File: LICENSE
-Requires-Dist: numpy>=1.23
-Requires-Dist: pandas>=2.0
-Requires-Dist: scikit-learn>=1.3
-Requires-Dist: openai>=1.0
-Requires-Dist: tqdm
-Requires-Dist: pyyaml
-Requires-Dist: chromadb>=0.4
-Requires-Dist: faiss-cpu; sys_platform != "darwin"
-Requires-Dist: optuna>=3.0
-Requires-Dist: pytest
-Requires-Dist: colorama
-Requires-Dist: google-generativeai>=0.8.0
-Requires-Dist: supabase>=2.4.0
-Dynamic: license-file
-
 # Ragmint
 
 ![Python](https://img.shields.io/badge/python-3.9%2B-blue)
@@ -38,7 +10,7 @@ Dynamic: license-file
 
 **Ragmint** (Retrieval-Augmented Generation Model Inspection & Tuning) is a modular, developer-friendly Python library for **evaluating, optimizing, and tuning RAG (Retrieval-Augmented Generation) pipelines**.
 
-It provides a complete toolkit for **retriever selection**, **embedding model tuning**, and **automated RAG evaluation** with support for **Optuna-based Bayesian optimization**, **Auto-RAG tuning**, and **explainability** through Gemini or Claude.
+It provides a complete toolkit for **retriever selection**, **embedding model tuning**, **automated RAG evaluation**, and **config-driven prebuilding** of pipelines with support for **Optuna-based Bayesian optimization**, **Auto-RAG tuning**, **chunking**, and **explainability** through Gemini or Claude.
 
 ---
 
@@ -49,10 +21,13 @@ It provides a complete toolkit for **retriever selection**, **embedding model tu
 - 🧠 **Explainability Layer** — interprets RAG performance via Gemini or Claude APIs  
 - 🏆 **Leaderboard Tracking** — stores and ranks experiment runs via JSON or external DB  
 - 🔍 **Built-in RAG evaluation metrics** — faithfulness, recall, BLEU, ROUGE, latency  
-- ⚙️ **Retrievers** — FAISS, Chroma, ElasticSearch  
-- 🧩 **Embeddings** — OpenAI, HuggingFace  
+- ⚙️ **Retrievers** — FAISS, Chroma, scikit-learn  
+- 🧩 **Embeddings** — Hugging Face  
 - 💾 **Caching, experiment tracking, and reproducibility** out of the box  
 - 🧰 **Clean modular structure** for easy integration in research and production setups  
+- 📦 **Chunking system** — automatic or configurable chunk_size and overlap for documents  
+- 🏗️ **Langchain Prebuilder** — prepares pipelines, applies chunking, embeddings, and vector store creation automatically  
+- ⚙️ **Config Adapter (LangchainConfigAdapter)** — normalizes configuration, fills defaults, validates retrievers
 
 ---
 
@@ -81,6 +56,8 @@ Example `configs/default.yaml`:
 ```yaml
 retriever: faiss
 embedding_model: text-embedding-3-small
+chunk_size: 500
+overlap: 100
 reranker:
   mode: mmr
   lambda_param: 0.5
@@ -94,15 +71,94 @@ optimization:
 ### 3️⃣ Manual Pipeline Usage
 
 ```python
-from ragmint.core.pipeline import RAGPipeline
+from ragmint.prebuilder import PreBuilder
+from ragmint.tuner import RAGMint
+
+# Prebuild pipeline (chunking, embeddings, vector store)
+prebuilder = PreBuilder(
+    docs_path="data/docs/",
+    config_path="configs/default.yaml"
+)
+pipeline = prebuilder.build_pipeline()
+
+# Initialize RAGMint with prebuilt components
+rag = RAGMint(pipeline=pipeline)
+
+# Run optimization
+best, results = rag.optimize(validation_set=None, metric="faithfulness", trials=3)
+print("Best configuration:", best)
+
+```
+---
+# 🧩 Embeddings and Retrievers
+
+**Ragmint** supports a flexible set of embeddings and retrievers, allowing you to adapt easily to various **RAG architectures**.
+
+---
+## 🧩 Chunking System
+
+* **Automatically splits documents** into chunks with `chunk_size` and `overlap` parameters.
+* **Supports default values** if not provided in configuration.
+* **Optimized** for downstream **retrieval and embeddings**.
+* **Enables adaptive chunking strategies** in future releases.
+
+---
+## 🧩 Langchain Config Adapter
+
+* **Ensures consistent configuration** across pipeline components.
+* **Normalizes retriever and embedding names** (e.g., `faiss`, `sentence-transformers/...`).
+* **Adds default chunk parameters** when missing.
+* **Validates retriever backends** and **raises clear errors** for unsupported options.
+
+---
+## 🧩 Langchain Prebuilder
+
+**Automates pipeline preparation:**
+1. Reads documents
+2. Applies chunking
+3. Creates embeddings
+4. Initializes retriever / vector store
+5. Returns ready-to-use pipeline** for RAGMint or custom usage.
 
-pipeline = RAGPipeline({
-    "embedding_model": "text-embedding-3-small",
-    "retriever": "faiss",
-})
+---
+
+## 🔤 Available Embeddings (Hugging Face)
+
+You can select from the following models:
+
+* `sentence-transformers/all-MiniLM-L6-v2` — **lightweight**, general-purpose
+* `sentence-transformers/all-mpnet-base-v2` — **higher accuracy**, slower
+* `BAAI/bge-base-en-v1.5` — **multilingual**, dense embeddings
+* `intfloat/multilingual-e5-base` — ideal for **multilingual corpora**
+
+
+
+### Configuration Example
+
+Use the following format in your config file to specify the embedding model:
+
+```yaml
+embedding_model: sentence-transformers/all-MiniLM-L6-v2
+```
+---
+
+## 🔍 Available Retrievers
+
+**Ragmint** integrates multiple **retrieval backends** to suit different needs:
+
+| Retriever | Description |
+| :--- | :--- |
+| **FAISS** | Fast vector similarity search; efficient for dense embeddings |
+| **Chroma** | Persistent vector DB; works well for incremental updates |
+| **scikit-learn (NearestNeighbors)** | Lightweight, zero-dependency local retriever |
 
-result = pipeline.run("What is retrieval-augmented generation?")
-print(result)
+
+### Configuration Example
+
+To specify the retriever in your configuration file, use the following format:
+
+```yaml
+retriever: faiss
 ```
 
 ---
@@ -174,8 +230,7 @@ lb.show_top(3)
 
 ## 🧠 Explainability with Gemini / Claude
 
-Compare two RAG configurations and receive natural language insights
-on **why** one performs better.
+Compare two RAG configurations and receive **natural language insights** on why one performs better.
 
 ```python
 from ragmint.explainer import explain_results
@@ -189,7 +244,7 @@ print(explanation)
 
 > Set your API keys in a `.env` file or via environment variables:
 > ```
-> export GOOGLE_API_KEY="your_gemini_key"
+> export GEMINI_API_KEY="your_gemini_key"
 > export ANTHROPIC_API_KEY="your_claude_key"
 > ```
 
@@ -203,8 +258,12 @@ ragmint/
 │   ├── pipeline.py
 │   ├── retriever.py
 │   ├── reranker.py
-│   ├── embedding.py
+│   ├── embeddings.py
+│   ├── chunking.py
 │   └── evaluation.py
+├── integration/
+│   ├── config_adapter.py
+│   └── langchain_prebuilder.py
 ├── autotuner.py
 ├── explainer.py
 ├── leaderboard.py
@@ -240,16 +299,42 @@ Your `pyproject.toml` includes all required dependencies:
 name = "ragmint"
 version = "0.1.0"
 dependencies = [
-    "numpy",
-    "optuna",
-    "scikit-learn",
-    "faiss-cpu",
-    "chromadb",
-    "pytest",
-    "openai",
-    "tqdm",
-    "google-generativeai",
-    "google-genai",
+  # Core ML + Embeddings
+  "numpy<2.0.0",
+  "pandas>=2.0",
+  "scikit-learn>=1.3",
+  "sentence-transformers>=2.2.2",
+
+  # Retrieval backends
+  "chromadb>=0.4",
+  "faiss-cpu; sys_platform != 'darwin'",       # For Linux/Windows
+  "faiss-cpu==1.7.4; sys_platform == 'darwin'", # Optional fix for macOS MPS
+  "rank-bm25>=0.2.2",                          # For BM25 retriever
+
+  # Optimization & evaluation
+  "optuna>=3.0",
+  "tqdm",
+  "colorama",
+
+  # RAG evaluation and data utils
+  "pyyaml",
+  "python-dotenv",
+
+  # Explainability and LLM APIs
+  "openai>=1.0.0",
+  "google-generativeai>=0.8.0",
+  "anthropic>=0.25.0",
+
+  # Integration / storage
+  "supabase>=2.4.0",
+
+  # Testing
+  "pytest",
+
+  # LangChain integration layer
+  "langchain>=0.2.5",
+  "langchain-community>=0.2.5",
+  "langchain-text-splitters>=0.2.1"
 ]
 ```
 
@@ -309,4 +394,4 @@ Licensed under the **Apache License 2.0** — free for personal, research, and c
 
 **André Oliveira**  
 [andyolivers.com](https://andyolivers.com)  
-Data Scientist | AI Engineer
+Data Scientist | AI Engineer

{ragmint-0.2.3 → ragmint-0.3.1}/pyproject.toml

@@ -4,31 +4,58 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "ragmint"
-version = "0.2.3"
+version = "0.3.1"
 description = "A modular framework for evaluating and optimizing RAG pipelines."
 readme = "README.md"
 license = { text = "Apache License 2.0" }
 authors = [
   { name = "Andre Oliveira", email = "oandreoliveira@outlook.com" }
 ]
-keywords = ["RAG", "LLM", "retrieval", "optimization", "AI", "evaluation"]
+keywords = ["RAG", "LLM", "retrieval", "optimization", "AI", "evaluation", "chunking", "autotuning"]
 requires-python = ">=3.9"
 dependencies = [
-  "numpy>=1.23",
+  # Core ML + Embeddings
+  "numpy<2.0.0",
   "pandas>=2.0",
   "scikit-learn>=1.3",
-  "openai>=1.0",
-  "tqdm",
-  "pyyaml",
-  "chromadb>=0.4",
-  "faiss-cpu; sys_platform != 'darwin'",
+  "sentence-transformers>=2.2.2",
+
+  # Retrieval backends
+  "chromadb>=0.3.1",
+  "faiss-cpu; sys_platform != 'darwin'",       # For Linux/Windows
+  "faiss-cpu==1.7.4; sys_platform == 'darwin'", # Optional fix for macOS MPS
+  "rank-bm25>=0.2.2",                          # For BM25 retriever
+
+  # Optimization & evaluation
   "optuna>=3.0",
-  "pytest",
+  "tqdm",
   "colorama",
+
+  # RAG evaluation and data utils
+  "pyyaml",
+  "python-dotenv",
+
+  # Explainability and LLM APIs
+  "openai>=1.0.0",
   "google-generativeai>=0.8.0",
-  "supabase>=2.4.0"
+  "anthropic>=0.25.0",
+
+  # Integration / storage
+  "supabase>=2.4.0",
+
+  # Testing
+  "pytest",
+
+  # LangChain integration layer
+  "langchain>=0.2.5",
+  "langchain-community>=0.2.5",
+  "langchain-text-splitters>=0.2.1"
 ]
 
+[project.optional-dependencies]
+dev = ["black", "flake8", "isort", "pytest-cov"]
+docs = ["mkdocs", "mkdocs-material"]
+
 [project.urls]
 Homepage = "https://github.com/andyolivers/ragmint"
 Documentation = "https://andyolivers.com"
@@ -49,5 +76,4 @@ ragmint = ["experiments/*.json"]
 
 [tool.pytest.ini_options]
 testpaths = ["tests"]
-addopts = "-v"
-
+addopts = "-v --tb=short"