ragmint 0.1.0__py3-none-any.whl → 0.2.0__py3-none-any.whl

ragmint/autotuner.py

@@ -0,0 +1,33 @@
+"""
+Auto-RAG Tuner
+--------------
+Recommends retriever–embedding pairs dynamically based on corpus size
+and dataset characteristics. Integrates seamlessly with RAGMint evaluator.
+"""
+
+from .core.evaluation import evaluate_config
+
+
+class AutoRAGTuner:
+    def __init__(self, corpus_stats: dict):
+        """
+        corpus_stats: dict
+            Example: {'size': 12000, 'avg_len': 240}
+        """
+        self.corpus_stats = corpus_stats
+
+    def recommend(self):
+        size = self.corpus_stats.get("size", 0)
+        avg_len = self.corpus_stats.get("avg_len", 0)
+
+        if size < 1000:
+            return {"retriever": "BM25", "embedding_model": "OpenAI"}
+        elif size < 10000:
+            return {"retriever": "Chroma", "embedding_model": "SentenceTransformers"}
+        else:
+            return {"retriever": "FAISS", "embedding_model": "InstructorXL"}
+
+    def auto_tune(self, validation_data):
+        config = self.recommend()
+        results = evaluate_config(config, validation_data)
+        return {"recommended": config, "results": results}

ragmint/core/evaluation.py

@@ -25,3 +25,14 @@ class Evaluator:
 
     def _similarity(self, a: str, b: str) -> float:
         return SequenceMatcher(None, a, b).ratio()
+
+def evaluate_config(config, validation_data):
+    evaluator = Evaluator()
+    results = []
+    for sample in validation_data:
+        query = sample.get("query", "")
+        answer = sample.get("answer", "")
+        context = sample.get("context", "")
+        results.append(evaluator.evaluate(query, answer, context))
+    return results
+

ragmint/explainer.py

@@ -0,0 +1,61 @@
+"""
+Interpretability Layer
+----------------------
+Uses Gemini or Anthropic Claude to explain why one RAG configuration
+outperforms another. Falls back gracefully if no API key is provided.
+"""
+
+import os
+import json
+
+
+def explain_results(results_a: dict, results_b: dict, model: str = "gemini-1.5-pro") -> str:
+    """
+    Generate a natural-language explanation comparing two RAG experiment results.
+    Priority:
+      1. Anthropic Claude (if ANTHROPIC_API_KEY is set)
+      2. Google Gemini (if GOOGLE_API_KEY is set)
+      3. Fallback text message
+    """
+    prompt = f"""
+    You are an AI evaluation expert.
+    Compare these two RAG experiment results and explain why one performs better.
+    Metrics A: {json.dumps(results_a, indent=2)}
+    Metrics B: {json.dumps(results_b, indent=2)}
+    Provide a concise, human-friendly explanation and practical improvement tips.
+    """
+
+    anthropic_key = os.getenv("ANTHROPIC_API_KEY")
+    google_key = os.getenv("GEMINI_API_KEY")
+
+
+    # 1️⃣ Try Anthropic Claude first
+    if anthropic_key:
+        try:
+            from anthropic import Anthropic
+            client = Anthropic(api_key=anthropic_key)
+            response = client.messages.create(
+                model="claude-3-opus-20240229",
+                max_tokens=300,
+                messages=[{"role": "user", "content": prompt}],
+            )
+            return response.content[0].text
+        except Exception as e:
+            return f"[Claude unavailable] {e}"
+
+    # 2️⃣ Fallback to Google Gemini
+    elif google_key:
+        try:
+            import google.generativeai as genai
+            genai.configure(api_key=google_key)
+            response = genai.GenerativeModel(model).generate_content(prompt)
+            return response.text
+        except Exception as e:
+            return f"[Gemini unavailable] {e}"
+
+    # 3️⃣ Fallback if neither key is available
+    else:
+        return (
+            "[No LLM available] Please set ANTHROPIC_API_KEY or GOOGLE_API_KEY "
+            "to enable interpretability via Claude or Gemini."
+        )

ragmint/leaderboard.py

@@ -0,0 +1,45 @@
+import os
+import json
+from datetime import datetime
+from typing import Dict, Any, Optional
+from supabase import create_client
+
+class Leaderboard:
+    def __init__(self, storage_path: Optional[str] = None):
+        self.storage_path = storage_path
+        url = os.getenv("SUPABASE_URL")
+        key = os.getenv("SUPABASE_KEY")
+        self.client = None
+        if url and key:
+            self.client = create_client(url, key)
+        elif not storage_path:
+            raise EnvironmentError("Set SUPABASE_URL/SUPABASE_KEY or pass storage_path")
+
+    def upload(self, run_id: str, config: Dict[str, Any], score: float):
+        data = {
+            "run_id": run_id,
+            "config": config,
+            "score": score,
+            "timestamp": datetime.utcnow().isoformat(),
+        }
+        if self.client:
+            return self.client.table("experiments").insert(data).execute()
+        else:
+            os.makedirs(os.path.dirname(self.storage_path), exist_ok=True)
+            with open(self.storage_path, "a", encoding="utf-8") as f:
+                f.write(json.dumps(data) + "\n")
+            return data
+
+    def top_results(self, limit: int = 10):
+        if self.client:
+            return (
+                self.client.table("experiments")
+                .select("*")
+                .order("score", desc=True)
+                .limit(limit)
+                .execute()
+            )
+        else:
+            with open(self.storage_path, "r", encoding="utf-8") as f:
+                lines = [json.loads(line) for line in f]
+            return sorted(lines, key=lambda x: x["score"], reverse=True)[:limit]

ragmint/tests/conftest.py

@@ -0,0 +1,16 @@
+# src/ragmint/tests/conftest.py
+import os
+from dotenv import load_dotenv
+import pytest
+
+# Load .env from project root
+load_dotenv(dotenv_path=os.path.join(os.path.dirname(__file__), "../../../.env"))
+
+def pytest_configure(config):
+    """Print which keys are loaded (debug)."""
+    google = os.getenv("GEMINI_API_KEY")
+    anthropic = os.getenv("ANTHROPIC_API_KEY")
+    if google:
+        print("✅ GOOGLE_API_KEY loaded")
+    if anthropic:
+        print("✅ ANTHROPIC_API_KEY loaded")

ragmint/tests/test_autotuner.py

@@ -0,0 +1,42 @@
+import pytest
+from ragmint.autotuner import AutoRAGTuner
+
+
+def test_autorag_recommend_small():
+    """Small corpus should trigger BM25 + OpenAI."""
+    tuner = AutoRAGTuner({"size": 500, "avg_len": 150})
+    rec = tuner.recommend()
+    assert rec["retriever"] == "BM25"
+    assert rec["embedding_model"] == "OpenAI"
+
+
+def test_autorag_recommend_medium():
+    """Medium corpus should trigger Chroma + SentenceTransformers."""
+    tuner = AutoRAGTuner({"size": 5000, "avg_len": 200})
+    rec = tuner.recommend()
+    assert rec["retriever"] == "Chroma"
+    assert rec["embedding_model"] == "SentenceTransformers"
+
+
+def test_autorag_recommend_large():
+    """Large corpus should trigger FAISS + InstructorXL."""
+    tuner = AutoRAGTuner({"size": 50000, "avg_len": 300})
+    rec = tuner.recommend()
+    assert rec["retriever"] == "FAISS"
+    assert rec["embedding_model"] == "InstructorXL"
+
+
+def test_autorag_auto_tune(monkeypatch):
+    """Test auto_tune with a mock validation dataset."""
+    tuner = AutoRAGTuner({"size": 12000, "avg_len": 250})
+
+    # Monkeypatch evaluate_config inside autotuner
+    import ragmint.autotuner as autotuner
+    def mock_eval(config, data):
+        return {"faithfulness": 0.9, "latency": 0.01}
+    monkeypatch.setattr(autotuner, "evaluate_config", mock_eval)
+
+    result = tuner.auto_tune([{"question": "What is AI?", "answer": "Artificial Intelligence"}])
+    assert "recommended" in result
+    assert "results" in result
+    assert isinstance(result["results"], dict)

ragmint/tests/test_explainer.py

@@ -0,0 +1,20 @@
+import pytest
+from ragmint.explainer import explain_results
+
+
+def test_explain_results_gemini():
+    """Gemini explanation should contain model-specific phrasing."""
+    config_a = {"retriever": "FAISS", "embedding_model": "OpenAI"}
+    config_b = {"retriever": "Chroma", "embedding_model": "SentenceTransformers"}
+    result = explain_results(config_a, config_b, model="gemini")
+    assert isinstance(result, str)
+    assert "Gemini" in result or "gemini" in result
+
+
+def test_explain_results_claude():
+    """Claude explanation should contain model-specific phrasing."""
+    config_a = {"retriever": "FAISS"}
+    config_b = {"retriever": "Chroma"}
+    result = explain_results(config_a, config_b, model="claude")
+    assert isinstance(result, str)
+    assert "Claude" in result or "claude" in result

ragmint/tests/test_explainer_integration.py

@@ -0,0 +1,18 @@
+import os
+import pytest
+from ragmint.explainer import explain_results
+
+
+@pytest.mark.integration
+def test_real_gemini_explanation():
+    """Run real Gemini call if GOOGLE_API_KEY is set."""
+    if not os.getenv("GEMINI_API_KEY"):
+        pytest.skip("GOOGLE_API_KEY not set")
+
+    config_a = {"retriever": "FAISS", "embedding_model": "OpenAI"}
+    config_b = {"retriever": "Chroma", "embedding_model": "SentenceTransformers"}
+
+    result = explain_results(config_a, config_b, model="gemini-1.5-pro")
+    assert isinstance(result, str)
+    assert len(result) > 0
+    print("\n[Gemini explanation]:", result[:200], "...")

ragmint/tests/test_integration_autotuner_ragmint.py

@@ -0,0 +1,60 @@
+import pytest
+from ragmint.tuner import RAGMint
+from ragmint.autotuner import AutoRAGTuner
+
+
+def test_integration_ragmint_autotune(monkeypatch, tmp_path):
+    """
+    Smoke test for integration between AutoRAGTuner and RAGMint.
+    Ensures end-to-end flow runs without real retrievers or embeddings.
+    """
+
+    # --- Mock corpus and validation data ---
+    corpus = tmp_path / "docs"
+    corpus.mkdir()
+    (corpus / "doc1.txt").write_text("This is an AI document.")
+    validation_data = [{"question": "What is AI?", "answer": "Artificial Intelligence"}]
+
+    # --- Mock RAGMint.optimize() to avoid real model work ---
+    def mock_optimize(self, validation_set=None, metric="faithfulness", trials=2):
+        return (
+            {"retriever": "FAISS", "embedding_model": "OpenAI", "score": 0.88},
+            [{"trial": 1, "score": 0.88}],
+        )
+
+    monkeypatch.setattr(RAGMint, "optimize", mock_optimize)
+
+    # --- Mock evaluation used by AutoRAGTuner ---
+    def mock_evaluate_config(config, data):
+        return {"faithfulness": 0.9, "latency": 0.01}
+
+    import ragmint.autotuner as autotuner
+    monkeypatch.setattr(autotuner, "evaluate_config", mock_evaluate_config)
+
+    # --- Create AutoRAGTuner and RAGMint instances ---
+    ragmint = RAGMint(
+        docs_path=str(corpus),
+        retrievers=["faiss", "chroma"],
+        embeddings=["text-embedding-3-small"],
+        rerankers=["mmr"],
+    )
+
+    tuner = AutoRAGTuner({"size": 2000, "avg_len": 150})
+
+    # --- Run Auto-Tune and RAG Optimization ---
+    recommendation = tuner.recommend()
+    assert "retriever" in recommendation
+    assert "embedding_model" in recommendation
+
+    tuning_results = tuner.auto_tune(validation_data)
+    assert "results" in tuning_results
+    assert isinstance(tuning_results["results"], dict)
+
+    # --- Run RAGMint optimization flow (mocked) ---
+    best_config, results = ragmint.optimize(validation_set=validation_data, trials=2)
+    assert isinstance(best_config, dict)
+    assert "score" in best_config
+    assert isinstance(results, list)
+
+    # --- Integration Success ---
+    print(f"Integration OK: AutoRAG recommended {recommendation}, RAGMint best {best_config}")

ragmint/tests/test_leaderboard.py

@@ -0,0 +1,39 @@
+import json
+import tempfile
+from pathlib import Path
+from ragmint.leaderboard import Leaderboard
+
+
+def test_leaderboard_add_and_top(tmp_path):
+    """Ensure local leaderboard persistence works without Supabase."""
+    file_path = tmp_path / "leaderboard.jsonl"
+    lb = Leaderboard(storage_path=str(file_path))
+
+    # Add two runs
+    lb.upload("run1", {"retriever": "FAISS"}, 0.91)
+    lb.upload("run2", {"retriever": "Chroma"}, 0.85)
+
+    # Verify file content
+    assert file_path.exists()
+    with open(file_path, "r", encoding="utf-8") as f:
+        lines = [json.loads(line) for line in f]
+    assert len(lines) == 2
+
+    # Get top results
+    top = lb.top_results(limit=1)
+    assert isinstance(top, list)
+    assert len(top) == 1
+    assert "score" in top[0]
+
+
+def test_leaderboard_append_existing(tmp_path):
+    """Ensure multiple uploads append properly."""
+    file_path = tmp_path / "leaderboard.jsonl"
+    lb = Leaderboard(storage_path=str(file_path))
+
+    for i in range(3):
+        lb.upload(f"run{i}", {"retriever": "BM25"}, 0.8 + i * 0.05)
+
+    top = lb.top_results(limit=2)
+    assert len(top) == 2
+    assert top[0]["score"] >= top[1]["score"]

ragmint/tests/test_tuner.py

@@ -1,9 +1,11 @@
 import os
 import json
+import pytest
 from ragmint.tuner import RAGMint
 
 
 def setup_validation_file(tmp_path):
+    """Create a temporary validation QA dataset."""
     data = [
         {"question": "What is AI?", "answer": "Artificial Intelligence"},
         {"question": "Define ML", "answer": "Machine Learning"}
@@ -15,6 +17,7 @@ def setup_validation_file(tmp_path):
 
 
 def setup_docs(tmp_path):
+    """Create a small document corpus for testing."""
     corpus = tmp_path / "corpus"
     corpus.mkdir()
     (corpus / "doc1.txt").write_text("This is about Artificial Intelligence.")
@@ -22,17 +25,47 @@ def setup_docs(tmp_path):
     return str(corpus)
 
 
-def test_optimize_random(tmp_path):
+@pytest.mark.parametrize("validation_mode", [
+    None,  # Built-in dataset
+    "data/custom_eval.json",  # Custom dataset path (mocked below)
+])
+def test_optimize_ragmint(tmp_path, validation_mode, monkeypatch):
+    """Test RAGMint.optimize() with different dataset modes."""
     docs_path = setup_docs(tmp_path)
     val_file = setup_validation_file(tmp_path)
 
+    # If using custom dataset, mock the path
+    if validation_mode and "custom_eval" in validation_mode:
+        custom_path = tmp_path / "custom_eval.json"
+        os.rename(val_file, custom_path)
+        validation_mode = str(custom_path)
+
+    metric = "faithfulness"
+
+    # Initialize RAGMint
     rag = RAGMint(
         docs_path=docs_path,
         retrievers=["faiss"],
-        embeddings=["openai/text-embedding-3-small"],
+        embeddings=["text-embedding-3-small"],
         rerankers=["mmr"]
     )
 
-    best, results = rag.optimize(validation_set=val_file, metric="faithfulness", trials=2)
-    assert isinstance(best, dict)
-    assert isinstance(results, list)
+    # Run optimization
+    best, results = rag.optimize(
+        validation_set=validation_mode,
+        metric=metric,
+        trials=2
+    )
+
+    # Validate results
+    assert isinstance(best, dict), "Best config should be a dict"
+    assert isinstance(results, list), "Results should be a list of trials"
+    assert len(results) > 0, "Optimization should produce results"
+
+    # The best result can expose either 'score' or the metric name (e.g. 'faithfulness')
+    assert any(k in best for k in ("score", metric)), \
+        f"Best config should include either 'score' or '{metric}'"
+
+    # Ensure the metric value is valid
+    assert best.get(metric, best.get("score")) >= 0, \
+        f"{metric} score should be non-negative"

ragmint/tuner.py

@@ -90,7 +90,7 @@ class RAGMint:
         search_type: str = "random",
         trials: int = 10,
     ) -> Tuple[Dict[str, Any], List[Dict[str, Any]]]:
-        validation = load_validation_set(validation_set)
+        validation = load_validation_set(validation_set or "default")
 
         search_space = {
             "retriever": self.retrievers,

ragmint/utils/data_loader.py

@@ -2,6 +2,14 @@ import json
 import csv
 from typing import List, Dict
 from pathlib import Path
+import os
+
+try:
+    from datasets import load_dataset
+except ImportError:
+    load_dataset = None  # optional dependency
+
+DEFAULT_VALIDATION_PATH = Path(__file__).parent.parent / "experiments" / "validation_qa.json"
 
 
 def load_json(path: str) -> List[Dict]:
@@ -19,10 +27,32 @@ def save_json(path: str, data: Dict):
     with open(path, "w", encoding="utf-8") as f:
         json.dump(data, f, ensure_ascii=False, indent=2)
 
-def load_validation_set(path: str) -> List[Dict]:
+def load_validation_set(path: str | None = None) -> List[Dict]:
     """
-    Loads a validation dataset (QA pairs) from JSON or CSV.
+    Loads a validation dataset (QA pairs) from:
+    - Built-in default JSON file
+    - User-provided JSON or CSV
+    - Hugging Face dataset by name
     """
+    # Default behavior
+    if path is None or path == "default":
+        if not DEFAULT_VALIDATION_PATH.exists():
+            raise FileNotFoundError(f"Default validation set not found at {DEFAULT_VALIDATION_PATH}")
+        return load_json(DEFAULT_VALIDATION_PATH)
+
+    # Hugging Face dataset
+    if not os.path.exists(path) and load_dataset:
+        try:
+            dataset = load_dataset(path, split="validation")
+            data = [
+                {"question": q, "answer": a}
+                for q, a in zip(dataset["question"], dataset["answers"])
+            ]
+            return data
+        except Exception:
+            pass  # fall through to file loading
+
+    # Local file
     p = Path(path)
     if not p.exists():
         raise FileNotFoundError(f"Validation file not found: {path}")
@@ -32,4 +62,4 @@ def load_validation_set(path: str) -> List[Dict]:
     elif p.suffix.lower() in [".csv", ".tsv"]:
         return load_csv(path)
     else:
-        raise ValueError("Unsupported validation set format. Use JSON or CSV.")
+        raise ValueError("Unsupported validation set format. Use JSON, CSV, or a Hugging Face dataset name.")

{ragmint-0.1.0.dist-info → ragmint-0.2.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ragmint
-Version: 0.1.0
+Version: 0.2.0
 Summary: A modular framework for evaluating and optimizing RAG pipelines.
 Author-email: Andre Oliveira <oandreoliveira@outlook.com>
 License: Apache License 2.0
@@ -22,6 +22,8 @@ Requires-Dist: faiss-cpu; sys_platform != "darwin"
 Requires-Dist: optuna>=3.0
 Requires-Dist: pytest
 Requires-Dist: colorama
+Requires-Dist: google-generativeai>=0.8.0
+Requires-Dist: supabase>=2.4.0
 Dynamic: license-file
 
 # Ragmint
@@ -36,17 +38,19 @@ Dynamic: license-file
 
 **Ragmint** (Retrieval-Augmented Generation Model Inspection & Tuning) is a modular, developer-friendly Python library for **evaluating, optimizing, and tuning RAG (Retrieval-Augmented Generation) pipelines**.
 
-It provides a complete toolkit for **retriever selection**, **embedding model tuning**, and **automated RAG evaluation** with support for **Optuna-based Bayesian optimization**.
+It provides a complete toolkit for **retriever selection**, **embedding model tuning**, and **automated RAG evaluation** with support for **Optuna-based Bayesian optimization**, **Auto-RAG tuning**, and **explainability** through Gemini or Claude.
 
 ---
 
 ## ✨ Features
 
 - ✅ **Automated hyperparameter optimization** (Grid, Random, Bayesian via Optuna)  
+- 🤖 **Auto-RAG Tuner** — dynamically recommends retriever–embedding pairs based on corpus size  
+- 🧠 **Explainability Layer** — interprets RAG performance via Gemini or Claude APIs  
+- 🏆 **Leaderboard Tracking** — stores and ranks experiment runs via JSON or external DB  
 - 🔍 **Built-in RAG evaluation metrics** — faithfulness, recall, BLEU, ROUGE, latency  
 - ⚙️ **Retrievers** — FAISS, Chroma, ElasticSearch  
 - 🧩 **Embeddings** — OpenAI, HuggingFace  
-- 🧠 **Rerankers** — MMR, CrossEncoder (extensible via plugin interface)  
 - 💾 **Caching, experiment tracking, and reproducibility** out of the box  
 - 🧰 **Clean modular structure** for easy integration in research and production setups  
 
@@ -103,47 +107,133 @@ print(result)
 
 ---
 
+## 🧪 Dataset Options
+
+Ragmint can automatically load evaluation datasets for your RAG pipeline:
+
+| Mode | Example | Description |
+|------|----------|-------------|
+| 🧱 **Default** | `validation_set=None` | Uses built-in `experiments/validation_qa.json` |
+| 📁 **Custom File** | `validation_set="data/my_eval.json"` | Load your own QA dataset (JSON or CSV) |
+| 🌐 **Hugging Face Dataset** | `validation_set="squad"` | Automatically downloads benchmark datasets (requires `pip install datasets`) |
+
+### Example
+
+```python
+from ragmint.tuner import RAGMint
+
+ragmint = RAGMint(
+    docs_path="data/docs/",
+    retrievers=["faiss", "chroma"],
+    embeddings=["text-embedding-3-small"],
+    rerankers=["mmr"],
+)
+
+# Use built-in default
+ragmint.optimize(validation_set=None)
+
+# Use Hugging Face benchmark
+ragmint.optimize(validation_set="squad")
+
+# Use your own dataset
+ragmint.optimize(validation_set="data/custom_qa.json")
+```
+
+---
+
+## 🧠 Auto-RAG Tuner
+
+The **AutoRAGTuner** automatically recommends retriever–embedding combinations
+based on corpus size and average document length.
+
+```python
+from ragmint.autotuner import AutoRAGTuner
+
+corpus_stats = {"size": 5000, "avg_len": 250}
+tuner = AutoRAGTuner(corpus_stats)
+recommendation = tuner.recommend()
+print(recommendation)
+# Example output: {"retriever": "Chroma", "embedding_model": "SentenceTransformers"}
+```
+
+---
+
+## 🏆 Leaderboard Tracking
+
+Track and visualize your best experiments across runs.
+
+```python
+from ragmint.leaderboard import Leaderboard
+
+lb = Leaderboard("experiments/leaderboard.json")
+lb.add_entry({"trial": 1, "faithfulness": 0.87, "latency": 0.12})
+lb.show_top(3)
+```
+
+---
+
+## 🧠 Explainability with Gemini / Claude
+
+Compare two RAG configurations and receive natural language insights
+on **why** one performs better.
+
+```python
+from ragmint.explainer import explain_results
+
+config_a = {"retriever": "FAISS", "embedding_model": "OpenAI"}
+config_b = {"retriever": "Chroma", "embedding_model": "SentenceTransformers"}
+
+explanation = explain_results(config_a, config_b, model="gemini")
+print(explanation)
+```
+
+> Set your API keys in a `.env` file or via environment variables:
+> ```
+> export GOOGLE_API_KEY="your_gemini_key"
+> export ANTHROPIC_API_KEY="your_claude_key"
+> ```
+
+---
+
 ## 🧩 Folder Structure
 
 ```
 ragmint/
 ├── core/
-│   ├── pipeline.py         # RAGPipeline implementation
-│   ├── retriever.py        # Retriever logic (FAISS, Chroma)
-│   ├── reranker.py         # MMR + CrossEncoder rerankers
-│   └── embedding.py        # Embedding backends
-├── tuner.py                # Grid, Random, Bayesian optimization (Optuna)
-├── utils/                  # Metrics, logging, caching helpers
-├── configs/                # Default experiment configs
-├── experiments/            # Saved experiment results
-├── tests/                  # Unit tests for all components
-├── main.py                 # CLI entrypoint for tuning
-└── pyproject.toml          # Project dependencies & build metadata
+│   ├── pipeline.py
+│   ├── retriever.py
+│   ├── reranker.py
+│   ├── embedding.py
+│   └── evaluation.py
+├── autotuner.py
+├── explainer.py
+├── leaderboard.py
+├── tuner.py
+├── utils/
+├── configs/
+├── experiments/
+├── tests/
+└── main.py
 ```
 
 ---
 
 ## 🧪 Running Tests
 
-To verify your setup:
-
 ```bash
 pytest -v
 ```
 
-Or to test a specific component (e.g., reranker):
-
+To include integration tests with Gemini or Claude APIs:
 ```bash
-pytest tests/test_reranker.py -v
+pytest -m integration
 ```
 
-All tests are designed for **Pytest** and run with lightweight mock data.
-
 ---
 
 ## ⚙️ Configuration via `pyproject.toml`
 
-Your `pyproject.toml` automatically includes:
+Your `pyproject.toml` includes all required dependencies:
 
 ```toml
 [project]
@@ -158,6 +248,8 @@ dependencies = [
     "pytest",
     "openai",
     "tqdm",
+    "google-generativeai",
+    "google-genai",
 ]
 ```
 
@@ -165,10 +257,10 @@ dependencies = [
 
 ## 📊 Example Experiment Workflow
 
-1. Define your retriever and reranker configuration in YAML  
-2. Launch an optimization search (Grid, Random, or Bayesian)  
-3. Ragmint evaluates combinations automatically and reports top results  
-4. Export best parameters for production pipelines  
+1. Define your retriever, embedding, and reranker setup  
+2. Launch optimization (Grid, Random, Bayesian) or AutoTune  
+3. Compare performance with explainability  
+4. Persist results to leaderboard for later inspection  
 
 ---
 
@@ -181,7 +273,7 @@ flowchart TD
     C --> D[Reranker]
     D --> E[Generator]
     E --> F[Evaluation]
-    F --> G[Optuna Tuner]
+    F --> G[Optuna / AutoRAGTuner]
     G -->|Best Params| B
 ```
 
@@ -191,8 +283,9 @@ flowchart TD
 
 ```
 [INFO] Starting Bayesian optimization with Optuna
-[INFO] Trial 7 finished: recall=0.83, latency=0.42s
+[INFO] Trial 7 finished: faithfulness=0.83, latency=0.42s
 [INFO] Best parameters: {'lambda_param': 0.6, 'retriever': 'faiss'}
+[INFO] AutoRAGTuner: Suggested retriever=Chroma for medium corpus
 ```
 
 ---
@@ -200,8 +293,9 @@ flowchart TD
 ## 🧠 Why Ragmint?
 
 - Built for **RAG researchers**, **AI engineers**, and **LLM ops**  
-- Works with **LangChain**, **LlamaIndex**, or standalone RAG setups  
-- Designed for **extensibility** — plug in your own models, retrievers, or metrics  
+- Works with **LangChain**, **LlamaIndex**, or standalone setups  
+- Designed for **extensibility** — plug in your own retrievers, models, or metrics  
+- Integrated **explainability and leaderboard** modules for research and production  
 
 ---
 

{ragmint-0.1.0.dist-info → ragmint-0.2.0.dist-info}/RECORD

@@ -1,10 +1,13 @@
 ragmint/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ragmint/__main__.py,sha256=q7hBn56Z1xAckbs03i8ynsuOzJVUXmod2qHddX7gkpc,729
-ragmint/tuner.py,sha256=sCUb-qGqk-lz4nUJboomwXFt3us7mYf3oJhwWV9Kzo4,4429
+ragmint/autotuner.py,sha256=eXEH4e_3Os9FPX9y_0N7GnIQsmoHxmFbtjj7xanu17g,1064
+ragmint/explainer.py,sha256=1glGNdC4GlwR6Qs8Bj1oOol7f5_db7Ksnh07HAp-A2c,2077
+ragmint/leaderboard.py,sha256=nILQ5QR63RpZtCrZ__RFfwHXy4bkUIMUcSfH92OQ93Y,1628
+ragmint/tuner.py,sha256=BLPZ66sVk3dh3Wj-GVUYRVmVtgXYTzv3oTQtKJeDlgE,4442
 ragmint/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ragmint/core/chunking.py,sha256=Dy9RYyapGSS6ik6Vg9lqbUPCFqSraU1JKpHbYUTkaFo,576
 ragmint/core/embeddings.py,sha256=6wJjfZ5ukr8G5bJJ1evjIqj0_FMbs_gq4xC-sBBqNlA,566
-ragmint/core/evaluation.py,sha256=LcR9AIsL9OyoENrUVSu0hhKzAItcBvEOy33V4i-0DtI,682
+ragmint/core/evaluation.py,sha256=3OFcZU2zZyaP53d9S2zdpknV0CYfTq0KoRB3a_dtjM4,1022
 ragmint/core/pipeline.py,sha256=2qwGKuG0Du7gtIpieLFn71h_RcwBpjcV-h9PQz2ZOsc,1169
 ragmint/core/reranker.py,sha256=B2-NDExqpd9jdXHkEHOXC0B_6-FMJm5vdi-_ZbxC3Os,2303
 ragmint/core/retriever.py,sha256=jbpKy_fGdDq736y0es_utQuLqY9eiWNd71Q8JbU0Sko,1259
@@ -12,17 +15,23 @@ ragmint/experiments/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSu
 ragmint/optimization/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ragmint/optimization/search.py,sha256=uiLJeoO_jaLCQEw99L6uI1rnqHHx_rTY81WxfMmlALs,1623
 ragmint/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ragmint/tests/conftest.py,sha256=QhYPp5nrQ_DbZlsVH3nLjDgjPATAnLwzJkwl-Y-xrmM,488
+ragmint/tests/test_autotuner.py,sha256=k5nsIH6MYB5zaocR_Wn1wTX-QDYfhH6ugx2chZu9Q8U,1500
+ragmint/tests/test_explainer.py,sha256=K_DRnGGl34WcTA2yaQGmfzWkVi1uEkzjpsTPeZxXeIg,802
+ragmint/tests/test_explainer_integration.py,sha256=tYT62fYqk616bjQ1VxHADVRfJ9vdF_CiF3cz4A9BdbE,620
+ragmint/tests/test_integration_autotuner_ragmint.py,sha256=YCGge0_KOijAdB7VNDGHl2VRJjiOyl_-sJNRLjAXGLw,2182
+ragmint/tests/test_leaderboard.py,sha256=ay81YK6KxAUU6mcG6n1_xV8GPYkBgjzJj9iAIyAzIzA,1163
 ragmint/tests/test_pipeline.py,sha256=MIMkEKelh-POlbXzbCc4ClMk8XCGzfuj569xXltziic,615
 ragmint/tests/test_retriever.py,sha256=Ag0uGW8-iMzKA4nJNnsjuzlQHa79sN-T-K1g1cdin-A,421
 ragmint/tests/test_search.py,sha256=FcC-DEnw9veAEyMnFoRw9DAwzqJC9F6-r63Nqo2nO58,598
-ragmint/tests/test_tuner.py,sha256=VFZ23og0dOypBpr3TxkRmSngilkNgyboZc6u9qB0pME,1101
+ragmint/tests/test_tuner.py,sha256=LOvtIxAbUsoRHQudZ23UVr60FYAU0a1SBNvAN0mLpfU,2322
 ragmint/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ragmint/utils/caching.py,sha256=LPE2JorOQ90BgVf6NUiS0-bdt-FGpNxDy7FnuwEHzy0,1060
-ragmint/utils/data_loader.py,sha256=Q3pBO77XZ1rl4fuMn3TK7x3mSM2eLdV_OJTyy_eL3Ys,988
+ragmint/utils/data_loader.py,sha256=GXU9Nc3o0UWxtBeRwiskD1aCjSiNNuRoAokIUODn7q8,2024
 ragmint/utils/logger.py,sha256=X7hTNb3st3fUeQIzSghuoV5B8FWXzm_O3DRkSfJvhmI,1033
 ragmint/utils/metrics.py,sha256=DR8mrdumHtQerK0VrugwYKIG1oNptEcsFqodXq3i2kY,717
-ragmint-0.1.0.dist-info/licenses/LICENSE,sha256=ahkhYfFLI8tGrdxdO2_GaT6OJW2eNwyFT3kYi85QQhc,692
-ragmint-0.1.0.dist-info/METADATA,sha256=BgMj5BxH2C2_5GweYpClkopepUBCVen5tWAFcOby8o8,5643
-ragmint-0.1.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-ragmint-0.1.0.dist-info/top_level.txt,sha256=K2ulzMHuvFm6xayvvJdGABeRJAvKDBn6M3EI-3SbYLw,8
-ragmint-0.1.0.dist-info/RECORD,,
+ragmint-0.2.0.dist-info/licenses/LICENSE,sha256=ahkhYfFLI8tGrdxdO2_GaT6OJW2eNwyFT3kYi85QQhc,692
+ragmint-0.2.0.dist-info/METADATA,sha256=uwavcr5XnbneN7d7kfKiBD-Uc5TIIZFThmVhNjGWb0o,7948
+ragmint-0.2.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+ragmint-0.2.0.dist-info/top_level.txt,sha256=K2ulzMHuvFm6xayvvJdGABeRJAvKDBn6M3EI-3SbYLw,8
+ragmint-0.2.0.dist-info/RECORD,,