ragmint 0.2.1__py3-none-any.whl

ragmint/__main__.py

@@ -0,0 +1,28 @@
+from pathlib import Path
+from ragmint.tuner import RAGMint
+
+def main():
+    # Dynamically resolve the path to the installed ragmint package
+    base_dir = Path(__file__).resolve().parent
+
+    docs_path = base_dir / "experiments" / "corpus"
+    validation_file = base_dir / "experiments" / "validation_qa.json"
+
+    rag = RAGMint(
+        docs_path=str(docs_path),
+        retrievers=["faiss"],
+        embeddings=["openai/text-embedding-3-small"],
+        rerankers=["mmr"],
+    )
+
+    best, results = rag.optimize(
+        validation_set=str(validation_file),
+        metric="faithfulness",
+        search_type="bayesian",
+        trials=10,
+    )
+
+    print("Best config found:\n", best)
+
+if __name__ == "__main__":
+    main()

ragmint/autotuner.py

@@ -0,0 +1,33 @@
+"""
+Auto-RAG Tuner
+--------------
+Recommends retriever–embedding pairs dynamically based on corpus size
+and dataset characteristics. Integrates seamlessly with RAGMint evaluator.
+"""
+
+from .core.evaluation import evaluate_config
+
+
+class AutoRAGTuner:
+    def __init__(self, corpus_stats: dict):
+        """
+        corpus_stats: dict
+            Example: {'size': 12000, 'avg_len': 240}
+        """
+        self.corpus_stats = corpus_stats
+
+    def recommend(self):
+        size = self.corpus_stats.get("size", 0)
+        avg_len = self.corpus_stats.get("avg_len", 0)
+
+        if size < 1000:
+            return {"retriever": "BM25", "embedding_model": "OpenAI"}
+        elif size < 10000:
+            return {"retriever": "Chroma", "embedding_model": "SentenceTransformers"}
+        else:
+            return {"retriever": "FAISS", "embedding_model": "InstructorXL"}
+
+    def auto_tune(self, validation_data):
+        config = self.recommend()
+        results = evaluate_config(config, validation_data)
+        return {"recommended": config, "results": results}

ragmint/core/chunking.py

@@ -0,0 +1,22 @@
+from typing import List
+
+
+class Chunker:
+    """
+    Handles text chunking and splitting strategies:
+    - Fixed size chunks
+    - Overlapping windows
+    """
+
+    def __init__(self, chunk_size: int = 500, overlap: int = 100):
+        self.chunk_size = chunk_size
+        self.overlap = overlap
+
+    def chunk_text(self, text: str) -> List[str]:
+        chunks = []
+        start = 0
+        while start < len(text):
+            end = start + self.chunk_size
+            chunks.append(text[start:end])
+            start += self.chunk_size - self.overlap
+        return chunks

ragmint/core/embeddings.py

@@ -0,0 +1,19 @@
+import numpy as np
+
+
+class EmbeddingModel:
+    """
+    Wrapper for embedding backends (OpenAI, HuggingFace, etc.)
+    """
+
+    def __init__(self, backend: str = "dummy"):
+        self.backend = backend
+
+    def encode(self, texts):
+        if self.backend == "openai":
+            # Example placeholder — integrate with actual OpenAI API
+            return [np.random.rand(768) for _ in texts]
+        elif self.backend == "huggingface":
+            return [np.random.rand(768) for _ in texts]
+        else:
+            return [np.random.rand(768) for _ in texts]

ragmint/core/evaluation.py

@@ -0,0 +1,38 @@
+import time
+from typing import Dict, Any
+from difflib import SequenceMatcher
+
+
+class Evaluator:
+    """
+    Simple evaluation of generated answers:
+      - Faithfulness (similarity between answer and context)
+      - Latency
+    """
+
+    def __init__(self):
+        pass
+
+    def evaluate(self, query: str, answer: str, context: str) -> Dict[str, Any]:
+        start = time.time()
+        faithfulness = self._similarity(answer, context)
+        latency = time.time() - start
+
+        return {
+            "faithfulness": faithfulness,
+            "latency": latency,
+        }
+
+    def _similarity(self, a: str, b: str) -> float:
+        return SequenceMatcher(None, a, b).ratio()
+
+def evaluate_config(config, validation_data):
+    evaluator = Evaluator()
+    results = []
+    for sample in validation_data:
+        query = sample.get("query", "")
+        answer = sample.get("answer", "")
+        context = sample.get("context", "")
+        results.append(evaluator.evaluate(query, answer, context))
+    return results
+

ragmint/core/pipeline.py

@@ -0,0 +1,38 @@
+from typing import Any, Dict, List
+from .retriever import Retriever
+from .reranker import Reranker
+from .evaluation import Evaluator
+
+
+class RAGPipeline:
+    """
+    Core Retrieval-Augmented Generation pipeline.
+    Simplified (no generator). It retrieves, reranks, and evaluates.
+    """
+
+    def __init__(self, retriever: Retriever, reranker: Reranker, evaluator: Evaluator):
+        self.retriever = retriever
+        self.reranker = reranker
+        self.evaluator = evaluator
+
+    def run(self, query: str, top_k: int = 5) -> Dict[str, Any]:
+        # Retrieve documents
+        retrieved_docs = self.retriever.retrieve(query, top_k=top_k)
+        # Rerank
+        reranked_docs = self.reranker.rerank(query, retrieved_docs)
+
+        # Use top document as pseudo-answer
+        if reranked_docs:
+            answer = reranked_docs[0]["text"]
+        else:
+            answer = ""
+
+        context = "\n".join([d["text"] for d in reranked_docs])
+        metrics = self.evaluator.evaluate(query, answer, context)
+
+        return {
+            "query": query,
+            "answer": answer,
+            "docs": reranked_docs,
+            "metrics": metrics,
+        }

ragmint/core/reranker.py

@@ -0,0 +1,62 @@
+from typing import List, Dict, Any
+import numpy as np
+
+
+class Reranker:
+    """
+    Supports:
+      - MMR (Maximal Marginal Relevance)
+      - Dummy CrossEncoder (for demonstration)
+    """
+
+    def __init__(self, mode: str = "mmr", lambda_param: float = 0.5, seed: int = 42):
+        self.mode = mode
+        self.lambda_param = lambda_param
+        np.random.seed(seed)
+
+    def rerank(self, query: str, docs: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        if not docs:
+            return []
+
+        if self.mode == "crossencoder":
+            return self._crossencoder_rerank(query, docs)
+        return self._mmr_rerank(query, docs)
+
+    def _mmr_rerank(self, query: str, docs: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        """Perform MMR reranking using dummy similarity scores."""
+        selected = []
+        remaining = docs.copy()
+
+        while remaining and len(selected) < len(docs):
+            if not selected:
+                # pick doc with highest base score
+                best = max(remaining, key=lambda d: d["score"])
+            else:
+                # MMR balancing between relevance and diversity
+                mmr_scores = []
+                for d in remaining:
+                    max_div = max(
+                        [self._similarity(d["text"], s["text"]) for s in selected],
+                        default=0,
+                    )
+                    mmr_score = (
+                        self.lambda_param * d["score"]
+                        - (1 - self.lambda_param) * max_div
+                    )
+                    mmr_scores.append(mmr_score)
+                best = remaining[int(np.argmax(mmr_scores))]
+            selected.append(best)
+            remaining.remove(best)
+
+        return selected
+
+    def _crossencoder_rerank(self, query: str, docs: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        """Adds a small random perturbation to simulate crossencoder reranking."""
+        for d in docs:
+            d["score"] += np.random.uniform(0, 0.1)
+        return sorted(docs, key=lambda d: d["score"], reverse=True)
+
+    def _similarity(self, a: str, b: str) -> float:
+        """Dummy similarity function between two strings."""
+        # Deterministic pseudo-similarity based on hash
+        return abs(hash(a + b)) % 100 / 100.0

ragmint/core/retriever.py

@@ -0,0 +1,33 @@
+from typing import List, Dict, Any
+import numpy as np
+
+
+class Retriever:
+    """
+    Simple vector retriever using cosine similarity.
+    """
+
+    def __init__(self, embeddings: List[np.ndarray], documents: List[str]):
+        if len(embeddings) == 0:
+            self.embeddings = np.zeros((1, 768))
+        else:
+            self.embeddings = np.array(embeddings)
+        self.documents = documents or [""]
+
+    def retrieve(self, query: str, top_k: int = 5) -> List[Dict[str, Any]]:
+        if self.embeddings.size == 0 or len(self.documents) == 0:
+            return [{"text": "", "score": 0.0}]
+
+        query_vec = self._embed(query)
+        scores = self._cosine_similarity(query_vec, self.embeddings)
+        top_indices = np.argsort(scores)[::-1][:min(top_k, len(scores))]
+        return [{"text": self.documents[i], "score": float(scores[i])} for i in top_indices]
+
+    def _embed(self, query: str) -> np.ndarray:
+        dim = self.embeddings.shape[1] if len(self.embeddings.shape) > 1 else 768
+        return np.random.rand(dim)
+
+    def _cosine_similarity(self, a: np.ndarray, b: np.ndarray) -> np.ndarray:
+        a_norm = a / np.linalg.norm(a)
+        b_norm = b / np.linalg.norm(b, axis=1, keepdims=True)
+        return np.dot(b_norm, a_norm)

ragmint/experiments/validation_qa.json

@@ -0,0 +1,14 @@
+[
+  {
+    "query": "What is Retrieval-Augmented Generation?",
+    "expected_answer": "A technique that combines information retrieval with language generation to improve factual accuracy."
+  },
+  {
+    "query": "What is the role of embeddings in a RAG system?",
+    "expected_answer": "They represent text as numerical vectors for similarity-based retrieval."
+  },
+  {
+    "query": "What is Maximal Marginal Relevance used for?",
+    "expected_answer": "To select diverse and relevant documents during reranking."
+  }
+]

ragmint/explainer.py

@@ -0,0 +1,61 @@
+"""
+Interpretability Layer
+----------------------
+Uses Gemini or Anthropic Claude to explain why one RAG configuration
+outperforms another. Falls back gracefully if no API key is provided.
+"""
+
+import os
+import json
+
+
+def explain_results(results_a: dict, results_b: dict, model: str = "gemini-1.5-pro") -> str:
+    """
+    Generate a natural-language explanation comparing two RAG experiment results.
+    Priority:
+      1. Anthropic Claude (if ANTHROPIC_API_KEY is set)
+      2. Google Gemini (if GOOGLE_API_KEY is set)
+      3. Fallback text message
+    """
+    prompt = f"""
+    You are an AI evaluation expert.
+    Compare these two RAG experiment results and explain why one performs better.
+    Metrics A: {json.dumps(results_a, indent=2)}
+    Metrics B: {json.dumps(results_b, indent=2)}
+    Provide a concise, human-friendly explanation and practical improvement tips.
+    """
+
+    anthropic_key = os.getenv("ANTHROPIC_API_KEY")
+    google_key = os.getenv("GEMINI_API_KEY")
+
+
+    # 1️⃣ Try Anthropic Claude first
+    if anthropic_key:
+        try:
+            from anthropic import Anthropic
+            client = Anthropic(api_key=anthropic_key)
+            response = client.messages.create(
+                model="claude-3-opus-20240229",
+                max_tokens=300,
+                messages=[{"role": "user", "content": prompt}],
+            )
+            return response.content[0].text
+        except Exception as e:
+            return f"[Claude unavailable] {e}"
+
+    # 2️⃣ Fallback to Google Gemini
+    elif google_key:
+        try:
+            import google.generativeai as genai
+            genai.configure(api_key=google_key)
+            response = genai.GenerativeModel(model).generate_content(prompt)
+            return response.text
+        except Exception as e:
+            return f"[Gemini unavailable] {e}"
+
+    # 3️⃣ Fallback if neither key is available
+    else:
+        return (
+            "[No LLM available] Please set ANTHROPIC_API_KEY or GOOGLE_API_KEY "
+            "to enable interpretability via Claude or Gemini."
+        )

ragmint/leaderboard.py

@@ -0,0 +1,45 @@
+import os
+import json
+from datetime import datetime
+from typing import Dict, Any, Optional
+from supabase import create_client
+
+class Leaderboard:
+    def __init__(self, storage_path: Optional[str] = None):
+        self.storage_path = storage_path
+        url = os.getenv("SUPABASE_URL")
+        key = os.getenv("SUPABASE_KEY")
+        self.client = None
+        if url and key:
+            self.client = create_client(url, key)
+        elif not storage_path:
+            raise EnvironmentError("Set SUPABASE_URL/SUPABASE_KEY or pass storage_path")
+
+    def upload(self, run_id: str, config: Dict[str, Any], score: float):
+        data = {
+            "run_id": run_id,
+            "config": config,
+            "score": score,
+            "timestamp": datetime.utcnow().isoformat(),
+        }
+        if self.client:
+            return self.client.table("experiments").insert(data).execute()
+        else:
+            os.makedirs(os.path.dirname(self.storage_path), exist_ok=True)
+            with open(self.storage_path, "a", encoding="utf-8") as f:
+                f.write(json.dumps(data) + "\n")
+            return data
+
+    def top_results(self, limit: int = 10):
+        if self.client:
+            return (
+                self.client.table("experiments")
+                .select("*")
+                .order("score", desc=True)
+                .limit(limit)
+                .execute()
+            )
+        else:
+            with open(self.storage_path, "r", encoding="utf-8") as f:
+                lines = [json.loads(line) for line in f]
+            return sorted(lines, key=lambda x: x["score"], reverse=True)[:limit]

ragmint/optimization/search.py

@@ -0,0 +1,48 @@
+import itertools
+import random
+import logging
+from typing import Dict, List, Iterator, Any
+
+logging.basicConfig(level=logging.INFO, format="[%(levelname)s] %(message)s")
+
+
+class GridSearch:
+    def __init__(self, search_space: Dict[str, List[Any]]):
+        keys = list(search_space.keys())
+        values = list(search_space.values())
+        self.combinations = [dict(zip(keys, v)) for v in itertools.product(*values)]
+
+    def __iter__(self) -> Iterator[Dict[str, Any]]:
+        for combo in self.combinations:
+            yield combo
+
+
+class RandomSearch:
+    def __init__(self, search_space: Dict[str, List[Any]], n_trials: int = 10):
+        self.search_space = search_space
+        self.n_trials = n_trials
+
+    def __iter__(self) -> Iterator[Dict[str, Any]]:
+        keys = list(self.search_space.keys())
+        for _ in range(self.n_trials):
+            yield {k: random.choice(self.search_space[k]) for k in keys}
+
+
+class BayesianSearch:
+    def __init__(self, search_space: Dict[str, List[Any]]):
+        try:
+            import optuna
+            self.optuna = optuna
+        except ImportError:
+            raise RuntimeError("Optuna not installed; use GridSearch or RandomSearch instead.")
+        self.search_space = search_space
+
+    def __iter__(self) -> Iterator[Dict[str, Any]]:
+        keys = list(self.search_space.keys())
+
+        def objective(trial):
+            return {k: trial.suggest_categorical(k, self.search_space[k]) for k in keys}
+
+        # Example static 5-trial yield for compatibility
+        for _ in range(5):
+            yield {k: random.choice(self.search_space[k]) for k in keys}

ragmint/tests/conftest.py

@@ -0,0 +1,16 @@
+# src/ragmint/tests/conftest.py
+import os
+from dotenv import load_dotenv
+import pytest
+
+# Load .env from project root
+load_dotenv(dotenv_path=os.path.join(os.path.dirname(__file__), "../../../.env"))
+
+def pytest_configure(config):
+    """Print which keys are loaded (debug)."""
+    google = os.getenv("GEMINI_API_KEY")
+    anthropic = os.getenv("ANTHROPIC_API_KEY")
+    if google:
+        print("✅ GOOGLE_API_KEY loaded")
+    if anthropic:
+        print("✅ ANTHROPIC_API_KEY loaded")

ragmint/tests/test_autotuner.py

@@ -0,0 +1,42 @@
+import pytest
+from ragmint.autotuner import AutoRAGTuner
+
+
+def test_autorag_recommend_small():
+    """Small corpus should trigger BM25 + OpenAI."""
+    tuner = AutoRAGTuner({"size": 500, "avg_len": 150})
+    rec = tuner.recommend()
+    assert rec["retriever"] == "BM25"
+    assert rec["embedding_model"] == "OpenAI"
+
+
+def test_autorag_recommend_medium():
+    """Medium corpus should trigger Chroma + SentenceTransformers."""
+    tuner = AutoRAGTuner({"size": 5000, "avg_len": 200})
+    rec = tuner.recommend()
+    assert rec["retriever"] == "Chroma"
+    assert rec["embedding_model"] == "SentenceTransformers"
+
+
+def test_autorag_recommend_large():
+    """Large corpus should trigger FAISS + InstructorXL."""
+    tuner = AutoRAGTuner({"size": 50000, "avg_len": 300})
+    rec = tuner.recommend()
+    assert rec["retriever"] == "FAISS"
+    assert rec["embedding_model"] == "InstructorXL"
+
+
+def test_autorag_auto_tune(monkeypatch):
+    """Test auto_tune with a mock validation dataset."""
+    tuner = AutoRAGTuner({"size": 12000, "avg_len": 250})
+
+    # Monkeypatch evaluate_config inside autotuner
+    import ragmint.autotuner as autotuner
+    def mock_eval(config, data):
+        return {"faithfulness": 0.9, "latency": 0.01}
+    monkeypatch.setattr(autotuner, "evaluate_config", mock_eval)
+
+    result = tuner.auto_tune([{"question": "What is AI?", "answer": "Artificial Intelligence"}])
+    assert "recommended" in result
+    assert "results" in result
+    assert isinstance(result["results"], dict)

ragmint/tests/test_explainer.py

@@ -0,0 +1,20 @@
+import pytest
+from ragmint.explainer import explain_results
+
+
+def test_explain_results_gemini():
+    """Gemini explanation should contain model-specific phrasing."""
+    config_a = {"retriever": "FAISS", "embedding_model": "OpenAI"}
+    config_b = {"retriever": "Chroma", "embedding_model": "SentenceTransformers"}
+    result = explain_results(config_a, config_b, model="gemini")
+    assert isinstance(result, str)
+    assert "Gemini" in result or "gemini" in result
+
+
+def test_explain_results_claude():
+    """Claude explanation should contain model-specific phrasing."""
+    config_a = {"retriever": "FAISS"}
+    config_b = {"retriever": "Chroma"}
+    result = explain_results(config_a, config_b, model="claude")
+    assert isinstance(result, str)
+    assert "Claude" in result or "claude" in result

ragmint/tests/test_explainer_integration.py

@@ -0,0 +1,18 @@
+import os
+import pytest
+from ragmint.explainer import explain_results
+
+
+@pytest.mark.integration
+def test_real_gemini_explanation():
+    """Run real Gemini call if GOOGLE_API_KEY is set."""
+    if not os.getenv("GEMINI_API_KEY"):
+        pytest.skip("GOOGLE_API_KEY not set")
+
+    config_a = {"retriever": "FAISS", "embedding_model": "OpenAI"}
+    config_b = {"retriever": "Chroma", "embedding_model": "SentenceTransformers"}
+
+    result = explain_results(config_a, config_b, model="gemini-1.5-pro")
+    assert isinstance(result, str)
+    assert len(result) > 0
+    print("\n[Gemini explanation]:", result[:200], "...")

ragmint/tests/test_integration_autotuner_ragmint.py

@@ -0,0 +1,60 @@
+import pytest
+from ragmint.tuner import RAGMint
+from ragmint.autotuner import AutoRAGTuner
+
+
+def test_integration_ragmint_autotune(monkeypatch, tmp_path):
+    """
+    Smoke test for integration between AutoRAGTuner and RAGMint.
+    Ensures end-to-end flow runs without real retrievers or embeddings.
+    """
+
+    # --- Mock corpus and validation data ---
+    corpus = tmp_path / "docs"
+    corpus.mkdir()
+    (corpus / "doc1.txt").write_text("This is an AI document.")
+    validation_data = [{"question": "What is AI?", "answer": "Artificial Intelligence"}]
+
+    # --- Mock RAGMint.optimize() to avoid real model work ---
+    def mock_optimize(self, validation_set=None, metric="faithfulness", trials=2):
+        return (
+            {"retriever": "FAISS", "embedding_model": "OpenAI", "score": 0.88},
+            [{"trial": 1, "score": 0.88}],
+        )
+
+    monkeypatch.setattr(RAGMint, "optimize", mock_optimize)
+
+    # --- Mock evaluation used by AutoRAGTuner ---
+    def mock_evaluate_config(config, data):
+        return {"faithfulness": 0.9, "latency": 0.01}
+
+    import ragmint.autotuner as autotuner
+    monkeypatch.setattr(autotuner, "evaluate_config", mock_evaluate_config)
+
+    # --- Create AutoRAGTuner and RAGMint instances ---
+    ragmint = RAGMint(
+        docs_path=str(corpus),
+        retrievers=["faiss", "chroma"],
+        embeddings=["text-embedding-3-small"],
+        rerankers=["mmr"],
+    )
+
+    tuner = AutoRAGTuner({"size": 2000, "avg_len": 150})
+
+    # --- Run Auto-Tune and RAG Optimization ---
+    recommendation = tuner.recommend()
+    assert "retriever" in recommendation
+    assert "embedding_model" in recommendation
+
+    tuning_results = tuner.auto_tune(validation_data)
+    assert "results" in tuning_results
+    assert isinstance(tuning_results["results"], dict)
+
+    # --- Run RAGMint optimization flow (mocked) ---
+    best_config, results = ragmint.optimize(validation_set=validation_data, trials=2)
+    assert isinstance(best_config, dict)
+    assert "score" in best_config
+    assert isinstance(results, list)
+
+    # --- Integration Success ---
+    print(f"Integration OK: AutoRAG recommended {recommendation}, RAGMint best {best_config}")

ragmint/tests/test_leaderboard.py

@@ -0,0 +1,39 @@
+import json
+import tempfile
+from pathlib import Path
+from ragmint.leaderboard import Leaderboard
+
+
+def test_leaderboard_add_and_top(tmp_path):
+    """Ensure local leaderboard persistence works without Supabase."""
+    file_path = tmp_path / "leaderboard.jsonl"
+    lb = Leaderboard(storage_path=str(file_path))
+
+    # Add two runs
+    lb.upload("run1", {"retriever": "FAISS"}, 0.91)
+    lb.upload("run2", {"retriever": "Chroma"}, 0.85)
+
+    # Verify file content
+    assert file_path.exists()
+    with open(file_path, "r", encoding="utf-8") as f:
+        lines = [json.loads(line) for line in f]
+    assert len(lines) == 2
+
+    # Get top results
+    top = lb.top_results(limit=1)
+    assert isinstance(top, list)
+    assert len(top) == 1
+    assert "score" in top[0]
+
+
+def test_leaderboard_append_existing(tmp_path):
+    """Ensure multiple uploads append properly."""
+    file_path = tmp_path / "leaderboard.jsonl"
+    lb = Leaderboard(storage_path=str(file_path))
+
+    for i in range(3):
+        lb.upload(f"run{i}", {"retriever": "BM25"}, 0.8 + i * 0.05)
+
+    top = lb.top_results(limit=2)
+    assert len(top) == 2
+    assert top[0]["score"] >= top[1]["score"]

ragmint/tests/test_pipeline.py

@@ -0,0 +1,19 @@
+import numpy as np
+from ragmint.core.pipeline import RAGPipeline
+from ragmint.core.retriever import Retriever
+from ragmint.core.reranker import Reranker
+from ragmint.core.evaluation import Evaluator
+
+
+def test_pipeline_run():
+    docs = ["doc1 text", "doc2 text"]
+    embeddings = [np.random.rand(4) for _ in range(2)]
+    retriever = Retriever(embeddings, docs)
+    reranker = Reranker("mmr")
+    evaluator = Evaluator()
+    pipeline = RAGPipeline(retriever, reranker, evaluator)
+
+    result = pipeline.run("what is doc1?")
+    assert "query" in result
+    assert "answer" in result
+    assert "metrics" in result

ragmint/tests/test_retriever.py

@@ -0,0 +1,14 @@
+import numpy as np
+from ragmint.core.retriever import Retriever
+
+
+def test_retrieve_basic():
+    embeddings = [np.random.rand(5) for _ in range(3)]
+    docs = ["doc A", "doc B", "doc C"]
+    retriever = Retriever(embeddings, docs)
+
+    results = retriever.retrieve("sample query", top_k=2)
+    assert isinstance(results, list)
+    assert len(results) == 2
+    assert "text" in results[0]
+    assert "score" in results[0]

ragmint/tests/test_search.py

@@ -0,0 +1,17 @@
+from ragmint.optimization.search import GridSearch, RandomSearch
+
+
+def test_grid_search_iterates():
+    space = {"retriever": ["faiss"], "embedding_model": ["openai"], "reranker": ["mmr"]}
+    search = GridSearch(space)
+    combos = list(search)
+    assert len(combos) == 1
+    assert "retriever" in combos[0]
+
+
+def test_random_search_n_trials():
+    space = {"retriever": ["faiss", "bm25"], "embedding_model": ["openai", "st"], "reranker": ["mmr"]}
+    search = RandomSearch(space, n_trials=5)
+    combos = list(search)
+    assert len(combos) == 5
+    assert all("retriever" in c for c in combos)

ragmint/tests/test_tuner.py

@@ -0,0 +1,71 @@
+import os
+import json
+import pytest
+from ragmint.tuner import RAGMint
+
+
+def setup_validation_file(tmp_path):
+    """Create a temporary validation QA dataset."""
+    data = [
+        {"question": "What is AI?", "answer": "Artificial Intelligence"},
+        {"question": "Define ML", "answer": "Machine Learning"}
+    ]
+    file = tmp_path / "validation_qa.json"
+    with open(file, "w", encoding="utf-8") as f:
+        json.dump(data, f)
+    return str(file)
+
+
+def setup_docs(tmp_path):
+    """Create a small document corpus for testing."""
+    corpus = tmp_path / "corpus"
+    corpus.mkdir()
+    (corpus / "doc1.txt").write_text("This is about Artificial Intelligence.")
+    (corpus / "doc2.txt").write_text("This text explains Machine Learning.")
+    return str(corpus)
+
+
+@pytest.mark.parametrize("validation_mode", [
+    None,  # Built-in dataset
+    "data/custom_eval.json",  # Custom dataset path (mocked below)
+])
+def test_optimize_ragmint(tmp_path, validation_mode, monkeypatch):
+    """Test RAGMint.optimize() with different dataset modes."""
+    docs_path = setup_docs(tmp_path)
+    val_file = setup_validation_file(tmp_path)
+
+    # If using custom dataset, mock the path
+    if validation_mode and "custom_eval" in validation_mode:
+        custom_path = tmp_path / "custom_eval.json"
+        os.rename(val_file, custom_path)
+        validation_mode = str(custom_path)
+
+    metric = "faithfulness"
+
+    # Initialize RAGMint
+    rag = RAGMint(
+        docs_path=docs_path,
+        retrievers=["faiss"],
+        embeddings=["text-embedding-3-small"],
+        rerankers=["mmr"]
+    )
+
+    # Run optimization
+    best, results = rag.optimize(
+        validation_set=validation_mode,
+        metric=metric,
+        trials=2
+    )
+
+    # Validate results
+    assert isinstance(best, dict), "Best config should be a dict"
+    assert isinstance(results, list), "Results should be a list of trials"
+    assert len(results) > 0, "Optimization should produce results"
+
+    # The best result can expose either 'score' or the metric name (e.g. 'faithfulness')
+    assert any(k in best for k in ("score", metric)), \
+        f"Best config should include either 'score' or '{metric}'"
+
+    # Ensure the metric value is valid
+    assert best.get(metric, best.get("score")) >= 0, \
+        f"{metric} score should be non-negative"

ragmint/tuner.py

@@ -0,0 +1,123 @@
+import os
+import json
+import logging
+from typing import Any, Dict, List, Tuple, Optional
+from time import perf_counter
+
+from .core.pipeline import RAGPipeline
+from .core.embeddings import EmbeddingModel
+from .core.retriever import Retriever
+from .core.reranker import Reranker
+from .core.evaluation import Evaluator
+from .optimization.search import GridSearch, RandomSearch, BayesianSearch
+
+from .utils.data_loader import load_validation_set
+
+logging.basicConfig(level=logging.INFO, format="[%(levelname)s] %(message)s")
+
+
+class RAGMint:
+    """
+    Main RAG pipeline optimizer and evaluator.
+    """
+
+    def __init__(
+        self,
+        docs_path: str,
+        retrievers: List[str],
+        embeddings: List[str],
+        rerankers: List[str],
+    ):
+        self.docs_path = docs_path
+        self.retrievers = retrievers
+        self.embeddings = embeddings
+        self.rerankers = rerankers
+
+        self.documents: List[str] = self._load_docs()
+        self.embeddings_cache: Dict[str, Any] = {}
+
+    def _load_docs(self) -> List[str]:
+        if not os.path.exists(self.docs_path):
+            logging.warning(f"Corpus path not found: {self.docs_path}")
+            return []
+        docs = []
+        for file in os.listdir(self.docs_path):
+            if file.endswith(".txt") or file.endswith(".md") or file.endswith(".rst"):
+                with open(os.path.join(self.docs_path, file), "r", encoding="utf-8") as f:
+                    docs.append(f.read())
+        logging.info(f"Loaded {len(docs)} documents from {self.docs_path}")
+        return docs
+
+    def _embed_docs(self, model_name: str):
+        if model_name in self.embeddings_cache:
+            return self.embeddings_cache[model_name]
+
+        model = EmbeddingModel(model_name)
+        embeddings = model.encode(self.documents)
+        self.embeddings_cache[model_name] = embeddings
+        return embeddings
+
+    def _build_pipeline(self, config: Dict[str, str]) -> RAGPipeline:
+        emb_model = EmbeddingModel(config["embedding_model"])
+        embeddings = self._embed_docs(config["embedding_model"])
+        retriever = Retriever(embeddings, self.documents)
+        reranker = Reranker(config["reranker"])
+        evaluator = Evaluator()
+        return RAGPipeline(retriever, reranker, evaluator)
+
+    def _evaluate_config(
+        self, config: Dict[str, Any], validation: List[Dict[str, str]], metric: str
+    ) -> Dict[str, float]:
+        pipeline = self._build_pipeline(config)
+
+        scores = []
+        start = perf_counter()
+        for sample in validation:
+            query = sample.get("question") or sample.get("query")
+            reference = sample.get("answer")
+            result = pipeline.run(query)
+            score = result["metrics"].get(metric, 0.0)
+            scores.append(score)
+        elapsed = perf_counter() - start
+
+        avg_score = sum(scores) / len(scores) if scores else 0.0
+        return {metric: avg_score, "latency": elapsed / max(1, len(validation))}
+
+    def optimize(
+        self,
+        validation_set: str,
+        metric: str = "faithfulness",
+        search_type: str = "random",
+        trials: int = 10,
+    ) -> Tuple[Dict[str, Any], List[Dict[str, Any]]]:
+        validation = load_validation_set(validation_set or "default")
+
+        search_space = {
+            "retriever": self.retrievers,
+            "embedding_model": self.embeddings,
+            "reranker": self.rerankers,
+        }
+
+        logging.info(f"Starting {search_type} optimization with {trials} trials")
+
+        try:
+            if search_type == "grid":
+                searcher = GridSearch(search_space)
+            elif search_type == "bayesian":
+                searcher = BayesianSearch(search_space)
+            else:
+                searcher = RandomSearch(search_space, n_trials=trials)
+        except Exception as e:
+            logging.warning(f"Falling back to RandomSearch due to missing deps: {e}")
+            searcher = RandomSearch(search_space, n_trials=trials)
+
+        results = []
+        for config in searcher:
+            metrics = self._evaluate_config(config, validation, metric)
+            result = {**config, **metrics}
+            results.append(result)
+            logging.info(f"Tested config: {config} -> {metrics}")
+
+        best = max(results, key=lambda r: r.get(metric, 0.0)) if results else {}
+        logging.info(f"✅ Best configuration found: {best}")
+        return best, results

ragmint/utils/caching.py

@@ -0,0 +1,37 @@
+import os
+import json
+import hashlib
+import pickle
+from typing import Any
+
+
+class Cache:
+    """
+    Simple file-based cache for embeddings or retrievals.
+    """
+
+    def __init__(self, cache_dir: str = ".ragmint_cache"):
+        self.cache_dir = cache_dir
+        os.makedirs(cache_dir, exist_ok=True)
+
+    def _hash_key(self, key: str) -> str:
+        return hashlib.md5(key.encode()).hexdigest()
+
+    def exists(self, key: str) -> bool:
+        return os.path.exists(os.path.join(self.cache_dir, self._hash_key(key)))
+
+    def get(self, key: str) -> Any:
+        path = os.path.join(self.cache_dir, self._hash_key(key))
+        if not os.path.exists(path):
+            return None
+        with open(path, "rb") as f:
+            return pickle.load(f)
+
+    def set(self, key: str, value: Any):
+        path = os.path.join(self.cache_dir, self._hash_key(key))
+        with open(path, "wb") as f:
+            pickle.dump(value, f)
+
+    def clear(self):
+        for file in os.listdir(self.cache_dir):
+            os.remove(os.path.join(self.cache_dir, file))

ragmint/utils/data_loader.py

@@ -0,0 +1,65 @@
+import json
+import csv
+from typing import List, Dict
+from pathlib import Path
+import os
+
+try:
+    from datasets import load_dataset
+except ImportError:
+    load_dataset = None  # optional dependency
+
+DEFAULT_VALIDATION_PATH = Path(__file__).parent.parent / "experiments" / "validation_qa.json"
+
+
+def load_json(path: str) -> List[Dict]:
+    with open(path, "r", encoding="utf-8") as f:
+        return json.load(f)
+
+
+def load_csv(path: str) -> List[Dict]:
+    with open(path, newline="", encoding="utf-8") as csvfile:
+        reader = csv.DictReader(csvfile)
+        return list(reader)
+
+
+def save_json(path: str, data: Dict):
+    with open(path, "w", encoding="utf-8") as f:
+        json.dump(data, f, ensure_ascii=False, indent=2)
+
+def load_validation_set(path: str | None = None) -> List[Dict]:
+    """
+    Loads a validation dataset (QA pairs) from:
+    - Built-in default JSON file
+    - User-provided JSON or CSV
+    - Hugging Face dataset by name
+    """
+    # Default behavior
+    if path is None or path == "default":
+        if not DEFAULT_VALIDATION_PATH.exists():
+            raise FileNotFoundError(f"Default validation set not found at {DEFAULT_VALIDATION_PATH}")
+        return load_json(DEFAULT_VALIDATION_PATH)
+
+    # Hugging Face dataset
+    if not os.path.exists(path) and load_dataset:
+        try:
+            dataset = load_dataset(path, split="validation")
+            data = [
+                {"question": q, "answer": a}
+                for q, a in zip(dataset["question"], dataset["answers"])
+            ]
+            return data
+        except Exception:
+            pass  # fall through to file loading
+
+    # Local file
+    p = Path(path)
+    if not p.exists():
+        raise FileNotFoundError(f"Validation file not found: {path}")
+
+    if p.suffix.lower() == ".json":
+        return load_json(path)
+    elif p.suffix.lower() in [".csv", ".tsv"]:
+        return load_csv(path)
+    else:
+        raise ValueError("Unsupported validation set format. Use JSON, CSV, or a Hugging Face dataset name.")

ragmint/utils/logger.py

@@ -0,0 +1,36 @@
+import logging
+from tqdm import tqdm
+
+
+class Logger:
+    """
+    Centralized logger with optional tqdm integration and color formatting.
+    """
+
+    def __init__(self, name: str = "ragmint", level: int = logging.INFO):
+        self.logger = logging.getLogger(name)
+        self.logger.setLevel(level)
+
+        if not self.logger.handlers:
+            handler = logging.StreamHandler()
+            formatter = logging.Formatter(
+                "\033[96m[%(asctime)s]\033[0m \033[93m%(levelname)s\033[0m: %(message)s",
+                "%H:%M:%S",
+            )
+            handler.setFormatter(formatter)
+            self.logger.addHandler(handler)
+
+    def info(self, msg: str):
+        self.logger.info(msg)
+
+    def warning(self, msg: str):
+        self.logger.warning(msg)
+
+    def error(self, msg: str):
+        self.logger.error(msg)
+
+    def progress(self, iterable, desc="Processing", total=None):
+        return tqdm(iterable, desc=desc, total=total)
+
+def get_logger(name: str = "ragmint") -> Logger:
+    return Logger(name)

ragmint/utils/metrics.py

@@ -0,0 +1,27 @@
+from typing import List
+import numpy as np
+from difflib import SequenceMatcher
+
+
+def bleu_score(reference: str, candidate: str) -> float:
+    """
+    Simple BLEU-like precision approximation.
+    """
+    ref_tokens = reference.split()
+    cand_tokens = candidate.split()
+    if not cand_tokens:
+        return 0.0
+
+    matches = sum(1 for token in cand_tokens if token in ref_tokens)
+    return matches / len(cand_tokens)
+
+
+def rouge_l(reference: str, candidate: str) -> float:
+    """
+    Approximation of ROUGE-L using sequence matcher ratio.
+    """
+    return SequenceMatcher(None, reference, candidate).ratio()
+
+
+def mean_score(scores: List[float]) -> float:
+    return float(np.mean(scores)) if scores else 0.0

ragmint-0.2.1.dist-info/METADATA

@@ -0,0 +1,27 @@
+Metadata-Version: 2.4
+Name: ragmint
+Version: 0.2.1
+Summary: A modular framework for evaluating and optimizing RAG pipelines.
+Author-email: Andre Oliveira <oandreoliveira@outlook.com>
+License: Apache License 2.0
+Project-URL: Homepage, https://github.com/andyolivers/ragmint
+Project-URL: Documentation, https://andyolivers.com
+Project-URL: Issues, https://github.com/andyolivers/ragmint/issues
+Keywords: RAG,LLM,retrieval,optimization,AI,evaluation
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: numpy>=1.23
+Requires-Dist: pandas>=2.0
+Requires-Dist: scikit-learn>=1.3
+Requires-Dist: openai>=1.0
+Requires-Dist: tqdm
+Requires-Dist: pyyaml
+Requires-Dist: chromadb>=0.4
+Requires-Dist: faiss-cpu; sys_platform != "darwin"
+Requires-Dist: optuna>=3.0
+Requires-Dist: pytest
+Requires-Dist: colorama
+Requires-Dist: google-generativeai>=0.8.0
+Requires-Dist: supabase>=2.4.0
+Dynamic: license-file

ragmint-0.2.1.dist-info/RECORD

@@ -0,0 +1,38 @@
+ragmint/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ragmint/__main__.py,sha256=q7hBn56Z1xAckbs03i8ynsuOzJVUXmod2qHddX7gkpc,729
+ragmint/autotuner.py,sha256=eXEH4e_3Os9FPX9y_0N7GnIQsmoHxmFbtjj7xanu17g,1064
+ragmint/explainer.py,sha256=1glGNdC4GlwR6Qs8Bj1oOol7f5_db7Ksnh07HAp-A2c,2077
+ragmint/leaderboard.py,sha256=nILQ5QR63RpZtCrZ__RFfwHXy4bkUIMUcSfH92OQ93Y,1628
+ragmint/tuner.py,sha256=BLPZ66sVk3dh3Wj-GVUYRVmVtgXYTzv3oTQtKJeDlgE,4442
+ragmint/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ragmint/core/chunking.py,sha256=Dy9RYyapGSS6ik6Vg9lqbUPCFqSraU1JKpHbYUTkaFo,576
+ragmint/core/embeddings.py,sha256=6wJjfZ5ukr8G5bJJ1evjIqj0_FMbs_gq4xC-sBBqNlA,566
+ragmint/core/evaluation.py,sha256=3OFcZU2zZyaP53d9S2zdpknV0CYfTq0KoRB3a_dtjM4,1022
+ragmint/core/pipeline.py,sha256=2qwGKuG0Du7gtIpieLFn71h_RcwBpjcV-h9PQz2ZOsc,1169
+ragmint/core/reranker.py,sha256=B2-NDExqpd9jdXHkEHOXC0B_6-FMJm5vdi-_ZbxC3Os,2303
+ragmint/core/retriever.py,sha256=jbpKy_fGdDq736y0es_utQuLqY9eiWNd71Q8JbU0Sko,1259
+ragmint/experiments/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ragmint/experiments/validation_qa.json,sha256=mQyGeMyvyAqN5yGjpjqW42JJ7FfhFYNki__paKVhQss,520
+ragmint/optimization/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ragmint/optimization/search.py,sha256=uiLJeoO_jaLCQEw99L6uI1rnqHHx_rTY81WxfMmlALs,1623
+ragmint/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ragmint/tests/conftest.py,sha256=QhYPp5nrQ_DbZlsVH3nLjDgjPATAnLwzJkwl-Y-xrmM,488
+ragmint/tests/test_autotuner.py,sha256=k5nsIH6MYB5zaocR_Wn1wTX-QDYfhH6ugx2chZu9Q8U,1500
+ragmint/tests/test_explainer.py,sha256=K_DRnGGl34WcTA2yaQGmfzWkVi1uEkzjpsTPeZxXeIg,802
+ragmint/tests/test_explainer_integration.py,sha256=tYT62fYqk616bjQ1VxHADVRfJ9vdF_CiF3cz4A9BdbE,620
+ragmint/tests/test_integration_autotuner_ragmint.py,sha256=YCGge0_KOijAdB7VNDGHl2VRJjiOyl_-sJNRLjAXGLw,2182
+ragmint/tests/test_leaderboard.py,sha256=ay81YK6KxAUU6mcG6n1_xV8GPYkBgjzJj9iAIyAzIzA,1163
+ragmint/tests/test_pipeline.py,sha256=MIMkEKelh-POlbXzbCc4ClMk8XCGzfuj569xXltziic,615
+ragmint/tests/test_retriever.py,sha256=Ag0uGW8-iMzKA4nJNnsjuzlQHa79sN-T-K1g1cdin-A,421
+ragmint/tests/test_search.py,sha256=FcC-DEnw9veAEyMnFoRw9DAwzqJC9F6-r63Nqo2nO58,598
+ragmint/tests/test_tuner.py,sha256=LOvtIxAbUsoRHQudZ23UVr60FYAU0a1SBNvAN0mLpfU,2322
+ragmint/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ragmint/utils/caching.py,sha256=LPE2JorOQ90BgVf6NUiS0-bdt-FGpNxDy7FnuwEHzy0,1060
+ragmint/utils/data_loader.py,sha256=GXU9Nc3o0UWxtBeRwiskD1aCjSiNNuRoAokIUODn7q8,2024
+ragmint/utils/logger.py,sha256=X7hTNb3st3fUeQIzSghuoV5B8FWXzm_O3DRkSfJvhmI,1033
+ragmint/utils/metrics.py,sha256=DR8mrdumHtQerK0VrugwYKIG1oNptEcsFqodXq3i2kY,717
+ragmint-0.2.1.dist-info/licenses/LICENSE,sha256=ahkhYfFLI8tGrdxdO2_GaT6OJW2eNwyFT3kYi85QQhc,692
+ragmint-0.2.1.dist-info/METADATA,sha256=sR31_9qGCaarWLNqTc2uNjO4aklKq9nxaGEjtcy5ipU,936
+ragmint-0.2.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+ragmint-0.2.1.dist-info/top_level.txt,sha256=K2ulzMHuvFm6xayvvJdGABeRJAvKDBn6M3EI-3SbYLw,8
+ragmint-0.2.1.dist-info/RECORD,,

ragmint-0.2.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.9.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

ragmint-0.2.1.dist-info/licenses/LICENSE

@@ -0,0 +1,19 @@
+Apache License
+Version 2.0, January 2004
+http://www.apache.org/licenses/
+
+TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+Copyright 2025 André Oliveira
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

ragmint-0.2.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+ragmint