rag-harness 0.1.0__py3-none-any.whl

rag_harness-0.1.0.dist-info/METADATA

@@ -0,0 +1,137 @@
+Metadata-Version: 2.4
+Name: rag-harness
+Version: 0.1.0
+Summary: ⚡ CLI tool to evaluate and compare RAG systems.
+Author: Abhishek Bevinkatti
+License: MIT
+Project-URL: Homepage, https://github.com/bevinkatti/rag-harness
+Project-URL: Repository, https://github.com/bevinkatti/rag-harness
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: typer>=0.12.3
+Requires-Dist: rich>=13.7.1
+Provides-Extra: dev
+Requires-Dist: pytest; extra == "dev"
+Dynamic: license-file
+
+# ⚡ RAG Harness
+
+> The fastest way to evaluate and compare RAG systems from your terminal.
+
+[![Python](https://img.shields.io/badge/python-3.10+-blue)]()
+[![License](https://img.shields.io/badge/license-MIT-green)]()
+[![CLI](https://img.shields.io/badge/interface-CLI-black)]()
+
+---
+
+## 🚀 Why this exists
+
+Evaluating RAG systems is messy.
+
+- Different metrics everywhere  
+- No standard CLI tools  
+- Hard to compare models  
+
+👉 **RAG Harness fixes that.**  
+## 🎥 Demo
+
+![Demo](docs/demo.gif)
+
+---
+
+## 🔥 Features
+
+- ⚡ One-command RAG evaluation
+- 📊 Exact Match + F1 + Context metrics
+- 🧠 RAGAS-style scoring (no API required)
+- ⚔️ Compare multiple RAG systems
+- 📁 Works with JSONL / JSON / CSV
+
+
+---
+
+## ⚡ Quick Start
+
+```bash
+git clone https://github.com/yourname/rag-harness.git
+cd rag-harness
+python -m venv .venv
+.\.venv\Scripts\activate
+pip install -e .  
+```  
+## ▶️ Run Evaluation
+
+```bash
+rag-harness evaluate examples/dataset.jsonl examples/predictions_a.jsonl
+```
+
+### Output
+
+```
+📊 RAG Evaluation Summary
+
+Exact Match        0.5000
+F1 Score           0.8333
+Context Precision  1.0000
+Context Recall     1.0000
+
+🧠 RAGAS Score     0.9000
+```
+
+---
+
+## ⚔️ Compare Systems
+
+```bash
+rag-harness compare examples/dataset.jsonl examples/predictions_a.jsonl examples/predictions_b.jsonl
+```
+
+### Output
+
+```
+⚔️ RAG Systems Comparison
+
+Metric              A        B
+--------------------------------
+Exact Match         0.50     0.00
+F1 Score            0.83     0.00
+RAGAS Score         0.90     0.00
+
+🏆 System A wins
+```
+
+---
+
+## 📁 Dataset Format
+
+### Dataset
+
+```json
+{"id":"1","question":"Who wrote Hamlet?","answer":"William Shakespeare","contexts":["William Shakespeare wrote Hamlet."]}
+```
+
+### Predictions
+
+```json
+{"id":"1","answer":"Shakespeare","contexts":["William Shakespeare wrote Hamlet."]}
+```
+
+---
+
+## 🧠 RAGAS-style Scoring
+
+We provide a lightweight RAGAS-inspired score:
+
+```
+RAGAS = 0.6 * F1 + 0.4 * Context Recall
+```
+
+* No API keys required
+* Fast and deterministic
+* Extendable to real RAGAS later
+
+---
+
+## 🤝 Contributing  
+PRs, Ideas, Improvements welcome.

rag_harness-0.1.0.dist-info/RECORD

@@ -0,0 +1,16 @@
+rag_harness-0.1.0.dist-info/licenses/LICENSE,sha256=p8L8i_iZPNhEW9j8kENiFiDhEip_fc33vbUgMQZiHe4,1097
+ragharness/__init__.py,sha256=Pru0BlFBASFCFo7McHdohtKkUtgMPDwbGfyUZlE2_Vw,21
+ragharness/__main__.py,sha256=Y6T9w7NIhVdtzCErAbKUA_of74_wsKcilck--LzHNIs,61
+ragharness/cli.py,sha256=EW3OGIRIouh326GXRe0rQeb9Cb96EGQYFwhkFSoypxE,2231
+ragharness/compare.py,sha256=mXxUezzo9vwA0kQXZaxIPJXPPrrCQm_RPRL2mGc59lI,1332
+ragharness/io.py,sha256=4wSZeDjz1ktIE57yCmbEGTvKiddjFjfOZmkwUn5d2mo,1963
+ragharness/metrics.py,sha256=xhgLIq8_6U3-QVHQCK3lh4uU1a-9JjyH6nB8ffLeiHM,2146
+ragharness/models.py,sha256=VEimOk0eH7jKlg8YHT2-dChFV3P0M1qIqCwShIg9rfY,854
+ragharness/report.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ragharness/runner.py,sha256=CIAcNS-KjM7VdPiy6SWVQMTosEagAntp-5np6XZHE-U,2076
+ragharness/utils.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+rag_harness-0.1.0.dist-info/METADATA,sha256=QsO7PvIpS9Y8ghGo7YOhf1PkY2514EIUWuRruUbNeDs,2790
+rag_harness-0.1.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+rag_harness-0.1.0.dist-info/entry_points.txt,sha256=eP4SKGSNxfrQCZkBPh0cmoNAaQ2tNekolpj_zQLudpQ,51
+rag_harness-0.1.0.dist-info/top_level.txt,sha256=F-7l2UusP7pXBkwZ8psjgc3FmSXfHururX34CRwtN0Q,11
+rag_harness-0.1.0.dist-info/RECORD,,

rag_harness-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

rag_harness-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+rag-harness = ragharness.cli:app

rag_harness-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Abhishek Bevinkatti
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

rag_harness-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+ragharness

ragharness/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

ragharness/__main__.py

@@ -0,0 +1,4 @@
+from .cli import app
+
+if __name__ == "__main__":
+    app()

ragharness/cli.py

@@ -0,0 +1,74 @@
+import typer
+from pathlib import Path
+from rich.console import Console
+from rich.table import Table
+
+from .compare import compare_models
+from .runner import evaluate
+
+app = typer.Typer(help="⚡ RAG Harness CLI", invoke_without_command=True)
+console = Console()
+
+
+@app.callback()
+def main():
+    """RAG Harness CLI"""
+    pass
+
+
+# -------------------------------
+# EVALUATE COMMAND
+# -------------------------------
+@app.command()
+def evaluate_cmd(
+    dataset: Path = typer.Argument(..., exists=True, help="Dataset file"),
+    predictions: Path = typer.Argument(..., exists=True, help="Predictions file"),
+):
+    """Evaluate a RAG system"""
+
+    rows, agg = evaluate(dataset, predictions)
+
+    # 🔹 Main Evaluation Table
+    table = Table(title="📊 RAG Evaluation Summary", show_lines=True)
+
+    table.add_column("Metric", style="bold magenta")
+    table.add_column("Value", justify="right", style="green")
+
+    table.add_row("Total", str(agg.total))
+    table.add_row("Matched", str(agg.matched))
+    table.add_row("Missing", str(agg.missing))
+    table.add_row("Exact Match", f"{agg.exact_match:.4f}")
+    table.add_row("F1 Score", f"{agg.f1:.4f}")
+    table.add_row("Context Precision", f"{agg.context_precision:.4f}")
+    table.add_row("Context Recall", f"{agg.context_recall:.4f}")
+
+    console.print(table)
+
+    # 🔹 Separate RAGAS Highlight Table (nice UX)
+    ragas_table = Table(title= None, show_lines=True)
+    ragas_table.add_column("Metric", style="bold cyan")
+    ragas_table.add_column("Value", justify="right", style="yellow")
+    ragas_table.add_row("RAGAS Score", f"{agg.ragas_score:.4f}")
+
+    console.print(ragas_table)
+
+
+# alias → allows `rag-harness evaluate`
+app.command(name="evaluate")(evaluate_cmd)
+
+# -------------------------------
+# COMPARE COMMAND (ADD HERE)
+# -------------------------------
+@app.command()
+def compare(
+    dataset: Path = typer.Argument(..., exists=True),
+    pred_a: Path = typer.Argument(..., help="Predictions A"),
+    pred_b: Path = typer.Argument(..., help="Predictions B"),
+):
+    """Compare two RAG systems"""
+
+    compare_models(dataset, pred_a, pred_b)
+
+
+if __name__ == "__main__":
+    app()

ragharness/compare.py

@@ -0,0 +1,40 @@
+from pathlib import Path
+from rich.table import Table
+from rich.console import Console
+
+from .runner import evaluate
+
+console = Console()
+
+
+def compare_models(dataset: Path, pred_a: Path, pred_b: Path):
+    rows_a, agg_a = evaluate(dataset, pred_a)
+    rows_b, agg_b = evaluate(dataset, pred_b)
+
+    table = Table(title="⚔️ RAG Systems Comparison", show_lines=True)
+
+    table.add_column("Metric", style="bold magenta")
+    table.add_column("System A", style="green")
+    table.add_column("System B", style="cyan")
+
+    def row(name, a, b):
+        table.add_row(name, f"{a:.4f}", f"{b:.4f}")
+
+    row("Exact Match", agg_a.exact_match, agg_b.exact_match)
+    row("F1 Score", agg_a.f1, agg_b.f1)
+    row("Context Precision", agg_a.context_precision, agg_b.context_precision)
+    row("Context Recall", agg_a.context_recall, agg_b.context_recall)
+    row("RAGAS Score", agg_a.ragas_score, agg_b.ragas_score)
+
+    console.print(table)
+
+    # 🏆 Winner logic
+    score_a = agg_a.f1 + agg_a.ragas_score
+    score_b = agg_b.f1 + agg_b.ragas_score
+
+    if score_a > score_b:
+        console.print("\n🏆 [bold green]System A wins[/bold green]")
+    elif score_b > score_a:
+        console.print("\n🏆 [bold cyan]System B wins[/bold cyan]")
+    else:
+        console.print("\n🤝 It's a tie")

ragharness/io.py

@@ -0,0 +1,81 @@
+import json
+import csv
+from pathlib import Path
+
+from .models import Example, Prediction
+
+
+def _parse_contexts(value):
+    if not value:
+        return []
+
+    if isinstance(value, list):
+        return [str(x).strip() for x in value]
+
+    if isinstance(value, str):
+        try:
+            # Try JSON list
+            parsed = json.loads(value)
+            if isinstance(parsed, list):
+                return [str(x).strip() for x in parsed]
+        except:
+            pass
+
+        # fallback: split by ||
+        return [x.strip() for x in value.split("||")]
+
+    return []
+
+
+def load_jsonl(path: Path):
+    data = []
+    with open(path, "r", encoding="utf-8") as f:
+        for line in f:
+            if line.strip():
+                data.append(json.loads(line))
+    return data
+
+
+def load_dataset(path: Path) -> list[Example]:
+    suffix = path.suffix.lower()
+
+    if suffix == ".jsonl":
+        rows = load_jsonl(path)
+    elif suffix == ".json":
+        rows = json.load(open(path))
+    elif suffix == ".csv":
+        rows = list(csv.DictReader(open(path)))
+    else:
+        raise ValueError("Unsupported format")
+
+    return [
+        Example(
+            id=str(r["id"]),
+            question=r["question"],
+            answer=r["answer"],
+            contexts=_parse_contexts(r.get("contexts")),
+        )
+        for r in rows
+    ]
+
+
+def load_predictions(path: Path) -> list[Prediction]:
+    suffix = path.suffix.lower()
+
+    if suffix == ".jsonl":
+        rows = load_jsonl(path)
+    elif suffix == ".json":
+        rows = json.load(open(path))
+    elif suffix == ".csv":
+        rows = list(csv.DictReader(open(path)))
+    else:
+        raise ValueError("Unsupported format")
+
+    return [
+        Prediction(
+            id=str(r["id"]),
+            answer=r["answer"],
+            contexts=_parse_contexts(r.get("contexts")),
+        )
+        for r in rows
+    ]

ragharness/metrics.py

@@ -0,0 +1,81 @@
+import re
+from collections import Counter
+
+
+# ---------- TEXT NORMALIZATION ----------
+def normalize(text: str) -> str:
+    text = text.lower().strip()
+    text = re.sub(r"[^a-z0-9\s]", " ", text)
+    text = re.sub(r"\b(a|an|the)\b", " ", text)
+    text = re.sub(r"\s+", " ", text)
+    return text.strip()
+
+
+# ---------- EXACT MATCH ----------
+def exact_match(pred: str, gold: str) -> float:
+    return 1.0 if normalize(pred) == normalize(gold) else 0.0
+
+
+# ---------- TOKEN F1 ----------
+def f1_score(pred: str, gold: str) -> float:
+    pred_tokens = normalize(pred).split()
+    gold_tokens = normalize(gold).split()
+
+    if not pred_tokens and not gold_tokens:
+        return 1.0
+    if not pred_tokens or not gold_tokens:
+        return 0.0
+
+    common = Counter(pred_tokens) & Counter(gold_tokens)
+    overlap = sum(common.values())
+
+    if overlap == 0:
+        return 0.0
+
+    precision = overlap / len(pred_tokens)
+    recall = overlap / len(gold_tokens)
+
+    return 2 * precision * recall / (precision + recall)
+
+
+# ---------- CONTEXT PRECISION ----------
+def context_precision(pred_ctx: list[str], gold_ctx: list[str]) -> float:
+    if not pred_ctx:
+        return 0.0
+
+    pred_set = {normalize(c) for c in pred_ctx}
+    gold_set = {normalize(c) for c in gold_ctx}
+
+    if not pred_set:
+        return 0.0
+
+    overlap = len(pred_set & gold_set)
+    return overlap / len(pred_set)
+
+
+# ---------- CONTEXT RECALL ----------
+def context_recall(pred_ctx: list[str], gold_ctx: list[str]) -> float:
+    if not gold_ctx:
+        return 0.0
+
+    pred_set = {normalize(c) for c in pred_ctx}
+    gold_set = {normalize(c) for c in gold_ctx}
+
+    if not gold_set:
+        return 0.0
+
+    overlap = len(pred_set & gold_set)
+    return overlap / len(gold_set)
+
+def ragas_score(pred: str, gold: str, pred_ctx: list[str], gold_ctx: list[str]) -> float:
+    """
+    Lightweight RAGAS-style score
+    Combines:
+    - Answer F1
+    - Context Recall
+    """
+
+    f1 = f1_score(pred, gold)
+    cr = context_recall(pred_ctx, gold_ctx)
+
+    return round((f1 * 0.6 + cr * 0.4), 4)

ragharness/models.py

@@ -0,0 +1,42 @@
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass
+class Example:
+    id: str
+    question: str
+    answer: str
+    contexts: list[str] = field(default_factory=list)
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class Prediction:
+    id: str
+    answer: str
+    contexts: list[str] = field(default_factory=list)
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class ExampleScore:
+    id: str
+    exact_match: float
+    f1: float
+    context_precision: float
+    context_recall: float
+    missing: bool = False
+    ragas_score: float = 0.0
+
+
+@dataclass
+class AggregateScore:
+    total: int
+    matched: int
+    missing: int
+    exact_match: float
+    f1: float
+    context_precision: float
+    context_recall: float
+    ragas_score: float

ragharness/runner.py

@@ -0,0 +1,65 @@
+from pathlib import Path
+
+from .io import load_dataset, load_predictions
+from .metrics import exact_match, f1_score, context_precision, context_recall, ragas_score
+from .models import ExampleScore, AggregateScore
+
+
+def evaluate(dataset_path: Path, predictions_path: Path):
+    dataset = load_dataset(dataset_path)
+    predictions = load_predictions(predictions_path)
+
+    pred_map = {p.id: p for p in predictions}
+
+    rows = []
+
+    for ex in dataset:
+        pred = pred_map.get(ex.id)
+
+        if pred is None:
+            rows.append(
+                ExampleScore(
+                    id=ex.id,
+                    exact_match=0.0,
+                    f1=0.0,
+                    context_precision=0.0,
+                    context_recall=0.0,
+                    ragas_score=0.0,
+                    missing=True,
+                )
+            )
+            continue
+
+        ragas = ragas_score(pred.answer, ex.answer, pred.contexts, ex.contexts)
+        
+        rows.append(
+            ExampleScore(
+                id=ex.id,
+                exact_match=exact_match(pred.answer, ex.answer),
+                f1=f1_score(pred.answer, ex.answer),
+                context_precision=context_precision(pred.contexts, ex.contexts),
+                context_recall=context_recall(pred.contexts, ex.contexts),
+                ragas_score=ragas,
+                missing=False,
+            )
+        )
+
+    total = len(rows)
+    matched = sum(1 for r in rows if not r.missing)
+    missing = total - matched
+
+    def avg(values):
+        return round(sum(values) / len(values), 4) if values else 0.0
+
+    aggregate = AggregateScore(
+        total=total,
+        matched=matched,
+        missing=missing,
+        exact_match=avg([r.exact_match for r in rows]),
+        f1=avg([r.f1 for r in rows]),
+        context_precision=avg([r.context_precision for r in rows]),
+        context_recall=avg([r.context_recall for r in rows]),
+        ragas_score=avg([r.ragas_score for r in rows]),
+    )
+
+    return rows, aggregate