rag-eval-lite 0.1.0__tar.gz

rag_eval_lite-0.1.0/PKG-INFO

@@ -0,0 +1,26 @@
+Metadata-Version: 2.4
+Name: rag-eval-lite
+Version: 0.1.0
+Summary: Lightweight evaluation metrics for RAG (Hit@k, Recall@k, MRR, nDCG)
+Author: Your Name
+Project-URL: Homepage, https://github.com/yourusername/rag-eval-lite
+Keywords: rag,evaluation,retrieval,ml,nlp
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+
+# rag-eval-lite
+
+Lightweight RAG evaluation metrics.
+
+## Install
+pip install rag-eval-lite
+
+## Usage
+```python
+from rag_eval import evaluate_dataset
+
+results = evaluate_dataset(data, k=3)
+print(results)

rag_eval_lite-0.1.0/README.md

@@ -0,0 +1,13 @@
+# rag-eval-lite
+
+Lightweight RAG evaluation metrics.
+
+## Install
+pip install rag-eval-lite
+
+## Usage
+```python
+from rag_eval import evaluate_dataset
+
+results = evaluate_dataset(data, k=3)
+print(results)

rag_eval_lite-0.1.0/pyproject.toml

@@ -0,0 +1,28 @@
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "rag-eval-lite"
+version = "0.1.0"
+description = "Lightweight evaluation metrics for RAG (Hit@k, Recall@k, MRR, nDCG)"
+authors = [
+    { name = "Your Name" }
+]
+readme = "README.md"
+requires-python = ">=3.8"
+dependencies = []
+
+keywords = ["rag", "evaluation", "retrieval", "ml", "nlp"]
+
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+]
+
+[project.urls]
+Homepage = "https://github.com/yourusername/rag-eval-lite"
+
+[tool.setuptools.packages.find]
+where = ["."]

rag_eval_lite-0.1.0/rag_eval/__init__.py

@@ -0,0 +1,3 @@
+from .metrics import evaluate_dataset
+
+__all__ = ["evaluate_dataset"]

rag_eval_lite-0.1.0/rag_eval/metrics.py

@@ -0,0 +1,113 @@
+import math
+from typing import List, Dict, Any
+
+
+# ---------- Metrics ----------
+
+def hit_at_k(golden, retrieved, k):
+    return 1.0 if any(cid in golden for cid in retrieved[:k]) else 0.0
+
+
+def precision_at_k(golden, retrieved, k):
+    if k == 0:
+        return 0.0
+    retrieved_k = retrieved[:k]
+    relevant = sum(1 for cid in retrieved_k if cid in golden)
+    return relevant / k
+
+
+def recall_at_k(golden, retrieved, k):
+    if not golden:
+        return 0.0
+    retrieved_k = retrieved[:k]
+    relevant = sum(1 for cid in retrieved_k if cid in golden)
+    return relevant / len(golden)
+
+
+def mrr(golden, retrieved):
+    for i, cid in enumerate(retrieved):
+        if cid in golden:
+            return 1.0 / (i + 1)
+    return 0.0
+
+
+def dcg_at_k(golden, retrieved, k):
+    score = 0.0
+    for i, cid in enumerate(retrieved[:k]):
+        if cid in golden:
+            score += 1.0 / math.log2(i + 2)
+    return score
+
+
+def ndcg_at_k(golden, retrieved, k):
+    dcg = dcg_at_k(golden, retrieved, k)
+    ideal_hits = min(len(golden), k)
+    idcg = sum(1.0 / math.log2(i + 2) for i in range(ideal_hits))
+    return dcg / idcg if idcg > 0 else 0.0
+
+
+# ---------- Main evaluator ----------
+
+def evaluate_dataset(
+    data: List[Dict[str, Any]],
+    k: int = 3,
+    return_failures: bool = True
+) -> Dict[str, Any]:
+
+    hit_scores, precision_scores, recall_scores = [], [], []
+    mrr_scores, ndcg_scores = [], []
+
+    failures = []
+
+    for idx, row in enumerate(data):
+        golden = row.get("golden_chunk_ids", [])
+        retrieved = row.get("retrieved_chunk_ids", [])
+
+        q = row.get("question", "")
+        qid = row.get("question_id", idx)
+
+        h = hit_at_k(golden, retrieved, k)
+        p = precision_at_k(golden, retrieved, k)
+        r = recall_at_k(golden, retrieved, k)
+        m = mrr(golden, retrieved)
+        n = ndcg_at_k(golden, retrieved, k)
+
+        hit_scores.append(h)
+        precision_scores.append(p)
+        recall_scores.append(r) 
+        mrr_scores.append(m)
+        ndcg_scores.append(n)
+
+        # Define "failure"
+        if h == 0:  # you can tweak this condition
+            failures.append({
+                "question_id": qid,
+                "question": q,
+                "golden_chunk_ids": golden,
+                "retrieved_chunk_ids": retrieved,
+                "metrics": {
+                    "hit": h,
+                    "precision": p,
+                    "recall": r,
+                    "mrr": m,
+                    "ndcg": n
+                }
+            })
+
+    def avg(lst):
+        return sum(lst) / len(lst) if lst else 0.0
+
+    result = {
+        f"hit@{k}": avg(hit_scores),
+        f"precision@{k}": avg(precision_scores),
+        f"recall@{k}": avg(recall_scores),
+        "mrr": avg(mrr_scores),
+        f"ndcg@{k}": avg(ndcg_scores),
+    }
+
+    if return_failures:
+        result["failures"] = failures
+        result["num_failures"] = len(failures)
+
+    return result
+    

rag_eval_lite-0.1.0/rag_eval_lite.egg-info/PKG-INFO

@@ -0,0 +1,26 @@
+Metadata-Version: 2.4
+Name: rag-eval-lite
+Version: 0.1.0
+Summary: Lightweight evaluation metrics for RAG (Hit@k, Recall@k, MRR, nDCG)
+Author: Your Name
+Project-URL: Homepage, https://github.com/yourusername/rag-eval-lite
+Keywords: rag,evaluation,retrieval,ml,nlp
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+
+# rag-eval-lite
+
+Lightweight RAG evaluation metrics.
+
+## Install
+pip install rag-eval-lite
+
+## Usage
+```python
+from rag_eval import evaluate_dataset
+
+results = evaluate_dataset(data, k=3)
+print(results)

rag_eval_lite-0.1.0/rag_eval_lite.egg-info/SOURCES.txt

@@ -0,0 +1,8 @@
+README.md
+pyproject.toml
+rag_eval/__init__.py
+rag_eval/metrics.py
+rag_eval_lite.egg-info/PKG-INFO
+rag_eval_lite.egg-info/SOURCES.txt
+rag_eval_lite.egg-info/dependency_links.txt
+rag_eval_lite.egg-info/top_level.txt

rag_eval_lite-0.1.0/rag_eval_lite.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

rag_eval_lite-0.1.0/rag_eval_lite.egg-info/top_level.txt

@@ -0,0 +1,2 @@
+dist
+rag_eval

rag_eval_lite-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+