rag-scorecard 0.1.0__tar.gz

rag_scorecard-0.1.0/PKG-INFO

@@ -0,0 +1,15 @@
+Metadata-Version: 2.4
+Name: rag-scorecard
+Version: 0.1.0
+Summary: A CLI and SDK tool to audit RAG pipelines for systemic infrastructure weak points.
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: pydantic>=2.4.0
+Requires-Dist: chromadb>=0.4.15
+Requires-Dist: typer>=0.9.0
+Requires-Dist: rich>=13.6.0
+Requires-Dist: pyyaml>=6.0.1
+
+# rag-scorecard
+
+A CLI and SDK tool to audit RAG pipelines.

rag_scorecard-0.1.0/README.md

@@ -0,0 +1,3 @@
+# rag-scorecard
+
+A CLI and SDK tool to audit RAG pipelines.

rag_scorecard-0.1.0/pyproject.toml

@@ -0,0 +1,23 @@
+[build-system]
+requires = ["setuptools>=61.0.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "rag-scorecard"
+version = "0.1.0"
+description = "A CLI and SDK tool to audit RAG pipelines for systemic infrastructure weak points."
+readme = "README.md"
+requires-python = ">=3.10"
+dependencies = [
+    "pydantic>=2.4.0",
+    "chromadb>=0.4.15",
+    "typer>=0.9.0",
+    "rich>=13.6.0",
+    "pyyaml>=6.0.1",
+]
+
+[project.scripts]
+rag-scorecard = "scorecard.cli:app"
+
+[tool.setuptools]
+packages = ["scorecard"]

rag_scorecard-0.1.0/rag_scorecard.egg-info/PKG-INFO

@@ -0,0 +1,15 @@
+Metadata-Version: 2.4
+Name: rag-scorecard
+Version: 0.1.0
+Summary: A CLI and SDK tool to audit RAG pipelines for systemic infrastructure weak points.
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: pydantic>=2.4.0
+Requires-Dist: chromadb>=0.4.15
+Requires-Dist: typer>=0.9.0
+Requires-Dist: rich>=13.6.0
+Requires-Dist: pyyaml>=6.0.1
+
+# rag-scorecard
+
+A CLI and SDK tool to audit RAG pipelines.

rag_scorecard-0.1.0/rag_scorecard.egg-info/SOURCES.txt

@@ -0,0 +1,11 @@
+README.md
+pyproject.toml
+rag_scorecard.egg-info/PKG-INFO
+rag_scorecard.egg-info/SOURCES.txt
+rag_scorecard.egg-info/dependency_links.txt
+rag_scorecard.egg-info/entry_points.txt
+rag_scorecard.egg-info/requires.txt
+rag_scorecard.egg-info/top_level.txt
+scorecard/__init__.py
+scorecard/models.py
+scorecard/reporter.py

rag_scorecard-0.1.0/rag_scorecard.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

rag_scorecard-0.1.0/rag_scorecard.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+rag-scorecard = scorecard.cli:app

rag_scorecard-0.1.0/rag_scorecard.egg-info/requires.txt

@@ -0,0 +1,5 @@
+pydantic>=2.4.0
+chromadb>=0.4.15
+typer>=0.9.0
+rich>=13.6.0
+pyyaml>=6.0.1

rag_scorecard-0.1.0/rag_scorecard.egg-info/top_level.txt

@@ -0,0 +1 @@
+scorecard

rag_scorecard-0.1.0/scorecard/__init__.py

@@ -0,0 +1 @@
+__version__ = '0.1.0'

rag_scorecard-0.1.0/scorecard/models.py

@@ -0,0 +1,23 @@
+
+from typing import List, Optional, Dict, Any
+from pydantic import BaseModel, Field, ConfigDict
+
+class RAGSample(BaseModel):
+    model_config = ConfigDict(strict=True)
+
+    query: str = Field(..., description="The user query or prompt.")
+    retrieved_contexts: List[str] = Field(..., description="List of document strings returned by the retriever.")
+    context_scores: List[float] = Field(..., description="Distance or similarity scores corresponding to retrieved contexts.")
+    generated_response: Optional[str] = Field(None, description="The final generation from the LLM, if applicable.")
+    ground_truth: Optional[str] = Field(None, description="The expected factual answer or exact context substring to match against.")
+
+class MetricResult(BaseModel):
+    metric_name: str
+    score: float
+    metadata: Dict[str, Any] = Field(default_factory=dict)
+
+class EvaluationReport(BaseModel):
+    samples: List[RAGSample]
+    aggregated_metrics: Dict[str, float]
+    latency_breakdown_ms: Dict[str, float]
+    estimated_cost_usd: float

rag_scorecard-0.1.0/scorecard/reporter.py

@@ -0,0 +1,24 @@
+from scorecard.models import EvaluationReport
+
+def print_terminal_scorecard(report: EvaluationReport):
+    print("\n--- RAG Scorecard Report ---")
+    print("Aggregated Metrics:")
+    for metric_name, score in report.aggregated_metrics.items():
+        print(f"  {metric_name}: {score:.4f}")
+    print(f"Estimated Cost (USD): {report.estimated_cost_usd:.6f}")
+    print("Latency Breakdown (ms):")
+    for stage, latency in report.latency_breakdown_ms.items():
+        print(f"  {stage}: {latency:.2f}")
+    print("---------------------------\n")
+
+def export_markdown_scorecard(report: EvaluationReport) -> str:
+    md = "# RAG Scorecard Report\n\n"
+    md += "## Aggregated Metrics\n"
+    for metric_name, score in report.aggregated_metrics.items():
+        md += f"- **{metric_name}**: {score:.4f}\n"
+    md += "\n## Cost & Latency\n"
+    md += f"- **Estimated Cost (USD)**: {report.estimated_cost_usd:.6f}\n"
+    md += "\n### Latency Breakdown\n"
+    for stage, latency in report.latency_breakdown_ms.items():
+        md += f"- **{stage}**: {latency:.2f} ms\n"
+    return md

rag_scorecard-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+