rag-forge-evaluator 0.1.0__tar.gz

rag_forge_evaluator-0.1.0/.gitignore

@@ -0,0 +1,62 @@
+# Dependencies
+node_modules/
+.pnpm-store/
+
+# Build outputs
+dist/
+build/
+*.tsbuildinfo
+
+# Turborepo
+.turbo/
+
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.egg-info/
+*.egg
+.venv/
+.python-version-local
+
+# Python tools
+.mypy_cache/
+.ruff_cache/
+.pytest_cache/
+htmlcov/
+.coverage
+.coverage.*
+
+# Environment variables
+.env
+.env.local
+.env.*.local
+
+# IDE
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+
+# OS
+.DS_Store
+Thumbs.db
+desktop.ini
+
+# Test & coverage
+coverage/
+*.lcov
+
+# Logs
+*.log
+npm-debug.log*
+pnpm-debug.log*
+
+.claude/
+
+# Next.js
+apps/*/.next
+apps/*/out
+apps/*/next-env.d.ts
+.vercel

rag_forge_evaluator-0.1.0/PKG-INFO

@@ -0,0 +1,66 @@
+Metadata-Version: 2.4
+Name: rag-forge-evaluator
+Version: 0.1.0
+Summary: Evaluation engine: RAGAS, DeepEval, LLM-as-Judge, and audit report generation
+Project-URL: Homepage, https://github.com/hallengray/rag-forge
+Project-URL: Repository, https://github.com/hallengray/rag-forge
+Project-URL: Issues, https://github.com/hallengray/rag-forge/issues
+Project-URL: Documentation, https://github.com/hallengray/rag-forge#readme
+Author: Femi Adedayo
+License-Expression: MIT
+Keywords: deepeval,evaluation,llm-as-judge,rag,ragas,rmm
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.11
+Requires-Dist: anthropic>=0.30
+Requires-Dist: jinja2>=3.1
+Requires-Dist: openai>=1.30
+Requires-Dist: pydantic>=2.0
+Provides-Extra: deepeval
+Requires-Dist: deepeval>=1.0; extra == 'deepeval'
+Provides-Extra: pdf
+Requires-Dist: playwright>=1.40; extra == 'pdf'
+Provides-Extra: ragas
+Requires-Dist: ragas>=0.2; extra == 'ragas'
+Description-Content-Type: text/markdown
+
+# rag-forge-evaluator
+
+RAG pipeline evaluation engine for the RAG-Forge toolkit: RAGAS, DeepEval, LLM-as-Judge, and the RAG Maturity Model.
+
+## Installation
+
+```bash
+pip install rag-forge-evaluator
+```
+
+## Usage
+
+```python
+from rag_forge_evaluator.assess import RMMAssessor
+
+assessor = RMMAssessor()
+result = assessor.assess(config={
+    "retrieval_strategy": "hybrid",
+    "input_guard_configured": True,
+    "output_guard_configured": True,
+})
+print(result.badge)  # e.g., "RMM-3 Better Trust"
+```
+
+## Features
+
+- RMM (RAG Maturity Model) scoring (levels 0-5)
+- RAGAS, DeepEval, and LLM-as-Judge evaluators
+- Golden set management with traffic sampling
+- Cost estimation
+- HTML and PDF report generation
+
+## License
+
+MIT

rag_forge_evaluator-0.1.0/README.md

@@ -0,0 +1,35 @@
+# rag-forge-evaluator
+
+RAG pipeline evaluation engine for the RAG-Forge toolkit: RAGAS, DeepEval, LLM-as-Judge, and the RAG Maturity Model.
+
+## Installation
+
+```bash
+pip install rag-forge-evaluator
+```
+
+## Usage
+
+```python
+from rag_forge_evaluator.assess import RMMAssessor
+
+assessor = RMMAssessor()
+result = assessor.assess(config={
+    "retrieval_strategy": "hybrid",
+    "input_guard_configured": True,
+    "output_guard_configured": True,
+})
+print(result.badge)  # e.g., "RMM-3 Better Trust"
+```
+
+## Features
+
+- RMM (RAG Maturity Model) scoring (levels 0-5)
+- RAGAS, DeepEval, and LLM-as-Judge evaluators
+- Golden set management with traffic sampling
+- Cost estimation
+- HTML and PDF report generation
+
+## License
+
+MIT

rag_forge_evaluator-0.1.0/pyproject.toml

@@ -0,0 +1,42 @@
+[project]
+name = "rag-forge-evaluator"
+version = "0.1.0"
+description = "Evaluation engine: RAGAS, DeepEval, LLM-as-Judge, and audit report generation"
+requires-python = ">=3.11"
+license = "MIT"
+authors = [{ name = "Femi Adedayo" }]
+keywords = ["rag", "evaluation", "ragas", "deepeval", "llm-as-judge", "rmm"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+]
+readme = "README.md"
+dependencies = [
+    "pydantic>=2.0",
+    "jinja2>=3.1",
+    "anthropic>=0.30",
+    "openai>=1.30",
+]
+
+[project.urls]
+Homepage = "https://github.com/hallengray/rag-forge"
+Repository = "https://github.com/hallengray/rag-forge"
+Issues = "https://github.com/hallengray/rag-forge/issues"
+Documentation = "https://github.com/hallengray/rag-forge#readme"
+
+[project.optional-dependencies]
+ragas = ["ragas>=0.2"]
+deepeval = ["deepeval>=1.0"]
+pdf = ["playwright>=1.40"]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/rag_forge_evaluator"]

rag_forge_evaluator-0.1.0/src/rag_forge_evaluator/__init__.py

@@ -0,0 +1,3 @@
+"""RAG-Forge Evaluator: Evaluation engine with RAGAS, DeepEval, and LLM-as-Judge."""
+
+__version__ = "0.1.0"

rag_forge_evaluator-0.1.0/src/rag_forge_evaluator/assess.py

@@ -0,0 +1,215 @@
+"""RMM assessment: score a pipeline against the RAG Maturity Model.
+
+Inspects configuration and optional audit data to determine the current
+RMM level (0-5) without running a full evaluation.
+"""
+
+import json
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+
+@dataclass
+class AssessmentCheck:
+    """A single check within an RMM level."""
+
+    description: str
+    passed: bool
+    source: str  # "config", "audit", or "unknown"
+
+
+@dataclass
+class AssessmentResult:
+    """Result of an RMM assessment."""
+
+    rmm_level: int
+    rmm_name: str
+    criteria: list[dict[str, Any]]
+    badge: str
+
+
+_LEVEL_NAMES = {
+    0: "Naive RAG",
+    1: "Better Recall",
+    2: "Better Precision",
+    3: "Better Trust",
+    4: "Better Workflow",
+    5: "Enterprise",
+}
+
+
+class RMMAssessor:
+    """Assesses a pipeline's RMM level from config and optional audit data."""
+
+    def load_audit_metrics(self, report_path: str) -> dict[str, float]:
+        """Load metric scores from an audit JSON report."""
+        path = Path(report_path)
+        with path.open() as f:
+            data = json.load(f)
+        metrics: dict[str, float] = {}
+        for m in data.get("metrics", []):
+            try:
+                metrics[str(m["name"])] = float(m["score"])
+            except (KeyError, ValueError, TypeError):
+                continue
+        return metrics
+
+    def assess(
+        self,
+        config: dict[str, Any],
+        audit_metrics: dict[str, float] | None = None,
+    ) -> AssessmentResult:
+        """Determine RMM level from pipeline configuration and audit data."""
+        metrics = audit_metrics or {}
+        all_criteria: list[dict[str, Any]] = []
+        current_level = 0
+
+        def _to_dict(checks: list[AssessmentCheck]) -> list[dict[str, Any]]:
+            return [
+                {"description": c.description, "passed": c.passed, "source": c.source}
+                for c in checks
+            ]
+
+        # RMM-0: Naive RAG — always passes
+        checks_0 = [AssessmentCheck("Pipeline exists", True, "config")]
+        all_criteria.append(
+            {"level": 0, "name": "Naive RAG", "passed": True, "checks": _to_dict(checks_0)}
+        )
+
+        # RMM-1: Better Recall
+        hybrid = config.get("retrieval_strategy") == "hybrid"
+        sparse = bool(config.get("sparse_index_configured", False))
+        recall_ok = metrics.get("recall_at_k", 0.0) >= 0.70
+        checks_1 = [
+            AssessmentCheck("Hybrid search configured", hybrid, "config"),
+            AssessmentCheck("Sparse index configured", sparse, "config"),
+            AssessmentCheck(
+                "Recall@5 >= 70%",
+                recall_ok,
+                "audit" if "recall_at_k" in metrics else "unknown",
+            ),
+        ]
+        level1_passed = hybrid and sparse and recall_ok
+        if level1_passed:
+            current_level = 1
+        all_criteria.append(
+            {
+                "level": 1,
+                "name": "Better Recall",
+                "passed": level1_passed,
+                "checks": _to_dict(checks_1),
+            }
+        )
+
+        # RMM-2: Better Precision
+        reranker = bool(config.get("reranker_configured", False))
+        ndcg_ok = metrics.get("ndcg_improvement", 0.0) >= 0.10
+        checks_2 = [
+            AssessmentCheck("Reranker active", reranker, "config"),
+            AssessmentCheck(
+                "nDCG@10 improvement >= 10%",
+                ndcg_ok,
+                "audit" if "ndcg_improvement" in metrics else "unknown",
+            ),
+        ]
+        level2_passed = level1_passed and reranker and ndcg_ok
+        if level2_passed:
+            current_level = 2
+        all_criteria.append(
+            {
+                "level": 2,
+                "name": "Better Precision",
+                "passed": level2_passed,
+                "checks": _to_dict(checks_2),
+            }
+        )
+
+        # RMM-3: Better Trust
+        input_guard = bool(config.get("input_guard_configured", False))
+        output_guard = bool(config.get("output_guard_configured", False))
+        faith_ok = metrics.get("faithfulness", 0.0) >= 0.85
+        ctx_ok = metrics.get("context_relevance", 0.0) >= 0.80
+        checks_3 = [
+            AssessmentCheck("InputGuard active", input_guard, "config"),
+            AssessmentCheck("OutputGuard active", output_guard, "config"),
+            AssessmentCheck(
+                "Faithfulness >= 85%",
+                faith_ok,
+                "audit" if "faithfulness" in metrics else "unknown",
+            ),
+            AssessmentCheck(
+                "Context relevance >= 80%",
+                ctx_ok,
+                "audit" if "context_relevance" in metrics else "unknown",
+            ),
+        ]
+        level3_passed = (
+            current_level >= 2 and input_guard and output_guard and faith_ok and ctx_ok
+        )
+        if level3_passed:
+            current_level = 3
+        all_criteria.append(
+            {
+                "level": 3,
+                "name": "Better Trust",
+                "passed": level3_passed,
+                "checks": _to_dict(checks_3),
+            }
+        )
+
+        # RMM-4: Better Workflow
+        caching = bool(config.get("caching_configured", False))
+        latency_ok = metrics.get("latency_p95", 99999.0) <= 4000
+        cost_tracked = bool(config.get("cost_tracking_configured", False))
+        checks_4 = [
+            AssessmentCheck("Semantic caching active", caching, "config"),
+            AssessmentCheck(
+                "P95 latency < 4s",
+                latency_ok,
+                "audit" if "latency_p95" in metrics else "unknown",
+            ),
+            AssessmentCheck("Cost per query tracked", cost_tracked, "config"),
+        ]
+        level4_passed = current_level >= 3 and caching and latency_ok and cost_tracked
+        if level4_passed:
+            current_level = 4
+        all_criteria.append(
+            {
+                "level": 4,
+                "name": "Better Workflow",
+                "passed": level4_passed,
+                "checks": _to_dict(checks_4),
+            }
+        )
+
+        # RMM-5: Enterprise
+        drift = bool(config.get("drift_detection_configured", False))
+        ci_gates = bool(config.get("ci_cd_gates_configured", False))
+        adversarial_ok = bool(config.get("adversarial_tests_passing", False))
+        checks_5 = [
+            AssessmentCheck("Drift detection live", drift, "config"),
+            AssessmentCheck("CI/CD evaluation gates configured", ci_gates, "config"),
+            AssessmentCheck("Adversarial tests green", adversarial_ok, "config"),
+        ]
+        level5_passed = level4_passed and drift and ci_gates and adversarial_ok
+        if level5_passed:
+            current_level = 5
+        all_criteria.append(
+            {
+                "level": 5,
+                "name": "Enterprise",
+                "passed": level5_passed,
+                "checks": _to_dict(checks_5),
+            }
+        )
+
+        name = _LEVEL_NAMES.get(current_level, "Unknown")
+        badge = f"RMM-{current_level} {name}"
+
+        return AssessmentResult(
+            rmm_level=current_level,
+            rmm_name=name,
+            criteria=all_criteria,
+            badge=badge,
+        )

rag_forge_evaluator-0.1.0/src/rag_forge_evaluator/audit.py

@@ -0,0 +1,153 @@
+"""Audit orchestrator: coordinates evaluation, history, and report generation."""
+
+from contextlib import nullcontext
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+from rag_forge_evaluator.engine import EvaluationResult
+from rag_forge_evaluator.engines import create_evaluator
+from rag_forge_evaluator.history import AuditHistory, AuditHistoryEntry
+from rag_forge_evaluator.input_loader import InputLoader
+from rag_forge_evaluator.judge.base import JudgeProvider
+from rag_forge_evaluator.judge.mock_judge import MockJudge
+from rag_forge_evaluator.maturity import RMMLevel, RMMScorer
+from rag_forge_evaluator.report.generator import ReportGenerator
+
+
+@dataclass
+class AuditConfig:
+    """Configuration for an audit run."""
+
+    input_path: Path | None = None
+    golden_set_path: Path | None = None
+    judge_model: str | None = None
+    output_dir: Path = Path("./reports")
+    generate_pdf: bool = False
+    thresholds: dict[str, float] | None = None
+    evaluator_engine: str = "llm-judge"
+    tracer: Any = None  # opentelemetry.trace.Tracer or None
+
+
+@dataclass
+class AuditReport:
+    """Complete audit report with evaluation results and RMM scoring."""
+
+    evaluation: EvaluationResult
+    rmm_level: RMMLevel
+    report_path: Path
+    json_report_path: Path
+    samples_evaluated: int
+    pdf_report_path: Path | None = None
+
+
+def _create_judge(model: str | None) -> JudgeProvider:
+    """Create a judge provider based on model name."""
+    if model == "mock" or model is None:
+        return MockJudge()
+    if model in ("claude", "claude-sonnet"):
+        from rag_forge_evaluator.judge.claude_judge import ClaudeJudge
+        return ClaudeJudge()
+    if model in ("openai", "gpt-4o"):
+        from rag_forge_evaluator.judge.openai_judge import OpenAIJudge
+        return OpenAIJudge()
+    return MockJudge()
+
+
+class AuditOrchestrator:
+    """Orchestrates the full audit pipeline."""
+
+    def __init__(self, config: AuditConfig) -> None:
+        self.config = config
+        self._tracer = config.tracer
+
+    def _span(self, name: str) -> Any:
+        """Return an active span context manager, or a no-op if no tracer is configured."""
+        if self._tracer is not None:
+            return self._tracer.start_as_current_span(name)
+        return nullcontext()
+
+    def run(self) -> AuditReport:
+        """Execute the full audit pipeline."""
+        with self._span("rag-forge.audit"):
+            # 1. Load input
+            with self._span("rag-forge.load_input") as span:
+                if self.config.input_path:
+                    samples = InputLoader.load_jsonl(self.config.input_path)
+                    source_type = "jsonl"
+                elif self.config.golden_set_path:
+                    samples = InputLoader.load_golden_set(self.config.golden_set_path)
+                    source_type = "golden_set"
+                else:
+                    msg = "Either input_path or golden_set_path must be provided"
+                    raise ValueError(msg)
+                if span is not None:
+                    span.set_attribute("sample_count", len(samples))
+                    span.set_attribute("source_type", source_type)
+
+            # 2. Create evaluator via factory
+            judge = _create_judge(self.config.judge_model)
+            evaluator = create_evaluator(
+                self.config.evaluator_engine,
+                judge=judge,
+                thresholds=self.config.thresholds,
+            )
+
+            # 3. Run evaluation
+            with self._span("rag-forge.evaluate") as span:
+                evaluation = evaluator.evaluate(samples)
+                if span is not None:
+                    span.set_attribute("engine", self.config.evaluator_engine)
+                    span.set_attribute("sample_count", evaluation.samples_evaluated)
+
+            # 4. Score against RMM
+            metric_map = {m.name: m.score for m in evaluation.metrics}
+            with self._span("rag-forge.score_rmm") as span:
+                rmm_level = RMMScorer().assess(metric_map)
+                if span is not None:
+                    span.set_attribute("rmm_level", int(rmm_level))
+
+            # 5. Load history and compute trends
+            history = AuditHistory(self.config.output_dir / "audit-history.json")
+            previous = history.get_previous()
+            trends = history.compute_trends(metric_map, previous)
+
+            # 6. Generate reports
+            generator = ReportGenerator(output_dir=self.config.output_dir)
+            with self._span("rag-forge.generate_report") as span:
+                report_path = generator.generate_html(
+                    evaluation, rmm_level,
+                    trends=trends,
+                    sample_results=evaluation.sample_results,
+                )
+                json_report_path = generator.generate_json(
+                    evaluation, rmm_level,
+                    sample_results=evaluation.sample_results,
+                )
+                if span is not None:
+                    span.set_attribute("report_path", str(report_path))
+
+            # 7. Generate PDF (optional)
+            pdf_report_path: Path | None = None
+            if self.config.generate_pdf:
+                from rag_forge_evaluator.report.pdf import PDFGenerator
+                pdf_report_path = PDFGenerator().generate(report_path)
+
+            # 8. Append to history (after all reports succeed)
+            history.append(AuditHistoryEntry(
+                timestamp=datetime.now(tz=UTC).strftime("%Y-%m-%dT%H:%M:%SZ"),
+                metrics=metric_map,
+                rmm_level=int(rmm_level),
+                overall_score=evaluation.overall_score,
+                passed=evaluation.passed,
+            ))
+
+            return AuditReport(
+                evaluation=evaluation,
+                rmm_level=rmm_level,
+                report_path=report_path,
+                json_report_path=json_report_path,
+                samples_evaluated=evaluation.samples_evaluated,
+                pdf_report_path=pdf_report_path,
+            )