minieval-pro 1.0.0__py3-none-any.whl

minieval_pro/__init__.py

@@ -0,0 +1,23 @@
+"""MiniEval - A minimal evaluation framework for LLM outputs."""
+
+# Import from the scorers folder
+from .scorers.faithfulness import FaithfulnessScorer, FaithfulnessResult
+from .scorers.relevance import RelevanceScorer, RelevanceResult
+from .scorers.toxicity import ToxicityScorer, ToxicityResult
+
+# Import from evaluator
+from .evaluator import Evaluator, EvalResult
+
+__version__ = "0.1.0"
+__all__ = [
+    # Main API
+    "Evaluator",
+    "EvalResult",
+    # Individual scorers
+    "FaithfulnessScorer",
+    "FaithfulnessResult",
+    "RelevanceScorer", 
+    "RelevanceResult",
+    "ToxicityScorer",
+    "ToxicityResult",
+]

minieval_pro/cli.py

@@ -0,0 +1,71 @@
+import argparse
+import sys
+import uvicorn
+from pathlib import Path
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        prog="minieval-pro",
+        description="MiniEval Pro — LLM hallucination detection dashboard"
+    )
+
+    parser.add_argument(
+        "--port",
+        type=int,
+        default=8000,
+        help="Port to run dashboard on (default: 8000)"
+    )
+    parser.add_argument(
+        "--host",
+        type=str,
+        default="127.0.0.1",
+        help="Host to bind to (default: 127.0.0.1)"
+    )
+    parser.add_argument(
+        "--version",
+        action="store_true",
+        help="Show version"
+    )
+    parser.add_argument(
+        "command",
+        nargs="?",
+        choices=["init", "version"],
+        help="Command to run"
+    )
+
+    args = parser.parse_args()
+
+    if args.version or args.command == "version":
+        print("minieval-pro v1.0.0")
+        sys.exit(0)
+
+    if args.command == "init":
+        print("Initializing MiniEval Pro database...")
+        try:
+            from minieval_pro.web.app import init_db
+            init_db()
+            print("Database initialized.")
+            print("Run 'minieval-pro' to start the dashboard.")
+        except Exception as e:
+            print(f"Init failed: {e}")
+            sys.exit(1)
+        return
+
+    # Default — start dashboard
+    print("=" * 45)
+    print("  MiniEval Pro — LLM Hallucination Detection")
+    print("=" * 45)
+    print(f"  Dashboard: http://{args.host}:{args.port}")
+    print("  Press Ctrl+C to stop")
+    print("=" * 45)
+
+    try:
+        uvicorn.run(
+            "minieval_pro.web.app:app",
+            host=args.host,
+            port=args.port,
+            reload=False
+        )
+    except KeyboardInterrupt:
+        print("\nStopped.")

minieval_pro/database/__init__.py

@@ -0,0 +1,29 @@
+# Empty file to make it a package.
+
+__version__ = "1.0.0"
+__author__ = "Preeti Soni"
+
+from minieval.scorers.faithfulness import FaithfulnessScorer
+from minieval.scorers.relevance import RelevanceScorer
+from minieval.scorers.toxicity import ToxicityScorer
+
+class Evaluator:
+    def __init__(self):
+        self.faithfulness = FaithfulnessScorer()
+        self.relevance = RelevanceScorer()
+        self.toxicity = ToxicityScorer()
+    
+    def score(self, question: str, context: str, answer: str):
+        faithfulness = self.faithfulness.score(context, answer)
+        relevance = self.relevance.score(question, answer)
+        toxicity = self.toxicity.score(answer)
+        overall = (faithfulness + relevance) / 2
+        
+        return type("Result", (), {
+            "faithfulness": faithfulness,
+            "relevance": relevance,
+            "toxicity": toxicity,
+            "overall_score": overall,
+            "passed": overall > 0.7,
+            "summary": lambda: f"Overall: {overall:.2f} | Faithfulness: {faithfulness:.2f} | Relevance: {relevance:.2f}"
+        })()

minieval_pro/database/db.py

@@ -0,0 +1,165 @@
+import sqlite3
+from datetime import datetime
+import os
+import uuid
+
+# Get the project root directory
+PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
+DB_PATH = os.path.join(PROJECT_ROOT, "minieval.db")
+
+def init_db():
+    """Initialize the database with the evaluations table"""
+    conn = sqlite3.connect(DB_PATH)
+    cursor = conn.cursor()
+    
+    cursor.execute('''
+        CREATE TABLE IF NOT EXISTS evaluations (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            run_id TEXT UNIQUE NOT NULL,
+            timestamp TEXT NOT NULL,
+            question TEXT NOT NULL,
+            answer TEXT NOT NULL,
+            context TEXT,
+            faithfulness REAL,
+            relevance REAL,
+            toxicity REAL,
+            overall_score REAL,
+            passed BOOLEAN,
+            failure_reason TEXT,
+            model_name TEXT,
+            model_temperature REAL,
+            prompt_template TEXT,
+            evaluation_duration REAL
+        )
+    ''')
+    
+    conn.commit()
+    conn.close()
+    print(f"[MiniEval] Database initialized at {DB_PATH}")
+
+def save_evaluation(
+    question, answer, context, 
+    faithfulness, relevance, toxicity, 
+    overall_score, passed, failure_reason=None,
+    model_name=None, model_temperature=None, 
+    prompt_template=None, evaluation_duration=None
+):
+    """Save a single evaluation to the database"""
+    conn = sqlite3.connect(DB_PATH)
+    cursor = conn.cursor()
+    
+    # Generate unique run_id
+    timestamp_str = datetime.now().strftime("%Y%m%d_%H%M%S")
+    unique_id = str(uuid.uuid4())[:8]
+    run_id = f"ev_{timestamp_str}_{unique_id}"
+    
+    cursor.execute('''
+        INSERT INTO evaluations (
+            run_id, timestamp, question, answer, context, 
+            faithfulness, relevance, toxicity, overall_score, 
+            passed, failure_reason, model_name, model_temperature, 
+            prompt_template, evaluation_duration
+        )
+        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    ''', (
+        run_id,
+        datetime.now().isoformat(),
+        question,
+        answer,
+        context,
+        faithfulness,
+        relevance,
+        toxicity,
+        overall_score,
+        passed,
+        failure_reason,
+        model_name,
+        model_temperature,
+        prompt_template,
+        evaluation_duration
+    ))
+    
+    eval_id = cursor.lastrowid
+    conn.commit()
+    conn.close()
+    
+    return eval_id, run_id
+
+def get_recent_evaluations(limit=100):
+    """Get the most recent evaluations"""
+    conn = sqlite3.connect(DB_PATH)
+    cursor = conn.cursor()
+    
+    cursor.execute('''
+        SELECT id, timestamp, question, answer, overall_score, passed, failure_reason
+        FROM evaluations
+        ORDER BY timestamp DESC
+        LIMIT ?
+    ''', (limit,))
+    
+    rows = cursor.fetchall()
+    conn.close()
+    
+    evaluations = []
+    for row in rows:
+        evaluations.append({
+            'id': row[0],
+            'timestamp': row[1],
+            'question': row[2],
+            'answer': row[3],
+            'overall_score': row[4],
+            'passed': bool(row[5]),
+            'failure_reason': row[6]
+        })
+    
+    return evaluations
+
+def get_evaluation_by_id(eval_id):
+    """Get a single evaluation by ID"""
+    conn = sqlite3.connect(DB_PATH)
+    cursor = conn.cursor()
+    
+    cursor.execute('''
+        SELECT 
+            id, run_id, timestamp, question, answer, context, 
+            faithfulness, relevance, toxicity, overall_score, 
+            passed, failure_reason, model_name, model_temperature, 
+            prompt_template, evaluation_duration
+        FROM evaluations 
+        WHERE id = ?
+    ''', (eval_id,))
+    
+    row = cursor.fetchone()
+    conn.close()
+    
+    if row:
+        return {
+            'id': row[0],
+            'run_id': row[1],
+            'timestamp': row[2],
+            'question': row[3],
+            'answer': row[4],
+            'context': row[5],
+            'faithfulness': row[6],
+            'relevance': row[7],
+            'toxicity': row[8],
+            'overall_score': row[9],
+            'passed': row[10],
+            'failure_reason': row[11],
+            'model_name': row[12],
+            'model_temperature': row[13],
+            'prompt_template': row[14],
+            'evaluation_duration': row[15]
+        }
+    return None
+def get_correlation_metrics():
+    """Get correlation data between MiniEval and GPT-4"""
+    import json
+    import os
+    
+    metrics_file = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), 'correlation_metrics.json')
+    
+    if os.path.exists(metrics_file):
+        with open(metrics_file, 'r') as f:
+            return json.load(f)
+    return None

minieval_pro/database/update_database.py

@@ -0,0 +1,59 @@
+import sqlite3
+
+conn = sqlite3.connect('minieval.db')
+cursor = conn.cursor()
+
+# Create datasets table
+cursor.execute('''
+    CREATE TABLE IF NOT EXISTS datasets (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        name TEXT UNIQUE NOT NULL,
+        description TEXT,
+        filename TEXT,
+        total_samples INTEGER,
+        created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+        is_active INTEGER DEFAULT 0
+    )
+''')
+
+# Create evaluations table with dataset_id
+cursor.execute('''
+    CREATE TABLE IF NOT EXISTS evaluation_results (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        dataset_id INTEGER,
+        question TEXT,
+        answer TEXT,
+        ground_truth TEXT,
+        faithfulness_score REAL,
+        relevance_score REAL,
+        overall_score REAL,
+        status TEXT,
+        evaluated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+        FOREIGN KEY (dataset_id) REFERENCES datasets (id)
+    )
+''')
+
+# Check if old evaluations table exists and migrate data
+cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='evaluations'")
+if cursor.fetchone():
+    # Create default dataset for old data
+    cursor.execute('''
+        INSERT OR IGNORE INTO datasets (name, description, is_active)
+        VALUES ('Legacy Evaluations', 'Imported from previous evaluations', 1)
+    ''')
+    cursor.execute("SELECT id FROM datasets WHERE name='Legacy Evaluations'")
+    dataset_id = cursor.fetchone()[0]
+    
+    # Migrate old data
+    cursor.execute('''
+        INSERT INTO evaluation_results (dataset_id, question, answer, faithfulness_score, 
+        relevance_score, overall_score, status, evaluated_at)
+        SELECT ?, question, answer, faithfulness_score, relevance_score, 
+        overall_score, status, evaluation_date
+        FROM evaluations
+    ''', (dataset_id,))
+    print("✅ Migrated legacy data")
+
+conn.commit()
+conn.close()
+print("✅ Database updated with dataset management features")

minieval_pro/evaluator.py

@@ -0,0 +1,246 @@
+from __future__ import annotations
+from dataclasses import dataclass, field
+from typing import Optional
+import time
+import re
+
+from minieval.scorers.faithfulness import FaithfulnessScorer, FaithfulnessResult
+from minieval.scorers.relevance import RelevanceScorer, RelevanceResult
+from minieval.scorers.toxicity import ToxicityScorer, ToxicityResult
+
+from minieval.database.db import save_evaluation
+
+
+@dataclass
+class EvalResult:
+    """Complete evaluation result for one LLM output."""
+    overall: float              # 0.0 to 1.0 — the single headline score
+
+    faithfulness: FaithfulnessResult
+    relevance: RelevanceResult
+    toxicity: ToxicityResult
+
+    # Weights used to calculate overall score
+    weights: dict = field(default_factory=lambda: {
+        "faithfulness": 0.5,
+        "relevance": 0.3,
+        "safety": 0.2,
+    })
+
+    def summary(self) -> str:
+        """Human-readable summary of the evaluation."""
+        lines = [
+            f"Overall Score:   {self.overall:.2f} / 1.00",
+            f"Faithfulness:    {self.faithfulness.score:.2f} ({self.faithfulness.label})",
+            f"Relevance:       {self.relevance.score:.2f} ({self.relevance.label})",
+            f"Toxicity:        {self.toxicity.score:.2f} ({self.toxicity.label})",
+            "",
+            "Details:",
+            f"  {self.faithfulness.explanation}",
+            f"  {self.relevance.explanation}",
+            f"  {self.toxicity.explanation}",
+        ]
+        return "\n".join(lines)
+
+    def passed(self, threshold: float = 0.6) -> bool:
+        """Quick check — did this output meet minimum quality?"""
+        return self.overall >= threshold and not self.toxicity.is_toxic
+
+
+class Evaluator:
+    """
+    Main MiniEval API. This is what users import and use.
+
+    Usage:
+        from minieval import Evaluator
+
+        ev = Evaluator()
+        result = ev.score(
+            question = "What is the capital of France?",
+            context  = "France is a country in Europe. Its capital is Paris.",
+            answer   = "The capital of France is Paris.",
+        )
+        print(result.summary())
+        print(result.passed())   # True
+    """
+
+    def __init__(
+        self,
+        faithfulness_weight: float = 0.5,
+        relevance_weight: float = 0.3,
+        safety_weight: float = 0.2,
+    ):
+        # Validate weights sum to 1.0
+        total = faithfulness_weight + relevance_weight + safety_weight
+        if abs(total - 1.0) > 0.01:
+            raise ValueError(
+                f"Weights must sum to 1.0, got {total:.2f}. "
+                f"Adjust faithfulness_weight, relevance_weight, safety_weight."
+            )
+
+        self.fw = faithfulness_weight
+        self.rw = relevance_weight
+        self.sw = safety_weight
+
+        # Create scorer instances — models load lazily on first use
+        self._faithfulness = FaithfulnessScorer()
+        self._relevance    = RelevanceScorer()
+        self._toxicity     = ToxicityScorer()
+
+    def _extract_contradiction(self, context: str, answer: str) -> str | None:
+        """Try to find what exactly contradicts between context and answer"""
+        # Check for number differences
+        context_numbers = re.findall(r'\d+', context)
+        answer_numbers = re.findall(r'\d+', answer)
+        
+        if context_numbers and answer_numbers and context_numbers[0] != answer_numbers[0]:
+            return f"Your source says {context_numbers[0]}, but the answer says {answer_numbers[0]}."
+        
+        # Check for percentage differences
+        context_percent = re.findall(r'(\d+)%', context)
+        answer_percent = re.findall(r'(\d+)%', answer)
+        
+        if context_percent and answer_percent and context_percent[0] != answer_percent[0]:
+            return f"Your source says {context_percent[0]}%, but the answer says {answer_percent[0]}%."
+        
+        return None
+
+    def score(
+        self,
+        question: str,   # what the user asked
+        context: str,    # what information the LLM was given
+        answer: str,     # what the LLM responded
+    ) -> EvalResult:
+        """
+        Evaluate one LLM output completely.
+        Runs all three scorers and returns a combined result.
+        """
+        # Start timing
+        start_time = time.time()
+        
+        # Run all three scorers
+        faith_result = self._faithfulness.score(context, answer)
+        rel_result   = self._relevance.score(question, answer)
+        tox_result   = self._toxicity.score(answer)
+
+        # safety_score: 1.0 = completely safe, 0.0 = completely toxic
+        # We invert toxicity because high toxicity = bad quality
+        safety_score = 1.0 - tox_result.score
+
+        # Weighted combination
+        overall = (
+            (faith_result.score * self.fw) +
+            (rel_result.score   * self.rw) +
+            (safety_score       * self.sw)
+        )
+        overall = round(max(0.0, min(1.0, overall)), 4)
+
+        # Calculate duration
+        evaluation_duration = round(time.time() - start_time, 3)
+
+        # Create EvalResult first
+        result = EvalResult(
+            overall=overall,
+            faithfulness=faith_result,
+            relevance=rel_result,
+            toxicity=tox_result,
+            weights={
+                "faithfulness": self.fw,
+                "relevance": self.rw,
+                "safety": self.sw,
+            },
+        )
+
+        # Save to database
+        threshold = 0.6
+        passed = result.passed(threshold)
+
+        # ========== PLAIN ENGLISH EXPLAINER ==========
+        failure_reason = None
+        if not passed:
+            reasons = []
+            
+            # Check for contradiction
+            contradiction = self._extract_contradiction(context, answer)
+            
+            # Faithfulness / Hallucination check
+            if faith_result.score < 0.5:
+                if contradiction:
+                    reasons.append(f"❌ HALLUCINATION: {contradiction}")
+                else:
+                    # Try to show what the answer said vs context
+                    context_preview = context[:150] if context else "your source document"
+                    answer_preview = answer[:150]
+                    reasons.append(f"❌ HALLUCINATION: The answer says \"{answer_preview}...\"")
+                    reasons.append(f"   But this information is NOT found in {context_preview}...")
+                reasons.append(f"   The LLM invented information that wasn't there. Confidence: {faith_result.score:.0%}")
+            
+            # Relevance check
+            if rel_result.score < 0.5:
+                reasons.append(f"❌ IRRELEVANT ANSWER: The answer doesn't address the question.")
+                reasons.append(f"   Question: \"{question[:100]}...\"")
+                reasons.append(f"   Answer talked about: \"{answer[:100]}...\"")
+                reasons.append(f"   Relevance score: {rel_result.score:.0%} (needs >50%)")
+            
+            # Toxicity check
+            if tox_result.is_toxic:
+                reasons.append(f"❌ TOXIC CONTENT: The answer contains unsafe or inappropriate language.")
+                reasons.append(f"   Offending text: \"{answer[:150]}...\"")
+                reasons.append(f"   Toxicity score: {tox_result.score:.0%} (should be <30%)")
+            
+            # Low overall score (but no specific issue found)
+            if overall < threshold and faith_result.score >= 0.5 and rel_result.score >= 0.5 and not tox_result.is_toxic:
+                reasons.append(f"⚠️ LOW QUALITY SCORE: {overall:.0%} (needs >{threshold:.0%})")
+                reasons.append(f"   The answer is faithful and relevant, but quality is below standard.")
+                reasons.append(f"   Consider improving answer clarity, adding more details, or making it more specific.")
+            
+            failure_reason = "\n\n".join(reasons)
+        
+        if not failure_reason and passed:
+            failure_reason = f"✅ PASSED: Quality score {overall:.0%} meets threshold (>70%). Answer is faithful, relevant, and safe."
+
+        # Hyperparameters
+        model_name = "minieval-v1"
+        model_temperature = 0.7
+        prompt_template = "default"
+
+        # Save to database with hyperparameters
+        save_evaluation(
+            question=question,
+            answer=answer,
+            context=context,
+            faithfulness=faith_result.score,
+            relevance=rel_result.score,
+            toxicity=tox_result.score,
+            overall_score=overall,
+            passed=passed,
+            failure_reason=failure_reason,
+            model_name=model_name,
+            model_temperature=model_temperature,
+            prompt_template=prompt_template,
+            evaluation_duration=evaluation_duration
+        )
+
+        return result
+
+    def score_batch(
+        self,
+        items: list[dict],  # list of {question, context, answer}
+    ) -> list[EvalResult]:
+        """
+        Evaluate multiple outputs at once.
+
+        Usage:
+            results = ev.score_batch([
+                {"question": "...", "context": "...", "answer": "..."},
+                {"question": "...", "context": "...", "answer": "..."},
+            ])
+        """
+        return [
+            self.score(
+                question=item["question"],
+                context=item["context"],
+                answer=item["answer"],
+            )
+            for item in items
+        ]

minieval_pro/scorers/__init__.py

@@ -0,0 +1,12 @@
+from .faithfulness import FaithfulnessScorer, FaithfulnessResult
+from .relevance import RelevanceScorer, RelevanceResult
+from .toxicity import ToxicityScorer, ToxicityResult
+
+__all__ = [
+    "FaithfulnessScorer",
+    "FaithfulnessResult",
+    "RelevanceScorer",
+    "RelevanceResult",
+    "ToxicityScorer",
+    "ToxicityResult",
+]