ragaeval 0.1.0__py3-none-any.whl

ragaeval/__init__.py

@@ -0,0 +1,34 @@
+"""
+ragaeval - RAG Evaluation Package
+
+A pip-installable Python library providing drop-in RAG evaluation capabilities using RAGAS.
+"""
+
+import sys
+
+__version__ = "0.1.0"
+
+# Setup VertexAI mock before other imports
+from .mock_setup import setup_vertexai_mock
+setup_vertexai_mock()
+
+# Set UTF-8 encoding on Windows
+if sys.platform == 'win32':
+    try:
+        sys.stdout.reconfigure(encoding='utf-8')
+        sys.stderr.reconfigure(encoding='utf-8')
+    except Exception:
+        pass
+
+# Import public API components
+from .interceptors import evaluate_rag, EvalSession, log
+from .config import ConfigManager
+
+# Public API exports
+__all__ = [
+    "__version__",
+    "evaluate_rag",
+    "EvalSession",
+    "log",
+    "ConfigManager",
+]

ragaeval/cli.py

@@ -0,0 +1,332 @@
+"""Command-line interface for the ragaeval package.
+
+This module provides the CLI with commands: run, configure, status, clear.
+"""
+
+import sys
+import argparse
+from pathlib import Path
+
+from . import __version__
+from .config import ConfigManager
+from .runner import EvaluationRunner
+from .report import ReportGenerator
+from .logger import Logger
+
+
+def main():
+    """
+    CLI entry point registered as console script.
+    
+    Subcommands: run, configure, status, clear
+    """
+    parser = argparse.ArgumentParser(
+        prog='ragaeval',
+        description='RAG Evaluation using RAGAS framework'
+    )
+    
+    parser.add_argument(
+        '--version',
+        action='version',
+        version=f'ragaeval {__version__}'
+    )
+    
+    subparsers = parser.add_subparsers(dest='command', help='Available commands')
+    
+    # Run command
+    run_parser = subparsers.add_parser('run', help='Execute evaluation on logged data')
+    run_parser.add_argument(
+        '--file',
+        default='.ragaeval_log.jsonl',
+        help='Log file path (default: .ragaeval_log.jsonl)'
+    )
+    run_parser.add_argument(
+        '--model',
+        default='gpt-4o',
+        help='Azure deployment name (default: gpt-4o)'
+    )
+    run_parser.add_argument(
+        '--output',
+        default='eval_results.csv',
+        help='CSV output path (default: eval_results.csv)'
+    )
+    
+    # Configure command
+    config_parser = subparsers.add_parser('configure', help='Interactive credential setup')
+    
+    # Status command
+    status_parser = subparsers.add_parser('status', help='Display configuration and log status')
+    
+    # Clear command
+    clear_parser = subparsers.add_parser('clear', help='Clear evaluation logs')
+    clear_parser.add_argument(
+        '--force',
+        action='store_true',
+        help='Skip confirmation prompt'
+    )
+    
+    args = parser.parse_args()
+    
+    if args.command == 'run':
+        sys.exit(cmd_run(args))
+    elif args.command == 'configure':
+        sys.exit(cmd_configure(args))
+    elif args.command == 'status':
+        sys.exit(cmd_status(args))
+    elif args.command == 'clear':
+        sys.exit(cmd_clear(args))
+    else:
+        parser.print_help()
+        sys.exit(0)
+
+
+def cmd_run(args):
+    """
+    Execute evaluation on logged data.
+    
+    Steps:
+    1. Initialize ConfigManager
+    2. Create EvaluationRunner
+    3. Load logs
+    4. Run evaluation
+    5. Generate report
+    
+    Args:
+        args: Parsed command-line arguments
+        
+    Returns:
+        Exit code (0 for success, 1 for error)
+    """
+    try:
+        print(f"Loading logs from {args.file}...")
+        
+        # Initialize config and runner
+        config = ConfigManager()
+        runner = EvaluationRunner(config, model=args.model)
+        
+        # Load logs
+        records = runner.load_logs(args.file)
+        print(f"Loaded {len(records)} records")
+        
+        # Run evaluation
+        print(f"\nRunning RAGAS evaluation with {args.model}...")
+        print("(This may take a few minutes depending on the number of records)")
+        
+        df = runner.run_evaluation(records, args.output)
+        
+        # Generate report
+        print("\n")
+        report = ReportGenerator(df)
+        report.print_terminal_report()
+        
+        print(f"\nResults saved to: {args.output}")
+        
+        return 0
+        
+    except Exception as e:
+        print(f"Error: {str(e)}", file=sys.stderr)
+        return 1
+
+
+def cmd_configure(args):
+    """
+    Interactive configuration setup.
+    
+    Steps:
+    1. Prompt for Azure OpenAI credentials
+    2. Prompt for optional OpenAI API key
+    3. Validate endpoint format
+    4. Offer to test credentials
+    5. Save to ~/.ragaeval/config.json
+    6. Display config path
+    
+    Args:
+        args: Parsed command-line arguments
+        
+    Returns:
+        Exit code (0 for success, 1 for error)
+    """
+    try:
+        print("=" * 80)
+        print("  ragaeval Configuration")
+        print("=" * 80)
+        print("\nEnter your API credentials. Press Enter to skip optional fields.\n")
+        
+        # Prompt for Azure OpenAI credentials
+        azure_api_key = input("Azure OpenAI API Key: ").strip()
+        azure_endpoint = input("Azure OpenAI Endpoint (e.g., https://xxx.openai.azure.com/): ").strip()
+        azure_deployment = input("Azure Deployment Name (e.g., gpt-4o): ").strip()
+        azure_version = input("Azure API Version (default: 2024-02-15-preview): ").strip() or "2024-02-15-preview"
+        
+        # Prompt for optional OpenAI API key
+        print("\nOptional: OpenAI API key for embeddings (or press Enter to use local)")
+        openai_key = input("OpenAI API Key: ").strip()
+        
+        # Build config dict
+        config_dict = {
+            "azure_api_key": azure_api_key,
+            "azure_endpoint": azure_endpoint,
+            "azure_deployment_name": azure_deployment,
+            "azure_api_version": azure_version,
+        }
+        
+        if openai_key:
+            config_dict["openai_api_key"] = openai_key
+        
+        # Save config
+        config = ConfigManager()
+        config_path = config.save_config(config_dict)
+        
+        print(f"\n✅ Configuration saved to: {config_path}")
+        print("\nYou can now run: ragaeval run")
+        
+        return 0
+        
+    except Exception as e:
+        print(f"Error: {str(e)}", file=sys.stderr)
+        return 1
+
+
+def cmd_status(args):
+    """
+    Display configuration and log status.
+    
+    Output:
+    - Log file status (exists, record count)
+    - Config file status (exists, which credentials configured)
+    - Environment variable status (which keys set)
+    - Embedding model selection
+    - Azure OpenAI completeness
+    
+    Args:
+        args: Parsed command-line arguments
+        
+    Returns:
+        Exit code (0 for success)
+    """
+    try:
+        print("=" * 80)
+        print("  ragaeval Status")
+        print("=" * 80)
+        print()
+        
+        # Log file status
+        logger = Logger()
+        if logger.log_exists():
+            count = logger.count_records()
+            print(f"📄 Log File: {logger.get_log_path()} ({count} records)")
+        else:
+            print(f"📄 Log File: Not found ({logger.get_log_path()})")
+        
+        print()
+        
+        # Config file status
+        config = ConfigManager()
+        if config.config_exists():
+            print(f"⚙️  Config File: {config.get_config_path()}")
+            
+            # Check which credentials are configured
+            cfg = config.load_config()
+            creds_configured = []
+            if cfg.get('azure_api_key'):
+                creds_configured.append("Azure OpenAI")
+            if cfg.get('openai_api_key'):
+                creds_configured.append("OpenAI (embeddings)")
+            
+            if creds_configured:
+                print(f"   Credentials: {', '.join(creds_configured)}")
+        else:
+            print(f"⚙️  Config File: Not found (run 'ragaeval configure')")
+        
+        print()
+        
+        # Environment variables
+        import os
+        env_vars = []
+        if os.getenv('AZURE_OPENAI_API_KEY'):
+            env_vars.append("AZURE_OPENAI_API_KEY")
+        if os.getenv('AZURE_OPENAI_ENDPOINT'):
+            env_vars.append("AZURE_OPENAI_ENDPOINT")
+        if os.getenv('AZURE_OPENAI_DEPLOYMENT_NAME'):
+            env_vars.append("AZURE_OPENAI_DEPLOYMENT_NAME")
+        if os.getenv('OPENAI_API_KEY'):
+            env_vars.append("OPENAI_API_KEY")
+        
+        if env_vars:
+            print(f"🔑 Environment Variables: {', '.join(env_vars)}")
+        else:
+            print("🔑 Environment Variables: None set")
+        
+        print()
+        
+        # Embedding model
+        openai_key = config.get_openai_api_key()
+        if openai_key and openai_key.startswith('sk-'):
+            print("🧮 Embeddings: OpenAI (text-embedding-3-small)")
+        else:
+            print("🧮 Embeddings: Local (sentence-transformers/all-MiniLM-L6-v2)")
+        
+        print()
+        
+        # Azure OpenAI completeness
+        try:
+            azure_config = config.get_azure_config()
+            print("✅ Azure OpenAI: Configured")
+        except Exception:
+            print("❌ Azure OpenAI: Incomplete (run 'ragaeval configure')")
+        
+        print()
+        print("=" * 80)
+        
+        return 0
+        
+    except Exception as e:
+        print(f"Error: {str(e)}", file=sys.stderr)
+        return 1
+
+
+def cmd_clear(args):
+    """
+    Clear evaluation logs.
+    
+    Steps:
+    1. Check if log file exists
+    2. Prompt for confirmation (unless --force)
+    3. Delete log file
+    4. Display confirmation with deleted record count
+    
+    Args:
+        args: Parsed command-line arguments
+        
+    Returns:
+        Exit code (0 for success, 1 for error)
+    """
+    try:
+        logger = Logger()
+        
+        if not logger.log_exists():
+            print(f"Nothing to clear. Log file not found: {logger.get_log_path()}")
+            return 0
+        
+        count = logger.count_records()
+        
+        # Confirm deletion
+        if not args.force:
+            response = input(f"Delete {count} records from {logger.get_log_path()}? (y/N): ")
+            if response.lower() != 'y':
+                print("Cancelled.")
+                return 0
+        
+        # Delete
+        logger.clear_log()
+        print(f"✅ Deleted {count} records from {logger.get_log_path()}")
+        
+        return 0
+        
+    except Exception as e:
+        print(f"Error: {str(e)}", file=sys.stderr)
+        return 1
+
+
+if __name__ == '__main__':
+    main()

ragaeval/config.py

@@ -0,0 +1,237 @@
+"""Configuration manager for the ragaeval package.
+
+Universal LLM provider support with auto-detection and smart defaults.
+"""
+
+import os
+import json
+from pathlib import Path
+from typing import Optional, Dict, Tuple
+from dotenv import load_dotenv
+
+from .exceptions import MissingCredentialsError, ConfigFileError
+
+
+# Supported LLM providers configuration
+SUPPORTED_PROVIDERS = {
+    "openrouter": {
+        "env_keys": ["OPENROUTER_API_KEY"],
+        "base_url": "https://openrouter.ai/api/v1",
+        "default_model": "openai/gpt-oss-120b:free",
+        "free_models": ["openai/gpt-oss-120b:free", "openai/gpt-oss-20b:free"],
+        "description": "OpenRouter (free): OPENROUTER_API_KEY=sk-or-v1-...",
+        "signup_url": "https://openrouter.ai/keys"
+    },
+    "openai": {
+        "env_keys": ["OPENAI_API_KEY"],
+        "base_url": "https://api.openai.com/v1",
+        "default_model": "gpt-4o-mini",
+        "description": "OpenAI: OPENAI_API_KEY=sk-...",
+        "signup_url": "https://platform.openai.com/api-keys"
+    },
+    "azure_openai": {
+        "env_keys": ["AZURE_OPENAI_API_KEY"],
+        "extra_env": ["AZURE_OPENAI_ENDPOINT", "AZURE_OPENAI_DEPLOYMENT_NAME"],
+        "default_model": "gpt-4o-mini",
+        "description": "Azure OpenAI: AZURE_OPENAI_API_KEY + AZURE_OPENAI_ENDPOINT + AZURE_OPENAI_DEPLOYMENT_NAME",
+        "signup_url": "https://azure.microsoft.com/en-us/products/ai-services/openai-service"
+    },
+    "anthropic": {
+        "env_keys": ["ANTHROPIC_API_KEY"],
+        "base_url": "https://api.anthropic.com",
+        "default_model": "claude-3-haiku-20240307",
+        "description": "Anthropic: ANTHROPIC_API_KEY=sk-ant-...",
+        "signup_url": "https://console.anthropic.com/keys"
+    },
+    "groq": {
+        "env_keys": ["GROQ_API_KEY"],
+        "base_url": "https://api.groq.com/openai/v1",
+        "default_model": "llama3-8b-8192",
+        "description": "Groq (free): GROQ_API_KEY=gsk_...",
+        "signup_url": "https://console.groq.com/keys"
+    },
+    "together": {
+        "env_keys": ["TOGETHER_API_KEY"],
+        "base_url": "https://api.together.xyz/v1",
+        "default_model": "mistralai/Mixtral-8x7B-Instruct-v0.1",
+        "description": "Together AI: TOGETHER_API_KEY=...",
+        "signup_url": "https://api.together.xyz"
+    },
+    "ollama": {
+        "env_keys": [],
+        "base_url": "http://localhost:11434/v1",
+        "default_model": "llama3",
+        "no_key_required": True,
+        "description": "Ollama (local): No key needed, just run Ollama",
+        "signup_url": "https://ollama.ai"
+    },
+    "custom": {
+        "env_keys": ["RAGAEVAL_API_KEY"],
+        "extra_env": ["RAGAEVAL_BASE_URL", "RAGAEVAL_MODEL"],
+        "description": "Custom: RAGAEVAL_API_KEY + RAGAEVAL_BASE_URL + RAGAEVAL_MODEL",
+        "signup_url": None
+    }
+}
+
+
+class ConfigManager:
+    """Manages LLM provider detection and configuration for ragaeval."""
+    
+    def __init__(self):
+        """Initialize config manager and load environment."""
+        # Load .env file if it exists in current directory
+        load_dotenv(dotenv_path=".env", override=False)
+        
+        # Load config from file
+        self._config = self.load_config()
+    
+    def detect_provider(self) -> Tuple[str, Optional[str], str, Optional[str]]:
+        """
+        Auto-detect LLM provider from environment.
+        
+        Scans in order: openrouter → openai → azure → anthropic → groq → together → ollama → custom
+        Sources checked: env vars → .env in cwd → ~/.ragaeval/config.json
+        
+        Returns:
+            Tuple of (provider_name, api_key, model, base_url)
+            
+        Raises:
+            MissingCredentialsError: If no provider is configured
+        """
+        # Scan providers in priority order
+        scan_order = ["openrouter", "openai", "azure_openai", "anthropic", 
+                      "groq", "together", "ollama", "custom"]
+        
+        for provider_name in scan_order:
+            provider_config = SUPPORTED_PROVIDERS[provider_name]
+            
+            # Check if this provider is configured
+            if provider_name == "ollama" and provider_config.get("no_key_required"):
+                # Ollama doesn't need a key - check if it's explicitly selected
+                if self._config.get("provider") == "ollama":
+                    model = self._config.get("model", provider_config["default_model"])
+                    self._print_provider_detected("ollama", model, "config file")
+                    return ("ollama", None, model, provider_config["base_url"])
+                continue
+            
+            # Check for API key
+            api_key = None
+            key_source = None
+            
+            for env_key in provider_config["env_keys"]:
+                api_key = os.getenv(env_key)
+                if api_key:
+                    key_source = ".env file" if Path(".env").exists() else "environment variable"
+                    break
+            
+            # If no key in env, check config file
+            if not api_key and provider_name in self._config.get("provider", ""):
+                api_key = self._config.get("api_key")
+                key_source = "config file"
+            
+            if api_key:
+                # Provider found! Check extra requirements (e.g., Azure needs endpoint)
+                if "extra_env" in provider_config:
+                    missing = []
+                    for extra_key in provider_config["extra_env"]:
+                        if not os.getenv(extra_key) and not self._config.get(extra_key.lower()):
+                            missing.append(extra_key)
+                    
+                    if missing:
+                        continue  # Skip this provider if extra keys missing
+                
+                # Determine model
+                model = self._config.get("model", provider_config.get("default_model", "unknown"))
+                
+                # Get base URL
+                base_url = provider_config.get("base_url")
+                if provider_name == "custom":
+                    base_url = os.getenv("RAGAEVAL_BASE_URL") or self._config.get("base_url")
+                elif provider_name == "azure_openai":
+                    base_url = os.getenv("AZURE_OPENAI_ENDPOINT") or self._config.get("azure_endpoint")
+                
+                self._print_provider_detected(provider_name, model, key_source)
+                return (provider_name, api_key, model, base_url)
+        
+        # No provider found - print help
+        self._print_no_provider_help()
+        raise MissingCredentialsError("No LLM provider configured. See message above for setup instructions.")
+    
+    def _print_provider_detected(self, provider: str, model: str, source: str):
+        """Print provider detection message."""
+        print(f"[ragaeval] Provider detected: {provider}")
+        print(f"[ragaeval] Model: {model}")
+        print(f"[ragaeval] Key source: {source}")
+    
+    def _print_no_provider_help(self):
+        """Print comprehensive help when no provider is detected."""
+        print("\n" + "=" * 70)
+        print("[ragaeval] No LLM provider configured.")
+        print("=" * 70)
+        print("\nAdd any ONE of these to your .env file or environment:\n")
+        
+        for provider_name in ["openrouter", "openai", "anthropic", "groq", 
+                               "together", "ollama", "custom"]:
+            config = SUPPORTED_PROVIDERS[provider_name]
+            print(f"  {config['description']}")
+            if config.get("signup_url"):
+                print(f"    {config['signup_url']}")
+            print()
+        
+        print("Or set globally:")
+        print("  ragaeval configure --provider openrouter --key sk-or-v1-...")
+        print("=" * 70 + "\n")
+    
+    def save_config(self, config_dict: Dict[str, str]) -> Path:
+        """
+        Save provider configuration to ~/.ragaeval/config.json.
+        
+        Note: API keys are NOT stored - only provider name and model.
+        
+        Args:
+            config_dict: Dictionary with provider settings
+            
+        Returns:
+            Path to config file
+        """
+        config_path = self.get_config_path()
+        
+        # Create directory if needed
+        config_path.parent.mkdir(parents=True, exist_ok=True)
+        
+        # Write config file
+        with open(config_path, 'w', encoding='utf-8') as f:
+            json.dump(config_dict, f, indent=2)
+        
+        # Reload config
+        self._config = self.load_config()
+        
+        return config_path
+    
+    def load_config(self) -> Dict:
+        """
+        Load configuration from ~/.ragaeval/config.json.
+        
+        Returns:
+            dict with config keys, empty dict if file doesn't exist
+        """
+        config_path = self.get_config_path()
+        
+        if not config_path.exists():
+            return {}
+        
+        try:
+            with open(config_path, 'r', encoding='utf-8') as f:
+                return json.load(f)
+        except json.JSONDecodeError as e:
+            raise ConfigFileError(f"Config file contains invalid JSON: {str(e)}")
+        except Exception as e:
+            raise ConfigFileError(f"Failed to read config file: {str(e)}")
+    
+    def get_config_path(self) -> Path:
+        """Return path to config file."""
+        return Path.home() / ".ragaeval" / "config.json"
+    
+    def config_exists(self) -> bool:
+        """Check if config file exists."""
+        return self.get_config_path().exists()

ragaeval/exceptions.py

@@ -0,0 +1,40 @@
+"""Custom exception classes for the ragaeval package.
+
+This module defines the exception hierarchy used throughout ragaeval for
+handling various error conditions during configuration, logging, and evaluation.
+"""
+
+
+class RagaevalError(Exception):
+    """Base exception for ragaeval package."""
+    pass
+
+
+class MissingCredentialsError(RagaevalError):
+    """Raised when required API credentials are missing."""
+    pass
+
+
+class ConfigFileError(RagaevalError):
+    """Raised when config file is malformed or inaccessible."""
+    pass
+
+
+class ValidationError(RagaevalError):
+    """Raised when log data validation fails."""
+    pass
+
+
+class LogFileError(RagaevalError):
+    """Raised when log file is missing or inaccessible."""
+    pass
+
+
+class FileFormatError(RagaevalError):
+    """Raised when log file contains malformed JSON."""
+    pass
+
+
+class EvaluationError(RagaevalError):
+    """Raised when RAGAS evaluation fails."""
+    pass