rag-sentinel 0.1.0__py3-none-any.whl → 0.1.2__py3-none-any.whl

rag_sentinel/__init__.py

@@ -1,5 +1,23 @@
 """
-RAGSentinel - RAG Evaluation Framework
+RAGSentinel - RAG Evaluation Framework using Ragas and MLflow.
+
+A pip-installable package for evaluating Retrieval-Augmented Generation (RAG)
+applications using Ragas metrics with MLflow tracking.
+
+Features:
+    - Evaluate RAG pipelines with Faithfulness, AnswerRelevancy,
+      ContextPrecision, and AnswerCorrectness metrics
+    - Support for Azure OpenAI, OpenAI, and Ollama LLM providers
+    - Automatic MLflow server management and result tracking
+    - Flexible authentication (Cookie, Bearer, Custom Header)
+
+Usage:
+    $ pip install rag-sentinel
+    $ rag-sentinel init
+    $ rag-sentinel run
+
+Author: RAGSentinel Team
+License: MIT
 """
 
 __version__ = "0.1.0"

rag_sentinel/cli.py

@@ -1,8 +1,19 @@
 """
-RAGSentinel CLI - Command Line Interface
+RAGSentinel CLI - Command Line Interface.
+
+This module provides the command-line interface for RAGSentinel.
+
+Commands:
+    init     - Initialize a new project with configuration templates
+    run      - Run RAG evaluation (auto-starts MLflow server)
+    validate - Validate configuration files
+
+Example:
+    $ rag-sentinel init
+    $ rag-sentinel run
+    $ rag-sentinel validate
 """
 
-import os
 import sys
 import shutil
 import socket
@@ -11,58 +22,103 @@ import time
 import argparse
 from pathlib import Path
 
-# Get the templates directory path
+
+# =============================================================================
+# Constants
+# =============================================================================
+
 TEMPLATES_DIR = Path(__file__).parent / "templates"
 
 
+# =============================================================================
+# Helper Functions
+# =============================================================================
+
 def is_port_in_use(host, port):
-    """Check if a port is in use."""
+    """
+    Check if a port is already in use.
+
+    Args:
+        host: The hostname to check (e.g., "127.0.0.1")
+        port: The port number to check
+
+    Returns:
+        bool: True if port is in use, False otherwise
+    """
     with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
         return s.connect_ex((host, port)) == 0
 
 
 def start_mlflow_server(host="127.0.0.1", port=5001):
-    """Start MLflow server as a background process."""
+    """
+    Start MLflow tracking server as a background process.
+
+    If the server is already running on the specified port, this function
+    will skip starting a new instance.
+
+    Args:
+        host: The hostname to bind the server to (default: "127.0.0.1")
+        port: The port number for the server (default: 5001)
+
+    Returns:
+        subprocess.Popen or None: The server process, or None if already running
+    """
     if is_port_in_use(host, port):
         print(f"✓ MLflow server already running at http://{host}:{port}")
         return None
-    
+
     print(f"🚀 Starting MLflow server at http://{host}:{port}...")
-    
+
+    # Start MLflow server as a detached background process
     process = subprocess.Popen(
         [sys.executable, "-m", "mlflow", "server", "--host", host, "--port", str(port)],
         stdout=subprocess.DEVNULL,
         stderr=subprocess.DEVNULL,
         creationflags=subprocess.CREATE_NEW_PROCESS_GROUP if sys.platform == "win32" else 0
     )
-    
-    # Wait for server to start
+
+    # Wait for server to become available (max 10 seconds)
     for _ in range(10):
         time.sleep(1)
         if is_port_in_use(host, port):
             print(f"✓ MLflow server started at http://{host}:{port}")
             return process
-    
+
     print("⚠ MLflow server may not have started properly")
     return process
 
 
+# =============================================================================
+# CLI Commands
+# =============================================================================
+
 def cmd_init(args):
-    """Initialize project with config templates."""
+    """
+    Initialize a new project with configuration templates.
+
+    Copies the following template files to the current directory:
+        - .env           : LLM and Embeddings API configuration
+        - config.ini     : Application settings (URLs, auth, dataset path)
+        - rag_eval_config.yaml : Master configuration with placeholders
+
+    Args:
+        args: Parsed command-line arguments (includes --force flag)
+    """
     print("=" * 50)
     print("RAGSentinel - Project Initialization")
     print("=" * 50)
-    
+
+    # Template files to copy: (source_name, destination_name)
     files_to_copy = [
         (".env.template", ".env"),
         ("config.ini.template", "config.ini"),
         ("rag_eval_config.yaml", "rag_eval_config.yaml"),
     ]
-    
+
     for src_name, dest_name in files_to_copy:
         src_path = TEMPLATES_DIR / src_name
         dest_path = Path.cwd() / dest_name
-        
+
         if dest_path.exists() and not args.force:
             print(f"⚠ {dest_name} already exists (use --force to overwrite)")
         else:
@@ -71,7 +127,8 @@ def cmd_init(args):
                 print(f"✓ Created {dest_name}")
             else:
                 print(f"✗ Template not found: {src_name}")
-    
+
+    # Display next steps for the user
     print("\n" + "=" * 50)
     print("Next steps:")
     print("  1. Edit .env with your LLM/Embeddings API keys")
@@ -82,49 +139,71 @@ def cmd_init(args):
 
 
 def cmd_run(args):
-    """Run the RAG evaluation."""
-    # Check if config files exist
+    """
+    Run the RAG evaluation pipeline.
+
+    This command:
+        1. Validates that all required config files exist
+        2. Starts MLflow server (unless --no-server is specified)
+        3. Runs the evaluation using the evaluator module
+
+    Args:
+        args: Parsed command-line arguments (includes --no-server flag)
+    """
+    # Check if all required configuration files exist
     required_files = [".env", "config.ini", "rag_eval_config.yaml"]
     missing_files = [f for f in required_files if not Path(f).exists()]
-    
+
     if missing_files:
         print("❌ Missing configuration files:")
         for f in missing_files:
             print(f"   - {f}")
         print("\nRun 'rag-sentinel init' first to create config files.")
         sys.exit(1)
-    
-    # Start MLflow server if not running
+
+    # Start MLflow server if not disabled
     if not args.no_server:
         start_mlflow_server()
-    
-    # Import and run evaluation
+
+    # Import and run the evaluation
     from rag_sentinel.evaluator import run_evaluation
     run_evaluation()
 
 
 def cmd_validate(args):
-    """Validate configuration files."""
+    """
+    Validate configuration files without running evaluation.
+
+    Checks:
+        - All required files exist (.env, config.ini, rag_eval_config.yaml)
+        - Configuration can be loaded and parsed correctly
+        - Displays key configuration values for verification
+
+    Args:
+        args: Parsed command-line arguments (unused, kept for consistency)
+    """
+    _ = args  # Unused, but required for command handler signature
+
     print("=" * 50)
     print("RAGSentinel - Configuration Validation")
     print("=" * 50)
-    
-    # Check files exist
+
+    # Check if all required files exist
     files_to_check = [".env", "config.ini", "rag_eval_config.yaml"]
     all_exist = True
-    
+
     for f in files_to_check:
         if Path(f).exists():
             print(f"✓ {f} exists")
         else:
             print(f"✗ {f} missing")
             all_exist = False
-    
+
     if not all_exist:
         print("\n❌ Some files are missing. Run 'rag-sentinel init' first.")
         return
-    
-    # Try to load config
+
+    # Attempt to load and validate configuration
     try:
         from rag_sentinel.evaluator import load_config
         config = load_config()
@@ -137,28 +216,38 @@ def cmd_validate(args):
         print(f"\n❌ Configuration error: {e}")
 
 
+# =============================================================================
+# Main Entry Point
+# =============================================================================
+
 def main():
-    """Main entry point for CLI."""
+    """
+    Main entry point for the RAGSentinel CLI.
+
+    Parses command-line arguments and dispatches to the appropriate
+    command handler (init, run, or validate).
+    """
     parser = argparse.ArgumentParser(
         prog="rag-sentinel",
         description="RAGSentinel - RAG Evaluation Framework using Ragas and MLflow"
     )
-    
+
     subparsers = parser.add_subparsers(dest="command", help="Available commands")
-    
-    # init command
+
+    # 'init' command - Initialize project with config templates
     init_parser = subparsers.add_parser("init", help="Initialize project with config templates")
     init_parser.add_argument("--force", "-f", action="store_true", help="Overwrite existing files")
-    
-    # run command
+
+    # 'run' command - Run RAG evaluation
     run_parser = subparsers.add_parser("run", help="Run RAG evaluation")
     run_parser.add_argument("--no-server", action="store_true", help="Don't start MLflow server")
-    
-    # validate command
-    validate_parser = subparsers.add_parser("validate", help="Validate configuration files")
-    
+
+    # 'validate' command - Validate configuration files
+    subparsers.add_parser("validate", help="Validate configuration files")
+
     args = parser.parse_args()
-    
+
+    # Dispatch to appropriate command handler
     if args.command == "init":
         cmd_init(args)
     elif args.command == "run":

rag_sentinel/evaluator.py

@@ -1,5 +1,9 @@
 """
-RAGSentinel Evaluator - Core evaluation logic
+RAGSentinel Evaluator - Core evaluation logic.
+
+This module contains the main evaluation pipeline for RAGSentinel.
+It handles configuration loading, LLM initialization, API communication,
+Ragas metrics evaluation, and MLflow result logging.
 """
 
 import os
@@ -15,19 +19,34 @@ from ragas import evaluate, RunConfig
 from ragas.metrics import faithfulness, answer_relevancy, context_precision, answer_correctness
 
 
+# =============================================================================
+# Configuration Loading
+# =============================================================================
+
+
 def resolve_placeholder(value, env_vars, ini_config):
-    """Resolve ${ENV:...} and ${INI:...} placeholders."""
+    """
+    Resolve ${ENV:...} and ${INI:...} placeholders in a string value.
+
+    Args:
+        value: String that may contain placeholders
+        env_vars: Dictionary of environment variables
+        ini_config: ConfigParser object with ini file contents
+
+    Returns:
+        str: Value with all placeholders resolved
+    """
     if not isinstance(value, str):
         return value
 
-    # Resolve ${ENV:VAR_NAME}
+    # Resolve ${ENV:VAR_NAME} - reads from environment variables
     env_pattern = r'\$\{ENV:([^}]+)\}'
     def env_replacer(match):
         var_name = match.group(1)
         return env_vars.get(var_name, '')
     value = re.sub(env_pattern, env_replacer, value)
 
-    # Resolve ${INI:section.key}
+    # Resolve ${INI:section.key} - reads from config.ini
     ini_pattern = r'\$\{INI:([^}]+)\}'
     def ini_replacer(match):
         path = match.group(1)
@@ -43,7 +62,17 @@ def resolve_placeholder(value, env_vars, ini_config):
 
 
 def resolve_config(obj, env_vars, ini_config):
-    """Recursively resolve all placeholders in config."""
+    """
+    Recursively resolve all placeholders in a configuration object.
+
+    Args:
+        obj: Configuration object (dict, list, or str)
+        env_vars: Dictionary of environment variables
+        ini_config: ConfigParser object
+
+    Returns:
+        Configuration object with all placeholders resolved
+    """
     if isinstance(obj, dict):
         return {k: resolve_config(v, env_vars, ini_config) for k, v in obj.items()}
     elif isinstance(obj, list):
@@ -54,13 +83,21 @@ def resolve_config(obj, env_vars, ini_config):
 
 
 def load_config():
-    """Load configuration from .env, config.ini, and rag_eval_config.yaml."""
+    """
+    Load and merge configuration from .env, config.ini, and rag_eval_config.yaml.
+
+    Returns:
+        dict: Fully resolved configuration dictionary
+    """
+    # Load environment variables from .env file
     load_dotenv('.env')
     env_vars = dict(os.environ)
 
+    # Load INI configuration
     ini_config = configparser.ConfigParser()
     ini_config.read('config.ini')
 
+    # Load YAML configuration and resolve all placeholders
     with open('rag_eval_config.yaml', 'r') as f:
         yaml_config = yaml.safe_load(f)
 
@@ -140,8 +177,22 @@ def get_metrics(config):
 
 
 def get_auth_headers_and_cookies(config):
-    """Get authentication headers and cookies."""
-    auth_config = config.get('auth', {})
+    """
+    Get authentication headers and cookies from backend config.
+
+    Supports three authentication types:
+        - cookie: Session cookie authentication
+        - bearer: Bearer token authentication
+        - header: Custom header authentication
+
+    Args:
+        config: Full configuration dictionary
+
+    Returns:
+        tuple: (headers dict, cookies dict)
+    """
+    # Auth config is nested under backend.auth in the YAML
+    auth_config = config.get('backend', {}).get('auth', {})
     auth_type = auth_config.get('type', 'none').lower()
     headers = {}
     cookies = {}

rag_sentinel/templates/.env.template

@@ -2,51 +2,48 @@
 # RAGSentinel Environment Variables
 # =============================================================================
 # This file contains sensitive LLM and Embeddings configuration.
-# Copy this file to .env and fill in your values.
-# DO NOT commit .env to version control!
-# =============================================================================
 
-# =============================================================================
+
 # LLM Configuration
-# =============================================================================
+
 # Provider options: azure, openai, ollama
-LLM_PROVIDER=azure
+LLM_PROVIDER=
 
 # Azure OpenAI LLM Configuration
 AZURE_LLM_API_KEY=
 AZURE_LLM_ENDPOINT=
 AZURE_LLM_DEPLOYMENT_NAME=
-AZURE_LLM_MODEL=gpt-4
+AZURE_LLM_MODEL=
 AZURE_LLM_TEMPERATURE=0.0
-AZURE_LLM_API_VERSION=2024-02-15-preview
+AZURE_LLM_API_VERSION=
 
 # OpenAI LLM Configuration
 OPENAI_LLM_API_KEY=
-OPENAI_LLM_MODEL=gpt-4
+OPENAI_LLM_MODEL=
 OPENAI_LLM_TEMPERATURE=0.0
 
 # Ollama LLM Configuration
-OLLAMA_LLM_BASE_URL=http://localhost:11434
-OLLAMA_LLM_MODEL=llama3
+OLLAMA_LLM_BASE_URL=
+OLLAMA_LLM_MODEL=
 OLLAMA_LLM_TEMPERATURE=0.0
 
-# =============================================================================
+
 # Embeddings Configuration
-# =============================================================================
+
 # Provider options: azure, openai, ollama
-EMBEDDINGS_PROVIDER=azure
+EMBEDDINGS_PROVIDER=
 
 # Azure OpenAI Embeddings Configuration
 AZURE_EMBEDDINGS_API_KEY=
 AZURE_EMBEDDINGS_ENDPOINT=
 AZURE_EMBEDDINGS_DEPLOYMENT_NAME=
-AZURE_EMBEDDINGS_API_VERSION=2024-02-15-preview
+AZURE_EMBEDDINGS_API_VERSION=
 
 # OpenAI Embeddings Configuration
 OPENAI_EMBEDDINGS_API_KEY=
-OPENAI_EMBEDDINGS_MODEL=text-embedding-3-small
+OPENAI_EMBEDDINGS_MODEL=
 
 # Ollama Embeddings Configuration
-OLLAMA_EMBEDDINGS_BASE_URL=http://localhost:11434
-OLLAMA_EMBEDDINGS_MODEL=nomic-embed-text
+OLLAMA_EMBEDDINGS_BASE_URL=
+OLLAMA_EMBEDDINGS_MODEL=
 

rag_sentinel/templates/config.ini.template

@@ -1,9 +1,5 @@
-# =============================================================================
 # RAGSentinel Configuration File
-# =============================================================================
-# This file contains non-sensitive configuration settings.
-# Copy this file to config.ini and fill in your values.
-# =============================================================================
+
 
 [mlflow]
 # MLflow tracking server URI
@@ -15,8 +11,8 @@ app_url =
 
 [endpoints]
 # API endpoint paths for context retrieval and answer generation
-context_path = /api/retrieve_context
-answer_path = /api/respond
+context_path = 
+answer_path = 
 
 [dataset]
 # Path to test dataset CSV file (columns: query, ground_truth, chat_id)

{rag_sentinel-0.1.0.dist-info → rag_sentinel-0.1.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rag-sentinel
-Version: 0.1.0
+Version: 0.1.2
 Summary: RAG Evaluation Framework using Ragas metrics and MLflow tracking
 Author: RAGSentinel Team
 License: MIT

rag_sentinel-0.1.2.dist-info/RECORD

@@ -0,0 +1,12 @@
+rag_sentinel/__init__.py,sha256=jrVnGWJh60e1fucEP8DuQtFaXQcls8Rsw9tuUfKI87Y,672
+rag_sentinel/cli.py,sha256=EacFBn_7TBsP1CPw9MmNUyGcAG-MO7c1UDz2v3wPkz4,8499
+rag_sentinel/evaluator.py,sha256=_5fMiUeQVK93C87svEH8WI5MPcH4nfA7FQo8xeh5dTE,13321
+rag_sentinel/templates/.env.template,sha256=FabB1i4pUkU8gdNLRt2D8mgltY4AudClq8rx6vS33xc,1120
+rag_sentinel/templates/config.ini.template,sha256=OeW21j4LXxXnFCPVvOZhdZOq1id0BQLgwS5ruXSrXBQ,1016
+rag_sentinel/templates/rag_eval_config.yaml,sha256=zRPMOngALsbhgQbkKeNvXc8VVxzDJrASpSIpGTpVKlk,3080
+rag_sentinel-0.1.2.dist-info/licenses/LICENSE,sha256=0bRNV4OZXZGaeA4PLR0CZKk1peLyIw977fV9K5jAGws,1074
+rag_sentinel-0.1.2.dist-info/METADATA,sha256=QQdGA5cyjoSVKpTbO3-qbQWii7aCO28g8JaEDO4k36s,2716
+rag_sentinel-0.1.2.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+rag_sentinel-0.1.2.dist-info/entry_points.txt,sha256=ZBQp5JLnfMLjgLX3UdzX2rainIgvqkq36Wtf_VLa9ak,55
+rag_sentinel-0.1.2.dist-info/top_level.txt,sha256=qk1BCc3wrLshA3-jf0Jb4bFlZF-ARIjZfYaqAW1Aq0E,13
+rag_sentinel-0.1.2.dist-info/RECORD,,

rag_sentinel-0.1.0.dist-info/RECORD

@@ -1,12 +0,0 @@
-rag_sentinel/__init__.py,sha256=kBMRLysR1HgWEkDM6Wlx908YUbmhDgqRJzM4AaHgQ9I,71
-rag_sentinel/cli.py,sha256=OtUHHOfAkspnMB_YYhs8YldzUhJv3Tqt79nm0A-bv5s,5415
-rag_sentinel/evaluator.py,sha256=QDr16pfIm1KEiJAQDMsuU4irGStQUmDC8WZTyVJcUsU,11719
-rag_sentinel/templates/.env.template,sha256=4gR-dIqErDMVvgvZe-frU3dvd1weeyQ_x0JUzaqvwCY,1751
-rag_sentinel/templates/config.ini.template,sha256=wVteTE_h1PHvs9I3hv37fSkiJ66fTHg9qOD_CCbYGXE,1403
-rag_sentinel/templates/rag_eval_config.yaml,sha256=zRPMOngALsbhgQbkKeNvXc8VVxzDJrASpSIpGTpVKlk,3080
-rag_sentinel-0.1.0.dist-info/licenses/LICENSE,sha256=0bRNV4OZXZGaeA4PLR0CZKk1peLyIw977fV9K5jAGws,1074
-rag_sentinel-0.1.0.dist-info/METADATA,sha256=xCc6G6RbJJR-Axom_ymWgiL2tpwh6-W9DqJydkvJ-ng,2716
-rag_sentinel-0.1.0.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
-rag_sentinel-0.1.0.dist-info/entry_points.txt,sha256=ZBQp5JLnfMLjgLX3UdzX2rainIgvqkq36Wtf_VLa9ak,55
-rag_sentinel-0.1.0.dist-info/top_level.txt,sha256=qk1BCc3wrLshA3-jf0Jb4bFlZF-ARIjZfYaqAW1Aq0E,13
-rag_sentinel-0.1.0.dist-info/RECORD,,