rag-sentinel 0.1.2__py3-none-any.whl → 0.1.4__py3-none-any.whl

rag_sentinel/__init__.py

@@ -20,5 +20,5 @@ Author: RAGSentinel Team
 License: MIT
 """
 
-__version__ = "0.1.0"
+__version__ = "0.1.2"
 

rag_sentinel/cli.py

@@ -20,7 +20,9 @@ import socket
 import subprocess
 import time
 import argparse
+import configparser
 from pathlib import Path
+from urllib.parse import urlparse
 
 
 # =============================================================================
@@ -34,6 +36,39 @@ TEMPLATES_DIR = Path(__file__).parent / "templates"
 # Helper Functions
 # =============================================================================
 
+def get_mlflow_host_port():
+    """
+    Read MLflow tracking_uri from config.ini and parse host and port.
+
+    Returns:
+        tuple: (host, port) - defaults to ("127.0.0.1", 5001) if not configured
+    """
+    default_host = "127.0.0.1"
+    default_port = 5000
+
+    config_path = Path("config.ini")
+    if not config_path.exists():
+        return default_host, default_port
+
+    try:
+        ini = configparser.ConfigParser()
+        ini.read(config_path)
+
+        tracking_uri = ini.get("mlflow", "tracking_uri", fallback=None)
+        if not tracking_uri:
+            return default_host, default_port
+
+        # Parse the URI (e.g., "http://192.168.1.100:5000")
+        parsed = urlparse(tracking_uri)
+
+        host = parsed.hostname or default_host
+        port = parsed.port or default_port
+
+        return host, port
+    except Exception:
+        return default_host, default_port
+
+
 def is_port_in_use(host, port):
     """
     Check if a port is already in use.
@@ -49,7 +84,7 @@ def is_port_in_use(host, port):
         return s.connect_ex((host, port)) == 0
 
 
-def start_mlflow_server(host="127.0.0.1", port=5001):
+def start_mlflow_server(host, port):
     """
     Start MLflow tracking server as a background process.
 
@@ -57,8 +92,8 @@ def start_mlflow_server(host="127.0.0.1", port=5001):
     will skip starting a new instance.
 
     Args:
-        host: The hostname to bind the server to (default: "127.0.0.1")
-        port: The port number for the server (default: 5001)
+        host: The hostname to bind the server to
+        port: The port number for the server
 
     Returns:
         subprocess.Popen or None: The server process, or None if already running
@@ -144,8 +179,9 @@ def cmd_run(args):
 
     This command:
         1. Validates that all required config files exist
-        2. Starts MLflow server (unless --no-server is specified)
-        3. Runs the evaluation using the evaluator module
+        2. Reads MLflow host/port from config.ini
+        3. Starts MLflow server (unless --no-server is specified)
+        4. Runs the evaluation using the evaluator module
 
     Args:
         args: Parsed command-line arguments (includes --no-server flag)
@@ -161,9 +197,10 @@ def cmd_run(args):
         print("\nRun 'rag-sentinel init' first to create config files.")
         sys.exit(1)
 
-    # Start MLflow server if not disabled
+    # Start MLflow server if not disabled (uses host/port from config.ini)
     if not args.no_server:
-        start_mlflow_server()
+        host, port = get_mlflow_host_port()
+        start_mlflow_server(host, port)
 
     # Import and run the evaluation
     from rag_sentinel.evaluator import run_evaluation

rag_sentinel/evaluator.py

@@ -15,8 +15,16 @@ import pandas as pd
 import mlflow
 from dotenv import load_dotenv
 from datasets import Dataset
-from ragas import evaluate, RunConfig
-from ragas.metrics import faithfulness, answer_relevancy, context_precision, answer_correctness
+from ragas import evaluate
+from ragas.run_config import RunConfig
+from ragas.metrics import (
+    Faithfulness,
+    AnswerRelevancy,
+    ContextPrecision,
+    AnswerCorrectness,
+)
+from langchain_openai import ChatOpenAI, OpenAIEmbeddings, AzureChatOpenAI, AzureOpenAIEmbeddings
+from langchain_ollama import ChatOllama, OllamaEmbeddings
 
 
 # =============================================================================
@@ -24,236 +32,196 @@ from ragas.metrics import faithfulness, answer_relevancy, context_precision, ans
 # =============================================================================
 
 
-def resolve_placeholder(value, env_vars, ini_config):
+def load_config(yaml_file='rag_eval_config.yaml'):
     """
-    Resolve ${ENV:...} and ${INI:...} placeholders in a string value.
-
-    Args:
-        value: String that may contain placeholders
-        env_vars: Dictionary of environment variables
-        ini_config: ConfigParser object with ini file contents
-
-    Returns:
-        str: Value with all placeholders resolved
-    """
-    if not isinstance(value, str):
-        return value
-
-    # Resolve ${ENV:VAR_NAME} - reads from environment variables
-    env_pattern = r'\$\{ENV:([^}]+)\}'
-    def env_replacer(match):
-        var_name = match.group(1)
-        return env_vars.get(var_name, '')
-    value = re.sub(env_pattern, env_replacer, value)
-
-    # Resolve ${INI:section.key} - reads from config.ini
-    ini_pattern = r'\$\{INI:([^}]+)\}'
-    def ini_replacer(match):
-        path = match.group(1)
-        parts = path.split('.')
-        if len(parts) == 2:
-            section, key = parts
-            if ini_config.has_option(section, key):
-                return ini_config.get(section, key)
-        return ''
-    value = re.sub(ini_pattern, ini_replacer, value)
-
-    return value
-
-
-def resolve_config(obj, env_vars, ini_config):
-    """
-    Recursively resolve all placeholders in a configuration object.
-
-    Args:
-        obj: Configuration object (dict, list, or str)
-        env_vars: Dictionary of environment variables
-        ini_config: ConfigParser object
-
-    Returns:
-        Configuration object with all placeholders resolved
-    """
-    if isinstance(obj, dict):
-        return {k: resolve_config(v, env_vars, ini_config) for k, v in obj.items()}
-    elif isinstance(obj, list):
-        return [resolve_config(item, env_vars, ini_config) for item in obj]
-    elif isinstance(obj, str):
-        return resolve_placeholder(obj, env_vars, ini_config)
-    return obj
-
-
-def load_config():
-    """
-    Load and merge configuration from .env, config.ini, and rag_eval_config.yaml.
+    Load configuration from YAML file with values resolved from .env and config.ini.
 
     Returns:
         dict: Fully resolved configuration dictionary
     """
-    # Load environment variables from .env file
     load_dotenv('.env')
-    env_vars = dict(os.environ)
-
-    # Load INI configuration
-    ini_config = configparser.ConfigParser()
-    ini_config.read('config.ini')
 
-    # Load YAML configuration and resolve all placeholders
-    with open('rag_eval_config.yaml', 'r') as f:
-        yaml_config = yaml.safe_load(f)
-
-    return resolve_config(yaml_config, env_vars, ini_config)
+    ini = configparser.ConfigParser()
+    ini.read('config.ini')
+
+    def resolve(obj):
+        if isinstance(obj, dict):
+            return {k: resolve(v) for k, v in obj.items()}
+        if isinstance(obj, list):
+            return [resolve(i) for i in obj]
+        if isinstance(obj, str):
+            # Resolve ${ENV:VAR} and ${INI:section.key} placeholders
+            result = re.sub(r'\$\{ENV:([^}]+)\}', lambda m: os.getenv(m.group(1), ''), obj)
+            result = re.sub(r'\$\{INI:([^.]+)\.([^}]+)\}',
+                           lambda m: ini.get(m.group(1), m.group(2), fallback=''), result)
+            # Convert types
+            if result.lower() == 'true': return True
+            if result.lower() == 'false': return False
+            try:
+                if '.' in result: return float(result)
+            except ValueError:
+                pass
+            return result
+        return obj
+
+    with open(yaml_file, 'r') as f:
+        return resolve(yaml.safe_load(f))
 
 
 def get_llm(config):
-    """Initialize LLM based on provider."""
-    llm_config = config['ragas']['llm']
-    provider = llm_config['provider'].lower()
+    """Initialize LLM based on config."""
+    provider = config['ragas']['llm']['provider']
 
-    if provider == 'azure':
-        from langchain_openai import AzureChatOpenAI
+    if provider == "azure":
+        azure_config = config['ragas']['llm']['azure']
         return AzureChatOpenAI(
-            azure_endpoint=llm_config['azure_endpoint'],
-            api_key=llm_config['api_key'],
-            api_version=llm_config.get('api_version', '2024-02-15-preview'),
-            deployment_name=llm_config['model'],
-            temperature=float(llm_config.get('temperature', 0.0))
+            azure_endpoint=azure_config['endpoint'],
+            api_key=azure_config['api_key'],
+            deployment_name=azure_config['deployment_name'],
+            model=azure_config['model'],
+            temperature=azure_config['temperature'],
+            api_version=azure_config['api_version']
         )
-    elif provider == 'openai':
-        from langchain_openai import ChatOpenAI
+    elif provider == "openai":
+        openai_config = config['ragas']['llm']['openai']
         return ChatOpenAI(
-            api_key=llm_config['api_key'],
-            model=llm_config['model'],
-            temperature=float(llm_config.get('temperature', 0.0))
+            model=openai_config['model'],
+            temperature=openai_config['temperature'],
+            api_key=openai_config['api_key']
         )
-    elif provider == 'ollama':
-        from langchain_ollama import ChatOllama
+    elif provider == "ollama":
+        ollama_config = config['ragas']['llm']['ollama']
         return ChatOllama(
-            base_url=llm_config.get('base_url', 'http://localhost:11434'),
-            model=llm_config['model'],
-            temperature=float(llm_config.get('temperature', 0.0))
+            base_url=ollama_config['base_url'],
+            model=ollama_config['model'],
+            temperature=ollama_config['temperature']
         )
     else:
-        raise ValueError(f"Unknown LLM provider: {provider}")
+        raise ValueError(f"Unsupported LLM provider: {provider}")
 
 
 def get_embeddings(config):
-    """Initialize embeddings based on provider."""
-    emb_config = config['ragas']['embeddings']
-    provider = emb_config['provider'].lower()
+    """Initialize embeddings based on config."""
+    provider = config['ragas']['embeddings']['provider']
 
-    if provider == 'azure':
-        from langchain_openai import AzureOpenAIEmbeddings
+    if provider == "azure":
+        azure_config = config['ragas']['embeddings']['azure']
         return AzureOpenAIEmbeddings(
-            azure_endpoint=emb_config['azure_endpoint'],
-            api_key=emb_config['api_key'],
-            api_version=emb_config.get('api_version', '2024-02-15-preview'),
-            deployment=emb_config['model']
+            azure_endpoint=azure_config['endpoint'],
+            api_key=azure_config['api_key'],
+            deployment=azure_config['deployment_name'],
+            api_version=azure_config['api_version']
         )
-    elif provider == 'openai':
-        from langchain_openai import OpenAIEmbeddings
+    elif provider == "openai":
+        openai_config = config['ragas']['embeddings']['openai']
         return OpenAIEmbeddings(
-            api_key=emb_config['api_key'],
-            model=emb_config['model']
+            model=openai_config['model'],
+            api_key=openai_config['api_key']
         )
-    elif provider == 'ollama':
-        from langchain_ollama import OllamaEmbeddings
+    elif provider == "ollama":
+        ollama_config = config['ragas']['embeddings']['ollama']
         return OllamaEmbeddings(
-            base_url=emb_config.get('base_url', 'http://localhost:11434'),
-            model=emb_config['model']
+            base_url=ollama_config['base_url'],
+            model=ollama_config['model']
         )
     else:
-        raise ValueError(f"Unknown embeddings provider: {provider}")
+        raise ValueError(f"Unsupported embeddings provider: {provider}")
 
 
 def get_metrics(config):
-    """Get list of Ragas metrics."""
+    """Get Ragas metrics based on config."""
     metric_map = {
-        'faithfulness': faithfulness,
-        'answer_relevancy': answer_relevancy,
-        'context_precision': context_precision,
-        'answer_correctness': answer_correctness
+        "Faithfulness": Faithfulness(),
+        "AnswerRelevancy": AnswerRelevancy(),
+        "ContextPrecision": ContextPrecision(),
+        "AnswerCorrectness": AnswerCorrectness(),
     }
-    return [metric_map[m] for m in config['ragas']['metrics'] if m in metric_map]
-
 
-def get_auth_headers_and_cookies(config):
-    """
-    Get authentication headers and cookies from backend config.
+    metric_names = config['ragas']['metrics']
+    return [metric_map[name] for name in metric_names if name in metric_map]
 
-    Supports three authentication types:
-        - cookie: Session cookie authentication
-        - bearer: Bearer token authentication
-        - header: Custom header authentication
 
-    Args:
-        config: Full configuration dictionary
+def get_auth_headers_and_cookies(config):
+    """Get authentication headers and cookies based on config."""
+    auth_config = config['backend']['auth']
+    auth_type = auth_config.get('type', 'none')
 
-    Returns:
-        tuple: (headers dict, cookies dict)
-    """
-    # Auth config is nested under backend.auth in the YAML
-    auth_config = config.get('backend', {}).get('auth', {})
-    auth_type = auth_config.get('type', 'none').lower()
     headers = {}
     cookies = {}
 
-    if auth_type == 'cookie':
-        cookie_name = auth_config.get('cookie_name', 'session')
-        cookie_value = auth_config.get('cookie_value', '')
-        if cookie_value:
-            cookies[cookie_name] = cookie_value
-    elif auth_type == 'bearer':
-        token = auth_config.get('bearer_token', '')
-        if token:
-            headers['Authorization'] = f'Bearer {token}'
-    elif auth_type == 'header':
-        header_name = auth_config.get('header_name', '')
-        header_value = auth_config.get('header_value', '')
-        if header_name and header_value:
-            headers[header_name] = header_value
+    if auth_type == "cookie":
+        cookies[auth_config['cookie_name']] = auth_config['cookie_value']
+    elif auth_type == "bearer":
+        headers['Authorization'] = f"Bearer {auth_config['bearer_token']}"
+    elif auth_type == "header":
+        headers[auth_config['header_name']] = auth_config['header_value']
 
     return headers, cookies
 
 
 def extract_response_data(response, endpoint_config):
     """Extract data from API response."""
-    data = response.json()
-    response_path = endpoint_config.get('response_path', '')
+    if endpoint_config.get('stream', False):
+        return "".join(chunk.decode() for chunk in response.iter_content(chunk_size=None))
 
-    if response_path:
-        for key in response_path.split('.'):
-            if isinstance(data, dict) and key in data:
-                data = data[key]
-            elif isinstance(data, list) and key.isdigit():
-                data = data[int(key)]
-            else:
-                return data
-    return data
+    # Try to parse as JSON first
+    try:
+        data = response.json()
+        response_key = endpoint_config.get('response_key')
+        if response_key:
+            return data.get(response_key)
+        return data
+    except:
+        # If JSON parsing fails, return as plain text
+        return response.text
 
 
 def make_api_request(base_url, endpoint_config, query, chat_id, auth_headers, auth_cookies, verify_ssl=True):
     """Make API request to backend."""
-    url = base_url.rstrip('/') + endpoint_config['path']
-    method = endpoint_config.get('method', 'POST').upper()
-
-    body = endpoint_config.get('body', {}).copy()
-    body['query'] = query
-    body['chat_id'] = chat_id
-
-    headers = {'Content-Type': 'application/json'}
-    headers.update(auth_headers)
-
-    if method == 'POST':
-        response = requests.post(url, json=body, headers=headers, cookies=auth_cookies, verify=verify_ssl)
+    url = base_url + endpoint_config['path']
+    method = endpoint_config.get('method', 'POST')
+
+    headers = {**endpoint_config.get('headers', {}), **auth_headers}
+
+    # Flexible body preparation
+    body = {}
+    for key, value in endpoint_config.get('body', {}).items():
+        if isinstance(value, str) and ("{query}" in value or "{chat_id}" in value):
+            body[key] = value.format(query=query, chat_id=chat_id)
+        elif key == "chat_id":
+            try:
+                body[key] = int(chat_id)
+            except (ValueError, TypeError):
+                body[key] = chat_id
+        else:
+            body[key] = value
+
+    if method.upper() == 'POST':
+        resp = requests.post(
+            url,
+            json=body,
+            headers=headers,
+            cookies=auth_cookies,
+            stream=endpoint_config.get('stream', False),
+            verify=verify_ssl
+        )
+    elif method.upper() == 'GET':
+        resp = requests.get(
+            url,
+            params=body,
+            headers=headers,
+            cookies=auth_cookies,
+            verify=verify_ssl
+        )
     else:
-        response = requests.get(url, params=body, headers=headers, cookies=auth_cookies, verify=verify_ssl)
+        raise ValueError(f"Unsupported HTTP method: {method}")
 
-    response.raise_for_status()
-    return response
+    resp.raise_for_status()
+    return resp
 
 
 def get_context(config, query, chat_id, auth_headers, auth_cookies):
-    """Get context from backend API."""
+    """Retrieve context from backend API."""
     base_url = config['backend']['base_url']
     endpoint_config = config['backend']['endpoints']['context']
     verify_ssl = config['backend'].get('verify_ssl', True)
@@ -261,9 +229,12 @@ def get_context(config, query, chat_id, auth_headers, auth_cookies):
     response = make_api_request(base_url, endpoint_config, query, chat_id, auth_headers, auth_cookies, verify_ssl)
     context = extract_response_data(response, endpoint_config)
 
-    if isinstance(context, list):
-        return [str(c) for c in context]
-    return [str(context)]
+    if isinstance(context, str):
+        return [context]
+    elif isinstance(context, list):
+        return context
+    else:
+        return [str(context)]
 
 
 def get_answer(config, query, chat_id, auth_headers, auth_cookies):

{rag_sentinel-0.1.2.dist-info → rag_sentinel-0.1.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rag-sentinel
-Version: 0.1.2
+Version: 0.1.4
 Summary: RAG Evaluation Framework using Ragas metrics and MLflow tracking
 Author: RAGSentinel Team
 License: MIT

rag_sentinel-0.1.4.dist-info/RECORD

@@ -0,0 +1,12 @@
+rag_sentinel/__init__.py,sha256=KS1u5vgA5NxVHPFebebP2cTgt26nRKlC8dfGPy_Y2kM,672
+rag_sentinel/cli.py,sha256=ed9j_vetSDIuqkr3VjqFMBoRetRwkXFAX3sK3u-UodA,9525
+rag_sentinel/evaluator.py,sha256=6HcVv5quf0Hnxz3UkauuaODFt4xAH9Bo61dBZCxGQSw,12296
+rag_sentinel/templates/.env.template,sha256=FabB1i4pUkU8gdNLRt2D8mgltY4AudClq8rx6vS33xc,1120
+rag_sentinel/templates/config.ini.template,sha256=OeW21j4LXxXnFCPVvOZhdZOq1id0BQLgwS5ruXSrXBQ,1016
+rag_sentinel/templates/rag_eval_config.yaml,sha256=zRPMOngALsbhgQbkKeNvXc8VVxzDJrASpSIpGTpVKlk,3080
+rag_sentinel-0.1.4.dist-info/licenses/LICENSE,sha256=0bRNV4OZXZGaeA4PLR0CZKk1peLyIw977fV9K5jAGws,1074
+rag_sentinel-0.1.4.dist-info/METADATA,sha256=73Q2_ultXj4_DC0203Hrb8_gZsLB2y5c7U9rVqnn_0k,2716
+rag_sentinel-0.1.4.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+rag_sentinel-0.1.4.dist-info/entry_points.txt,sha256=ZBQp5JLnfMLjgLX3UdzX2rainIgvqkq36Wtf_VLa9ak,55
+rag_sentinel-0.1.4.dist-info/top_level.txt,sha256=qk1BCc3wrLshA3-jf0Jb4bFlZF-ARIjZfYaqAW1Aq0E,13
+rag_sentinel-0.1.4.dist-info/RECORD,,

rag_sentinel-0.1.2.dist-info/RECORD

@@ -1,12 +0,0 @@
-rag_sentinel/__init__.py,sha256=jrVnGWJh60e1fucEP8DuQtFaXQcls8Rsw9tuUfKI87Y,672
-rag_sentinel/cli.py,sha256=EacFBn_7TBsP1CPw9MmNUyGcAG-MO7c1UDz2v3wPkz4,8499
-rag_sentinel/evaluator.py,sha256=_5fMiUeQVK93C87svEH8WI5MPcH4nfA7FQo8xeh5dTE,13321
-rag_sentinel/templates/.env.template,sha256=FabB1i4pUkU8gdNLRt2D8mgltY4AudClq8rx6vS33xc,1120
-rag_sentinel/templates/config.ini.template,sha256=OeW21j4LXxXnFCPVvOZhdZOq1id0BQLgwS5ruXSrXBQ,1016
-rag_sentinel/templates/rag_eval_config.yaml,sha256=zRPMOngALsbhgQbkKeNvXc8VVxzDJrASpSIpGTpVKlk,3080
-rag_sentinel-0.1.2.dist-info/licenses/LICENSE,sha256=0bRNV4OZXZGaeA4PLR0CZKk1peLyIw977fV9K5jAGws,1074
-rag_sentinel-0.1.2.dist-info/METADATA,sha256=QQdGA5cyjoSVKpTbO3-qbQWii7aCO28g8JaEDO4k36s,2716
-rag_sentinel-0.1.2.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
-rag_sentinel-0.1.2.dist-info/entry_points.txt,sha256=ZBQp5JLnfMLjgLX3UdzX2rainIgvqkq36Wtf_VLa9ak,55
-rag_sentinel-0.1.2.dist-info/top_level.txt,sha256=qk1BCc3wrLshA3-jf0Jb4bFlZF-ARIjZfYaqAW1Aq0E,13
-rag_sentinel-0.1.2.dist-info/RECORD,,