ragaai-catalyst 2.0.7b1__tar.gz → 2.0.7.1__tar.gz

{ragaai_catalyst-2.0.7b1/ragaai_catalyst.egg-info → ragaai_catalyst-2.0.7.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ragaai_catalyst
-Version: 2.0.7b1
+Version: 2.0.7.1
 Summary: RAGA AI CATALYST
 Author-email: Kiran Scaria <kiran.scaria@raga.ai>, Kedar Gaikwad <kedar.gaikwad@raga.ai>, Dushyant Mahajan <dushyant.mahajan@raga.ai>, Siddhartha Kosti <siddhartha.kosti@raga.ai>, Ritika Goel <ritika.goel@raga.ai>, Vijay Chaurasia <vijay.chaurasia@raga.ai>
 Requires-Python: >=3.9
@@ -18,7 +18,7 @@ Requires-Dist: opentelemetry-instrumentation-langchain~=0.24.0
 Requires-Dist: opentelemetry-instrumentation-openai~=0.24.0
 Requires-Dist: langchain-core>=0.2.11
 Requires-Dist: langchain>=0.2.11
-Requires-Dist: openai>=1.35.10
+Requires-Dist: openai>=1.57.0
 Requires-Dist: pandas>=2.1.1
 Requires-Dist: groq>=0.11.0
 Requires-Dist: PyPDF2>=3.0.1

{ragaai_catalyst-2.0.7b1 → ragaai_catalyst-2.0.7.1}/pyproject.toml

@@ -8,7 +8,7 @@ description = "RAGA AI CATALYST"
 readme = "README.md"
 requires-python = ">=3.9"
 # license = {file = "LICENSE"}
-version = "2.0.7.beta.1"
+version = "2.0.7.1"
 authors = [
     {name = "Kiran Scaria", email = "kiran.scaria@raga.ai"},
     {name = "Kedar Gaikwad", email = "kedar.gaikwad@raga.ai"},
@@ -32,7 +32,7 @@ dependencies = [
     "opentelemetry-instrumentation-openai~=0.24.0",
     "langchain-core>=0.2.11",
     "langchain>=0.2.11",
-    "openai>=1.35.10",
+    "openai>=1.57.0",
     "pandas>=2.1.1",
     "groq>=0.11.0",
     "PyPDF2>=3.0.1",

{ragaai_catalyst-2.0.7b1 → ragaai_catalyst-2.0.7.1/ragaai_catalyst.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ragaai_catalyst
-Version: 2.0.7b1
+Version: 2.0.7.1
 Summary: RAGA AI CATALYST
 Author-email: Kiran Scaria <kiran.scaria@raga.ai>, Kedar Gaikwad <kedar.gaikwad@raga.ai>, Dushyant Mahajan <dushyant.mahajan@raga.ai>, Siddhartha Kosti <siddhartha.kosti@raga.ai>, Ritika Goel <ritika.goel@raga.ai>, Vijay Chaurasia <vijay.chaurasia@raga.ai>
 Requires-Python: >=3.9
@@ -18,7 +18,7 @@ Requires-Dist: opentelemetry-instrumentation-langchain~=0.24.0
 Requires-Dist: opentelemetry-instrumentation-openai~=0.24.0
 Requires-Dist: langchain-core>=0.2.11
 Requires-Dist: langchain>=0.2.11
-Requires-Dist: openai>=1.35.10
+Requires-Dist: openai>=1.57.0
 Requires-Dist: pandas>=2.1.1
 Requires-Dist: groq>=0.11.0
 Requires-Dist: PyPDF2>=3.0.1

{ragaai_catalyst-2.0.7b1 → ragaai_catalyst-2.0.7.1}/ragaai_catalyst.egg-info/SOURCES.txt

@@ -36,4 +36,9 @@ ragaai_catalyst/tracers/instrumentators/langchain.py
 ragaai_catalyst/tracers/instrumentators/llamaindex.py
 ragaai_catalyst/tracers/instrumentators/openai.py
 ragaai_catalyst/tracers/utils/__init__.py
-ragaai_catalyst/tracers/utils/utils.py
+ragaai_catalyst/tracers/utils/utils.py
+test/test_catalyst/test_configuration.py
+test/test_catalyst/test_dataset.py
+test/test_catalyst/test_evaluation.py
+test/test_catalyst/test_prompt_manager.py
+test/test_catalyst/test_synthetic_data_generation.py

{ragaai_catalyst-2.0.7b1 → ragaai_catalyst-2.0.7.1}/ragaai_catalyst.egg-info/requires.txt

@@ -11,7 +11,7 @@ opentelemetry-instrumentation-langchain~=0.24.0
 opentelemetry-instrumentation-openai~=0.24.0
 langchain-core>=0.2.11
 langchain>=0.2.11
-openai>=1.35.10
+openai>=1.57.0
 pandas>=2.1.1
 groq>=0.11.0
 PyPDF2>=3.0.1

ragaai_catalyst-2.0.7.1/test/test_catalyst/test_configuration.py

@@ -0,0 +1,199 @@
+import pytest
+import os
+import requests
+from unittest.mock import patch, MagicMock
+import dotenv
+dotenv.load_dotenv()
+import os
+
+from ragaai_catalyst import RagaAICatalyst
+
+
+# Mock environment variables for testing
+@pytest.fixture
+def mock_env_vars():
+    original_environ = os.environ.copy()
+    RAGAAI_CATALYST_ACCESS_KEY = os.getenv("RAGAAI_CATALYST_ACCESS_KEY")
+    RAGAAI_CATALYST_SECRET_KEY = os.getenv("RAGAAI_CATALYST_SECRET_KEY")
+    RAGAAI_CATALYST_BASE_URL = os.getenv("RAGAAI_CATALYST_BASE_URL")
+    
+    yield
+    
+    os.environ.clear()
+    os.environ.update(original_environ)
+
+@pytest.fixture
+def raga_catalyst(mock_env_vars):
+    with patch('ragaai_catalyst.RagaAICatalyst.get_token', return_value='test_token'):
+        catalyst = RagaAICatalyst(
+            os.getenv("RAGAAI_CATALYST_ACCESS_KEY"),  
+            os.getenv("RAGAAI_CATALYST_SECRET_KEY")  
+        )
+    return catalyst
+
+
+
+def test_project_use_cases():
+        catalyst = RagaAICatalyst(
+            access_key=os.getenv("RAGAAI_CATALYST_ACCESS_KEY"),
+            secret_key=os.getenv("RAGAAI_CATALYST_SECRET_KEY"),
+            base_url=os.getenv("RAGAAI_CATALYST_BASE_URL")
+        )
+        use_case = catalyst.project_use_cases()
+        assert use_case ==['Chatbot', 'Text2SQL', 'Q/A', 'Code Generation', 'Others']
+
+
+def test_list_project():
+        catalyst = RagaAICatalyst(
+            access_key=os.getenv("RAGAAI_CATALYST_ACCESS_KEY"),
+            secret_key=os.getenv("RAGAAI_CATALYST_SECRET_KEY"),
+            base_url=os.getenv("RAGAAI_CATALYST_BASE_URL")
+        )
+        use_case = catalyst.list_projects()
+        assert use_case is not None  # Check if the result is not None
+
+
+def test_existing_projectname():
+        with pytest.raises(ValueError, match="already exists. Please choose a different name."):
+            catalyst = RagaAICatalyst(
+                access_key=os.getenv("RAGAAI_CATALYST_ACCESS_KEY"),
+                secret_key=os.getenv("RAGAAI_CATALYST_SECRET_KEY"),
+                base_url=os.getenv("RAGAAI_CATALYST_BASE_URL")
+            )
+            project = catalyst.create_project(
+                project_name="prompt_metric_dataset3",
+                usecase="Chatbot"
+            )
+
+def test_initialization_missing_credentials():
+    """Test initialization with missing credentials"""
+    with pytest.raises(ValueError, match="RAGAAI_CATALYST_ACCESS_KEY and RAGAAI_CATALYST_SECRET_KEY environment variables must be set"):
+        RagaAICatalyst('', '')
+
+@patch('requests.post')
+def test_get_token_success(mock_post, mock_env_vars):
+    """Test token retrieval success"""
+    mock_response = MagicMock()
+    mock_response.status_code = 200
+    mock_response.json.return_value = {
+        'success': True,
+        'data': {'token': 'test_token'}
+    }
+    mock_post.return_value = mock_response
+
+    token = RagaAICatalyst.get_token()
+    assert token == 'test_token'
+    assert os.getenv('RAGAAI_CATALYST_TOKEN') == 'test_token'
+
+@patch('requests.post')
+def test_get_token_failure(mock_post, mock_env_vars):
+    """Test token retrieval failure"""
+    mock_response = MagicMock()
+    mock_response.status_code = 400
+    mock_response.json.return_value = {
+        'message': 'Please enter valid credentials'
+    }
+    mock_post.return_value = mock_response
+
+    with pytest.raises(Exception, match="Authentication failed"):
+        RagaAICatalyst.get_token()
+
+@patch('requests.get')
+def test_project_use_cases_success(mock_get, raga_catalyst):
+    """Test retrieving project use cases"""
+    mock_response = MagicMock()
+    mock_response.status_code = 200
+    mock_response.json.return_value = {
+        'data': {'usecase': ['Q/A', 'Chatbot', 'Summarization']}
+    }
+    mock_get.return_value = mock_response
+
+    use_cases = raga_catalyst.project_use_cases()
+    assert use_cases == ['Q/A', 'Chatbot', 'Summarization']
+
+@patch('requests.get')
+def test_project_use_cases_failure(mock_get, raga_catalyst):
+    """Test project use cases retrieval failure"""
+    mock_get.side_effect = requests.exceptions.RequestException("Network Error")
+
+    use_cases = raga_catalyst.project_use_cases()
+    assert use_cases == []
+
+@patch('requests.post')
+@patch('ragaai_catalyst.RagaAICatalyst.list_projects')
+def test_create_project_success(mock_list_projects, mock_post, raga_catalyst):
+    """Test successful project creation"""
+    mock_list_projects.return_value = []  # No existing projects
+    mock_post_response = MagicMock()
+    mock_post_response.status_code = 200
+    mock_post_response.json.return_value = {
+        'data': {'name': 'TestProject'}
+    }
+    mock_post.return_value = mock_post_response
+
+    with patch('ragaai_catalyst.RagaAICatalyst.project_use_cases', return_value=['Q/A']):
+        result = raga_catalyst.create_project('TestProject')
+        assert 'Project Created Successfully' in result
+
+@patch('requests.post')
+@patch('ragaai_catalyst.RagaAICatalyst.list_projects')
+def test_create_project_duplicate(mock_list_projects, mock_post, raga_catalyst):
+    """Test project creation with duplicate name"""
+    mock_list_projects.return_value = ['TestProject']
+
+    with pytest.raises(ValueError, match="Project name 'TestProject' already exists"):
+        raga_catalyst.create_project('TestProject')
+
+@patch('requests.get')
+def test_list_projects_success(mock_get, raga_catalyst):
+    """Test successful project listing"""
+    mock_response = MagicMock()
+    mock_response.status_code = 200
+    mock_response.json.return_value = {
+        'data': {
+            'content': [
+                {'name': 'Project1'},
+                {'name': 'Project2'}
+            ]
+        }
+    }
+    mock_get.return_value = mock_response
+
+    projects = raga_catalyst.list_projects()
+    assert projects == ['Project1', 'Project2']
+
+@patch('requests.get')
+def test_list_metrics_success(mock_get):
+    """Test successful metrics listing"""
+    with patch.dict(os.environ, {'RAGAAI_CATALYST_TOKEN': 'test_token'}):
+        mock_response = MagicMock()
+        mock_response.status_code = 200
+        mock_response.json.return_value = {
+            'data': {
+                'metrics': [
+                    {'name': 'hallucination', 'category': 'quality'},
+                    {'name': 'toxicity', 'category': 'safety'}
+                ]
+            }
+        }
+        mock_get.return_value = mock_response
+
+        metrics = RagaAICatalyst.list_metrics()
+        assert metrics == ['hallucination', 'toxicity']
+
+def test_initialization_invalid_credentials():
+    """Test initialization with invalid credentials"""
+    with pytest.raises(Exception, match="Authentication failed. Invalid credentials provided."):
+        RagaAICatalyst(
+            access_key=os.getenv("RAGAAI_CATALYST_ACCESS_KEY")+"a",
+            secret_key=os.getenv("RAGAAI_CATALYST_SECRET_KEY"),
+            base_url=os.getenv("RAGAAI_CATALYST_BASE_URL")
+        )
+
+def test_initialization_invalid_base_url():
+    with pytest.raises(ConnectionError, match="The provided base_url is not accessible. Please re-check the base_url."):
+        RagaAICatalyst(
+            access_key=os.getenv("RAGAAI_CATALYST_ACCESS_KEY"),
+            secret_key=os.getenv("RAGAAI_CATALYST_SECRET_KEY"),
+            base_url=os.getenv("RAGAAI_CATALYST_BASE_URL") +"a",
+        )

ragaai_catalyst-2.0.7.1/test/test_catalyst/test_dataset.py

@@ -0,0 +1,170 @@
+import pytest
+import os
+import dotenv
+dotenv.load_dotenv()
+import pandas as pd
+from datetime import datetime
+from typing import Dict, List
+from unittest.mock import patch, Mock
+import requests
+from ragaai_catalyst import Dataset,RagaAICatalyst
+
+
+@pytest.fixture
+def base_url():
+    return "https://catalyst.raga.ai/api"
+
+@pytest.fixture
+def access_keys():
+    return {
+        "access_key": os.getenv("RAGAAI_CATALYST_ACCESS_KEY"),
+        "secret_key": os.getenv("RAGAAI_CATALYST_SECRET_KEY")}
+
+@pytest.fixture
+def dataset(base_url, access_keys):
+    """Create evaluation instance with specific project and dataset"""
+    os.environ["RAGAAI_CATALYST_BASE_URL"] = base_url
+    catalyst = RagaAICatalyst(
+        access_key=access_keys["access_key"],
+        secret_key=access_keys["secret_key"]
+    )
+    return Dataset(project_name="prompt_metric_dataset")
+
+def test_list_dataset(dataset) -> List[str]:
+    datasets = dataset.list_datasets()
+    return datasets
+
+
+def test_get_dataset_columns(dataset)  -> List[str]:
+    dataset_column = dataset.get_dataset_columns(dataset_name="ritika_dataset")
+    return dataset_column
+
+def test_incorrect_dataset(dataset):
+    with pytest.raises(ValueError, match="Please enter a valid dataset name"):
+        dataset.get_dataset_columns(dataset_name="ritika_datset")
+
+def test_get_schema_mapping(dataset):
+    schema_mapping_columns= dataset.get_schema_mapping()
+    return schema_mapping_columns
+
+
+def test_upload_csv(dataset):
+    project_name = 'prompt_metric_dataset'
+
+    schema_mapping = {
+        'Query': 'prompt',
+        'Response': 'response',
+        'Context': 'context',
+        'ExpectedResponse': 'expected_response',
+    }
+
+    csv_path= "/Users/siddharthakosti/Downloads/catalyst_error_handling/catalyst_v2/catalyst_v2_new_1/data/prompt_metric_dataset_v1.csv"
+
+    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") 
+    dataset_name = f"schema_metric_dataset_ritika_{timestamp}"  
+
+    
+
+    dataset.create_from_csv(
+        csv_path=csv_path,
+        dataset_name=dataset_name,
+        schema_mapping=schema_mapping
+    )
+
+def test_upload_csv_repeat_dataset(dataset):
+    with pytest.raises(ValueError, match="already exists"):
+        project_name = 'prompt_metric_dataset'
+
+        schema_mapping = {
+            'Query': 'prompt',
+            'Response': 'response',
+            'Context': 'context',
+            'ExpectedResponse': 'expected_response',
+        }
+
+        csv_path= "/Users/siddharthakosti/Downloads/catalyst_error_handling/catalyst_v2/catalyst_v2_new_1/data/prompt_metric_dataset_v1.csv"
+
+        dataset.create_from_csv(
+            csv_path=csv_path,
+            dataset_name="schema_metric_dataset_ritika_3",
+            schema_mapping=schema_mapping
+        )
+
+
+def test_upload_csv_no_schema_mapping(dataset):
+    with pytest.raises(TypeError, match="missing 1 required positional argument"):
+        project_name = 'prompt_metric_dataset'
+
+        schema_mapping = {
+            'Query': 'prompt',
+            'Response': 'response',
+            'Context': 'context',
+            'ExpectedResponse': 'expected_response',
+        }
+
+        csv_path= "/Users/siddharthakosti/Downloads/catalyst_error_handling/catalyst_v2/catalyst_v2_new_1/data/prompt_metric_dataset_v1.csv"
+
+        dataset.create_from_csv(
+            csv_path=csv_path,
+            dataset_name="schema_metric_dataset_ritika_3",
+        )
+
+def test_upload_csv_empty_csv_path(dataset):
+    with pytest.raises(FileNotFoundError, match="No such file or directory"):
+        project_name = 'prompt_metric_dataset'
+
+        schema_mapping = {
+            'Query': 'prompt',
+            'Response': 'response',
+            'Context': 'context',
+            'ExpectedResponse': 'expected_response',
+        }
+
+        csv_path= "/Users/siddharthakosti/Downloads/catalyst_error_handling/catalyst_v2/catalyst_v2_new_1/data/prompt_metric_dataset_v1.csv"
+
+        dataset.create_from_csv(
+            csv_path="",
+            dataset_name="schema_metric_dataset_ritika_12",
+            schema_mapping=schema_mapping
+
+        )
+
+def test_upload_csv_empty_schema_mapping(dataset):
+    with pytest.raises(AttributeError):
+        project_name = 'prompt_metric_dataset'
+
+        schema_mapping = {
+            'Query': 'prompt',
+            'Response': 'response',
+            'Context': 'context',
+            'ExpectedResponse': 'expected_response',
+        }
+
+        csv_path= "/Users/siddharthakosti/Downloads/catalyst_error_handling/catalyst_v2/catalyst_v2_new_1/data/prompt_metric_dataset_v1.csv"
+
+        dataset.create_from_csv(
+            csv_path=csv_path,
+            dataset_name="schema_metric_dataset_ritika_12",
+            schema_mapping=""
+
+        )
+
+
+def test_upload_csv_invalid_schema(dataset):
+    with pytest.raises(ValueError, match="Invalid schema mapping provided"):
+
+        project_name = 'prompt_metric_dataset'
+
+        schema_mapping={
+            'prompt': 'prompt',
+            'response': 'response',
+            'chatId': 'chatId',
+            'chatSequence': 'chatSequence'
+        }
+
+        csv_path= "/Users/siddharthakosti/Downloads/catalyst_error_handling/catalyst_v2/catalyst_v2_new_1/data/prompt_metric_dataset_v1.csv"
+
+        dataset.create_from_csv(
+            csv_path=csv_path,
+            dataset_name="schema_metric_dataset_ritika_12",
+            schema_mapping=schema_mapping)

ragaai_catalyst-2.0.7.1/test/test_catalyst/test_evaluation.py

@@ -0,0 +1,503 @@
+
+from unittest.mock import patch
+import time
+import pytest
+import os
+import dotenv
+dotenv.load_dotenv()
+import pandas as pd
+from datetime import datetime 
+from typing import Dict, List
+from ragaai_catalyst import Evaluation, RagaAICatalyst
+GEMINI_API_KEY = os.environ.get("GEMINI_API_KEY")
+OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY")
+# Define model configurations
+MODEL_CONFIGS = [
+    # OpenAI Models
+    {
+        "provider": "openai",
+        "model": "gpt-4",
+        "suffix": "gpt4"
+    },
+    {
+        "provider": "openai",
+        "model": "gpt-4o",
+        "suffix": "gpt4o"
+    },
+    {
+        "provider": "openai",
+        "model": "gpt-4o-mini",
+        "suffix": "gpt4o_mini"
+    },
+    {
+        "provider": "openai",
+        "model": "gpt-3.5-turbo",
+        "suffix": "gpt35"
+    },
+    # Gemini Models
+    {
+        "provider": "gemini",
+        "model": "gemini-1.5-flash",
+        "suffix": "gemini15_flash"
+    },
+    {
+        "provider": "gemini",
+        "model": "gemini-1.5-pro",
+        "suffix": "gemini15_pro"
+    },
+    # Azure OpenAI Models
+    {
+        "provider": "azure",
+        "model": "gpt-4",
+        "suffix": "azure_gpt4"
+    },
+    {
+        "provider": "azure",
+        "model": "gpt-35-turbo",
+        "suffix": "azure_gpt35"
+    }
+]
+
+@pytest.fixture
+def base_url():
+    return "https://catalyst.raga.ai/api"
+
+@pytest.fixture
+def access_keys():
+    return {
+        "access_key": os.getenv("RAGAAI_CATALYST_ACCESS_KEY"),
+        "secret_key": os.getenv("RAGAAI_CATALYST_SECRET_KEY")}
+    
+
+@pytest.fixture
+def evaluation(base_url, access_keys):
+    """Create evaluation instance with specific project and dataset"""
+    os.environ["RAGAAI_CATALYST_BASE_URL"] = base_url
+    catalyst = RagaAICatalyst(
+        access_key=access_keys["access_key"],
+        secret_key=access_keys["secret_key"]
+    )
+    return Evaluation(project_name="prompt_metric_dataset", dataset_name="ritika_dataset")
+
+@pytest.fixture
+def chat_evaluation(base_url, access_keys):
+    """Create evaluation instance with specific project and dataset"""
+    os.environ["RAGAAI_CATALYST_BASE_URL"] = base_url
+    catalyst = RagaAICatalyst(
+        access_key=access_keys["access_key"],
+        secret_key=access_keys["secret_key"]
+    )
+    return Evaluation(project_name="chat_demo_sk_v1", dataset_name="chat_metric_dataset_ritika")
+
+def test_evaluation_initialization(evaluation):
+    """Test if evaluation is initialized correctly"""
+    assert evaluation.project_name == "prompt_metric_dataset"
+    assert evaluation.dataset_name == "ritika_dataset"
+    assert evaluation.base_url == "https://catalyst.raga.ai/api"
+    assert evaluation.timeout == 10
+    assert evaluation.jobId is None
+
+def test_project_does_not_exist():
+    """Test initialization with non-existent project"""
+    with pytest.raises(ValueError, match="Project not found. Please enter a valid project name"):
+        Evaluation(project_name="non_existent_project", dataset_name="prompt_metric_dataset_v1")
+
+def test_dataset_does_not_exist():
+    """Test initialization with non-existent dataset"""
+    with pytest.raises(ValueError, match="Dataset not found. Please enter a valid dataset name"):
+        Evaluation(project_name="prompt_metric_dataset", dataset_name="non_existent_dataset")
+
+def test_list_metrics(evaluation) -> List[str]:
+    """Test if it lists all the metrics correctly"""
+    metrics = evaluation.list_metrics()
+    return metrics
+
+@pytest.mark.parametrize("provider_config", MODEL_CONFIGS)
+def test_invalid_schema_mapping(evaluation, provider_config):
+    """Wrong schema mapping for different providers"""
+    with pytest.raises(ValueError, match="Map"):
+        schema_mapping={
+            'Query': 'Prompt',
+            'Context': 'Context',
+        }
+        metrics = [{
+            "name": "Hallucination", 
+            "config": {
+                "model": provider_config["model"], 
+                "provider": provider_config["provider"]
+            }, 
+            "column_name": f"Hallucination_{provider_config['suffix']}", 
+            "schema_mapping": schema_mapping
+        }]
+        evaluation.add_metrics(metrics=metrics)
+
+@pytest.mark.parametrize("provider_config", MODEL_CONFIGS)
+def test_missing_schema_mapping(evaluation, provider_config):
+    """schema_mapping not present for different providers"""
+    with pytest.raises(ValueError, match="{'schema_mapping'} required for each metric evaluation."):
+        metrics = [{
+            "name": "Hallucination", 
+            "config": {
+                "model": provider_config["model"], 
+                "provider": provider_config["provider"]
+            }, 
+            "column_name": f"Hallucination_{provider_config['suffix']}"
+        }]
+        evaluation.add_metrics(metrics=metrics)
+
+@pytest.mark.parametrize("provider_config", MODEL_CONFIGS)
+def test_missing_column_name(evaluation, provider_config):
+    """column_name not present for different providers"""
+    with pytest.raises(ValueError, match="{'column_name'} required for each metric evaluation."):
+        schema_mapping={
+            'Query': 'Prompt',
+            'Response': 'Response',
+            'Context': 'Context',
+        }
+        metrics = [{
+            "name": "Hallucination", 
+            "config": {
+                "model": provider_config["model"], 
+                "provider": provider_config["provider"]
+            },
+            "schema_mapping": schema_mapping
+        }]
+        evaluation.add_metrics(metrics=metrics)
+
+@pytest.mark.parametrize("provider_config", MODEL_CONFIGS)
+def test_missing_metric_name(evaluation, provider_config):
+    """metric name missing for different providers"""
+    with pytest.raises(ValueError, match="{'name'} required for each metric evaluation."):
+        schema_mapping={
+            'Query': 'Prompt',
+            'Response': 'Response',
+            'Context': 'Context',
+        }
+        metrics = [{
+            "config": {
+                "model": provider_config["model"], 
+                "provider": provider_config["provider"]
+            }, 
+            "column_name": f"Hallucination_{provider_config['suffix']}", 
+            "schema_mapping": schema_mapping
+        }]
+        evaluation.add_metrics(metrics=metrics)
+
+@pytest.mark.parametrize("provider_config", MODEL_CONFIGS)
+def test_column_name_already_exists(evaluation, provider_config):
+    """Column name already exists for different providers"""
+    with pytest.raises(ValueError, match="already exists."):
+        schema_mapping={
+            'Query': 'Prompt',
+            'Response': 'Response',
+            'Context': 'Context',
+        }
+        metrics = [{
+            "name": "Hallucination", 
+            "config": {
+                "model": provider_config["model"], 
+                "provider": provider_config["provider"]
+            }, 
+            "column_name": "Hallucination_column3", 
+            "schema_mapping": schema_mapping
+        }]
+        evaluation.add_metrics(metrics=metrics)
+
+def test_missing_config(evaluation):
+    with pytest.raises(ValueError, match="{'config'} required for each metric evaluation."):
+        schema_mapping={
+            'Query': 'Prompt',
+            'Response': 'Response',
+            'Context': 'Context',
+        }
+        metrics = [{"name": "Hallucination", "column_name": "Hallucination5", "schema_mapping": schema_mapping}]
+        evaluation.add_metrics(metrics=metrics)
+
+
+
+@pytest.mark.parametrize("metric_name", ['Hallucination',
+ 'Faithfulness',
+ 'SQL Prompt Injection',
+ 'Response Correctness',
+ 'Response Completeness',
+ 'False Refusal',
+ 'Context Precision',
+ 'Context Recall',
+ 'Context Relevancy'
+ 'SQL Response Correctness',
+ 'SQL Prompt Ambiguity',
+ 'SQL Context Sufficiency',
+ 'SQL Context Ambiguity'])
+
+def test_metric_initialization_gemini(evaluation, metric_name: str,capfd):
+    """Test if adding each metric and tracking its completion works correctly"""
+    schema_mapping = {
+        'Query': 'prompt',
+        'Response': 'response',
+        'Context': 'context',
+        'ExpectedResponse': 'expectedresponse',
+    }
+    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")  # Format: YYYYMMDD_HHMMSS
+    metrics = [{
+        "name": metric_name,
+        "config": {
+            "model": "gemini-1.5-flash",
+            "provider": "gemini"
+        },
+        "column_name": f"{metric_name}_column_{timestamp}",
+        "schema_mapping": schema_mapping
+    }]
+    
+    # Add metrics and capture the printed output
+    evaluation.add_metrics(metrics=metrics)
+    out, err = capfd.readouterr()
+    print(f"Add metrics output: {out}")  # Debug print
+    
+    # Verify the success message for metric addition
+    assert "Metric Evaluation Job scheduled successfully" in out, f"Failed to schedule job for metric: {metric_name}"
+    
+    # Store the jobId for status checking
+    assert evaluation.jobId is not None, "Job ID was not set after adding metrics"
+    print(f"Job ID: {evaluation.jobId}")  # Debug print
+    
+    # Check job status with timeout
+    max_wait_time = 180  # Increased timeout to 3 minutes
+    poll_interval = 5    # Check every 5 seconds
+    start_time = time.time()
+    status_checked = False
+    last_status = None
+    
+    print(f"Starting job status checks for {metric_name}...")  # Debug print
+    
+    while (time.time() - start_time) < max_wait_time:
+        try:
+            evaluation.get_status()
+            out, err = capfd.readouterr()
+            print(f"Status check output: {out}")  # Debug print
+            
+            if "Job completed" in out:
+                status_checked = True
+                print(f"Job completed for {metric_name}")  # Debug print
+                break
+                
+            if "Job failed" in out:
+                pytest.fail(f"Job failed for metric: {metric_name}")
+                
+            last_status = out
+            time.sleep(poll_interval)
+            
+        except Exception as e:
+            print(f"Error checking status: {str(e)}")  # Debug print
+            time.sleep(poll_interval)
+    
+    if not status_checked:
+        print(f"Last known status: {last_status}")  # Debug print
+        if last_status and "In Progress" in last_status:
+            pytest.skip(f"Job still in progress after {max_wait_time} seconds for {metric_name}. This is not a failure, but took longer than expected.")
+        else:
+            assert False, f"Job did not complete within {max_wait_time} seconds for metric: {metric_name}. Last status: {last_status}"
+    
+    # Only check results if the job completed successfully
+    if status_checked:
+        try:
+            results = evaluation.get_results()
+            assert isinstance(results, pd.DataFrame), "Results should be returned as a DataFrame"
+            assert not results.empty, "Results DataFrame should not be empty"
+            column_name = f"{metric_name}_column25"
+            assert column_name in results.columns, f"Expected column {column_name} not found in results. Available columns: {results.columns.tolist()}"
+        except Exception as e:
+            pytest.fail(f"Error getting results for {metric_name} with provider: gemini and model: gemini-1.5-flash: {str(e)}")
+
+
+
+@pytest.mark.parametrize("metric_name", ['Hallucination',
+ 'Faithfulness',
+ 'SQL Prompt Injection',
+ 'Response Correctness',
+ 'Response Completeness',
+ 'False Refusal',
+ 'Context Precision',
+ 'Context Recall',
+ 'Context Relevancy',
+ 'SQL Response Correctness',
+ 'SQL Prompt Ambiguity',
+ 'SQL Context Sufficiency',
+ 'SQL Context Ambiguity'])
+
+def test_metric_initialization_openai(evaluation, metric_name: str,capfd):
+    """Test if adding each metric and tracking its completion works correctly"""
+    schema_mapping = {
+        'Query': 'prompt',
+        'Response': 'response',
+        'Context': 'context',
+        'ExpectedResponse': 'expectedresponse',
+    }
+
+    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")  # Format: YYYYMMDD_HHMMSS
+
+    metrics = [{
+        "name": metric_name,
+        "config": {
+            "model": "gpt-4o-mini",
+            "provider": "openai"
+        },
+        "column_name": f"{metric_name}_column_{timestamp}", 
+        "schema_mapping": schema_mapping
+    }]
+    
+    # Add metrics and capture the printed output
+    evaluation.add_metrics(metrics=metrics)
+    out, err = capfd.readouterr()
+    print(f"Add metrics output: {out}")  # Debug print
+    
+    # Verify the success message for metric addition
+    assert "Metric Evaluation Job scheduled successfully" in out, f"Failed to schedule job for metric: {metric_name}"
+    
+    # Store the jobId for status checking
+    assert evaluation.jobId is not None, "Job ID was not set after adding metrics"
+    print(f"Job ID: {evaluation.jobId}")  # Debug print
+    
+    # Check job status with timeout
+    max_wait_time = 300  # Increased timeout to 3 minutes
+    poll_interval = 5    # Check every 5 seconds
+    start_time = time.time()
+    status_checked = False
+    last_status = None
+    
+    print(f"Starting job status checks for {metric_name}...")  # Debug print
+    
+    while (time.time() - start_time) < max_wait_time:
+        try:
+            evaluation.get_status()
+            out, err = capfd.readouterr()
+            print(f"Status check output: {out}")  # Debug print
+            
+            if "Job completed" in out:
+                status_checked = True
+                print(f"Job completed for {metric_name}")  # Debug print
+                break
+                
+            if "Job failed" in out:
+                pytest.fail(f"Job failed for metric: {metric_name}")
+                
+            last_status = out
+            time.sleep(poll_interval)
+            
+        except Exception as e:
+            print(f"Error checking status: {str(e)}")  # Debug print
+            time.sleep(poll_interval)
+    
+    if not status_checked:
+        print(f"Last known status: {last_status}")  # Debug print
+        if last_status and "In Progress" in last_status:
+            pytest.skip(f"Job still in progress after {max_wait_time} seconds for {metric_name}. This is not a failure, but took longer than expected.")
+        else:
+            assert False, f"Job did not complete within {max_wait_time} seconds for metric: {metric_name}. Last status: {last_status}"
+    
+    # Only check results if the job completed successfully
+    if status_checked:
+        try:
+            results = evaluation.get_results()
+            assert isinstance(results, pd.DataFrame), "Results should be returned as a DataFrame"
+            assert not results.empty, "Results DataFrame should not be empty"
+            column_name = f"{metric_name}_column26"
+            assert column_name in results.columns, f"Expected column {column_name} not found in results. Available columns: {results.columns.tolist()}"
+        except Exception as e:
+            pytest.fail(f"Error getting results for {metric_name} with provider: oprnai and model: gpt-4o-mini: {str(e)}")
+
+
+
+# Add a counter to keep track of the test iterations
+counter = 30
+
+@pytest.mark.parametrize("metric_name", ['Agent Quality',
+ 'User Chat Quality',
+ 'Instruction Adherence'])
+@pytest.mark.parametrize("model_config", [
+    {"model": "gpt-4o-mini", "provider": "openai"},
+    {"model": "gpt-4", "provider": "openai"},
+    {"model": "gpt-3.5-turbo", "provider": "openai"},
+    {"model":"gemini-1.5-flash", "provider": "gemini"}
+])
+def test_metric_initialization_openai_chatmetric(chat_evaluation, model_config, metric_name: str, capfd):
+    """Test if adding each metric and tracking its completion works correctly"""
+    global counter  # Use the global counter
+    schema_mapping = {
+        'ChatID': 'ChatID',
+        'Chat': 'Chat',
+        'Instructions': 'Instructions',
+        'System Prompt': 'systemprompt',
+    }
+    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")  # Format: YYYYMMDD_HHMMSS
+    metrics = [{
+        "name": metric_name,
+        "config": model_config,
+        "column_name": f"{metric_name}_column_{timestamp}_{counter}",  # Use counter for unique column name
+        "schema_mapping": schema_mapping
+    }]
+    
+    # Increment the counter after each test
+    counter += 1
+    
+    # Add metrics and capture the printed output
+    chat_evaluation.add_metrics(metrics=metrics)
+    out, err = capfd.readouterr()
+    print(f"Add metrics output: {out}")  # Debug print
+    
+    # Verify the success message for metric addition
+    assert "Metric Evaluation Job scheduled successfully" in out, f"Failed to schedule job for metric: {metric_name} and {model_config}"
+    
+    # Store the jobId for status checking
+    assert chat_evaluation.jobId is not None, "Job ID was not set after adding metrics"
+    print(f"Job ID: {chat_evaluation.jobId}")  # Debug print
+    
+    # Check job status with timeout
+    max_wait_time = 600  # Increased timeout to 3 minutes
+    poll_interval = 5    # Check every 5 seconds
+    start_time = time.time()
+    status_checked = False
+    last_status = None
+    
+    print(f"Starting job status checks for {metric_name}...")  # Debug print
+    
+    while (time.time() - start_time) < max_wait_time:
+        try:
+            chat_evaluation.get_status()
+            out, err = capfd.readouterr()
+            print(f"Status check output: {out}")  # Debug print
+            
+            if "Job completed" in out:
+                status_checked = True
+                print(f"Job completed for {metric_name}")  # Debug print
+                break
+                
+            if "Job failed" in out:
+                pytest.fail(f"Job failed for metric: {metric_name}{model_config}")
+                
+            last_status = out
+            time.sleep(poll_interval)
+            
+        except Exception as e:
+            print(f"Error checking status: {str(e)}")  # Debug print
+            time.sleep(poll_interval)
+    
+    if not status_checked:
+        print(f"Last known status: {last_status}")  # Debug print
+        if last_status and "In Progress" in last_status:
+            pytest.skip(f"Job still in progress after {max_wait_time} seconds {model_config} for {metric_name}. This is not a failure, but took longer than expected.")
+        else:
+            assert False, f"Job did not complete within {max_wait_time} seconds {model_config} for metric: {metric_name}. Last status: {last_status}"
+    
+    # Only check results if the job completed successfully
+    if status_checked:
+        try:
+            results = chat_evaluation.get_results()
+            assert isinstance(results, pd.DataFrame), "Results should be returned as a DataFrame"
+            assert not results.empty, "Results DataFrame should not be empty"
+            column_name = f"{metric_name}_column_{counter - 1}"  # Use the last counter value
+            assert column_name in results.columns, f"Expected column {column_name} not found in results. Available columns: {results.columns.tolist()}"
+        except Exception as e:
+            pytest.fail(f"Error getting results for {metric_name} with {model_config}: {str(e)}")
+
+
+

ragaai_catalyst-2.0.7.1/test/test_catalyst/test_prompt_manager.py

@@ -0,0 +1,88 @@
+import os
+import pytest
+import copy
+from ragaai_catalyst import PromptManager, RagaAICatalyst
+import dotenv
+import openai
+dotenv.load_dotenv()
+
+
+@pytest.fixture
+def base_url():
+    return "https://catalyst.raga.ai/api"
+
+@pytest.fixture
+def access_keys():
+    return {
+        "access_key": os.getenv("RAGAAI_CATALYST_ACCESS_KEY"),
+        "secret_key": os.getenv("RAGAAI_CATALYST_SECRET_KEY")}
+
+
+@pytest.fixture
+def prompt_manager(base_url, access_keys):
+    """Create evaluation instance with specific project and dataset"""
+    os.environ["RAGAAI_CATALYST_BASE_URL"] = base_url
+    catalyst = RagaAICatalyst(
+        access_key=access_keys["access_key"],
+        secret_key=access_keys["secret_key"]
+    )
+    return PromptManager(project_name="prompt_metric_dataset")
+
+def test_prompt_initialistaion(prompt_manager):
+    prompt_list= prompt_manager.list_prompts()
+    assert prompt_list ==['test','test2']
+
+def test_list_prompt_version(prompt_manager):
+    prompt_version_list = prompt_manager.list_prompt_versions(prompt_name="test2")
+    assert len(prompt_version_list.keys()) == 2
+
+def test_missing_prompt_name(prompt_manager):
+    with pytest.raises(ValueError, match="Please enter a valid prompt name"):
+        prompt = prompt_manager.get_prompt(prompt_name="", version="v1")
+
+def test_get_variable(prompt_manager):
+    prompt = prompt_manager.get_prompt(prompt_name="test2", version="v2")
+    prompt_variable = prompt.get_variables()
+    assert prompt_variable == ['system1', 'system2']
+
+def test_get_model_parameters(prompt_manager):
+    prompt = prompt_manager.get_prompt(prompt_name="test2", version="v2")
+    model_parameter = prompt.get_model_parameters()
+    assert model_parameter== {'frequency_penalty': 0.4,'max_tokens': 1038,'presence_penalty': 0.1,'temperature': 0.7,'model': 'gpt-4o-mini'}
+
+def test_compile_prompt(prompt_manager):
+    prompt = prompt_manager.get_prompt(prompt_name="test2", version="v2")
+    compiled_prompt = prompt.compile(
+    system1='What is chocolate?',
+    system2 = "How it is made")
+    def get_openai_response(prompt):
+        client = openai.OpenAI()
+        response = client.chat.completions.create(
+            model="gpt-4o-mini",
+            messages=prompt
+        )
+        return response.choices[0].message.content
+    get_openai_response(compiled_prompt)
+
+def test_compile_prompt_no_modelname(prompt_manager):
+    with pytest.raises(openai.BadRequestError,match="you must provide a model parameter"):
+
+        prompt = prompt_manager.get_prompt(prompt_name="test2", version="v2")
+        compiled_prompt = prompt.compile(
+        system1='What is chocolate?',
+        system2 = "How it is made")
+        def get_openai_response(prompt):
+            client = openai.OpenAI()
+            response = client.chat.completions.create(
+                model="",
+                messages=prompt
+            )
+            return response.choices[0].message.content
+        get_openai_response(compiled_prompt)
+
+
+
+
+
+
+

ragaai_catalyst-2.0.7.1/test/test_catalyst/test_synthetic_data_generation.py

@@ -0,0 +1,151 @@
+import sys
+# sys.path.append('/Users/ritikagoel/workspace/synthetic-catalyst-internal-api2/ragaai-catalyst')
+
+import pytest
+from ragaai_catalyst import SyntheticDataGeneration
+import os
+
+GEMINI_API_KEY = os.environ.get("GEMINI_API_KEY")
+OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY")
+
+@pytest.fixture
+def synthetic_gen():
+    return SyntheticDataGeneration()
+
+@pytest.fixture
+def sample_text(synthetic_gen):
+    text_file = "/Users/siddharthakosti/Downloads/catalyst_error_handling/catalyst_v2/catalyst_v2_new_1/data/ai_document_061023_2.pdf"  # Update this path as needed
+    return synthetic_gen.process_document(input_data=text_file)
+
+def test_invalid_csv_processing(synthetic_gen):
+    """Test processing an invalid CSV file"""
+    with pytest.raises(Exception):
+        synthetic_gen.process_document(input_data="/Users/siddharthakosti/Downloads/catalyst_error_handling/catalyst_v2/catalyst_v2_new_1/data/OG1.csv")
+
+def test_special_chars_csv_processing(synthetic_gen):
+    """Test processing CSV with special characters"""
+    with pytest.raises(Exception):
+        synthetic_gen.process_document(input_data="/Users/siddharthakosti/Downloads/catalyst_error_handling/catalyst_v2/catalyst_v2_new_1/data/OG1.csv")
+
+
+
+def test_missing_llm_proxy(synthetic_gen, sample_text):
+    """Test behavior when internal_llm_proxy is not provided"""
+    print('-'*10)
+    print(OPENAI_API_KEY)
+    print('-'*10)
+    with pytest.raises(ValueError, match="API key must be provided"):
+        synthetic_gen.generate_qna(
+            text=sample_text,
+            question_type='mcq',
+            model_config={"provider": "openai", "model": "gpt-4o-mini"},
+            n=20,
+            user_id="1"
+        )
+
+def test_llm_proxy(synthetic_gen, sample_text):
+    result = synthetic_gen.generate_qna(
+            text=sample_text,
+            question_type='mcq',
+            model_config={"provider": "gemini", "model": "gemini-1.5-flash"},
+            n=15,
+            internal_llm_proxy="http://4.247.138.221:4000/chat/completions",
+            user_id="1"
+        )
+    assert len(result) == 15 
+
+    
+
+def test_invalid_llm_proxy(synthetic_gen, sample_text):
+    """Test behavior with invalid internal_llm_proxy URL"""
+    with pytest.raises(Exception, match="No connection adapters were found for"):
+        synthetic_gen.generate_qna(
+            text=sample_text,
+            question_type='mcq',
+            model_config={"provider": "openai", "model": "gpt-4o-mini"},
+            n=2,
+            internal_llm_proxy="tp://invalid.url",
+            user_id="1"
+        )
+
+def test_missing_model_config(synthetic_gen, sample_text):
+    """Test behavior when model_config is not provided"""
+    with pytest.raises(ValueError, match="Model configuration must be provided with a valid provider and model"):
+        synthetic_gen.generate_qna(
+            text=sample_text,
+            question_type='mcq',
+            n=2,
+            internal_llm_proxy="http://20.244.126.4:4000/chat/completions",
+            user_id="1"
+        )
+
+def test_missing_api_key_for_external_provider(synthetic_gen, sample_text):
+    """Test behavior when API key is missing for external provider"""
+    with pytest.raises(ValueError, match="API key must be provided"):
+        synthetic_gen.generate_qna(
+            text=sample_text,
+            question_type='mcq',
+            model_config={"provider": "gemini", "model": "gemini/gemini-1.5-flash"},
+            n=5
+        )
+
+def test_invalid_api_key(synthetic_gen, sample_text):
+    """Test behavior with invalid API key"""
+    with pytest.raises(Exception, match="Failed to generate valid response after 3 attempts: Invalid API key provided"):
+        synthetic_gen.generate_qna(
+            text=sample_text,
+            question_type='mcq',
+            model_config={"provider": "gemini", "model": "gemini/gemini-1.5-flash"},
+            n=5,
+            api_key='invalid_key'
+        )
+
+def test_default_question_count(synthetic_gen, sample_text):
+    """Test default number of questions when n is not provided"""
+    result = synthetic_gen.generate_qna(
+        text=sample_text,
+        question_type='mcq',
+        model_config={"provider": "openai", "model": "gpt-4o-mini"},
+        internal_llm_proxy="http://4.247.138.221:4000/chat/completions",
+        user_id="1"
+    )
+    assert len(result) == 5  # Default should be 5 questions
+
+def test_default_question_type(synthetic_gen, sample_text):
+    """Test default question type when question_type is not provided"""
+    result = synthetic_gen.generate_qna(
+        text=sample_text,
+        model_config={"provider": "openai", "model": "gpt-4o-mini"},
+        n=5,
+        internal_llm_proxy="http://20.244.126.4:4000/chat/completions",
+        user_id="1"
+    )
+    # Verify result contains simple Q/A format without multiple choice options
+    assert all('options' not in qa for qa in result)
+
+def test_question_count_matches_n(synthetic_gen, sample_text):
+    """Test if number of generated questions matches n"""
+    n = 2
+    result = synthetic_gen.generate_qna(
+        text=sample_text,
+        question_type='mcq',
+        model_config={"provider": "openai", "model": "gpt-4o-mini"},
+        n=n,
+        internal_llm_proxy="http://4.247.138.221:4000/chat/completions",
+        user_id="1"
+    )
+    assert len(result) == n
+
+def test_proxy_call_check(synthetic_gen,sample_text):
+    """Test compatibility when proxy script called"""
+
+    result = synthetic_gen.generate_qna(
+            text=sample_text,
+            question_type='simple',
+            model_config={"provider": "gemini", "model": "gemini-1.5-flash", "api_base": "http://172.172.11.158:8000/v1alpha1/v1alpha1/predictions"},
+            n=5
+        )
+    assert len(result) == 5 
+
+
+