ragaai-catalyst 2.0.6b2__tar.gz → 2.0.7__tar.gz

{ragaai_catalyst-2.0.6b2/ragaai_catalyst.egg-info → ragaai_catalyst-2.0.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ragaai_catalyst
-Version: 2.0.6b2
+Version: 2.0.7
 Summary: RAGA AI CATALYST
 Author-email: Kiran Scaria <kiran.scaria@raga.ai>, Kedar Gaikwad <kedar.gaikwad@raga.ai>, Dushyant Mahajan <dushyant.mahajan@raga.ai>, Siddhartha Kosti <siddhartha.kosti@raga.ai>, Ritika Goel <ritika.goel@raga.ai>, Vijay Chaurasia <vijay.chaurasia@raga.ai>
 Requires-Python: >=3.9

{ragaai_catalyst-2.0.6b2 → ragaai_catalyst-2.0.7}/pyproject.toml

@@ -8,7 +8,7 @@ description = "RAGA AI CATALYST"
 readme = "README.md"
 requires-python = ">=3.9"
 # license = {file = "LICENSE"}
-version = "2.0.6.beta.2"
+version = "2.0.7"
 authors = [
     {name = "Kiran Scaria", email = "kiran.scaria@raga.ai"},
     {name = "Kedar Gaikwad", email = "kedar.gaikwad@raga.ai"},

{ragaai_catalyst-2.0.6b2 → ragaai_catalyst-2.0.7}/ragaai_catalyst/internal_api_completion.py

@@ -36,7 +36,7 @@ def api_completion(messages, model_config, kwargs):
                     raise ValueError(response["error"]["message"])
                 else:
                     result=  response["choices"][0]["message"]["content"]
-                    response1 = result.replace('\n', '')
+                    response1 = result.replace('\n', '').replace('```json','').replace('```', '').strip()
                     try:
                         json_data = json.loads(response1)
                         df = pd.DataFrame(json_data)

{ragaai_catalyst-2.0.6b2 → ragaai_catalyst-2.0.7}/ragaai_catalyst/prompt_manager.py

@@ -432,7 +432,12 @@ class PromptObject:
         Returns:
             dict: A dictionary of parameters found in the prompt text.
         """
-        parameters = {param["name"]: self._convert_value(param["value"], param["type"]) for param in self.parameters}
+        parameters = {}
+        for param in self.parameters:
+            if "value" in param:
+                parameters[param["name"]] = self._convert_value(param["value"], param["type"])
+            else:
+                parameters[param["name"]] = ""
         parameters["model"] = self.model
         return parameters    
     

{ragaai_catalyst-2.0.6b2 → ragaai_catalyst-2.0.7}/ragaai_catalyst/synthetic_data_generation.py

@@ -289,6 +289,7 @@ class SyntheticDataGeneration:
 
         # Extract the content from the response
         content = response.choices[0].message.content
+        content = content.replace('\n', '').replace('```json','').replace('```', '').strip()
 
         # Clean the response if needed (remove any prefix before the JSON list)
         list_start_index = content.find('[')

{ragaai_catalyst-2.0.6b2 → ragaai_catalyst-2.0.7/ragaai_catalyst.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ragaai_catalyst
-Version: 2.0.6b2
+Version: 2.0.7
 Summary: RAGA AI CATALYST
 Author-email: Kiran Scaria <kiran.scaria@raga.ai>, Kedar Gaikwad <kedar.gaikwad@raga.ai>, Dushyant Mahajan <dushyant.mahajan@raga.ai>, Siddhartha Kosti <siddhartha.kosti@raga.ai>, Ritika Goel <ritika.goel@raga.ai>, Vijay Chaurasia <vijay.chaurasia@raga.ai>
 Requires-Python: >=3.9

{ragaai_catalyst-2.0.6b2 → ragaai_catalyst-2.0.7}/ragaai_catalyst.egg-info/SOURCES.txt

@@ -36,4 +36,9 @@ ragaai_catalyst/tracers/instrumentators/langchain.py
 ragaai_catalyst/tracers/instrumentators/llamaindex.py
 ragaai_catalyst/tracers/instrumentators/openai.py
 ragaai_catalyst/tracers/utils/__init__.py
-ragaai_catalyst/tracers/utils/utils.py
+ragaai_catalyst/tracers/utils/utils.py
+test/test_catalyst/test_configuration.py
+test/test_catalyst/test_dataset.py
+test/test_catalyst/test_evaluation.py
+test/test_catalyst/test_prompt_manager.py
+test/test_catalyst/test_synthetic_data_generation.py

{ragaai_catalyst-2.0.6b2 → ragaai_catalyst-2.0.7}/requirements.txt

@@ -1,5 +1,5 @@
 toml==0.10.2
-aiohttp==3.9.5
+aiohttp==3.10.2
 opentelemetry-api==1.25.0
 opentelemetry-sdk==1.25.0
 opentelemetry-instrumentation==0.46b0

ragaai_catalyst-2.0.7/test/test_catalyst/test_configuration.py

@@ -0,0 +1,199 @@
+import pytest
+import os
+import requests
+from unittest.mock import patch, MagicMock
+import dotenv
+dotenv.load_dotenv()
+import os
+
+from ragaai_catalyst import RagaAICatalyst
+
+
+# Mock environment variables for testing
+@pytest.fixture
+def mock_env_vars():
+    original_environ = os.environ.copy()
+    RAGAAI_CATALYST_ACCESS_KEY = os.getenv("RAGAAI_CATALYST_ACCESS_KEY")
+    RAGAAI_CATALYST_SECRET_KEY = os.getenv("RAGAAI_CATALYST_SECRET_KEY")
+    RAGAAI_CATALYST_BASE_URL = os.getenv("RAGAAI_CATALYST_BASE_URL")
+    
+    yield
+    
+    os.environ.clear()
+    os.environ.update(original_environ)
+
+@pytest.fixture
+def raga_catalyst(mock_env_vars):
+    with patch('ragaai_catalyst.RagaAICatalyst.get_token', return_value='test_token'):
+        catalyst = RagaAICatalyst(
+            os.getenv("RAGAAI_CATALYST_ACCESS_KEY"),  
+            os.getenv("RAGAAI_CATALYST_SECRET_KEY")  
+        )
+    return catalyst
+
+
+
+def test_project_use_cases():
+        catalyst = RagaAICatalyst(
+            access_key=os.getenv("RAGAAI_CATALYST_ACCESS_KEY"),
+            secret_key=os.getenv("RAGAAI_CATALYST_SECRET_KEY"),
+            base_url=os.getenv("RAGAAI_CATALYST_BASE_URL")
+        )
+        use_case = catalyst.project_use_cases()
+        assert use_case ==['Chatbot', 'Text2SQL', 'Q/A', 'Code Generation', 'Others']
+
+
+def test_list_project():
+        catalyst = RagaAICatalyst(
+            access_key=os.getenv("RAGAAI_CATALYST_ACCESS_KEY"),
+            secret_key=os.getenv("RAGAAI_CATALYST_SECRET_KEY"),
+            base_url=os.getenv("RAGAAI_CATALYST_BASE_URL")
+        )
+        use_case = catalyst.list_projects()
+        assert use_case is not None  # Check if the result is not None
+
+
+def test_existing_projectname():
+        with pytest.raises(ValueError, match="already exists. Please choose a different name."):
+            catalyst = RagaAICatalyst(
+                access_key=os.getenv("RAGAAI_CATALYST_ACCESS_KEY"),
+                secret_key=os.getenv("RAGAAI_CATALYST_SECRET_KEY"),
+                base_url=os.getenv("RAGAAI_CATALYST_BASE_URL")
+            )
+            project = catalyst.create_project(
+                project_name="prompt_metric_dataset3",
+                usecase="Chatbot"
+            )
+
+def test_initialization_missing_credentials():
+    """Test initialization with missing credentials"""
+    with pytest.raises(ValueError, match="RAGAAI_CATALYST_ACCESS_KEY and RAGAAI_CATALYST_SECRET_KEY environment variables must be set"):
+        RagaAICatalyst('', '')
+
+@patch('requests.post')
+def test_get_token_success(mock_post, mock_env_vars):
+    """Test token retrieval success"""
+    mock_response = MagicMock()
+    mock_response.status_code = 200
+    mock_response.json.return_value = {
+        'success': True,
+        'data': {'token': 'test_token'}
+    }
+    mock_post.return_value = mock_response
+
+    token = RagaAICatalyst.get_token()
+    assert token == 'test_token'
+    assert os.getenv('RAGAAI_CATALYST_TOKEN') == 'test_token'
+
+@patch('requests.post')
+def test_get_token_failure(mock_post, mock_env_vars):
+    """Test token retrieval failure"""
+    mock_response = MagicMock()
+    mock_response.status_code = 400
+    mock_response.json.return_value = {
+        'message': 'Please enter valid credentials'
+    }
+    mock_post.return_value = mock_response
+
+    with pytest.raises(Exception, match="Authentication failed"):
+        RagaAICatalyst.get_token()
+
+@patch('requests.get')
+def test_project_use_cases_success(mock_get, raga_catalyst):
+    """Test retrieving project use cases"""
+    mock_response = MagicMock()
+    mock_response.status_code = 200
+    mock_response.json.return_value = {
+        'data': {'usecase': ['Q/A', 'Chatbot', 'Summarization']}
+    }
+    mock_get.return_value = mock_response
+
+    use_cases = raga_catalyst.project_use_cases()
+    assert use_cases == ['Q/A', 'Chatbot', 'Summarization']
+
+@patch('requests.get')
+def test_project_use_cases_failure(mock_get, raga_catalyst):
+    """Test project use cases retrieval failure"""
+    mock_get.side_effect = requests.exceptions.RequestException("Network Error")
+
+    use_cases = raga_catalyst.project_use_cases()
+    assert use_cases == []
+
+@patch('requests.post')
+@patch('ragaai_catalyst.RagaAICatalyst.list_projects')
+def test_create_project_success(mock_list_projects, mock_post, raga_catalyst):
+    """Test successful project creation"""
+    mock_list_projects.return_value = []  # No existing projects
+    mock_post_response = MagicMock()
+    mock_post_response.status_code = 200
+    mock_post_response.json.return_value = {
+        'data': {'name': 'TestProject'}
+    }
+    mock_post.return_value = mock_post_response
+
+    with patch('ragaai_catalyst.RagaAICatalyst.project_use_cases', return_value=['Q/A']):
+        result = raga_catalyst.create_project('TestProject')
+        assert 'Project Created Successfully' in result
+
+@patch('requests.post')
+@patch('ragaai_catalyst.RagaAICatalyst.list_projects')
+def test_create_project_duplicate(mock_list_projects, mock_post, raga_catalyst):
+    """Test project creation with duplicate name"""
+    mock_list_projects.return_value = ['TestProject']
+
+    with pytest.raises(ValueError, match="Project name 'TestProject' already exists"):
+        raga_catalyst.create_project('TestProject')
+
+@patch('requests.get')
+def test_list_projects_success(mock_get, raga_catalyst):
+    """Test successful project listing"""
+    mock_response = MagicMock()
+    mock_response.status_code = 200
+    mock_response.json.return_value = {
+        'data': {
+            'content': [
+                {'name': 'Project1'},
+                {'name': 'Project2'}
+            ]
+        }
+    }
+    mock_get.return_value = mock_response
+
+    projects = raga_catalyst.list_projects()
+    assert projects == ['Project1', 'Project2']
+
+@patch('requests.get')
+def test_list_metrics_success(mock_get):
+    """Test successful metrics listing"""
+    with patch.dict(os.environ, {'RAGAAI_CATALYST_TOKEN': 'test_token'}):
+        mock_response = MagicMock()
+        mock_response.status_code = 200
+        mock_response.json.return_value = {
+            'data': {
+                'metrics': [
+                    {'name': 'hallucination', 'category': 'quality'},
+                    {'name': 'toxicity', 'category': 'safety'}
+                ]
+            }
+        }
+        mock_get.return_value = mock_response
+
+        metrics = RagaAICatalyst.list_metrics()
+        assert metrics == ['hallucination', 'toxicity']
+
+def test_initialization_invalid_credentials():
+    """Test initialization with invalid credentials"""
+    with pytest.raises(Exception, match="Authentication failed. Invalid credentials provided."):
+        RagaAICatalyst(
+            access_key=os.getenv("RAGAAI_CATALYST_ACCESS_KEY")+"a",
+            secret_key=os.getenv("RAGAAI_CATALYST_SECRET_KEY"),
+            base_url=os.getenv("RAGAAI_CATALYST_BASE_URL")
+        )
+
+def test_initialization_invalid_base_url():
+    with pytest.raises(ConnectionError, match="The provided base_url is not accessible. Please re-check the base_url."):
+        RagaAICatalyst(
+            access_key=os.getenv("RAGAAI_CATALYST_ACCESS_KEY"),
+            secret_key=os.getenv("RAGAAI_CATALYST_SECRET_KEY"),
+            base_url=os.getenv("RAGAAI_CATALYST_BASE_URL") +"a",
+        )

ragaai_catalyst-2.0.7/test/test_catalyst/test_dataset.py

@@ -0,0 +1,170 @@
+import pytest
+import os
+import dotenv
+dotenv.load_dotenv()
+import pandas as pd
+from datetime import datetime
+from typing import Dict, List
+from unittest.mock import patch, Mock
+import requests
+from ragaai_catalyst import Dataset,RagaAICatalyst
+
+
+@pytest.fixture
+def base_url():
+    return "https://catalyst.raga.ai/api"
+
+@pytest.fixture
+def access_keys():
+    return {
+        "access_key": os.getenv("RAGAAI_CATALYST_ACCESS_KEY"),
+        "secret_key": os.getenv("RAGAAI_CATALYST_SECRET_KEY")}
+
+@pytest.fixture
+def dataset(base_url, access_keys):
+    """Create evaluation instance with specific project and dataset"""
+    os.environ["RAGAAI_CATALYST_BASE_URL"] = base_url
+    catalyst = RagaAICatalyst(
+        access_key=access_keys["access_key"],
+        secret_key=access_keys["secret_key"]
+    )
+    return Dataset(project_name="prompt_metric_dataset")
+
+def test_list_dataset(dataset) -> List[str]:
+    datasets = dataset.list_datasets()
+    return datasets
+
+
+def test_get_dataset_columns(dataset)  -> List[str]:
+    dataset_column = dataset.get_dataset_columns(dataset_name="ritika_dataset")
+    return dataset_column
+
+def test_incorrect_dataset(dataset):
+    with pytest.raises(ValueError, match="Please enter a valid dataset name"):
+        dataset.get_dataset_columns(dataset_name="ritika_datset")
+
+def test_get_schema_mapping(dataset):
+    schema_mapping_columns= dataset.get_schema_mapping()
+    return schema_mapping_columns
+
+
+def test_upload_csv(dataset):
+    project_name = 'prompt_metric_dataset'
+
+    schema_mapping = {
+        'Query': 'prompt',
+        'Response': 'response',
+        'Context': 'context',
+        'ExpectedResponse': 'expected_response',
+    }
+
+    csv_path= "/Users/siddharthakosti/Downloads/catalyst_error_handling/catalyst_v2/catalyst_v2_new_1/data/prompt_metric_dataset_v1.csv"
+
+    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") 
+    dataset_name = f"schema_metric_dataset_ritika_{timestamp}"  
+
+    
+
+    dataset.create_from_csv(
+        csv_path=csv_path,
+        dataset_name=dataset_name,
+        schema_mapping=schema_mapping
+    )
+
+def test_upload_csv_repeat_dataset(dataset):
+    with pytest.raises(ValueError, match="already exists"):
+        project_name = 'prompt_metric_dataset'
+
+        schema_mapping = {
+            'Query': 'prompt',
+            'Response': 'response',
+            'Context': 'context',
+            'ExpectedResponse': 'expected_response',
+        }
+
+        csv_path= "/Users/siddharthakosti/Downloads/catalyst_error_handling/catalyst_v2/catalyst_v2_new_1/data/prompt_metric_dataset_v1.csv"
+
+        dataset.create_from_csv(
+            csv_path=csv_path,
+            dataset_name="schema_metric_dataset_ritika_3",
+            schema_mapping=schema_mapping
+        )
+
+
+def test_upload_csv_no_schema_mapping(dataset):
+    with pytest.raises(TypeError, match="missing 1 required positional argument"):
+        project_name = 'prompt_metric_dataset'
+
+        schema_mapping = {
+            'Query': 'prompt',
+            'Response': 'response',
+            'Context': 'context',
+            'ExpectedResponse': 'expected_response',
+        }
+
+        csv_path= "/Users/siddharthakosti/Downloads/catalyst_error_handling/catalyst_v2/catalyst_v2_new_1/data/prompt_metric_dataset_v1.csv"
+
+        dataset.create_from_csv(
+            csv_path=csv_path,
+            dataset_name="schema_metric_dataset_ritika_3",
+        )
+
+def test_upload_csv_empty_csv_path(dataset):
+    with pytest.raises(FileNotFoundError, match="No such file or directory"):
+        project_name = 'prompt_metric_dataset'
+
+        schema_mapping = {
+            'Query': 'prompt',
+            'Response': 'response',
+            'Context': 'context',
+            'ExpectedResponse': 'expected_response',
+        }
+
+        csv_path= "/Users/siddharthakosti/Downloads/catalyst_error_handling/catalyst_v2/catalyst_v2_new_1/data/prompt_metric_dataset_v1.csv"
+
+        dataset.create_from_csv(
+            csv_path="",
+            dataset_name="schema_metric_dataset_ritika_12",
+            schema_mapping=schema_mapping
+
+        )
+
+def test_upload_csv_empty_schema_mapping(dataset):
+    with pytest.raises(AttributeError):
+        project_name = 'prompt_metric_dataset'
+
+        schema_mapping = {
+            'Query': 'prompt',
+            'Response': 'response',
+            'Context': 'context',
+            'ExpectedResponse': 'expected_response',
+        }
+
+        csv_path= "/Users/siddharthakosti/Downloads/catalyst_error_handling/catalyst_v2/catalyst_v2_new_1/data/prompt_metric_dataset_v1.csv"
+
+        dataset.create_from_csv(
+            csv_path=csv_path,
+            dataset_name="schema_metric_dataset_ritika_12",
+            schema_mapping=""
+
+        )
+
+
+def test_upload_csv_invalid_schema(dataset):
+    with pytest.raises(ValueError, match="Invalid schema mapping provided"):
+
+        project_name = 'prompt_metric_dataset'
+
+        schema_mapping={
+            'prompt': 'prompt',
+            'response': 'response',
+            'chatId': 'chatId',
+            'chatSequence': 'chatSequence'
+        }
+
+        csv_path= "/Users/siddharthakosti/Downloads/catalyst_error_handling/catalyst_v2/catalyst_v2_new_1/data/prompt_metric_dataset_v1.csv"
+
+        dataset.create_from_csv(
+            csv_path=csv_path,
+            dataset_name="schema_metric_dataset_ritika_12",
+            schema_mapping=schema_mapping)