ragaai-catalyst 1.0.7b3__py3-none-any.whl → 1.0.8b1__py3-none-any.whl

ragaai_catalyst/dataset.py

@@ -4,6 +4,7 @@ from .utils import response_checker
 from typing import Union
 import logging
 from .ragaai_catalyst import RagaAICatalyst
+import pandas as pd
 
 logger = logging.getLogger(__name__)
 get_token = RagaAICatalyst.get_token
@@ -11,7 +12,7 @@ get_token = RagaAICatalyst.get_token
 
 class Dataset:
     BASE_URL = None
-    TIMEOUT = 10
+    TIMEOUT = 30
 
     def __init__(self, project_name):
         self.project_name = project_name
@@ -63,7 +64,7 @@ class Dataset:
         sub_datasets = [dataset["name"] for dataset in datasets]
         return sub_datasets
 
-    def create_dataset(self, dataset_name, filter_list):
+    def create_from_trace(self, dataset_name, filter_list):
         """
         Creates a new dataset with the given `dataset_name` and `filter_list`.
 
@@ -79,7 +80,7 @@ class Dataset:
 
         """
 
-        def make_request():
+        def request_trace_creation():
             headers = {
                 "Content-Type": "application/json",
                 "Authorization": f"Bearer {os.getenv('RAGAAI_CATALYST_TOKEN')}",
@@ -98,12 +99,129 @@ class Dataset:
             )
             return response
 
-        response = make_request()
+        response = request_trace_creation()
         response_checker(response, "Dataset.create_dataset")
         if response.status_code == 401:
             get_token()  # Fetch a new token and set it in the environment
-            response = make_request()  # Retry the request
+            response = request_trace_creation()  # Retry the request
         if response.status_code != 200:
             return response.json()["message"]
         message = response.json()["message"]
         return f"{message} {dataset_name}"
+    
+
+
+###################### CSV Upload APIs ###################
+
+    def get_csv_schema(self):
+        headers = {
+            "Authorization": f"Bearer {os.getenv('RAGAAI_CATALYST_TOKEN')}",
+            "X-Project-Name": self.project_name,
+        }
+        response = requests.get(
+                f"{Dataset.BASE_URL}/v1/llm/schema-elements",
+                headers=headers,
+                timeout=Dataset.TIMEOUT,
+            )
+        
+        response_data = response.json()
+        if not response_data['success']:
+            raise ValueError('Unable to fetch Schema Elements for the CSV')
+        
+        # chema_elements = response['data']['schemaElements']
+        return response_data
+
+
+    def create_from_csv(self, csv_path, dataset_name, schema_mapping):
+
+        ## check the validity of schema_mapping
+        df = pd.read_csv(csv_path)
+        keys = list(df.columns)
+        values = self.get_csv_schema()['data']['schemaElements']
+        print(type(values), values)
+        for k in schema_mapping.keys():
+            if k not in keys:
+                raise ValueError(f'--{k}-- column is not present in csv column but present in schema_mapping. Plase provide the right schema_mapping.')
+        for k in schema_mapping.values():
+            if k not in values:
+                raise ValueError(f'--{k}-- is not present in the schema_elements but present in schema_mapping. Plase provide the right schema_mapping.')
+        
+
+        #### get presigned URL
+        def get_presignedUrl():
+            headers = {
+                "Authorization": f"Bearer {os.getenv('RAGAAI_CATALYST_TOKEN')}",
+                "X-Project-Name": self.project_name,
+            }
+            response = requests.get(
+                f"{Dataset.BASE_URL}/v1/llm/presignedUrl/test-url",
+                headers=headers,
+                timeout=Dataset.TIMEOUT,
+            )
+            return response.json()
+        
+        presignedUrl = get_presignedUrl()
+        if presignedUrl['success']:
+            url = presignedUrl['data']['presignedUrl']
+            filename = presignedUrl['data']['fileName']
+            print('-- PresignedUrl fetched Succussfuly --')
+            print('filename: ', filename)
+        else:
+            raise ValueError('Unable to fetch presignedUrl')
+        
+
+
+        #### put csv to presigned URL
+        def put_csv_to_presignedUrl(url):
+            headers = {
+                'Content-Type': 'text/csv',
+                'x-ms-blob-type': 'BlockBlob',
+            }
+            with open(csv_path, 'rb') as file:
+                response = requests.put(
+                    url,
+                    headers=headers,
+                    data=file,  
+                    timeout=Dataset.TIMEOUT,
+                )
+            return response
+        
+        
+        
+        put_csv_response = put_csv_to_presignedUrl(url)
+        if put_csv_response.status_code != 201:
+            raise ValueError('Unable to put csv to the presignedUrl')
+        else:
+            print('-- csv put to presignedUrl Succussfuly --')
+        
+
+
+        ## Upload csv to elastic
+        def upload_csv_to_elastic(data):
+            header = {
+                'Authorization': f"Bearer {os.getenv('RAGAAI_CATALYST_TOKEN')}",
+                'X-Project-Name': self.project_name
+            }
+            response = requests.post(
+                f"{Dataset.BASE_URL}/v1/llm/csv-dataset",
+                headers=header,
+                json=data,
+                timeout=Dataset.TIMEOUT,
+            )
+
+            return response.json()
+        
+        data = {
+            "datasetName": dataset_name,
+            "fileName": filename,
+            "schemaMapping": schema_mapping
+        }
+        print(data)
+
+        upload_csv_response = upload_csv_to_elastic(data)
+        print(type(upload_csv_response), upload_csv_response)
+        if not upload_csv_response['success']:
+            raise ValueError('Unable to upload csv')
+        else:
+            print(upload_csv_response['message'])
+       

ragaai_catalyst/experiment.py

@@ -14,6 +14,7 @@ get_token = RagaAICatalyst.get_token
 class Experiment:
     BASE_URL = None
     TIMEOUT = 10
+    NUM_PROJECTS = 100
 
     def __init__(
         self, project_name, experiment_name, experiment_description, dataset_name
@@ -42,6 +43,28 @@ class Experiment:
         self.experiment_id = None
         self.job_id = None
 
+        params = {
+            "size": str(self.NUM_PROJECTS),
+            "page": "0",
+            "type": "llm",
+        }
+        headers = {
+            "Content-Type": "application/json",
+            "Authorization": f'Bearer {os.getenv("RAGAAI_CATALYST_TOKEN")}',
+        }
+        response = requests.get(
+            f"{RagaAICatalyst.BASE_URL}/projects",
+            params=params,
+            headers=headers,
+            timeout=10,
+        )
+        response.raise_for_status()
+        # logger.debug("Projects list retrieved successfully")
+        experiment_list = [exp["name"] for project in response.json()["data"]["content"] if project["name"] == self.project_name for exp in project["experiments"]]
+        # print(experiment_list)
+        if self.experiment_name in experiment_list:
+            raise ValueError("The experiment name already exists in the project. Enter a unique experiment name.")
+
         self.access_key = os.getenv("RAGAAI_CATALYST_ACCESS_KEY")
         self.secret_key = os.getenv("RAGAAI_CATALYST_SECRET_KEY")
 
@@ -50,8 +73,73 @@ class Experiment:
             if os.getenv("RAGAAI_CATALYST_TOKEN") is not None
             else get_token()
         )
+        
+        if not self._check_if_project_exists(project_name=project_name):
+            raise ValueError(f"Project '{project_name}' not found. Please enter a valid project name")
+        
+        if not self._check_if_dataset_exists(project_name=project_name,dataset_name=dataset_name):
+            raise ValueError(f"dataset '{dataset_name}' not found. Please enter a valid dataset name")
+
+
         self.metrics = []
+    def _check_if_dataset_exists(self,project_name,dataset_name):
+        headers = {
+            "X-Project-Name":project_name,
+            # "accept":"application/json, text/plain, */*",
+            "Authorization": f'Bearer {os.getenv("RAGAAI_CATALYST_TOKEN")}',
+        }
+        response = requests.get(
+            f"{RagaAICatalyst.BASE_URL}/v1/llm/sub-datasets",
+            headers=headers,
+            timeout=self.TIMEOUT,
+        )
+        response.raise_for_status()
+        logger.debug("dataset list retrieved successfully")
+        dataset_list = [
+            item['name'] for item in response.json()['data']['content']
+        ]
+        exists = dataset_name in dataset_list
+        if exists:
+            logger.info(f"dataset '{dataset_name}' exists.")
+        else:
+            logger.info(f"dataset '{dataset_name}' does not exist.")
+        return exists
+
+
 
+
+    def _check_if_project_exists(self,project_name,num_projects=100):
+        # TODO: 1. List All projects
+        params = {
+            "size": str(num_projects),
+            "page": "0",
+            "type": "llm",
+        }
+        headers = {
+            "Content-Type": "application/json",
+            "Authorization": f'Bearer {os.getenv("RAGAAI_CATALYST_TOKEN")}',
+        }
+        response = requests.get(
+            f"{RagaAICatalyst.BASE_URL}/projects",
+            params=params,
+            headers=headers,
+            timeout=self.TIMEOUT,
+        )
+        response.raise_for_status()
+        logger.debug("Projects list retrieved successfully")
+        project_list = [
+            project["name"] for project in response.json()["data"]["content"]
+        ]
+        
+        # TODO: 2. Check if the given project_name exists
+        # TODO: 3. Return bool (True / False output)
+        exists = project_name in project_list
+        if exists:
+            logger.info(f"Project '{project_name}' exists.")
+        else:
+            logger.info(f"Project '{project_name}' does not exist.")
+        return exists
+        
     def list_experiments(self):
         """
         Retrieves a list of experiments associated with the current project.
@@ -115,6 +203,15 @@ class Experiment:
             metrics = [metrics]
         else:
             metrics_list = metrics
+        sub_providers = ["openai","azure","gemini","groq"]
+        sub_metrics = RagaAICatalyst.list_metrics()  
+        for metric in metrics_list:
+            provider = metric.get('config', {}).get('provider', '').lower()
+            if provider and provider not in sub_providers:
+                raise ValueError("Enter a valid provider name. The following Provider names are supported: OpenAI, Azure, Gemini, Groq")
+
+            if metric['name'] not in sub_metrics:
+                raise ValueError("Enter a valid metric name. Refer to RagaAI Metric Library to select a valid metric")
 
         json_data = {
             "projectName": self.project_name,
@@ -274,14 +371,15 @@ class Experiment:
             "projectId": self.project_id,
             "filterList": [],
         }
+        base_url_without_api = Experiment.BASE_URL.removesuffix('/api')
 
         status_json = self.get_status(job_id_to_use)
         if status_json == "Failed":
             return print("Job failed. No results to fetch.")
         elif status_json == "In Progress":
-            return print("Job in progress. Please wait while the job completes.")
+            return print(f"Job in progress. Please wait while the job completes.\n Visit Job Status: {base_url_without_api}/home/job-status to track")
         elif status_json == "Completed":
-            print("Job completed. fetching results")
+            print(f"Job completed. fetching results.\n Visit Job Status: {base_url_without_api}/home/job-status to track")
 
         response = requests.post(
             f"{Experiment.BASE_URL}/v1/llm/docs",
@@ -339,11 +437,50 @@ class Experiment:
         Parse the response to get the results
         """
         try:
-            parsed_response_df = pd.DataFrame(response["data"]["docs"])
-            return True, parsed_response_df
+            x = pd.DataFrame(response["data"]["docs"])
+
+            column_names_to_replace = [
+                {item["columnName"]: item["displayName"]}
+                for item in response["data"]["columns"]
+            ]
+
+            if column_names_to_replace:
+                for item in column_names_to_replace:
+                    x = x.rename(columns=item)
+
+                dict_cols = [
+                    col
+                    for col in x.columns
+                    if x[col].dtype == "object"
+                    and x[col].apply(lambda y: isinstance(y, dict)).any()
+                ]
+
+                for dict_col in dict_cols:
+                    x[f"{dict_col}_reason"] = x[dict_col].apply(
+                        lambda y: y.get("reason") if isinstance(y, dict) else None
+                    )
+                    x[f"{dict_col}_metric_config"] = x[dict_col].apply(
+                        lambda y: (
+                            y.get("metric_config") if isinstance(y, dict) else None
+                        )
+                    )
+                    x[f"{dict_col}_status"] = x[dict_col].apply(
+                        lambda y: y.get("status") if isinstance(y, dict) else None
+                    )
+
+                    x = x.drop(columns=[dict_col])
+
+            x.columns = x.columns.str.replace("_reason_reason", "_reason")
+            x.columns = x.columns.str.replace("_reason_metric_config", "_metric_config")
+            x.columns = x.columns.str.replace("_reason_status", "_status")
+
+            x = x.drop(columns=["trace_uri"])
+
+            return True, x
+
         except Exception as e:
-            logger.error(f"Failed to parse response: {e}")
-            return False, ""
+            logger.error(f"Failed to parse response: {e}", exc_info=True)
+            return False, pd.DataFrame()
 
 
 class FailedToRetrieveResults(Exception):

ragaai_catalyst/prompt_manager.py

@@ -0,0 +1,382 @@
+import os
+import requests
+import json
+import re
+from .ragaai_catalyst import RagaAICatalyst
+import pdb
+
+class PromptManager:
+    NUM_PROJECTS = 100
+    TIMEOUT = 10
+
+    def __init__(self, project_name):
+        """
+        Initialize the PromptManager with a project name.
+
+        Args:
+            project_name (str): The name of the project.
+
+        Raises:
+            requests.RequestException: If there's an error with the API request.
+            ValueError: If the project is not found.
+        """
+        self.project_name = project_name
+        self.headers = {
+                "Content-Type": "application/json",
+                "Authorization": f'Bearer {os.getenv("RAGAAI_CATALYST_TOKEN")}',
+                "X-Project-Name": self.project_name
+            }
+        self.base_url = f"{RagaAICatalyst.BASE_URL}/playground/prompt"
+        self.timeout = 10
+
+        try:
+            response = requests.get(
+                f"{RagaAICatalyst.BASE_URL}/projects",
+                params={
+                    "size": str(self.NUM_PROJECTS),
+                    "page": "0",
+                    "type": "llm",
+                },
+                headers={
+                    "Content-Type": "application/json",
+                    "Authorization": f'Bearer {os.getenv("RAGAAI_CATALYST_TOKEN")}',
+                },
+                timeout=self.TIMEOUT,
+            )
+            response.raise_for_status()
+        except requests.RequestException as e:
+            raise requests.RequestException(f"Error fetching projects: {str(e)}")
+
+        try:
+            project_list = [
+                project["name"] for project in response.json()["data"]["content"]
+            ]
+        except (KeyError, json.JSONDecodeError) as e:
+            raise ValueError(f"Error parsing project list: {str(e)}")
+
+        if self.project_name not in project_list:
+            raise ValueError("Project not found. Please enter a valid project name")
+
+
+    def list_prompts(self):
+        """
+        List all available prompts.
+
+        Returns:
+            list: A list of prompt names.
+
+        Raises:
+            requests.RequestException: If there's an error with the API request.
+        """
+        prompt = Prompt()
+        try:
+            prompt_list = prompt.list_prompts(self.base_url, self.headers, self.timeout)
+            return prompt_list
+        except requests.RequestException as e:
+            raise requests.RequestException(f"Error listing prompts: {str(e)}")
+    
+    def get_prompt(self, prompt_name, version=None):
+        """
+        Get a specific prompt.
+
+        Args:
+            prompt_name (str): The name of the prompt.
+            version (str, optional): The version of the prompt. Defaults to None.
+
+        Returns:
+            PromptObject: An object representing the prompt.
+
+        Raises:
+            ValueError: If the prompt or version is not found.
+            requests.RequestException: If there's an error with the API request.
+        """
+        try:
+            prompt_list = self.list_prompts()
+        except requests.RequestException as e:
+            raise requests.RequestException(f"Error fetching prompt list: {str(e)}")
+
+        if prompt_name not in prompt_list:
+            raise ValueError("Prompt not found. Please enter a valid prompt name")
+
+        try:
+            prompt_versions = self.list_prompt_versions(prompt_name)
+        except requests.RequestException as e:
+            raise requests.RequestException(f"Error fetching prompt versions: {str(e)}")
+
+        if version and version not in prompt_versions.keys():
+            raise ValueError("Version not found. Please enter a valid version name")
+
+        prompt = Prompt()
+        try:
+            prompt_object = prompt.get_prompt(self.base_url, self.headers, self.timeout, prompt_name, version)
+            return prompt_object
+        except requests.RequestException as e:
+            raise requests.RequestException(f"Error fetching prompt: {str(e)}")
+
+    def list_prompt_versions(self, prompt_name):
+        """
+        List all versions of a specific prompt.
+
+        Args:
+            prompt_name (str): The name of the prompt.
+
+        Returns:
+            dict: A dictionary mapping version names to prompt texts.
+
+        Raises:
+            ValueError: If the prompt is not found.
+            requests.RequestException: If there's an error with the API request.
+        """
+        try:
+            prompt_list = self.list_prompts()
+        except requests.RequestException as e:
+            raise requests.RequestException(f"Error fetching prompt list: {str(e)}")
+
+        if prompt_name not in prompt_list:
+            raise ValueError("Prompt not found. Please enter a valid prompt name")
+        
+        prompt = Prompt()
+        try:
+            prompt_versions = prompt.list_prompt_versions(self.base_url, self.headers, self.timeout, prompt_name)
+            return prompt_versions
+        except requests.RequestException as e:
+            raise requests.RequestException(f"Error fetching prompt versions: {str(e)}")
+
+
+class Prompt:
+    def __init__(self):
+        """
+        Initialize the Prompt class.
+        """
+        pass
+
+    def list_prompts(self, url, headers, timeout):
+        """
+        List all available prompts.
+
+        Args:
+            url (str): The base URL for the API.
+            headers (dict): The headers to be used in the request.
+            timeout (int): The timeout for the request.
+
+        Returns:
+            list: A list of prompt names.
+
+        Raises:
+            requests.RequestException: If there's an error with the API request.
+        """
+        try:
+            response = requests.get(url, headers=headers, timeout=timeout)
+            response.raise_for_status()
+            prompt_list = [prompt["name"] for prompt in response.json()["data"]]                        
+            return prompt_list
+        except requests.RequestException as e:
+            raise requests.RequestException(f"Error listing prompts: {str(e)}")
+        except (KeyError, json.JSONDecodeError) as e:
+            raise ValueError(f"Error parsing prompt list: {str(e)}")
+
+    def get_response_by_version(self, base_url, headers, timeout, prompt_name, version):
+        """
+        Get a specific version of a prompt.
+
+        Args:
+            base_url (str): The base URL for the API.
+            headers (dict): The headers to be used in the request.
+            timeout (int): The timeout for the request.
+            prompt_name (str): The name of the prompt.
+            version (str): The version of the prompt."""
+        try:
+            response = requests.get(f"{base_url}/version/{prompt_name}?version={version}",
+                                    headers=headers, timeout=timeout)
+            response.raise_for_status()
+        except requests.RequestException as e:
+            raise requests.RequestException(f"Error fetching prompt version: {str(e)}")
+        except (KeyError, json.JSONDecodeError, IndexError) as e:
+            raise ValueError(f"Error parsing prompt version: {str(e)}")
+        return response
+
+    def get_response(self, base_url, headers, timeout, prompt_name, version):
+        try:
+            response = requests.get(f"{base_url}/version/{prompt_name}",
+                                headers=headers, timeout=timeout)
+            response.raise_for_status()
+        except requests.RequestException as e:
+            raise requests.RequestException(f"Error fetching prompt version: {str(e)}")
+        except (KeyError, json.JSONDecodeError, IndexError) as e:
+            raise ValueError(f"Error parsing prompt version: {str(e)}")
+        return response
+
+    def get_prompt_by_version(self, base_url, headers, timeout, prompt_name, version):
+        """
+        Get a specific version of a prompt.
+
+        Args:
+            base_url (str): The base URL for the API.
+            headers (dict): The headers to be used in the request.
+            timeout (int): The timeout for the request.
+            prompt_name (str): The name of the prompt.
+            version (str): The version of the prompt.
+
+        Returns:
+            str: The text of the prompt.
+
+        Raises:
+            requests.RequestException: If there's an error with the API request.
+        """
+        response = self.get_response_by_version(base_url, headers, timeout, prompt_name, version)
+        prompt_text = response.json()["data"]["docs"][0]["textFields"]
+        return prompt_text
+
+    def get_prompt(self, base_url, headers, timeout, prompt_name, version=None):
+        """
+        Get a prompt, optionally specifying a version.
+
+        Args:
+            base_url (str): The base URL for the API.
+            headers (dict): The headers to be used in the request.
+            timeout (int): The timeout for the request.
+            prompt_name (str): The name of the prompt.
+            version (str, optional): The version of the prompt. Defaults to None.
+
+        Returns:
+            PromptObject: An object representing the prompt.
+
+        Raises:
+            requests.RequestException: If there's an error with the API request.
+        """
+        if version:
+            response = self.get_response_by_version(base_url, headers, timeout, prompt_name, version)
+            prompt_text = response.json()["data"]["docs"][0]["textFields"]
+            prompt_parameters = response.json()["data"]["docs"][0]["modelSpecs"]["parameters"]
+            model = response.json()["data"]["docs"][0]["modelSpecs"]["model"]
+        else:
+            response = self.get_response(base_url, headers, timeout, prompt_name)
+            prompt_text = response.json()["data"]["docs"][0]["textFields"]
+            prompt_parameters = response.json()["data"]["docs"][0]["modelSpecs"]["parameters"]
+            model = response.json()["data"]["docs"][0]["modelSpecs"]["model"]
+        return PromptObject(prompt_text, prompt_parameters, model)
+
+
+    def list_prompt_versions(self, base_url, headers, timeout, prompt_name):
+        """
+        List all versions of a specific prompt.
+
+        Args:
+            base_url (str): The base URL for the API.
+            headers (dict): The headers to be used in the request.
+            timeout (int): The timeout for the request.
+            prompt_name (str): The name of the prompt.
+
+        Returns:
+            dict: A dictionary mapping version names to prompt texts.
+
+        Raises:
+            requests.RequestException: If there's an error with the API request.
+        """
+        try:
+            response = requests.get(f"{base_url}/{prompt_name}/version",
+                                    headers=headers, timeout=timeout)
+            response.raise_for_status()
+            version_names = [version["name"] for version in response.json()["data"]]
+            prompt_versions = {}
+            for version in version_names:
+                prompt_versions[version] = self.get_prompt_by_version(base_url, headers, timeout, prompt_name, version)
+            return prompt_versions
+        except requests.RequestException as e:
+            raise requests.RequestException(f"Error listing prompt versions: {str(e)}")
+        except (KeyError, json.JSONDecodeError) as e:
+            raise ValueError(f"Error parsing prompt versions: {str(e)}")
+
+
+class PromptObject:
+    def __init__(self, text, parameters, model):
+        """
+        Initialize a PromptObject with the given text.
+
+        Args:
+            text (str): The text of the prompt.
+            parameters (dict): The parameters of the prompt.
+            model (str): The model of the prompt.
+        """
+        self.text = text
+        self.variables = self._extract_variables()
+        self.parameters = parameters
+        self.model = model
+
+    def _extract_variables(self):
+        """
+        Extract variables from the prompt text.
+
+        Returns:
+            list: A list of variable names found in the prompt text.
+        """
+        user_content = next(item["content"] for item in self.text if item["role"] == "user")
+        return [var.strip('{}') for var in user_content.split('{{')[1:]]
+    
+    def compile(self, **kwargs):
+        """
+        Compile the prompt by replacing variables with provided values.
+
+        Args:
+            **kwargs: Keyword arguments where keys are variable names and values are their replacements.
+
+        Returns:
+            str: The compiled prompt with variables replaced.
+
+        Raises:
+            ValueError: If there are missing or extra variables, or if a value is not a string.
+        """
+        required_variables = set(self.get_variables())
+        provided_variables = set(kwargs.keys())
+
+        missing_variables = required_variables - provided_variables
+        extra_variables = provided_variables - required_variables
+
+        if missing_variables:
+            raise ValueError(f"Missing variable(s): {', '.join(missing_variables)}")
+        if extra_variables:
+            raise ValueError(f"Extra variable(s) provided: {', '.join(extra_variables)}")
+        # pdb.set_trace()
+
+        # compiled_prompt = self.text
+        user_content = next(item["content"] for item in self.text if item["role"] == "user")
+
+        for key, value in kwargs.items():
+            if not isinstance(value, str):
+                raise ValueError(f"Value for variable '{key}' must be a string, not {type(value).__name__}")
+            user_content = user_content.replace(f"{{{{{key}}}}}", value)
+        compiled_prompt = [{"content": user_content if item["role"] == "user" else item["content"], "role": item["role"]} for item in self.text]
+        return compiled_prompt
+    
+    def get_variables(self):
+        """
+        Get all variables in the prompt text.
+
+        Returns:
+            list: A list of variable names found in the prompt text.
+        """
+        pattern = r'\{\{(.*?)\}\}'
+        user_content = next(item["content"] for item in self.text if item["role"] == "user")
+        matches = re.findall(pattern, user_content)
+        return [match.strip() for match in matches if '"' not in match]
+    
+    # Function to convert value based on type
+    def convert_value(self, value, type_):
+        if type_ == "float":
+            return float(value)
+        elif type_ == "int":
+            return int(value)
+        return value  # Default case, return as is
+
+    def get_parameters(self):
+        """
+        Get all parameters in the prompt text.
+
+        Returns:
+            dict: A dictionary of parameters found in the prompt text.
+        """
+        parameters = {param["name"]: self.convert_value(param["value"], param["type"]) for param in self.parameters}
+        parameters["model"] = self.model
+        return parameters
+    
+    

ragaai_catalyst/ragaai_catalyst.py

@@ -24,16 +24,15 @@ class RagaAICatalyst:
             access_key (str): The access key for the RagaAICatalyst.
             secret_key (str): The secret key for the RagaAICatalyst.
             api_keys (Optional[Dict[str, str]]): A dictionary of API keys for different services. Defaults to None.
+            base_url (Optional[str]): The base URL for the RagaAICatalyst API. Defaults to None.
 
         Raises:
             ValueError: If the RAGAAI_CATALYST_ACCESS_KEY and RAGAAI_CATALYST_SECRET_KEY environment variables are not set.
+            ConnectionError: If the provided base_url is not accessible.
 
         Returns:
             None
         """
-        if base_url:
-            RagaAICatalyst.BASE_URL = base_url
-            os.environ["RAGAAI_CATALYST_BASE_URL"] = base_url
 
         if not access_key or not secret_key:
             logger.error(
@@ -46,22 +45,25 @@ class RagaAICatalyst:
         self.access_key, self.secret_key = self._set_access_key_secret_key(
             access_key, secret_key
         )
+
         RagaAICatalyst.BASE_URL = (
             os.getenv("RAGAAI_CATALYST_BASE_URL")
             if os.getenv("RAGAAI_CATALYST_BASE_URL")
-            else "https://llm-platform.dev4.ragaai.ai/api"
+            else "https://catalyst.raga.ai/api"
         )
-        os.environ["RAGAAI_CATALYST_ACCESS_KEY"] = access_key
-        os.environ["RAGAAI_CATALYST_SECRET_KEY"] = secret_key
 
         self.api_keys = api_keys or {}
-        self.get_token()
+
         if self.api_keys:
             self._upload_keys()
 
         if base_url:
             RagaAICatalyst.BASE_URL = base_url
-            os.environ["RAGAAI_CATALYST_BASE_URL"] = base_url
+            try:
+                self.get_token()
+                os.environ["RAGAAI_CATALYST_BASE_URL"] = base_url
+            except requests.exceptions.RequestException:
+                raise ConnectionError("The provided base_url is not accessible. Please re-check the base_url.")
 
     def _set_access_key_secret_key(self, access_key, secret_key):
         os.environ["RAGAAI_CATALYST_ACCESS_KEY"] = access_key
@@ -130,7 +132,7 @@ class RagaAICatalyst:
         Raises:
             - requests.exceptions.HTTPError: If there is an HTTP error while retrieving the token.
             - requests.exceptions.RequestException: If there is an error while retrieving the token.
-            - ValueError: If there is a JSON decoding error.
+            - ValueError: If there is a JSON decoding error or if authentication fails.
             - Exception: If there is an unexpected error while retrieving the token.
         """
         access_key = os.getenv("RAGAAI_CATALYST_ACCESS_KEY")
@@ -145,54 +147,43 @@ class RagaAICatalyst:
         headers = {"Content-Type": "application/json"}
         json_data = {
             "accessKey": access_key,
-            "secretKey": secret_key,
+            "secretKey": secret_key
         }
 
-        try:
-            response = requests.post(
-                f"{ RagaAICatalyst.BASE_URL}/token",
-                headers=headers,
-                json=json_data,
-                timeout=RagaAICatalyst.TIMEOUT,
-            )
-            response.raise_for_status()
+        response = requests.post(
+            f"{ RagaAICatalyst.BASE_URL}/token",
+            headers=headers,
+            json=json_data,
+            timeout=RagaAICatalyst.TIMEOUT,
+        )
 
+        # Handle specific status codes before raising an error
+        if response.status_code == 400:
             token_response = response.json()
+            if token_response.get("message") == "Please enter valid credentials":
+                raise Exception("Authentication failed. Invalid credentials provided. Please check your Access key and Secret key. \nTo view or create new keys, navigate to Settings -> Authenticate in the RagaAI Catalyst dashboard.")
 
-            if not token_response.get("success", False):
-                logger.error(
-                    "Token retrieval was not successful: %s",
-                    token_response.get("message", "Unknown error"),
-                )
-                return None
+        response.raise_for_status()
 
-            token = token_response.get("data", {}).get("token")
-            if token:
-                os.environ["RAGAAI_CATALYST_TOKEN"] = token
-                print("Token(s) set successfully")
-                return token
-            else:
-                logger.error("Token(s) not set")
-                return None
+        token_response = response.json()
 
-        except requests.exceptions.HTTPError as http_err:
+        if not token_response.get("success", False):
             logger.error(
-                "HTTP error occurred while retrieving token: %s", str(http_err)
+                "Token retrieval was not successful: %s",
+                token_response.get("message", "Unknown error"),
             )
-            if response.status_code == 500:
-                error_message = response.json().get("message", "Unknown server error")
-                logger.error("Server error: %s", error_message)
-            return None
-        except requests.exceptions.RequestException as req_err:
-            logger.error("Error occurred while retrieving token: %s", str(req_err))
-            return None
-        except ValueError as json_err:
-            logger.error("JSON decoding error: %s", str(json_err))
             return None
-        except Exception as e:
-            logger.error("Unexpected error occurred while retrieving token: %s", str(e))
+
+        token = token_response.get("data", {}).get("token")
+        if token:
+            os.environ["RAGAAI_CATALYST_TOKEN"] = token
+            print("Token(s) set successfully")
+            return token
+        else:
+            logger.error("Token(s) not set")
             return None
 
+
     def create_project(self, project_name, type="llm", description=""):
         """
         Creates a project with the given project_name, type, and description.
@@ -262,6 +253,7 @@ class RagaAICatalyst:
                 "Unexpected error while creating project: %s", str(general_err1)
             )
             return "An unexpected error occurred while creating the project"
+        
 
     def list_projects(self, num_projects=100):
         """
@@ -344,6 +336,10 @@ class RagaAICatalyst:
             return "An unexpected error occurred while listing projects"
 
     def list_metrics(self):
+        return RagaAICatalyst.list_metrics()
+
+    @staticmethod
+    def list_metrics():
         headers = {
             "Content-Type": "application/json",
             "Authorization": f'Bearer {os.getenv("RAGAAI_CATALYST_TOKEN")}',
@@ -352,7 +348,7 @@ class RagaAICatalyst:
             response = requests.get(
                 f"{RagaAICatalyst.BASE_URL}/v1/llm/llm-metrics",
                 headers=headers,
-                timeout=self.TIMEOUT,
+                timeout=RagaAICatalyst.TIMEOUT,
             )
             response.raise_for_status()
             logger.debug("Metrics list retrieved successfully")

ragaai_catalyst/tracers/exporters/raga_exporter.py

@@ -44,7 +44,7 @@ class RagaExporter:
         RagaExporter.BASE_URL = (
             os.getenv("RAGAAI_CATALYST_BASE_URL")
             if os.getenv("RAGAAI_CATALYST_BASE_URL")
-            else "https://llm-platform.dev4.ragaai.ai/api"
+            else "https://catalyst.raga.ai/api"
         )
         self.access_key = os.getenv("RAGAAI_CATALYST_ACCESS_KEY")
         self.secret_key = os.getenv("RAGAAI_CATALYST_SECRET_KEY")
@@ -92,6 +92,23 @@ class RagaExporter:
                 timeout=RagaExporter.TIMEOUT,
             )
             return response
+    
+
+        def compare_schemas(base_schema, project_schema):
+            differences = []
+            for key, base_value in base_schema.items():
+                if key not in project_schema:
+                    differences.append(f"Key '{key}' is missing in new schema.")
+                else:
+                    # Remove everything after '_' in the new schema value
+                    new_value = project_schema[key].split('_')[0]
+                    if base_value != new_value:
+                        differences.append(f"Value mismatch for key '{key}': base = '{base_value}', new = '{new_value}'.")
+
+            if differences:
+                return False, differences
+            return True, []
+
 
         response = make_request()
         if response.status_code == 401:
@@ -99,6 +116,13 @@ class RagaExporter:
             response = make_request()  # Retry the request
         if response.status_code != 200:
             return response.status_code
+        if response.status_code == 200:
+            project_schema = response.json()["data"]
+            base_schema = RagaExporter.SCHEMA_MAPPING
+            is_same, _ = compare_schemas(base_schema, project_schema)
+            if not is_same:
+                raise Exception(f"Trace cannot be logged to this Project because of schema difference. Create a new project to log trace")
+            return response.status_code
         return response.status_code
 
     def _create_schema(self):

ragaai_catalyst/tracers/tracer.py

@@ -18,11 +18,13 @@ from .instrumentators import (
 )
 from .utils import get_unique_key
 
+from ..ragaai_catalyst import RagaAICatalyst
+
 logger = logging.getLogger(__name__)
 
 
 class Tracer:
-
+    NUM_PROJECTS = 100
     def __init__(
         self,
         project_name,
@@ -53,6 +55,31 @@ class Tracer:
         self.description = description
         self.upload_timeout = upload_timeout
 
+        params = {
+            "size": str(self.NUM_PROJECTS),
+            "page": "0",
+            "type": "llm",
+        }
+        headers = {
+            "Content-Type": "application/json",
+            "Authorization": f'Bearer {os.getenv("RAGAAI_CATALYST_TOKEN")}',
+        }
+        response = requests.get(
+            f"{RagaAICatalyst.BASE_URL}/projects",
+            params=params,
+            headers=headers,
+            timeout=10,
+        )
+        response.raise_for_status()
+        # logger.debug("Projects list retrieved successfully")
+        project_list = [
+            project["name"] for project in response.json()["data"]["content"]
+        ]
+
+        if self.project_name not in project_list:
+            raise ValueError("Project not found. Please enter a valid project name")
+
+
         self.raga_client = RagaExporter(project_name=self.project_name)
 
         self._tracer_provider = self._setup_provider()

{ragaai_catalyst-1.0.7b3.dist-info → ragaai_catalyst-1.0.8b1.dist-info}/METADATA

@@ -1,8 +1,8 @@
 Metadata-Version: 2.1
 Name: ragaai_catalyst
-Version: 1.0.7b3
+Version: 1.0.8b1
 Summary: RAGA AI CATALYST
-Author-email: Kiran Scaria <kiran.scaria@raga.ai>, Kedar Gaikwad <kedar.gaikwad@raga.ai>, Dushyant Mahajan <dushyant.mahajan@raga.ai>
+Author-email: Kiran Scaria <kiran.scaria@raga.ai>, Kedar Gaikwad <kedar.gaikwad@raga.ai>, Dushyant Mahajan <dushyant.mahajan@raga.ai>, Siddhartha Kosti <siddhartha.kosti@raga.ai>, Ritika Goel <ritika.goel@raga.ai>, Vijay Chaurasia <vijay.chaurasia@raga.ai>
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 Requires-Dist: aiohttp>=3.10.2
@@ -44,6 +44,7 @@ RagaAI Catalyst is a powerful tool for managing and optimizing LLM projects. It
     - [Trace Management](#trace-management)
     - [Experiment Management](#experiment-management)
     - [Dataset Management](#dataset-management)
+    - [Prompt Management](#prompt-management)
 
 ## Installation
 
@@ -63,9 +64,11 @@ from ragaai_catalyst import RagaAICatalyst
 catalyst = RagaAICatalyst(
     access_key="YOUR_ACCESS_KEY",
     secret_key="YOUR_SECRET_KEY",
-    api_keys={"OPENAI_API_KEY": "YOUR_OPENAI_API_KEY"}
+    base_url="BASE_URL"
 )
 ```
+**Note**: Authetication to RagaAICatalyst is necessary to perform any operations below 
+
 
 ## Usage
 
@@ -102,12 +105,12 @@ tracer = Tracer(
         "vector_store": "faiss",
         "embed_model": "text-embedding-ada-002",
     }
-).start_trace()
+).start()
 
 # Your code here
 
 # Stop the trace recording
-tracer.stop_trace()
+tracer.stop()
 
 # Alternatively, use a context manager
 with tracer.trace():
@@ -153,34 +156,69 @@ print("Experiment Status:", status)
 results = experiment_manager.get_results()
 print("Experiment Results:", results)
 ```
-### Dataset Management
 
-Create and manage trace datasets for your projects.
 
 
-```python
+## Dataset Management
+Manage datasets efficiently for your projects:
+
+```py
 from ragaai_catalyst import Dataset
 
 # Initialize Dataset management for a specific project
-dataset_manager = Dataset(project_name="Test-RAG-App-1")
+dataset_manager = Dataset(project_name="project_name")
 
 # List existing datasets
 datasets = dataset_manager.list_datasets()
-print("Exisiting Datasets:", datasets)
+print("Existing Datasets:", datasets)
 
-# Create a new dataset with filters
-dataset_manager.create_dataset(
+# Create a dataset from trace
+dataset_manager.create_from_trace(
     dataset_name='Test-dataset-1',
     filter_list=[
-        {
-            "name": "llm_model",
-            "values": ["gpt-3.5-turbo", "gpt-4"]
-        },
-        {
-            "name": "prompt_length",
-            "lte": 27,
-            "gte": 23
-        }
+        {"name": "llm_model", "values": ["gpt-3.5-turbo", "gpt-4"]},
+        {"name": "prompt_length", "lte": 27, "gte": 23}
     ]
 )
+
+# Create a dataset from CSV
+dataset_manager.create_from_csv(
+    csv_path='path/to/your.csv',
+    dataset_name='MyDataset',
+    schema_mapping={'column1': 'schema_element1', 'column2': 'schema_element2'}
+)
 ```
+
+For more detailed information on Dataset Management, including CSV schema handling and advanced usage, please refer to the [Dataset Management documentation](docs/dataset_management.md).
+
+## Prompt Management
+
+Manage and use prompts efficiently in your projects:
+
+```py
+from ragaai_catalyst.prompt_manager import PromptManager
+
+# Initialize PromptManager
+prompt_manager = PromptManager("your-project-name")
+
+# List available prompts
+prompts = prompt_manager.list_prompts()
+print("Available prompts:", prompts)
+
+# Get a specific prompt
+prompt_name = "your_prompt_name"
+prompt = prompt_manager.get_prompt(prompt_name)
+
+# Compile a prompt with variables
+compiled_prompt = prompt.compile(query="What's the weather?", context="sunny", llm_response="It's sunny today")
+print("Compiled prompt:", compiled_prompt)
+
+# Get prompt parameters
+parameters = prompt.get_parameters()
+print("Prompt parameters:", parameters)
+```
+
+For more detailed information on Prompt Management, please refer to the [Prompt Management documentation](docs/prompt_management.md).
+
+
+

{ragaai_catalyst-1.0.7b3.dist-info → ragaai_catalyst-1.0.8b1.dist-info}/RECORD

@@ -1,21 +1,22 @@
 ragaai_catalyst/__init__.py,sha256=0hf_H1xkhrpGnUWTbjvYOXXQta1S4JO2yud1Eqydt-I,226
 ragaai_catalyst/_version.py,sha256=JKt9KaVNOMVeGs8ojO6LvIZr7ZkMzNN-gCcvryy4x8E,460
-ragaai_catalyst/dataset.py,sha256=g0YH0QGDpCiJ4VULB3tukGFNLpWGw0ZRwLMP07ay2IE,3560
-ragaai_catalyst/experiment.py,sha256=2TOyHknEcNuWbDDO6cgDVAWc6wuhSqGCGYfKDOndqts,13161
-ragaai_catalyst/ragaai_catalyst.py,sha256=DqEApmjxQufZZT4ap-IzoBmp3AX6RWMq7HXQ5kO0wsI,16468
+ragaai_catalyst/dataset.py,sha256=xEun-MqTgT0JGyGB2t0BwwC6M7t1aUT7x5X12C-_4pI,7644
+ragaai_catalyst/experiment.py,sha256=f2Z-UJSXPqlcH59oB1ThSzc7VMrlfyFQnzlbX-BCm4Q,18823
+ragaai_catalyst/prompt_manager.py,sha256=3jEcgS0JRZzjHww_ityOkq9SMvi7DQ75PlKu-17dXW4,14729
+ragaai_catalyst/ragaai_catalyst.py,sha256=LJECRdCmW8DtF3Lk4wZ6lXJgsYkh6Oome45DzONyaVU,16228
 ragaai_catalyst/utils.py,sha256=TlhEFwLyRU690HvANbyoRycR3nQ67lxVUQoUOfTPYQ0,3772
 ragaai_catalyst/tracers/__init__.py,sha256=NppmJhD3sQ5R1q6teaZLS7rULj08Gb6JT8XiPRIe_B0,49
-ragaai_catalyst/tracers/tracer.py,sha256=hTLW1nj_TWv7yo77d8EVFv2SLYB78VE0jA0rMTrrJQQ,8299
+ragaai_catalyst/tracers/tracer.py,sha256=LZQNKQF6hRfzxDu_Ljs2HBxQinjihKQoH189gIsO4w4,9171
 ragaai_catalyst/tracers/exporters/__init__.py,sha256=kVA8zp05h3phu4e-iHSlnznp_PzMRczB7LphSsZgUjg,138
 ragaai_catalyst/tracers/exporters/file_span_exporter.py,sha256=E1uSBZ8JfUBLLtkhN3lIy_AWmPcZDCInsOFhPoWP9mU,6362
-ragaai_catalyst/tracers/exporters/raga_exporter.py,sha256=M2O-Hi4wLzezc7WrfoBauoHGnvdQYq7dfdsLhCSHSSE,16365
+ragaai_catalyst/tracers/exporters/raga_exporter.py,sha256=yxLadWzGCTr6AMh9n0U_1xNlw80aAcLE0KthktC2xMU,17487
 ragaai_catalyst/tracers/instrumentators/__init__.py,sha256=FgnMQupoRTzmVsG9YKsLQera2Pfs-AluZv8CxwavoyQ,253
 ragaai_catalyst/tracers/instrumentators/langchain.py,sha256=yMN0qVF0pUVk6R5M1vJoUXezDo1ejs4klCFRlE8x4vE,574
 ragaai_catalyst/tracers/instrumentators/llamaindex.py,sha256=SMrRlR4xM7k9HK43hakE8rkrWHxMlmtmWD-AX6TeByc,416
 ragaai_catalyst/tracers/instrumentators/openai.py,sha256=14R4KW9wQCR1xysLfsP_nxS7cqXrTPoD8En4MBAaZUU,379
 ragaai_catalyst/tracers/utils/__init__.py,sha256=KeMaZtYaTojilpLv65qH08QmpYclfpacDA0U3wg6Ybw,64
 ragaai_catalyst/tracers/utils/utils.py,sha256=ViygfJ7vZ7U0CTSA1lbxVloHp4NSlmfDzBRNCJuMhis,2374
-ragaai_catalyst-1.0.7b3.dist-info/METADATA,sha256=RBab5g5fSrrXqD4a-VWIuXdGQ33fZbE2qI4PUtLgTM8,5121
-ragaai_catalyst-1.0.7b3.dist-info/WHEEL,sha256=UvcQYKBHoFqaQd6LKyqHw9fxEolWLQnlzP0h_LgJAfI,91
-ragaai_catalyst-1.0.7b3.dist-info/top_level.txt,sha256=HpgsdRgEJMk8nqrU6qdCYk3di7MJkDL0B19lkc7dLfM,16
-ragaai_catalyst-1.0.7b3.dist-info/RECORD,,
+ragaai_catalyst-1.0.8b1.dist-info/METADATA,sha256=HdN7_Zxg_NAJwulu7LJZAEBHRljZsk-v83-mCRw1VeM,6499
+ragaai_catalyst-1.0.8b1.dist-info/WHEEL,sha256=cVxcB9AmuTcXqmwrtPhNK88dr7IR_b6qagTj0UvIEbY,91
+ragaai_catalyst-1.0.8b1.dist-info/top_level.txt,sha256=HpgsdRgEJMk8nqrU6qdCYk3di7MJkDL0B19lkc7dLfM,16
+ragaai_catalyst-1.0.8b1.dist-info/RECORD,,

{ragaai_catalyst-1.0.7b3.dist-info → ragaai_catalyst-1.0.8b1.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (74.0.0)
+Generator: setuptools (74.1.2)
 Root-Is-Purelib: true
 Tag: py3-none-any