ragaai-catalyst 2.0.5__py3-none-any.whl → 2.1b0__py3-none-any.whl

ragaai_catalyst/__init__.py

@@ -6,6 +6,7 @@ from .dataset import Dataset
 from .prompt_manager import PromptManager
 from .evaluation import Evaluation
 from .synthetic_data_generation import SyntheticDataGeneration
+from .guardrails_manager import GuardrailsManager
 
 
-__all__ = ["Experiment", "RagaAICatalyst", "Tracer", "PromptManager", "Evaluation","SyntheticDataGeneration"]
+__all__ = ["Experiment", "RagaAICatalyst", "Tracer", "PromptManager", "Evaluation","SyntheticDataGeneration", "GuardrailsManager"]

ragaai_catalyst/dataset.py

@@ -99,82 +99,71 @@ class Dataset:
             raise
 
     def get_schema_mapping(self):
-        return ["traceid", "prompt", "context", "response", "expected_response", "expected_context", "timestamp", "metadata", "pipeline", "cost", "feedBack", "latency", "sanitized_response", "system_prompt", "traceUri"]
-
-    def create_from_trace(self, dataset_name, filter_list):
-        """
-        Creates a new dataset with the given `dataset_name` and `filter_list`.
-
-        Args:
-            dataset_name (str): The name of the dataset to be created.
-            filter_list (list): A list of filters to be applied to the dataset.
-
-        Returns:
-            str: A message indicating the success of the dataset creation and the name of the created dataset.
-
-        Raises:
-            None
-
-        """
-
-        def request_trace_creation():
-            headers = {
-                "Content-Type": "application/json",
-                "Authorization": f"Bearer {os.getenv('RAGAAI_CATALYST_TOKEN')}",
-                "X-Project-Name": self.project_name,
-            }
-            json_data = {
-                "projectName": self.project_name,
-                "subDatasetName": dataset_name,
-                "filterList": filter_list,
-            }
-            try:
-                response = requests.post(
-                    f"{Dataset.BASE_URL}/v1/llm/sub-dataset",
-                    headers=headers,
-                    json=json_data,
-                    timeout=Dataset.TIMEOUT,
-                )
-                response.raise_for_status()
-                return response
-            except requests.exceptions.RequestException as e:
-                logger.error(f"Failed to create dataset from trace: {e}")
-                raise
-
+        headers = {
+            "Authorization": f"Bearer {os.getenv('RAGAAI_CATALYST_TOKEN')}",
+            "X-Project-Name": self.project_name,
+        }
         try:
-            response = request_trace_creation()
-            response_checker(response, "Dataset.create_dataset")
-            if response.status_code == 401:
-                get_token()  # Fetch a new token and set it in the environment
-                response = request_trace_creation()  # Retry the request
-            if response.status_code != 200:
-                return response.json()["message"]
-            message = response.json()["message"]
-            return f"{message} {dataset_name}"
-        except Exception as e:
-            logger.error(f"Error in create_from_trace: {e}")
+            response = requests.get(
+                f"{Dataset.BASE_URL}/v1/llm/schema-elements",
+                headers=headers,
+                timeout=Dataset.TIMEOUT,
+            )
+            response.raise_for_status()
+            response_data = response.json()["data"]["schemaElements"]
+            if not response.json()['success']:
+                raise ValueError('Unable to fetch Schema Elements for the CSV')
+            return response_data
+        except requests.exceptions.RequestException as e:
+            logger.error(f"Failed to get CSV schema: {e}")
             raise
 
     ###################### CSV Upload APIs ###################
 
-    def get_csv_schema(self):
+    def get_dataset_columns(self, dataset_name):
+        list_dataset = self.list_datasets()
+        if dataset_name not in list_dataset:
+            raise ValueError(f"Dataset {dataset_name} does not exists. Please enter a valid dataset name")
+
         headers = {
             "Authorization": f"Bearer {os.getenv('RAGAAI_CATALYST_TOKEN')}",
             "X-Project-Name": self.project_name,
         }
+        headers = {
+                'Content-Type': 'application/json',
+                "Authorization": f"Bearer {os.getenv('RAGAAI_CATALYST_TOKEN')}",
+                "X-Project-Id": str(self.project_id),
+            }
+        json_data = {"size": 12, "page": "0", "projectId": str(self.project_id), "search": ""}
+        try:
+            response = requests.post(
+                f"{Dataset.BASE_URL}/v2/llm/dataset",
+                headers=headers,
+                json=json_data,
+                timeout=Dataset.TIMEOUT,
+            )
+            response.raise_for_status()
+            datasets = response.json()["data"]["content"]
+            dataset_id = [dataset["id"] for dataset in datasets if dataset["name"]==dataset_name][0]
+        except requests.exceptions.RequestException as e:
+            logger.error(f"Failed to list datasets: {e}")
+            raise
+
         try:
             response = requests.get(
-                f"{Dataset.BASE_URL}/v1/llm/schema-elements",
+                f"{Dataset.BASE_URL}/v2/llm/dataset/{dataset_id}?initialCols=0",
                 headers=headers,
                 timeout=Dataset.TIMEOUT,
             )
             response.raise_for_status()
-            response_data = response.json()
-            if not response_data['success']:
-                raise ValueError('Unable to fetch Schema Elements for the CSV')
-            return response_data
+            dataset_columns = response.json()["data"]["datasetColumnsResponses"]
+            dataset_columns = [item["displayName"] for item in dataset_columns]
+            dataset_columns = [data for data in dataset_columns if not data.startswith('_')]
+            if not response.json()['success']:
+                raise ValueError('Unable to fetch details of for the CSV')
+            return dataset_columns
         except requests.exceptions.RequestException as e:
-            logger.error(f"Failed to get CSV schema: {e}")
+            logger.error(f"Failed to get CSV columns: {e}")
             raise
 
     def create_from_csv(self, csv_path, dataset_name, schema_mapping):

ragaai_catalyst/evaluation.py

@@ -97,14 +97,45 @@ class Evaluation:
             logger.error(f"An unexpected error occurred: {e}")
             return []
 
-    def _get_dataset_schema(self):
+    def _get_dataset_id_based_on_dataset_type(self, metric_to_evaluate):
+        try:
+            headers = {
+                'Content-Type': 'application/json',
+                "Authorization": f"Bearer {os.getenv('RAGAAI_CATALYST_TOKEN')}",
+                "X-Project-Id": str(self.project_id),
+            }
+            json_data = {"size": 12, "page": "0", "projectId": str(self.project_id), "search": ""}
+            response = requests.post(
+                f"{self.base_url}/v2/llm/dataset",
+                headers=headers,
+                json=json_data,
+                timeout=self.timeout,
+            )
+            
+            response.raise_for_status()
+            datasets_content = response.json()["data"]["content"]
+            dataset = [dataset for dataset in datasets_content if dataset["name"]==self.dataset_name][0]
+            if (dataset["datasetType"]=="prompt" and metric_to_evaluate=="prompt") or (dataset["datasetType"]=="chat" and metric_to_evaluate=="chat") or dataset["datasetType"]==None:
+                return dataset["id"]
+            else:
+                return dataset["derivedDatasetId"]
+        except requests.exceptions.RequestException as e:
+            logger.error(f"Failed to retrieve dataset list: {e}")
+            raise
+
+
+    def _get_dataset_schema(self, metric_to_evaluate=None):
+        #this dataset_id is based on which type of metric_to_evaluate  
+        data_set_id=self._get_dataset_id_based_on_dataset_type(metric_to_evaluate)
+        self.dataset_id=data_set_id
+
         headers = {
             "Authorization": f"Bearer {os.getenv('RAGAAI_CATALYST_TOKEN')}",
             'Content-Type': 'application/json',
             'X-Project-Id': str(self.project_id),
         }
         data = {
-            "datasetId": str(self.dataset_id),
+            "datasetId": str(data_set_id),
             "fields": [],
             "rowFilterList": []
         }
@@ -129,29 +160,9 @@ class Evaluation:
             logger.error(f"An unexpected error occurred: {e}")
         return {}
 
-    def _get_variablename_from_dataset_schema(self, schemaName, metric_name):
-        # pdb.set_trace()
-        # print(schemaName)
-        dataset_schema = self._get_dataset_schema()
-        variableName = None
-        for column in dataset_schema:
-            columnName = column["columnType"]
-            displayName = column["displayName"]
-            # print(columnName, displayName)
-            if "".join(columnName.split("_")).lower() == schemaName.lower():
-                variableName = displayName
-                break
-        return variableName
-        # print(variableName)
-        # if variableName:
-        #     return variableName
-        # else:
-        #     raise ValueError(f"'{schemaName}' column is required for {metric_name} metric evaluation, but not found in dataset")
-
-
-    def _get_variablename_from_user_schema_mapping(self, schemaName, metric_name, schema_mapping):
-        # pdb.set_trace()
-        user_dataset_schema = self._get_dataset_schema()
+
+    def _get_variablename_from_user_schema_mapping(self, schemaName, metric_name, schema_mapping, metric_to_evaluate):
+        user_dataset_schema = self._get_dataset_schema(metric_to_evaluate)
         user_dataset_columns = [item["displayName"] for item in user_dataset_schema]
         variableName = None
         for key, val in schema_mapping.items():
@@ -159,7 +170,7 @@ class Evaluation:
                 if key in user_dataset_columns:
                     variableName=key
                 else:
-                    raise ValueError(f"Column '{key}' is not present in {self.dataset_name}")
+                    raise ValueError(f"Column '{key}' is not present in '{self.dataset_name}' dataset")
         if variableName:
             return variableName
         else:
@@ -172,10 +183,17 @@ class Evaluation:
         for schema in metrics_schema:
             if schema["name"]==metric_name:
                 requiredFields = schema["config"]["requiredFields"]
+
+                #this is added to check if "Chat" column is required for metric evaluation
+                required_variables = [_["name"].lower() for _ in requiredFields]
+                if "chat" in required_variables:
+                    metric_to_evaluate = "chat"
+                else:
+                    metric_to_evaluate = "prompt"
+
                 for field in requiredFields:
                     schemaName = field["name"]
-                    # variableName = self._get_variablename_from_dataset_schema(schemaName, metric_name)
-                    variableName = self._get_variablename_from_user_schema_mapping(schemaName.lower(), metric_name, schema_mapping)
+                    variableName = self._get_variablename_from_user_schema_mapping(schemaName.lower(), metric_name, schema_mapping, metric_to_evaluate)
                     mapping.append({"schemaName": schemaName, "variableName": variableName})
         return mapping
 
@@ -223,7 +241,6 @@ class Evaluation:
             return []
 
     def _update_base_json(self, metrics):
-        metric_schema_mapping = {"datasetId":self.dataset_id}
         metrics_schema_response = self._get_metrics_schema_response()
         sub_providers = ["openai","azure","gemini","groq"]
         metricParams = []
@@ -253,6 +270,7 @@ class Evaluation:
             mappings = self._get_mapping(metric["name"], metrics_schema_response, metric["schema_mapping"])
             base_json["metricSpec"]["config"]["mappings"] = mappings
             metricParams.append(base_json)
+        metric_schema_mapping = {"datasetId":self.dataset_id}
         metric_schema_mapping["metricParams"] = metricParams
         return metric_schema_mapping
 

ragaai_catalyst/guardrails_manager.py

@@ -0,0 +1,233 @@
+import requests
+import json
+import os
+from .ragaai_catalyst import RagaAICatalyst
+
+
+class GuardrailsManager:
+    def __init__(self, project_name):
+        """
+        Initialize the GuardrailsManager with the given project name.
+        
+        :param project_name: The name of the project to manage guardrails for.
+        """
+        self.project_name = project_name
+        self.timeout = 10
+        self.num_projects = 100
+        self.deployment_name = "NA"
+        self.deployment_id = "NA"
+        self.base_url = f"{RagaAICatalyst.BASE_URL}"
+        list_projects, project_name_with_id = self._get_project_list()
+        if project_name not in list_projects:
+            raise ValueError(f"Project '{self.project_name}' does not exists")
+        
+        self.project_id = [_["id"] for _ in project_name_with_id if _["name"]==self.project_name][0]
+
+
+    def _get_project_list(self):
+        """
+        Retrieve the list of projects and their IDs from the API.
+        
+        :return: A tuple containing a list of project names and a list of dictionaries with project IDs and names.
+        """
+        headers = {'Authorization': f'Bearer {os.getenv("RAGAAI_CATALYST_TOKEN")}'}
+        response = requests.request("GET", f"{self.base_url}/v2/llm/projects?size=12&page=0", headers=headers, timeout=self.timeout)
+        project_content = response.json()["data"]["content"]
+        list_project = [_["name"] for _ in project_content]
+        project_name_with_id = [{"id": _["id"], "name": _["name"]} for _ in project_content]
+        return list_project, project_name_with_id
+
+
+    def list_deployment_ids(self):
+        """
+        List all deployment IDs and their names for the current project.
+        
+        :return: A list of dictionaries containing deployment IDs and names.
+        """
+        payload = {}
+        headers = {
+                'Authorization': f'Bearer {os.getenv("RAGAAI_CATALYST_TOKEN")}',
+                'X-Project-Id': str(self.project_id)
+                }
+        response = requests.request("GET", f"{self.base_url}/guardrail/deployment?size={self.num_projects}&page=0&sort=lastUsedAt,desc", headers=headers, data=payload, timeout=self.timeout)
+        deployment_ids_content = response.json()["data"]["content"]
+        deployment_ids_content = [{"id": _["id"], "name": _["name"]} for _ in deployment_ids_content]
+        return deployment_ids_content
+
+
+    def get_deployment(self, deployment_id):
+        """
+        Get details of a specific deployment ID, including its name and guardrails.
+        
+        :param deployment_id: The ID of the deployment to retrieve details for.
+        :return: A dictionary containing the deployment name and a list of guardrails.
+        """
+        payload = {}
+        headers = {
+                'Authorization': f'Bearer {os.getenv("RAGAAI_CATALYST_TOKEN")}',
+                'X-Project-Id': str(self.project_id)
+                }
+        response = requests.request("GET", f"{self.base_url}/guardrail/deployment/{deployment_id}", headers=headers, data=payload, timeout=self.timeout)
+        deployment_id_name = response.json()["data"]["name"]
+        deployment_id_guardrails = response.json()["data"]["guardrailsResponse"]
+        guardrails_list_deployment_id = [{_["type"]:_["name"]} for _ in deployment_id_guardrails]
+        return {"deployment_name":deployment_id_name, "guardrails_list":guardrails_list_deployment_id}
+
+
+    def list_guardrails(self):
+        """
+        List all available guardrails for the current project.
+        
+        :return: A list of guardrail names.
+        """
+        payload = {}
+        headers = {
+                'Authorization': f'Bearer {os.getenv("RAGAAI_CATALYST_TOKEN")}',
+                'X-Project-Id': str(self.project_id)
+                }
+        response = requests.request("GET", f"{self.base_url}/v1/llm/llm-metrics?category=Guardrail", headers=headers, data=payload, timeout=self.timeout)
+        list_guardrails_content = response.json()["data"]["metrics"]
+        list_guardrails = [_["name"] for _ in list_guardrails_content]
+        return list_guardrails
+
+
+    def list_fail_condition(self):
+        """
+        List all fail conditions for the current project's deployments.
+        
+        :return: A list of fail conditions.
+        """
+        payload = {}
+        headers = {
+                'Authorization': f'Bearer {os.getenv("RAGAAI_CATALYST_TOKEN")}',
+                'X-Project-Id': str(self.project_id)
+                }
+        response = requests.request("GET", f"{self.base_url}/guardrail/deployment/configurations", headers=headers, data=payload, timeout=self.timeout)
+        return response.json()["data"]
+
+    
+    def create_deployment(self, deployment_name):
+        """
+        Create a new deployment ID with the given name.
+        
+        :param deployment_name: The name of the new deployment.
+        :raises ValueError: If a deployment with the given name already exists.
+        """
+        self.deployment_name = deployment_name
+        list_deployment_ids = self.list_deployment_ids()
+        list_deployment_names = [_["name"] for _ in list_deployment_ids]
+        if deployment_name in list_deployment_names:
+            raise ValueError(f"Deployment with '{deployment_name}' already exists, choose a unique name")
+        
+        payload = json.dumps({"name": str(deployment_name)})
+        headers = {
+                'Authorization': f'Bearer {os.getenv("RAGAAI_CATALYST_TOKEN")}',
+                'Content-Type': 'application/json',
+                'X-Project-Id': str(self.project_id)
+                }
+        response = requests.request("POST", f"{self.base_url}/guardrail/deployment", headers=headers, data=payload, timeout=self.timeout)
+        if response.status_code == 409:
+            raise ValueError(f"Data with '{deployment_name}' already exists, choose a unique name")
+        if response.json()["success"]:
+            print(response.json()["message"])
+            deployment_ids = self.list_deployment_ids()
+            self.deployment_id = [_["id"] for _ in deployment_ids if _["name"]==self.deployment_name][0]
+        else:
+            print(response)
+            
+
+    def add_guardrails(self, guardrails, guardrails_config={}):
+        """
+        Add guardrails to the current deployment.
+        
+        :param guardrails: A list of guardrails to add.
+        :param guardrails_config: Configuration settings for the guardrails.
+        :raises ValueError: If a guardrail name or type is invalid.
+        """
+        # Checking if guardrails names given already exist or not
+        _, guardrails_type_name_exists = self.get_deployment(self.deployment_id)
+        guardrails_type_name_exists = [list(d.values())[0] for d in guardrails_type_name_exists]
+        user_guardrails_name_list = [_["name"] for _ in guardrails]
+        for g_name in user_guardrails_name_list:
+            if g_name in guardrails_type_name_exists:
+                raise ValueError(f"Guardrail with '{g_name} already exists, choose a unique name'")
+
+        # Checking if guardrails type is correct or not
+        available_guardrails_list = self.list_guardrails()
+        user_guardrails_type_list = [_["type"] for _ in guardrails]
+        for g_type in user_guardrails_type_list:
+            if g_type not in available_guardrails_list:
+                raise ValueError(f"Guardrail type '{g_type} does not exists, choose a correct type'")
+
+        payload = self._get_guardrail_config_payload(guardrails_config)
+        payload["guardrails"] = self._get_guardrail_list_payload(guardrails)
+        payload = json.dumps(payload)
+        headers = {
+                'Authorization': f'Bearer {os.getenv("RAGAAI_CATALYST_TOKEN")}',
+                'Content-Type': 'application/json',
+                'X-Project-Id': str(self.project_id)
+                }
+        response = requests.request("POST", f"{self.base_url}/guardrail/deployment/{str(self.deployment_id)}/configure", headers=headers, data=payload)
+        if response.json()["success"]:
+            print(response.json()["message"])
+
+    def _get_guardrail_config_payload(self, guardrails_config):
+        """
+        Construct the payload for guardrail configuration.
+        
+        :param guardrails_config: Configuration settings for the guardrails.
+        :return: A dictionary representing the guardrail configuration payload.
+        """
+        data = {
+            "isActive": guardrails_config.get("isActive",False),
+            "guardrailFailConditions": guardrails_config.get("guardrailFailConditions",["FAIL"]),
+            "deploymentFailCondition": guardrails_config.get("deploymentFailCondition","ONE_FAIL"),
+            "failAction": {
+                "action": "ALTERNATE_RESPONSE",
+                "args": f'{{\"alternateResponse\": \"{guardrails_config.get("alternateResponse","This is the Alternate Response")}\"}}'
+                },
+            "guardrails" : []
+            }
+        return data
+
+    def _get_guardrail_list_payload(self, guardrails):
+        """
+        Construct the payload for a list of guardrails.
+        
+        :param guardrails: A list of guardrails to include in the payload.
+        :return: A list of dictionaries representing each guardrail's data.
+        """
+        guardrails_list_payload = []
+        for guardrail in guardrails:
+            guardrails_list_payload.append(self._get_one_guardrail_data(guardrail))
+        return guardrails_list_payload
+
+    def _get_one_guardrail_data(self, guardrail):
+        """
+        Construct the data for a single guardrail.
+        
+        :param guardrail: A dictionary containing the guardrail's attributes.
+        :return: A dictionary representing the guardrail's data.
+        """
+        data = {
+            "name": guardrail["name"],
+            "type": guardrail["type"],
+            "isHighRisk": guardrail.get("isHighRisk", False),
+            "isActive": guardrail.get("isActive", False),
+            "threshold": {}
+        }
+        if "lte" in guardrail["threshold"]:
+            data["threshold"]["lte"] = guardrail["threshold"]["lte"]
+        elif "gte" in guardrail["threshold"]:
+            data["threshold"]["gte"] = guardrail["threshold"]["gte"]
+        elif "eq" in guardrail["threshold"]:
+            data["threshold"]["eq"] = guardrail["threshold"]["eq"]
+        else:
+            data["threshold"]["gte"] = 0.0
+        return data
+
+
+    def _run(self, **kwargs):
+        """
+        Execute the guardrail checks with the provided variables.
+        """

ragaai_catalyst/internal_api_completion.py

@@ -0,0 +1,83 @@
+import requests
+import json
+import subprocess
+import logging
+import traceback
+import pandas as pd
+
+logger = logging.getLogger(__name__)
+
+def api_completion(messages, model_config, kwargs):
+    attempts = 0
+    while attempts < 3:
+
+        user_id = kwargs.get('user_id', '1')
+        internal_llm_proxy = kwargs.get('internal_llm_proxy', -1)
+            
+            
+        job_id = model_config.get('job_id',-1)
+        converted_message = convert_input(messages,model_config, user_id)
+        payload = json.dumps(converted_message)
+        headers = {
+            'Content-Type': 'application/json',
+            # 'Wd-PCA-Feature-Key':f'your_feature_key, $(whoami)'
+        }
+        try:
+            response = requests.request("POST", internal_llm_proxy, headers=headers, data=payload)
+            if model_config.get('log_level','')=='debug':
+                logger.info(f'Model response Job ID {job_id} {response.text}')
+            if response.status_code!=200:
+                # logger.error(f'Error in model response Job ID {job_id}:',str(response.text))
+                raise ValueError(str(response.text))
+            
+            if response.status_code==200:
+                response = response.json()                
+                if "error" in response:
+                    raise ValueError(response["error"]["message"])
+                else:
+                    result=  response["choices"][0]["message"]["content"]
+                    response1 = result.replace('\n', '')
+                    try:
+                        json_data = json.loads(response1)
+                        df = pd.DataFrame(json_data)
+                        return(df)
+                    except json.JSONDecodeError:
+                        attempts += 1  # Increment attempts if JSON parsing fails
+                        if attempts == 3:
+                            raise Exception("Failed to generate a valid response after multiple attempts.")
+
+        except Exception as e:
+            raise ValueError(f"{e}")
+
+
+def get_username():
+    result = subprocess.run(['whoami'], capture_output=True, text=True)
+    result = result.stdout
+    return result
+
+
+def convert_input(messages, model_config, user_id):
+    doc_input = {
+      "model": model_config.get('model'),
+      **model_config,
+      "messages": messages,
+      "user_id": user_id
+    }
+    return doc_input
+
+
+if __name__=='__main__':
+    messages = [
+        {
+            "role": "system",
+            "content": "you are a poet well versed in shakespeare literature"
+        },
+        {
+          "role": "user",
+          "content": "write a poem on pirates and penguins"
+        }
+      ]
+    kwargs = {"internal_llm_proxy": "http://13.200.11.66:4000/chat/completions", "user_id": 1}
+    model_config = {"model": "workday_gateway", "provider":"openai", "max_tokens": 10}
+    answer = api_completion(messages, model_config, kwargs)
+    print(answer)

ragaai_catalyst/proxy_call.py

@@ -23,7 +23,7 @@ def api_completion(model,messages, api_base='http://127.0.0.1:8000',
         if model_config.get('log_level','')=='debug':
             logger.info(f'Model response Job ID {job_id} {response.text}')
         if response.status_code!=200:
-            logger.error(f'Error in model response Job ID {job_id}:',str(response.text))
+            # logger.error(f'Error in model response Job ID {job_id}:',str(response.text))
             raise ValueError(str(response.text))
     except Exception as e:
         logger.error(f'Error in calling api Job ID {job_id}:',str(e))