ragaai-catalyst 2.0.4__py3-none-any.whl → 2.0.6__py3-none-any.whl

ragaai_catalyst/__init__.py

@@ -6,6 +6,8 @@ from .dataset import Dataset
 from .prompt_manager import PromptManager
 from .evaluation import Evaluation
 from .synthetic_data_generation import SyntheticDataGeneration
+from .guardrails_manager import GuardrailsManager
+from .guard_executor import GuardExecutor
 
 
-__all__ = ["Experiment", "RagaAICatalyst", "Tracer", "PromptManager", "Evaluation","SyntheticDataGeneration"]
+__all__ = ["Experiment", "RagaAICatalyst", "Tracer", "PromptManager", "Evaluation","SyntheticDataGeneration", "GuardrailsManager"]

ragaai_catalyst/dataset.py

@@ -16,7 +16,7 @@ class Dataset:
 
     def __init__(self, project_name):
         self.project_name = project_name
-        self.num_projects = 100
+        self.num_projects = 99999
         Dataset.BASE_URL = (
             os.getenv("RAGAAI_CATALYST_BASE_URL")
             if os.getenv("RAGAAI_CATALYST_BASE_URL")
@@ -99,82 +99,71 @@ class Dataset:
             raise
 
     def get_schema_mapping(self):
-        return ["traceid", "prompt", "context", "response", "expected_response", "expected_context", "timestamp", "metadata", "pipeline", "cost", "feedBack", "latency", "sanitized_response", "system_prompt", "traceUri"]
-
-    def create_from_trace(self, dataset_name, filter_list):
-        """
-        Creates a new dataset with the given `dataset_name` and `filter_list`.
-
-        Args:
-            dataset_name (str): The name of the dataset to be created.
-            filter_list (list): A list of filters to be applied to the dataset.
-
-        Returns:
-            str: A message indicating the success of the dataset creation and the name of the created dataset.
-
-        Raises:
-            None
-
-        """
-
-        def request_trace_creation():
-            headers = {
-                "Content-Type": "application/json",
-                "Authorization": f"Bearer {os.getenv('RAGAAI_CATALYST_TOKEN')}",
-                "X-Project-Name": self.project_name,
-            }
-            json_data = {
-                "projectName": self.project_name,
-                "subDatasetName": dataset_name,
-                "filterList": filter_list,
-            }
-            try:
-                response = requests.post(
-                    f"{Dataset.BASE_URL}/v1/llm/sub-dataset",
-                    headers=headers,
-                    json=json_data,
-                    timeout=Dataset.TIMEOUT,
-                )
-                response.raise_for_status()
-                return response
-            except requests.exceptions.RequestException as e:
-                logger.error(f"Failed to create dataset from trace: {e}")
-                raise
-
+        headers = {
+            "Authorization": f"Bearer {os.getenv('RAGAAI_CATALYST_TOKEN')}",
+            "X-Project-Name": self.project_name,
+        }
         try:
-            response = request_trace_creation()
-            response_checker(response, "Dataset.create_dataset")
-            if response.status_code == 401:
-                get_token()  # Fetch a new token and set it in the environment
-                response = request_trace_creation()  # Retry the request
-            if response.status_code != 200:
-                return response.json()["message"]
-            message = response.json()["message"]
-            return f"{message} {dataset_name}"
-        except Exception as e:
-            logger.error(f"Error in create_from_trace: {e}")
+            response = requests.get(
+                f"{Dataset.BASE_URL}/v1/llm/schema-elements",
+                headers=headers,
+                timeout=Dataset.TIMEOUT,
+            )
+            response.raise_for_status()
+            response_data = response.json()["data"]["schemaElements"]
+            if not response.json()['success']:
+                raise ValueError('Unable to fetch Schema Elements for the CSV')
+            return response_data
+        except requests.exceptions.RequestException as e:
+            logger.error(f"Failed to get CSV schema: {e}")
             raise
 
     ###################### CSV Upload APIs ###################
 
-    def get_csv_schema(self):
+    def get_dataset_columns(self, dataset_name):
+        list_dataset = self.list_datasets()
+        if dataset_name not in list_dataset:
+            raise ValueError(f"Dataset {dataset_name} does not exists. Please enter a valid dataset name")
+
         headers = {
             "Authorization": f"Bearer {os.getenv('RAGAAI_CATALYST_TOKEN')}",
             "X-Project-Name": self.project_name,
         }
+        headers = {
+                'Content-Type': 'application/json',
+                "Authorization": f"Bearer {os.getenv('RAGAAI_CATALYST_TOKEN')}",
+                "X-Project-Id": str(self.project_id),
+            }
+        json_data = {"size": 12, "page": "0", "projectId": str(self.project_id), "search": ""}
+        try:
+            response = requests.post(
+                f"{Dataset.BASE_URL}/v2/llm/dataset",
+                headers=headers,
+                json=json_data,
+                timeout=Dataset.TIMEOUT,
+            )
+            response.raise_for_status()
+            datasets = response.json()["data"]["content"]
+            dataset_id = [dataset["id"] for dataset in datasets if dataset["name"]==dataset_name][0]
+        except requests.exceptions.RequestException as e:
+            logger.error(f"Failed to list datasets: {e}")
+            raise
+
         try:
             response = requests.get(
-                f"{Dataset.BASE_URL}/v1/llm/schema-elements",
+                f"{Dataset.BASE_URL}/v2/llm/dataset/{dataset_id}?initialCols=0",
                 headers=headers,
                 timeout=Dataset.TIMEOUT,
             )
             response.raise_for_status()
-            response_data = response.json()
-            if not response_data['success']:
-                raise ValueError('Unable to fetch Schema Elements for the CSV')
-            return response_data
+            dataset_columns = response.json()["data"]["datasetColumnsResponses"]
+            dataset_columns = [item["displayName"] for item in dataset_columns]
+            dataset_columns = [data for data in dataset_columns if not data.startswith('_')]
+            if not response.json()['success']:
+                raise ValueError('Unable to fetch details of for the CSV')
+            return dataset_columns
         except requests.exceptions.RequestException as e:
-            logger.error(f"Failed to get CSV schema: {e}")
+            logger.error(f"Failed to get CSV columns: {e}")
             raise
 
     def create_from_csv(self, csv_path, dataset_name, schema_mapping):

ragaai_catalyst/evaluation.py

@@ -16,7 +16,7 @@ class Evaluation:
         self.base_url = f"{RagaAICatalyst.BASE_URL}"
         self.timeout = 10
         self.jobId = None
-        self.num_projects=100
+        self.num_projects=99999
 
         try:
             response = requests.get(
@@ -80,7 +80,8 @@ class Evaluation:
         try:
             response = requests.get(
                 f'{self.base_url}/v1/llm/llm-metrics', 
-                headers=headers)
+                headers=headers,
+                timeout=self.timeout)
             response.raise_for_status()
             metric_names = [metric["name"] for metric in response.json()["data"]["metrics"]]
             return metric_names
@@ -96,14 +97,45 @@ class Evaluation:
             logger.error(f"An unexpected error occurred: {e}")
             return []
 
-    def _get_dataset_schema(self):
+    def _get_dataset_id_based_on_dataset_type(self, metric_to_evaluate):
+        try:
+            headers = {
+                'Content-Type': 'application/json',
+                "Authorization": f"Bearer {os.getenv('RAGAAI_CATALYST_TOKEN')}",
+                "X-Project-Id": str(self.project_id),
+            }
+            json_data = {"size": 12, "page": "0", "projectId": str(self.project_id), "search": ""}
+            response = requests.post(
+                f"{self.base_url}/v2/llm/dataset",
+                headers=headers,
+                json=json_data,
+                timeout=self.timeout,
+            )
+            
+            response.raise_for_status()
+            datasets_content = response.json()["data"]["content"]
+            dataset = [dataset for dataset in datasets_content if dataset["name"]==self.dataset_name][0]
+            if (dataset["datasetType"]=="prompt" and metric_to_evaluate=="prompt") or (dataset["datasetType"]=="chat" and metric_to_evaluate=="chat") or dataset["datasetType"]==None:
+                return dataset["id"]
+            else:
+                return dataset["derivedDatasetId"]
+        except requests.exceptions.RequestException as e:
+            logger.error(f"Failed to retrieve dataset list: {e}")
+            raise
+
+
+    def _get_dataset_schema(self, metric_to_evaluate=None):
+        #this dataset_id is based on which type of metric_to_evaluate  
+        data_set_id=self._get_dataset_id_based_on_dataset_type(metric_to_evaluate)
+        self.dataset_id=data_set_id
+
         headers = {
             "Authorization": f"Bearer {os.getenv('RAGAAI_CATALYST_TOKEN')}",
             'Content-Type': 'application/json',
             'X-Project-Id': str(self.project_id),
         }
         data = {
-            "datasetId": str(self.dataset_id),
+            "datasetId": str(data_set_id),
             "fields": [],
             "rowFilterList": []
         }
@@ -111,7 +143,8 @@ class Evaluation:
             response = requests.post(
                 f'{self.base_url}/v1/llm/docs', 
                 headers=headers,
-                json=data)
+                json=data,
+                timeout=self.timeout)
             response.raise_for_status()
             if response.status_code == 200:
                 return response.json()["data"]["columns"]
@@ -127,29 +160,9 @@ class Evaluation:
             logger.error(f"An unexpected error occurred: {e}")
         return {}
 
-    def _get_variablename_from_dataset_schema(self, schemaName, metric_name):
-        # pdb.set_trace()
-        # print(schemaName)
-        dataset_schema = self._get_dataset_schema()
-        variableName = None
-        for column in dataset_schema:
-            columnName = column["columnType"]
-            displayName = column["displayName"]
-            # print(columnName, displayName)
-            if "".join(columnName.split("_")).lower() == schemaName.lower():
-                variableName = displayName
-                break
-        return variableName
-        # print(variableName)
-        # if variableName:
-        #     return variableName
-        # else:
-        #     raise ValueError(f"'{schemaName}' column is required for {metric_name} metric evaluation, but not found in dataset")
-
-
-    def _get_variablename_from_user_schema_mapping(self, schemaName, metric_name, schema_mapping):
-        # pdb.set_trace()
-        user_dataset_schema = self._get_dataset_schema()
+
+    def _get_variablename_from_user_schema_mapping(self, schemaName, metric_name, schema_mapping, metric_to_evaluate):
+        user_dataset_schema = self._get_dataset_schema(metric_to_evaluate)
         user_dataset_columns = [item["displayName"] for item in user_dataset_schema]
         variableName = None
         for key, val in schema_mapping.items():
@@ -157,7 +170,7 @@ class Evaluation:
                 if key in user_dataset_columns:
                     variableName=key
                 else:
-                    raise ValueError(f"Column '{key}' is not present in {self.dataset_name}")
+                    raise ValueError(f"Column '{key}' is not present in '{self.dataset_name}' dataset")
         if variableName:
             return variableName
         else:
@@ -170,10 +183,17 @@ class Evaluation:
         for schema in metrics_schema:
             if schema["name"]==metric_name:
                 requiredFields = schema["config"]["requiredFields"]
+
+                #this is added to check if "Chat" column is required for metric evaluation
+                required_variables = [_["name"].lower() for _ in requiredFields]
+                if "chat" in required_variables:
+                    metric_to_evaluate = "chat"
+                else:
+                    metric_to_evaluate = "prompt"
+
                 for field in requiredFields:
                     schemaName = field["name"]
-                    # variableName = self._get_variablename_from_dataset_schema(schemaName, metric_name)
-                    variableName = self._get_variablename_from_user_schema_mapping(schemaName.lower(), metric_name, schema_mapping)
+                    variableName = self._get_variablename_from_user_schema_mapping(schemaName.lower(), metric_name, schema_mapping, metric_to_evaluate)
                     mapping.append({"schemaName": schemaName, "variableName": variableName})
         return mapping
 
@@ -203,7 +223,8 @@ class Evaluation:
         try:
             response = requests.get(
                 f'{self.base_url}/v1/llm/llm-metrics', 
-                headers=headers)
+                headers=headers,
+                timeout=self.timeout)
             response.raise_for_status()
             metrics_schema = [metric for metric in response.json()["data"]["metrics"]]
             return metrics_schema
@@ -220,7 +241,6 @@ class Evaluation:
             return []
 
     def _update_base_json(self, metrics):
-        metric_schema_mapping = {"datasetId":self.dataset_id}
         metrics_schema_response = self._get_metrics_schema_response()
         sub_providers = ["openai","azure","gemini","groq"]
         metricParams = []
@@ -233,8 +253,15 @@ class Evaluation:
                 #checking if provider is one of the allowed providers
                 if key.lower()=="provider" and value.lower() not in sub_providers:
                     raise ValueError("Enter a valid provider name. The following Provider names are supported: OpenAI, Azure, Gemini, Groq")
-                
-                base_json["metricSpec"]["config"]["params"][key] = {"value": value}
+    
+                if key.lower()=="threshold":
+                    if len(value)>1:
+                        raise ValueError("'threshold' can only take one argument gte/lte/eq")
+                    else:
+                        for key_thres, value_thres in value.items():
+                            base_json["metricSpec"]["config"]["params"][key] = {f"{key_thres}":value_thres}
+                else:
+                    base_json["metricSpec"]["config"]["params"][key] = {"value": value}
 
 
             # if metric["config"]["model"]:
@@ -243,6 +270,7 @@ class Evaluation:
             mappings = self._get_mapping(metric["name"], metrics_schema_response, metric["schema_mapping"])
             base_json["metricSpec"]["config"]["mappings"] = mappings
             metricParams.append(base_json)
+        metric_schema_mapping = {"datasetId":self.dataset_id}
         metric_schema_mapping["metricParams"] = metricParams
         return metric_schema_mapping
 
@@ -253,12 +281,15 @@ class Evaluation:
         }
         try:
             response = requests.get(
-                f'{self.base_url}/v1/llm/filter?datasetId={str(self.dataset_id)}', 
-                headers=headers
-                )
+                f"{self.base_url}/v2/llm/dataset/{str(self.dataset_id)}?initialCols=0",
+                headers=headers,
+                timeout=self.timeout,
+            )
             response.raise_for_status()
-            executed_metric_response = response.json()["data"]["filter"]
-            executed_metric_list = [item["displayName"] for item in executed_metric_response]
+            dataset_columns = response.json()["data"]["datasetColumnsResponses"]
+            dataset_columns = [item["displayName"] for item in dataset_columns]
+            executed_metric_list = [data for data in dataset_columns if not data.startswith('_')]
+
             return executed_metric_list
         except requests.exceptions.HTTPError as http_err:
             logger.error(f"HTTP error occurred: {http_err}")
@@ -301,7 +332,8 @@ class Evaluation:
             response = requests.post(
                 f'{self.base_url}/playground/metric-evaluation', 
                 headers=headers, 
-                json=metric_schema_mapping
+                json=metric_schema_mapping,
+                timeout=self.timeout
                 )
             if response.status_code == 400:
                 raise ValueError(response.json()["message"])
@@ -327,14 +359,14 @@ class Evaluation:
             "Authorization": f"Bearer {os.getenv('RAGAAI_CATALYST_TOKEN')}",
             'X-Project-Id': str(self.project_id),
         }
-        data = {"jobId": self.jobId}
         try:
-            response = requests.post(
+            response = requests.get(
                 f'{self.base_url}/job/status', 
                 headers=headers, 
-                json=data)
+                timeout=self.timeout)
             response.raise_for_status()
-            status_json = response.json()["data"]["status"]
+            if response.json()["success"]:
+                status_json = [item["status"] for item in response.json()["data"]["content"] if item["id"]==self.jobId][0]
             if status_json == "Failed":
                 return print("Job failed. No results to fetch.")
             elif status_json == "In Progress":
@@ -373,7 +405,8 @@ class Evaluation:
                 response = requests.post(
                     f'{self.base_url}/v1/llm/docs', 
                     headers=headers, 
-                    json=data)
+                    json=data,
+                    timeout=self.timeout)
                 response.raise_for_status()
                 return response.json()
             except requests.exceptions.HTTPError as http_err:
@@ -392,7 +425,7 @@ class Evaluation:
             try:
                 response = get_presignedUrl()
                 preSignedURL = response["data"]["preSignedURL"]
-                response = requests.get(preSignedURL)
+                response = requests.get(preSignedURL, timeout=self.timeout)
                 response.raise_for_status()
                 return response.text
             except requests.exceptions.HTTPError as http_err:

ragaai_catalyst/guard_executor.py

@@ -0,0 +1,97 @@
+import litellm
+import json
+import requests
+import os
+import logging
+logger = logging.getLogger('LiteLLM')
+logger.setLevel(logging.ERROR)
+
+class GuardExecutor:
+
+    def __init__(self,id,guard_manager,field_map={}):
+        self.deployment_id = id
+        self.field_map = field_map
+        self.guard_manager = guard_manager
+        self.deployment_details = self.guard_manager.get_deployment(id)
+        if not self.deployment_details:
+            raise ValueError('Error in getting deployment details')
+        self.base_url = guard_manager.base_url
+        for key in field_map.keys():
+            if key not in ['prompt','context','response','instruction']:
+                print('Keys in field map should be in ["prompt","context","response","instruction"]')
+
+    def execute_deployment(self,payload):
+        api = self.base_url + f'/guardrail/deployment/{self.deployment_id}/ingest'
+
+        payload = json.dumps(payload)
+        headers = {
+            'x-project-id': str(self.guard_manager.project_id),
+            'Content-Type': 'application/json',
+            'Authorization': f'Bearer {os.getenv("RAGAAI_CATALYST_TOKEN")}'
+        }
+        try:
+            response = requests.request("POST", api, headers=headers, data=payload,timeout=self.guard_manager.timeout)
+        except Exception as e:
+            print('Failed running guardrail: ',str(e))
+            return None
+        if response.status_code!=200:
+            print('Error in running deployment ',response.json()['message'])
+        if response.json()['success']:
+            return response.json()
+        else:
+            print(response.json()['message'])
+            return None
+
+    def llm_executor(self,messages,model_params,llm_caller):
+        if llm_caller == 'litellm':
+            model_params['messages'] = messages
+            response = litellm.completion(**model_params)
+            return response
+        else:
+            print(f"{llm_caller} not supported currently, use litellm as llm caller")
+
+    
+    def __call__(self,messages,prompt_params,model_params,llm_caller='litellm'):
+        for key in self.field_map:
+            if key not in ['prompt','response']:
+                if self.field_map[key] not in prompt_params:
+                    raise ValueError(f'{key} added as field map but not passed as prompt parameter')
+        context_var = self.field_map.get('context',None)
+        prompt = None
+        for msg in messages:
+            if 'role' in msg:
+                if msg['role'] == 'user':
+                    prompt = msg['content']
+                    if not context_var:
+                        msg['content'] += '\n' + prompt_params[context_var]
+        doc = dict()
+        doc['prompt'] = prompt
+        doc['context'] = prompt_params[context_var]
+        
+        # inactive the guardrails that needs Response variable
+        #deployment_response = self.execute_deployment(doc)
+        
+        # activate only guardrails that require response
+        try:
+            llm_response = self.llm_executor(messages,model_params,llm_caller)
+        except Exception as e:
+            print('Error in running llm:',str(e))
+            return None
+        doc['response'] = llm_response['choices'][0].message.content
+        if 'instruction' in self.field_map:
+            instruction = prompt_params[self.field_map['instruction']]
+            doc['instruction'] = instruction
+        response = self.execute_deployment(doc)
+        if response and response['data']['status'] == 'FAIL':
+            print('Guardrail deployment run retured failed status, replacing with alternate response')
+            return response['data']['alternateResponse'],llm_response,response
+        else:
+            return None,llm_response,response
+
+
+
+
+        
+
+
+