ragaai-catalyst 2.1b0__py3-none-any.whl → 2.1b2__py3-none-any.whl

ragaai_catalyst/__init__.py

@@ -7,6 +7,7 @@ from .prompt_manager import PromptManager
 from .evaluation import Evaluation
 from .synthetic_data_generation import SyntheticDataGeneration
 from .guardrails_manager import GuardrailsManager
+from .guard_executor import GuardExecutor
 
 
 __all__ = ["Experiment", "RagaAICatalyst", "Tracer", "PromptManager", "Evaluation","SyntheticDataGeneration", "GuardrailsManager"]

ragaai_catalyst/dataset.py

@@ -5,7 +5,6 @@ from typing import Union
 import logging
 from .ragaai_catalyst import RagaAICatalyst
 import pandas as pd
-import pdb
 logger = logging.getLogger(__name__)
 get_token = RagaAICatalyst.get_token
 
@@ -16,7 +15,7 @@ class Dataset:
 
     def __init__(self, project_name):
         self.project_name = project_name
-        self.num_projects = 100
+        self.num_projects = 99999
         Dataset.BASE_URL = (
             os.getenv("RAGAAI_CATALYST_BASE_URL")
             if os.getenv("RAGAAI_CATALYST_BASE_URL")
@@ -202,7 +201,6 @@ class Dataset:
 
         #### put csv to presigned URL
         def put_csv_to_presignedUrl(url):
-            # pdb.set_trace()
             headers = {
                 'Content-Type': 'text/csv',
                 'x-ms-blob-type': 'BlockBlob',
@@ -224,7 +222,6 @@ class Dataset:
         try:
 
             put_csv_response = put_csv_to_presignedUrl(url)
-            # pdb.set_trace()
             print(put_csv_response)
             if put_csv_response.status_code not in (200, 201):
                 raise ValueError('Unable to put csv to the presignedUrl')

ragaai_catalyst/evaluation.py

@@ -4,7 +4,6 @@ import pandas as pd
 import io
 from .ragaai_catalyst import RagaAICatalyst
 import logging
-import pdb
 
 logger = logging.getLogger(__name__)
 
@@ -16,7 +15,7 @@ class Evaluation:
         self.base_url = f"{RagaAICatalyst.BASE_URL}"
         self.timeout = 10
         self.jobId = None
-        self.num_projects=100
+        self.num_projects=99999
 
         try:
             response = requests.get(
@@ -242,7 +241,7 @@ class Evaluation:
 
     def _update_base_json(self, metrics):
         metrics_schema_response = self._get_metrics_schema_response()
-        sub_providers = ["openai","azure","gemini","groq"]
+        sub_providers = ["openai","azure","gemini","groq","anthropic","bedrock"]
         metricParams = []
         for metric in metrics:
             base_json = self._get_metricParams()
@@ -252,7 +251,7 @@ class Evaluation:
             for key, value in metric["config"].items():
                 #checking if provider is one of the allowed providers
                 if key.lower()=="provider" and value.lower() not in sub_providers:
-                    raise ValueError("Enter a valid provider name. The following Provider names are supported: OpenAI, Azure, Gemini, Groq")
+                    raise ValueError("Enter a valid provider name. The following Provider names are supported: openai, azure, gemini, groq, anthropic, bedrock")
     
                 if key.lower()=="threshold":
                     if len(value)>1:
@@ -330,7 +329,7 @@ class Evaluation:
         metric_schema_mapping = self._update_base_json(metrics)
         try:
             response = requests.post(
-                f'{self.base_url}/playground/metric-evaluation', 
+                f'{self.base_url}/v2/llm/metric-evaluation',
                 headers=headers, 
                 json=metric_schema_mapping,
                 timeout=self.timeout

ragaai_catalyst/guard_executor.py

@@ -0,0 +1,97 @@
+import litellm
+import json
+import requests
+import os
+import logging
+logger = logging.getLogger('LiteLLM')
+logger.setLevel(logging.ERROR)
+
+class GuardExecutor:
+
+    def __init__(self,id,guard_manager,field_map={}):
+        self.deployment_id = id
+        self.field_map = field_map
+        self.guard_manager = guard_manager
+        self.deployment_details = self.guard_manager.get_deployment(id)
+        if not self.deployment_details:
+            raise ValueError('Error in getting deployment details')
+        self.base_url = guard_manager.base_url
+        for key in field_map.keys():
+            if key not in ['prompt','context','response','instruction']:
+                print('Keys in field map should be in ["prompt","context","response","instruction"]')
+
+    def execute_deployment(self,payload):
+        api = self.base_url + f'/guardrail/deployment/{self.deployment_id}/ingest'
+
+        payload = json.dumps(payload)
+        headers = {
+            'x-project-id': str(self.guard_manager.project_id),
+            'Content-Type': 'application/json',
+            'Authorization': f'Bearer {os.getenv("RAGAAI_CATALYST_TOKEN")}'
+        }
+        try:
+            response = requests.request("POST", api, headers=headers, data=payload,timeout=self.guard_manager.timeout)
+        except Exception as e:
+            print('Failed running guardrail: ',str(e))
+            return None
+        if response.status_code!=200:
+            print('Error in running deployment ',response.json()['message'])
+        if response.json()['success']:
+            return response.json()
+        else:
+            print(response.json()['message'])
+            return None
+
+    def llm_executor(self,messages,model_params,llm_caller):
+        if llm_caller == 'litellm':
+            model_params['messages'] = messages
+            response = litellm.completion(**model_params)
+            return response
+        else:
+            print(f"{llm_caller} not supported currently, use litellm as llm caller")
+
+    
+    def __call__(self,messages,prompt_params,model_params,llm_caller='litellm'):
+        for key in self.field_map:
+            if key not in ['prompt','response']:
+                if self.field_map[key] not in prompt_params:
+                    raise ValueError(f'{key} added as field map but not passed as prompt parameter')
+        context_var = self.field_map.get('context',None)
+        prompt = None
+        for msg in messages:
+            if 'role' in msg:
+                if msg['role'] == 'user':
+                    prompt = msg['content']
+                    if not context_var:
+                        msg['content'] += '\n' + prompt_params[context_var]
+        doc = dict()
+        doc['prompt'] = prompt
+        doc['context'] = prompt_params[context_var]
+        
+        # inactive the guardrails that needs Response variable
+        #deployment_response = self.execute_deployment(doc)
+        
+        # activate only guardrails that require response
+        try:
+            llm_response = self.llm_executor(messages,model_params,llm_caller)
+        except Exception as e:
+            print('Error in running llm:',str(e))
+            return None
+        doc['response'] = llm_response['choices'][0].message.content
+        if 'instruction' in self.field_map:
+            instruction = prompt_params[self.field_map['instruction']]
+            doc['instruction'] = instruction
+        response = self.execute_deployment(doc)
+        if response and response['data']['status'] == 'FAIL':
+            print('Guardrail deployment run retured failed status, replacing with alternate response')
+            return response['data']['alternateResponse'],llm_response,response
+        else:
+            return None,llm_response,response
+
+
+
+
+        
+
+
+

ragaai_catalyst/guardrails_manager.py

@@ -13,7 +13,7 @@ class GuardrailsManager:
         """
         self.project_name = project_name
         self.timeout = 10
-        self.num_projects = 100
+        self.num_projects = 99999
         self.deployment_name = "NA"
         self.deployment_id = "NA"
         self.base_url = f"{RagaAICatalyst.BASE_URL}"
@@ -31,7 +31,7 @@ class GuardrailsManager:
         :return: A tuple containing a list of project names and a list of dictionaries with project IDs and names.
         """
         headers = {'Authorization': f'Bearer {os.getenv("RAGAAI_CATALYST_TOKEN")}'}
-        response = requests.request("GET", f"{self.base_url}/v2/llm/projects?size=12&page=0", headers=headers, timeout=self.timeout)
+        response = requests.request("GET", f"{self.base_url}/v2/llm/projects?size={self.num_projects}", headers=headers, timeout=self.timeout)
         project_content = response.json()["data"]["content"]
         list_project = [_["name"] for _ in project_content]
         project_name_with_id = [{"id": _["id"], "name": _["name"]} for _ in project_content]
@@ -68,10 +68,11 @@ class GuardrailsManager:
                 'X-Project-Id': str(self.project_id)
                 }
         response = requests.request("GET", f"{self.base_url}/guardrail/deployment/{deployment_id}", headers=headers, data=payload, timeout=self.timeout)
-        deployment_id_name = response.json()["data"]["name"]
-        deployment_id_guardrails = response.json()["data"]["guardrailsResponse"]
-        guardrails_list_deployment_id = [{_["type"]:_["name"]} for _ in deployment_id_guardrails]
-        return {"deployment_name":deployment_id_name, "guardrails_list":guardrails_list_deployment_id}
+        if response.json()['success']:
+            return response.json()
+        else:
+            print('Error in retrieving deployment details:',response.json()['message'])
+            return None
 
 
     def list_guardrails(self):
@@ -132,11 +133,12 @@ class GuardrailsManager:
             print(response.json()["message"])
             deployment_ids = self.list_deployment_ids()
             self.deployment_id = [_["id"] for _ in deployment_ids if _["name"]==self.deployment_name][0]
+            return self.deployment_id
         else:
             print(response)
             
 
-    def add_guardrails(self, guardrails, guardrails_config={}):
+    def add_guardrails(self, deployment_id, guardrails, guardrails_config={}):
         """
         Add guardrails to the current deployment.
         
@@ -145,16 +147,21 @@ class GuardrailsManager:
         :raises ValueError: If a guardrail name or type is invalid.
         """
         # Checking if guardrails names given already exist or not
-        _, guardrails_type_name_exists = self.get_deployment(self.deployment_id)
+        self.deployment_id = deployment_id
+        deployment_details = self.get_deployment(self.deployment_id)
+        if not deployment_details:
+            return None
+        deployment_id_name = deployment_details["data"]["name"]
+        deployment_id_guardrails = deployment_details["data"]["guardrailsResponse"]
+        guardrails_type_name_exists = [{_['metricSpec']["name"]:_['metricSpec']["displayName"]} for _ in deployment_id_guardrails]
         guardrails_type_name_exists = [list(d.values())[0] for d in guardrails_type_name_exists]
         user_guardrails_name_list = [_["name"] for _ in guardrails]
         for g_name in user_guardrails_name_list:
             if g_name in guardrails_type_name_exists:
                 raise ValueError(f"Guardrail with '{g_name} already exists, choose a unique name'")
-
         # Checking if guardrails type is correct or not
         available_guardrails_list = self.list_guardrails()
-        user_guardrails_type_list = [_["type"] for _ in guardrails]
+        user_guardrails_type_list = [_["name"] for _ in guardrails]
         for g_type in user_guardrails_type_list:
             if g_type not in available_guardrails_list:
                 raise ValueError(f"Guardrail type '{g_type} does not exists, choose a correct type'")
@@ -170,6 +177,8 @@ class GuardrailsManager:
         response = requests.request("POST", f"{self.base_url}/guardrail/deployment/{str(self.deployment_id)}/configure", headers=headers, data=payload)
         if response.json()["success"]:
             print(response.json()["message"])
+        else:
+            print('Error updating guardrail ',response.json()['message'])
 
     def _get_guardrail_config_payload(self, guardrails_config):
         """
@@ -209,13 +218,30 @@ class GuardrailsManager:
         :param guardrail: A dictionary containing the guardrail's attributes.
         :return: A dictionary representing the guardrail's data.
         """
+        if 'config' in guardrail:
+            if 'mappings' in guardrail.get('config'):
+                for mapping in guardrail.get('config',{}).get('mappings',{}):
+                    if mapping['schemaName'] not in ['Text','Prompt','Context','Response']:
+                        raise(ValueError('Invalid schemaName in guardrail mapping schema'))
+                    if mapping['variableName'] not in ['Instruction','Prompt','Context','Response']:
+                        raise(ValueError('Invalid variableName in guardrail mapping schema'))
+            if 'model' in guardrail.get('config'):
+                if guardrail.get('config',{}).get('model','') not in ['gpt-4o-mini','gpt-4o','gpt-4-turbo']:
+                    raise(ValueError('Invalid model name in guardrail model schema'))
+            if 'params' not in guardrail.get('config'):
+                guardrail['config']['params'] = {
+                    "isActive": {"value": False},
+                    "isHighRisk": {"value": False},
+                    "threshold": {"lt": 1}
+                }
+
+
         data = {
+            "displayName": guardrail["displayName"],
             "name": guardrail["name"],
-            "type": guardrail["type"],
-            "isHighRisk": guardrail.get("isHighRisk", False),
-            "isActive": guardrail.get("isActive", False),
-            "threshold": {}
+            "config": guardrail.get("config", {})
         }
+        '''
         if "lte" in guardrail["threshold"]:
             data["threshold"]["lte"] = guardrail["threshold"]["lte"]
         elif "gte" in guardrail["threshold"]:
@@ -223,7 +249,7 @@ class GuardrailsManager:
         elif "eq" in guardrail["threshold"]:
             data["threshold"]["eq"] = guardrail["threshold"]["eq"]
         else:
-            data["threshold"]["gte"] = 0.0
+            data["threshold"]["gte"] = 0.0'''
         return data
 
 

ragaai_catalyst/internal_api_completion.py

@@ -36,7 +36,7 @@ def api_completion(messages, model_config, kwargs):
                     raise ValueError(response["error"]["message"])
                 else:
                     result=  response["choices"][0]["message"]["content"]
-                    response1 = result.replace('\n', '')
+                    response1 = result.replace('\n', '').replace('```json','').replace('```', '').strip()
                     try:
                         json_data = json.loads(response1)
                         df = pd.DataFrame(json_data)

ragaai_catalyst/prompt_manager.py

@@ -23,7 +23,7 @@ class PromptManager:
         self.project_name = project_name
         self.base_url = f"{RagaAICatalyst.BASE_URL}/playground/prompt"
         self.timeout = 10
-        self.size = 100 #Number of projects to fetch
+        self.size = 99999 #Number of projects to fetch
 
         try:
             response = requests.get(
@@ -432,7 +432,12 @@ class PromptObject:
         Returns:
             dict: A dictionary of parameters found in the prompt text.
         """
-        parameters = {param["name"]: self._convert_value(param["value"], param["type"]) for param in self.parameters}
+        parameters = {}
+        for param in self.parameters:
+            if "value" in param:
+                parameters[param["name"]] = self._convert_value(param["value"], param["type"])
+            else:
+                parameters[param["name"]] = ""
         parameters["model"] = self.model
         return parameters    
     

ragaai_catalyst/ragaai_catalyst.py

@@ -287,7 +287,7 @@ class RagaAICatalyst:
     def get_project_id(self, project_name):
         pass
 
-    def list_projects(self, num_projects=100):
+    def list_projects(self, num_projects=99999):
         """
         Retrieves a list of projects with the specified number of projects.
 

ragaai_catalyst/synthetic_data_generation.py

@@ -141,6 +141,9 @@ class SyntheticDataGeneration:
 
     def _initialize_client(self, provider, api_key, api_base=None, internal_llm_proxy=None):
         """Initialize the appropriate client based on provider."""
+        if not provider:
+            raise ValueError("Model configuration must be provided with a valid provider and model.")
+
         if provider == "groq":
             if api_key is None and os.getenv("GROQ_API_KEY") is None:
                 raise ValueError("API key must be provided for Groq.")
@@ -155,6 +158,9 @@ class SyntheticDataGeneration:
             if api_key is None and os.getenv("OPENAI_API_KEY") is None and internal_llm_proxy is None:
                 raise ValueError("API key must be provided for OpenAI.")
             openai.api_key = api_key or os.getenv("OPENAI_API_KEY")
+        
+        else:
+            raise ValueError(f"Provider is not recognized.")
 
     def _generate_batch_response(self, text, system_message, provider, model_config, api_key, api_base):
         """Generate a batch of responses using the specified provider."""
@@ -283,6 +289,7 @@ class SyntheticDataGeneration:
 
         # Extract the content from the response
         content = response.choices[0].message.content
+        content = content.replace('\n', '').replace('```json','').replace('```', '').strip()
 
         # Clean the response if needed (remove any prefix before the JSON list)
         list_start_index = content.find('[')

ragaai_catalyst/tracers/__init__.py

@@ -1,3 +1,3 @@
 from .tracer import Tracer
 
-__all__ = ["Tracer"]
+__all__ = ["Tracer"] 

ragaai_catalyst/tracers/agentic_tracing/__init__.py

@@ -0,0 +1,3 @@
+from .agentic_tracing import AgenticTracing
+
+__all__ = ['AgenticTracing']