ragaai-catalyst 2.0.7.1__py3-none-any.whl → 2.0.7.2__py3-none-any.whl

ragaai_catalyst/evaluation.py

@@ -8,6 +8,7 @@ import pdb
 
 logger = logging.getLogger(__name__)
 
+
 class Evaluation:
 
     def __init__(self, project_name, dataset_name):
@@ -16,7 +17,7 @@ class Evaluation:
         self.base_url = f"{RagaAICatalyst.BASE_URL}"
         self.timeout = 10
         self.jobId = None
-        self.num_projects=99999
+        self.num_projects = 99999
 
         try:
             response = requests.get(
@@ -34,9 +35,11 @@ class Evaluation:
             ]
             if project_name not in project_list:
                 raise ValueError("Project not found. Please enter a valid project name")
-            
+
             self.project_id = [
-                project["id"] for project in response.json()["data"]["content"] if project["name"] == project_name
+                project["id"]
+                for project in response.json()["data"]["content"]
+                if project["name"] == project_name
             ][0]
 
         except requests.exceptions.RequestException as e:
@@ -46,44 +49,55 @@ class Evaluation:
         try:
 
             headers = {
-                'Content-Type': 'application/json',
+                "Content-Type": "application/json",
                 "Authorization": f"Bearer {os.getenv('RAGAAI_CATALYST_TOKEN')}",
                 "X-Project-Id": str(self.project_id),
             }
-            json_data = {"size": 12, "page": "0", "projectId": str(self.project_id), "search": ""}
+            json_data = {
+                "size": 12,
+                "page": "0",
+                "projectId": str(self.project_id),
+                "search": "",
+            }
             response = requests.post(
                 f"{self.base_url}/v2/llm/dataset",
                 headers=headers,
                 json=json_data,
                 timeout=self.timeout,
             )
-            
+
             response.raise_for_status()
             datasets_content = response.json()["data"]["content"]
             dataset_list = [dataset["name"] for dataset in datasets_content]
 
             if dataset_name not in dataset_list:
                 raise ValueError("Dataset not found. Please enter a valid dataset name")
-                
-            self.dataset_id = [dataset["id"] for dataset in datasets_content if dataset["name"]==dataset_name][0]
+
+            self.dataset_id = [
+                dataset["id"]
+                for dataset in datasets_content
+                if dataset["name"] == dataset_name
+            ][0]
 
         except requests.exceptions.RequestException as e:
             logger.error(f"Failed to retrieve dataset list: {e}")
             raise
 
-    
     def list_metrics(self):
         headers = {
             "Authorization": f"Bearer {os.getenv('RAGAAI_CATALYST_TOKEN')}",
-            'X-Project-Id': str(self.project_id),
+            "X-Project-Id": str(self.project_id),
         }
         try:
             response = requests.get(
-                f'{self.base_url}/v1/llm/llm-metrics', 
+                f"{self.base_url}/v1/llm/llm-metrics",
                 headers=headers,
-                timeout=self.timeout)
+                timeout=self.timeout,
+            )
             response.raise_for_status()
-            metric_names = [metric["name"] for metric in response.json()["data"]["metrics"]]
+            metric_names = [
+                metric["name"] for metric in response.json()["data"]["metrics"]
+            ]
             return metric_names
         except requests.exceptions.HTTPError as http_err:
             logger.error(f"HTTP error occurred: {http_err}")
@@ -100,22 +114,35 @@ class Evaluation:
     def _get_dataset_id_based_on_dataset_type(self, metric_to_evaluate):
         try:
             headers = {
-                'Content-Type': 'application/json',
+                "Content-Type": "application/json",
                 "Authorization": f"Bearer {os.getenv('RAGAAI_CATALYST_TOKEN')}",
                 "X-Project-Id": str(self.project_id),
             }
-            json_data = {"size": 12, "page": "0", "projectId": str(self.project_id), "search": ""}
+            json_data = {
+                "size": 12,
+                "page": "0",
+                "projectId": str(self.project_id),
+                "search": "",
+            }
             response = requests.post(
                 f"{self.base_url}/v2/llm/dataset",
                 headers=headers,
                 json=json_data,
                 timeout=self.timeout,
             )
-            
+
             response.raise_for_status()
             datasets_content = response.json()["data"]["content"]
-            dataset = [dataset for dataset in datasets_content if dataset["name"]==self.dataset_name][0]
-            if (dataset["datasetType"]=="prompt" and metric_to_evaluate=="prompt") or (dataset["datasetType"]=="chat" and metric_to_evaluate=="chat") or dataset["datasetType"]==None:
+            dataset = [
+                dataset
+                for dataset in datasets_content
+                if dataset["name"] == self.dataset_name
+            ][0]
+            if (
+                (dataset["datasetType"] == "prompt" and metric_to_evaluate == "prompt")
+                or (dataset["datasetType"] == "chat" and metric_to_evaluate == "chat")
+                or dataset["datasetType"] == None
+            ):
                 return dataset["id"]
             else:
                 return dataset["derivedDatasetId"]
@@ -123,28 +150,24 @@ class Evaluation:
             logger.error(f"Failed to retrieve dataset list: {e}")
             raise
 
-
     def _get_dataset_schema(self, metric_to_evaluate=None):
-        #this dataset_id is based on which type of metric_to_evaluate  
-        data_set_id=self._get_dataset_id_based_on_dataset_type(metric_to_evaluate)
-        self.dataset_id=data_set_id
+        # this dataset_id is based on which type of metric_to_evaluate
+        data_set_id = self._get_dataset_id_based_on_dataset_type(metric_to_evaluate)
+        self.dataset_id = data_set_id
 
         headers = {
             "Authorization": f"Bearer {os.getenv('RAGAAI_CATALYST_TOKEN')}",
-            'Content-Type': 'application/json',
-            'X-Project-Id': str(self.project_id),
-        }
-        data = {
-            "datasetId": str(data_set_id),
-            "fields": [],
-            "rowFilterList": []
+            "Content-Type": "application/json",
+            "X-Project-Id": str(self.project_id),
         }
+        data = {"datasetId": str(data_set_id), "fields": [], "rowFilterList": []}
         try:
             response = requests.post(
-                f'{self.base_url}/v1/llm/docs', 
+                f"{self.base_url}/v1/llm/docs",
                 headers=headers,
                 json=data,
-                timeout=self.timeout)
+                timeout=self.timeout,
+            )
             response.raise_for_status()
             if response.status_code == 200:
                 return response.json()["data"]["columns"]
@@ -160,31 +183,35 @@ class Evaluation:
             logger.error(f"An unexpected error occurred: {e}")
         return {}
 
-
-    def _get_variablename_from_user_schema_mapping(self, schemaName, metric_name, schema_mapping, metric_to_evaluate):
+    def _get_variablename_from_user_schema_mapping(
+        self, schemaName, metric_name, schema_mapping, metric_to_evaluate
+    ):
         user_dataset_schema = self._get_dataset_schema(metric_to_evaluate)
         user_dataset_columns = [item["displayName"] for item in user_dataset_schema]
         variableName = None
         for key, val in schema_mapping.items():
-            if "".join(val.split("_")).lower()==schemaName:
+            if "".join(val.split("_")).lower() == schemaName:
                 if key in user_dataset_columns:
-                    variableName=key
+                    variableName = key
                 else:
-                    raise ValueError(f"Column '{key}' is not present in '{self.dataset_name}' dataset")
+                    raise ValueError(
+                        f"Column '{key}' is not present in '{self.dataset_name}' dataset"
+                    )
         if variableName:
             return variableName
         else:
-            raise ValueError(f"Map '{schemaName}' column in schema_mapping for {metric_name} metric evaluation")
-
+            raise ValueError(
+                f"Map '{schemaName}' column in schema_mapping for {metric_name} metric evaluation"
+            )
 
     def _get_mapping(self, metric_name, metrics_schema, schema_mapping):
-        
+
         mapping = []
         for schema in metrics_schema:
-            if schema["name"]==metric_name:
+            if schema["name"] == metric_name:
                 requiredFields = schema["config"]["requiredFields"]
 
-                #this is added to check if "Chat" column is required for metric evaluation
+                # this is added to check if "Chat" column is required for metric evaluation
                 required_variables = [_["name"].lower() for _ in requiredFields]
                 if "chat" in required_variables:
                     metric_to_evaluate = "chat"
@@ -193,38 +220,42 @@ class Evaluation:
 
                 for field in requiredFields:
                     schemaName = field["name"]
-                    variableName = self._get_variablename_from_user_schema_mapping(schemaName.lower(), metric_name, schema_mapping, metric_to_evaluate)
-                    mapping.append({"schemaName": schemaName, "variableName": variableName})
+                    variableName = self._get_variablename_from_user_schema_mapping(
+                        schemaName.lower(),
+                        metric_name,
+                        schema_mapping,
+                        metric_to_evaluate,
+                    )
+                    mapping.append(
+                        {"schemaName": schemaName, "variableName": variableName}
+                    )
         return mapping
 
     def _get_metricParams(self):
         return {
-                "metricSpec": {
-                    "name": "metric_to_evaluate",
-                    "config": {
-                        "model": "null",
-                        "params": {
-                            "model": {
-                                "value": ""
-                            }
-                        },
-                        "mappings": "mappings"
-                    },
-                    "displayName": "displayName"
+            "metricSpec": {
+                "name": "metric_to_evaluate",
+                "config": {
+                    "model": "null",
+                    "params": {"model": {"value": ""}},
+                    "mappings": "mappings",
                 },
-                "rowFilterList": []
-            }
-    
+                "displayName": "displayName",
+            },
+            "rowFilterList": [],
+        }
+
     def _get_metrics_schema_response(self):
         headers = {
             "Authorization": f"Bearer {os.getenv('RAGAAI_CATALYST_TOKEN')}",
-            'X-Project-Id': str(self.project_id),
+            "X-Project-Id": str(self.project_id),
         }
         try:
             response = requests.get(
-                f'{self.base_url}/v1/llm/llm-metrics', 
+                f"{self.base_url}/v1/llm/llm-metrics",
                 headers=headers,
-                timeout=self.timeout)
+                timeout=self.timeout,
+            )
             response.raise_for_status()
             metrics_schema = [metric for metric in response.json()["data"]["metrics"]]
             return metrics_schema
@@ -242,42 +273,49 @@ class Evaluation:
 
     def _update_base_json(self, metrics):
         metrics_schema_response = self._get_metrics_schema_response()
-        sub_providers = ["openai","azure","gemini","groq"]
+        sub_providers = ["openai", "azure", "gemini", "groq", "anthropic", "bedrock"]
         metricParams = []
         for metric in metrics:
             base_json = self._get_metricParams()
             base_json["metricSpec"]["name"] = metric["name"]
-            
-            #pasing model configuration
+
+            # pasing model configuration
             for key, value in metric["config"].items():
-                #checking if provider is one of the allowed providers
-                if key.lower()=="provider" and value.lower() not in sub_providers:
-                    raise ValueError("Enter a valid provider name. The following Provider names are supported: OpenAI, Azure, Gemini, Groq")
-    
-                if key.lower()=="threshold":
-                    if len(value)>1:
-                        raise ValueError("'threshold' can only take one argument gte/lte/eq")
+                # checking if provider is one of the allowed providers
+                if key.lower() == "provider" and value.lower() not in sub_providers:
+                    raise ValueError(
+                        "Enter a valid provider name. The following Provider names are supported: openai, azure, gemini, groq, anthropic, bedrock"
+                    )
+
+                if key.lower() == "threshold":
+                    if len(value) > 1:
+                        raise ValueError(
+                            "'threshold' can only take one argument gte/lte/eq"
+                        )
                     else:
                         for key_thres, value_thres in value.items():
-                            base_json["metricSpec"]["config"]["params"][key] = {f"{key_thres}":value_thres}
+                            base_json["metricSpec"]["config"]["params"][key] = {
+                                f"{key_thres}": value_thres
+                            }
                 else:
                     base_json["metricSpec"]["config"]["params"][key] = {"value": value}
 
-
             # if metric["config"]["model"]:
             #     base_json["metricSpec"]["config"]["params"]["model"]["value"] = metric["config"]["model"]
             base_json["metricSpec"]["displayName"] = metric["column_name"]
-            mappings = self._get_mapping(metric["name"], metrics_schema_response, metric["schema_mapping"])
+            mappings = self._get_mapping(
+                metric["name"], metrics_schema_response, metric["schema_mapping"]
+            )
             base_json["metricSpec"]["config"]["mappings"] = mappings
             metricParams.append(base_json)
-        metric_schema_mapping = {"datasetId":self.dataset_id}
+        metric_schema_mapping = {"datasetId": self.dataset_id}
         metric_schema_mapping["metricParams"] = metricParams
         return metric_schema_mapping
 
     def _get_executed_metrics_list(self):
         headers = {
             "Authorization": f"Bearer {os.getenv('RAGAAI_CATALYST_TOKEN')}",
-            'X-Project-Id': str(self.project_id),
+            "X-Project-Id": str(self.project_id),
         }
         try:
             response = requests.get(
@@ -288,7 +326,9 @@ class Evaluation:
             response.raise_for_status()
             dataset_columns = response.json()["data"]["datasetColumnsResponses"]
             dataset_columns = [item["displayName"] for item in dataset_columns]
-            executed_metric_list = [data for data in dataset_columns if not data.startswith('_')]
+            executed_metric_list = [
+                data for data in dataset_columns if not data.startswith("_")
+            ]
 
             return executed_metric_list
         except requests.exceptions.HTTPError as http_err:
@@ -304,7 +344,7 @@ class Evaluation:
             return []
 
     def add_metrics(self, metrics):
-        #Handle required key if missing
+        # Handle required key if missing
         required_keys = {"name", "config", "column_name", "schema_mapping"}
         for metric in metrics:
             missing_keys = required_keys - metric.keys()
@@ -323,18 +363,18 @@ class Evaluation:
                 raise ValueError(f"Column name '{column_name}' already exists.")
 
         headers = {
-            'Content-Type': 'application/json',
+            "Content-Type": "application/json",
             "Authorization": f"Bearer {os.getenv('RAGAAI_CATALYST_TOKEN')}",
-            'X-Project-Id': str(self.project_id),
+            "X-Project-Id": str(self.project_id),
         }
         metric_schema_mapping = self._update_base_json(metrics)
         try:
             response = requests.post(
-                f'{self.base_url}/playground/metric-evaluation', 
-                headers=headers, 
+                f"{self.base_url}/playground/metric-evaluation",
+                headers=headers,
                 json=metric_schema_mapping,
-                timeout=self.timeout
-                )
+                timeout=self.timeout,
+            )
             if response.status_code == 400:
                 raise ValueError(response.json()["message"])
             response.raise_for_status()
@@ -355,24 +395,31 @@ class Evaluation:
 
     def get_status(self):
         headers = {
-            'Content-Type': 'application/json',
+            "Content-Type": "application/json",
             "Authorization": f"Bearer {os.getenv('RAGAAI_CATALYST_TOKEN')}",
-            'X-Project-Id': str(self.project_id),
+            "X-Project-Id": str(self.project_id),
         }
         try:
             response = requests.get(
-                f'{self.base_url}/job/status', 
-                headers=headers, 
-                timeout=self.timeout)
+                f"{self.base_url}/job/status", headers=headers, timeout=self.timeout
+            )
             response.raise_for_status()
             if response.json()["success"]:
-                status_json = [item["status"] for item in response.json()["data"]["content"] if item["id"]==self.jobId][0]
+                status_json = [
+                    item["status"]
+                    for item in response.json()["data"]["content"]
+                    if item["id"] == self.jobId
+                ][0]
             if status_json == "Failed":
                 return print("Job failed. No results to fetch.")
             elif status_json == "In Progress":
-                return print(f"Job in progress. Please wait while the job completes.\nVisit Job Status: {self.base_url.removesuffix('/api')}/projects/job-status?projectId={self.project_id} to track")
+                return print(
+                    f"Job in progress. Please wait while the job completes.\nVisit Job Status: {self.base_url.removesuffix('/api')}/projects/job-status?projectId={self.project_id} to track"
+                )
             elif status_json == "Completed":
-                print(f"Job completed. Fetching results.\nVisit Job Status: {self.base_url.removesuffix('/api')}/projects/job-status?projectId={self.project_id} to check")
+                print(
+                    f"Job completed. Fetching results.\nVisit Job Status: {self.base_url.removesuffix('/api')}/projects/job-status?projectId={self.project_id} to check"
+                )
         except requests.exceptions.HTTPError as http_err:
             logger.error(f"HTTP error occurred: {http_err}")
         except requests.exceptions.ConnectionError as conn_err:
@@ -388,25 +435,24 @@ class Evaluation:
 
         def get_presignedUrl():
             headers = {
-                'Content-Type': 'application/json',
+                "Content-Type": "application/json",
                 "Authorization": f"Bearer {os.getenv('RAGAAI_CATALYST_TOKEN')}",
-                'X-Project-Id': str(self.project_id),
-                }
-            
+                "X-Project-Id": str(self.project_id),
+            }
+
             data = {
-                "fields": [
-                    "*"
-                ],
+                "fields": ["*"],
                 "datasetId": str(self.dataset_id),
                 "rowFilterList": [],
-                "export": True
-                }
-            try:    
+                "export": True,
+            }
+            try:
                 response = requests.post(
-                    f'{self.base_url}/v1/llm/docs', 
-                    headers=headers, 
+                    f"{self.base_url}/v1/llm/docs",
+                    headers=headers,
                     json=data,
-                    timeout=self.timeout)
+                    timeout=self.timeout,
+                )
                 response.raise_for_status()
                 return response.json()
             except requests.exceptions.HTTPError as http_err:
@@ -445,8 +491,8 @@ class Evaluation:
             df = pd.read_csv(io.StringIO(response_text))
 
             column_list = df.columns.to_list()
-            column_list = [col for col in column_list if not col.startswith('_')]
-            column_list = [col for col in column_list if '.' not in col]
+            column_list = [col for col in column_list if not col.startswith("_")]
+            column_list = [col for col in column_list if "." not in col]
             return df[column_list]
         else:
             return pd.DataFrame()

{ragaai_catalyst-2.0.7.1.dist-info → ragaai_catalyst-2.0.7.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ragaai_catalyst
-Version: 2.0.7.1
+Version: 2.0.7.2
 Summary: RAGA AI CATALYST
 Author-email: Kiran Scaria <kiran.scaria@raga.ai>, Kedar Gaikwad <kedar.gaikwad@raga.ai>, Dushyant Mahajan <dushyant.mahajan@raga.ai>, Siddhartha Kosti <siddhartha.kosti@raga.ai>, Ritika Goel <ritika.goel@raga.ai>, Vijay Chaurasia <vijay.chaurasia@raga.ai>
 Requires-Python: >=3.9

{ragaai_catalyst-2.0.7.1.dist-info → ragaai_catalyst-2.0.7.2.dist-info}/RECORD

@@ -1,7 +1,7 @@
 ragaai_catalyst/__init__.py,sha256=BdIJ_UUre0uEnRTsLw_hE0C0muWk6XWNZqdVOel22R4,537
 ragaai_catalyst/_version.py,sha256=JKt9KaVNOMVeGs8ojO6LvIZr7ZkMzNN-gCcvryy4x8E,460
 ragaai_catalyst/dataset.py,sha256=j_vu3Xkp_8qYW0J9Qnn53Uyh98MsugqYl5zxhOv9EOg,10731
-ragaai_catalyst/evaluation.py,sha256=EZGISQH6vGB7vP5OmUBRXmEqkqZPIYN_eEUUz1jTrJ8,20285
+ragaai_catalyst/evaluation.py,sha256=hKwa8C8TE4IhCpkf5vrLhY2Y3TLajqa5s2-JqVGHoQo,21087
 ragaai_catalyst/experiment.py,sha256=8KvqgJg5JVnt9ghhGDJvdb4mN7ETBX_E5gNxBT0Nsn8,19010
 ragaai_catalyst/guard_executor.py,sha256=llPbE3DyVtrybojXknzBZj8-dtUrGBQwi9-ZiPJxGRo,3762
 ragaai_catalyst/guardrails_manager.py,sha256=DILMOAASK57FH9BLq_8yC1AQzRJ8McMFLwCXgYwNAd4,11904
@@ -23,7 +23,7 @@ ragaai_catalyst/tracers/instrumentators/llamaindex.py,sha256=SMrRlR4xM7k9HK43hak
 ragaai_catalyst/tracers/instrumentators/openai.py,sha256=14R4KW9wQCR1xysLfsP_nxS7cqXrTPoD8En4MBAaZUU,379
 ragaai_catalyst/tracers/utils/__init__.py,sha256=KeMaZtYaTojilpLv65qH08QmpYclfpacDA0U3wg6Ybw,64
 ragaai_catalyst/tracers/utils/utils.py,sha256=ViygfJ7vZ7U0CTSA1lbxVloHp4NSlmfDzBRNCJuMhis,2374
-ragaai_catalyst-2.0.7.1.dist-info/METADATA,sha256=MdLOjU46-aocYQAOxj8QxLNxSb0hTyV8OACWSuhGdkE,11240
-ragaai_catalyst-2.0.7.1.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-ragaai_catalyst-2.0.7.1.dist-info/top_level.txt,sha256=HpgsdRgEJMk8nqrU6qdCYk3di7MJkDL0B19lkc7dLfM,16
-ragaai_catalyst-2.0.7.1.dist-info/RECORD,,
+ragaai_catalyst-2.0.7.2.dist-info/METADATA,sha256=-t4xrOLS05B78m-JHQanSAflEBxHDLjgYwQA80J60b0,11240
+ragaai_catalyst-2.0.7.2.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+ragaai_catalyst-2.0.7.2.dist-info/top_level.txt,sha256=HpgsdRgEJMk8nqrU6qdCYk3di7MJkDL0B19lkc7dLfM,16
+ragaai_catalyst-2.0.7.2.dist-info/RECORD,,