ragaai-catalyst 2.0.7.2__py3-none-any.whl → 2.0.7.2b0__py3-none-any.whl

ragaai_catalyst/evaluation.py

@@ -8,7 +8,6 @@ import pdb
 
 logger = logging.getLogger(__name__)
 
-
 class Evaluation:
 
     def __init__(self, project_name, dataset_name):
@@ -17,7 +16,7 @@ class Evaluation:
         self.base_url = f"{RagaAICatalyst.BASE_URL}"
         self.timeout = 10
         self.jobId = None
-        self.num_projects = 99999
+        self.num_projects=99999
 
         try:
             response = requests.get(
@@ -35,11 +34,9 @@ class Evaluation:
             ]
             if project_name not in project_list:
                 raise ValueError("Project not found. Please enter a valid project name")
-
+            
             self.project_id = [
-                project["id"]
-                for project in response.json()["data"]["content"]
-                if project["name"] == project_name
+                project["id"] for project in response.json()["data"]["content"] if project["name"] == project_name
             ][0]
 
         except requests.exceptions.RequestException as e:
@@ -49,55 +46,44 @@ class Evaluation:
         try:
 
             headers = {
-                "Content-Type": "application/json",
+                'Content-Type': 'application/json',
                 "Authorization": f"Bearer {os.getenv('RAGAAI_CATALYST_TOKEN')}",
                 "X-Project-Id": str(self.project_id),
             }
-            json_data = {
-                "size": 12,
-                "page": "0",
-                "projectId": str(self.project_id),
-                "search": "",
-            }
+            json_data = {"size": 12, "page": "0", "projectId": str(self.project_id), "search": ""}
             response = requests.post(
                 f"{self.base_url}/v2/llm/dataset",
                 headers=headers,
                 json=json_data,
                 timeout=self.timeout,
             )
-
+            
             response.raise_for_status()
             datasets_content = response.json()["data"]["content"]
             dataset_list = [dataset["name"] for dataset in datasets_content]
 
             if dataset_name not in dataset_list:
                 raise ValueError("Dataset not found. Please enter a valid dataset name")
-
-            self.dataset_id = [
-                dataset["id"]
-                for dataset in datasets_content
-                if dataset["name"] == dataset_name
-            ][0]
+                
+            self.dataset_id = [dataset["id"] for dataset in datasets_content if dataset["name"]==dataset_name][0]
 
         except requests.exceptions.RequestException as e:
             logger.error(f"Failed to retrieve dataset list: {e}")
             raise
 
+    
     def list_metrics(self):
         headers = {
             "Authorization": f"Bearer {os.getenv('RAGAAI_CATALYST_TOKEN')}",
-            "X-Project-Id": str(self.project_id),
+            'X-Project-Id': str(self.project_id),
         }
         try:
             response = requests.get(
-                f"{self.base_url}/v1/llm/llm-metrics",
+                f'{self.base_url}/v1/llm/llm-metrics', 
                 headers=headers,
-                timeout=self.timeout,
-            )
+                timeout=self.timeout)
             response.raise_for_status()
-            metric_names = [
-                metric["name"] for metric in response.json()["data"]["metrics"]
-            ]
+            metric_names = [metric["name"] for metric in response.json()["data"]["metrics"]]
             return metric_names
         except requests.exceptions.HTTPError as http_err:
             logger.error(f"HTTP error occurred: {http_err}")
@@ -114,35 +100,22 @@ class Evaluation:
     def _get_dataset_id_based_on_dataset_type(self, metric_to_evaluate):
         try:
             headers = {
-                "Content-Type": "application/json",
+                'Content-Type': 'application/json',
                 "Authorization": f"Bearer {os.getenv('RAGAAI_CATALYST_TOKEN')}",
                 "X-Project-Id": str(self.project_id),
             }
-            json_data = {
-                "size": 12,
-                "page": "0",
-                "projectId": str(self.project_id),
-                "search": "",
-            }
+            json_data = {"size": 12, "page": "0", "projectId": str(self.project_id), "search": ""}
             response = requests.post(
                 f"{self.base_url}/v2/llm/dataset",
                 headers=headers,
                 json=json_data,
                 timeout=self.timeout,
             )
-
+            
             response.raise_for_status()
             datasets_content = response.json()["data"]["content"]
-            dataset = [
-                dataset
-                for dataset in datasets_content
-                if dataset["name"] == self.dataset_name
-            ][0]
-            if (
-                (dataset["datasetType"] == "prompt" and metric_to_evaluate == "prompt")
-                or (dataset["datasetType"] == "chat" and metric_to_evaluate == "chat")
-                or dataset["datasetType"] == None
-            ):
+            dataset = [dataset for dataset in datasets_content if dataset["name"]==self.dataset_name][0]
+            if (dataset["datasetType"]=="prompt" and metric_to_evaluate=="prompt") or (dataset["datasetType"]=="chat" and metric_to_evaluate=="chat") or dataset["datasetType"]==None:
                 return dataset["id"]
             else:
                 return dataset["derivedDatasetId"]
@@ -150,24 +123,28 @@ class Evaluation:
             logger.error(f"Failed to retrieve dataset list: {e}")
             raise
 
+
     def _get_dataset_schema(self, metric_to_evaluate=None):
-        # this dataset_id is based on which type of metric_to_evaluate
-        data_set_id = self._get_dataset_id_based_on_dataset_type(metric_to_evaluate)
-        self.dataset_id = data_set_id
+        #this dataset_id is based on which type of metric_to_evaluate  
+        data_set_id=self._get_dataset_id_based_on_dataset_type(metric_to_evaluate)
+        self.dataset_id=data_set_id
 
         headers = {
             "Authorization": f"Bearer {os.getenv('RAGAAI_CATALYST_TOKEN')}",
-            "Content-Type": "application/json",
-            "X-Project-Id": str(self.project_id),
+            'Content-Type': 'application/json',
+            'X-Project-Id': str(self.project_id),
+        }
+        data = {
+            "datasetId": str(data_set_id),
+            "fields": [],
+            "rowFilterList": []
         }
-        data = {"datasetId": str(data_set_id), "fields": [], "rowFilterList": []}
         try:
             response = requests.post(
-                f"{self.base_url}/v1/llm/docs",
+                f'{self.base_url}/v1/llm/docs', 
                 headers=headers,
                 json=data,
-                timeout=self.timeout,
-            )
+                timeout=self.timeout)
             response.raise_for_status()
             if response.status_code == 200:
                 return response.json()["data"]["columns"]
@@ -183,35 +160,31 @@ class Evaluation:
             logger.error(f"An unexpected error occurred: {e}")
         return {}
 
-    def _get_variablename_from_user_schema_mapping(
-        self, schemaName, metric_name, schema_mapping, metric_to_evaluate
-    ):
+
+    def _get_variablename_from_user_schema_mapping(self, schemaName, metric_name, schema_mapping, metric_to_evaluate):
         user_dataset_schema = self._get_dataset_schema(metric_to_evaluate)
         user_dataset_columns = [item["displayName"] for item in user_dataset_schema]
         variableName = None
         for key, val in schema_mapping.items():
-            if "".join(val.split("_")).lower() == schemaName:
+            if "".join(val.split("_")).lower()==schemaName:
                 if key in user_dataset_columns:
-                    variableName = key
+                    variableName=key
                 else:
-                    raise ValueError(
-                        f"Column '{key}' is not present in '{self.dataset_name}' dataset"
-                    )
+                    raise ValueError(f"Column '{key}' is not present in '{self.dataset_name}' dataset")
         if variableName:
             return variableName
         else:
-            raise ValueError(
-                f"Map '{schemaName}' column in schema_mapping for {metric_name} metric evaluation"
-            )
+            raise ValueError(f"Map '{schemaName}' column in schema_mapping for {metric_name} metric evaluation")
 
-    def _get_mapping(self, metric_name, metrics_schema, schema_mapping):
 
+    def _get_mapping(self, metric_name, metrics_schema, schema_mapping):
+        
         mapping = []
         for schema in metrics_schema:
-            if schema["name"] == metric_name:
+            if schema["name"]==metric_name:
                 requiredFields = schema["config"]["requiredFields"]
 
-                # this is added to check if "Chat" column is required for metric evaluation
+                #this is added to check if "Chat" column is required for metric evaluation
                 required_variables = [_["name"].lower() for _ in requiredFields]
                 if "chat" in required_variables:
                     metric_to_evaluate = "chat"
@@ -220,42 +193,38 @@ class Evaluation:
 
                 for field in requiredFields:
                     schemaName = field["name"]
-                    variableName = self._get_variablename_from_user_schema_mapping(
-                        schemaName.lower(),
-                        metric_name,
-                        schema_mapping,
-                        metric_to_evaluate,
-                    )
-                    mapping.append(
-                        {"schemaName": schemaName, "variableName": variableName}
-                    )
+                    variableName = self._get_variablename_from_user_schema_mapping(schemaName.lower(), metric_name, schema_mapping, metric_to_evaluate)
+                    mapping.append({"schemaName": schemaName, "variableName": variableName})
         return mapping
 
     def _get_metricParams(self):
         return {
-            "metricSpec": {
-                "name": "metric_to_evaluate",
-                "config": {
-                    "model": "null",
-                    "params": {"model": {"value": ""}},
-                    "mappings": "mappings",
+                "metricSpec": {
+                    "name": "metric_to_evaluate",
+                    "config": {
+                        "model": "null",
+                        "params": {
+                            "model": {
+                                "value": ""
+                            }
+                        },
+                        "mappings": "mappings"
+                    },
+                    "displayName": "displayName"
                 },
-                "displayName": "displayName",
-            },
-            "rowFilterList": [],
-        }
-
+                "rowFilterList": []
+            }
+    
     def _get_metrics_schema_response(self):
         headers = {
             "Authorization": f"Bearer {os.getenv('RAGAAI_CATALYST_TOKEN')}",
-            "X-Project-Id": str(self.project_id),
+            'X-Project-Id': str(self.project_id),
         }
         try:
             response = requests.get(
-                f"{self.base_url}/v1/llm/llm-metrics",
+                f'{self.base_url}/v1/llm/llm-metrics', 
                 headers=headers,
-                timeout=self.timeout,
-            )
+                timeout=self.timeout)
             response.raise_for_status()
             metrics_schema = [metric for metric in response.json()["data"]["metrics"]]
             return metrics_schema
@@ -273,49 +242,42 @@ class Evaluation:
 
     def _update_base_json(self, metrics):
         metrics_schema_response = self._get_metrics_schema_response()
-        sub_providers = ["openai", "azure", "gemini", "groq", "anthropic", "bedrock"]
+        sub_providers = ["openai","azure","gemini","groq","anthropic","bedrock"]
         metricParams = []
         for metric in metrics:
             base_json = self._get_metricParams()
             base_json["metricSpec"]["name"] = metric["name"]
-
-            # pasing model configuration
+            
+            #pasing model configuration
             for key, value in metric["config"].items():
-                # checking if provider is one of the allowed providers
-                if key.lower() == "provider" and value.lower() not in sub_providers:
-                    raise ValueError(
-                        "Enter a valid provider name. The following Provider names are supported: openai, azure, gemini, groq, anthropic, bedrock"
-                    )
-
-                if key.lower() == "threshold":
-                    if len(value) > 1:
-                        raise ValueError(
-                            "'threshold' can only take one argument gte/lte/eq"
-                        )
+                #checking if provider is one of the allowed providers
+                if key.lower()=="provider" and value.lower() not in sub_providers:
+                    raise ValueError("Enter a valid provider name. The following Provider names are supported: openai, azure, gemini, groq, anthropic, bedrock")
+    
+                if key.lower()=="threshold":
+                    if len(value)>1:
+                        raise ValueError("'threshold' can only take one argument gte/lte/eq")
                     else:
                         for key_thres, value_thres in value.items():
-                            base_json["metricSpec"]["config"]["params"][key] = {
-                                f"{key_thres}": value_thres
-                            }
+                            base_json["metricSpec"]["config"]["params"][key] = {f"{key_thres}":value_thres}
                 else:
                     base_json["metricSpec"]["config"]["params"][key] = {"value": value}
 
+
             # if metric["config"]["model"]:
             #     base_json["metricSpec"]["config"]["params"]["model"]["value"] = metric["config"]["model"]
             base_json["metricSpec"]["displayName"] = metric["column_name"]
-            mappings = self._get_mapping(
-                metric["name"], metrics_schema_response, metric["schema_mapping"]
-            )
+            mappings = self._get_mapping(metric["name"], metrics_schema_response, metric["schema_mapping"])
             base_json["metricSpec"]["config"]["mappings"] = mappings
             metricParams.append(base_json)
-        metric_schema_mapping = {"datasetId": self.dataset_id}
+        metric_schema_mapping = {"datasetId":self.dataset_id}
         metric_schema_mapping["metricParams"] = metricParams
         return metric_schema_mapping
 
     def _get_executed_metrics_list(self):
         headers = {
             "Authorization": f"Bearer {os.getenv('RAGAAI_CATALYST_TOKEN')}",
-            "X-Project-Id": str(self.project_id),
+            'X-Project-Id': str(self.project_id),
         }
         try:
             response = requests.get(
@@ -326,9 +288,7 @@ class Evaluation:
             response.raise_for_status()
             dataset_columns = response.json()["data"]["datasetColumnsResponses"]
             dataset_columns = [item["displayName"] for item in dataset_columns]
-            executed_metric_list = [
-                data for data in dataset_columns if not data.startswith("_")
-            ]
+            executed_metric_list = [data for data in dataset_columns if not data.startswith('_')]
 
             return executed_metric_list
         except requests.exceptions.HTTPError as http_err:
@@ -344,7 +304,7 @@ class Evaluation:
             return []
 
     def add_metrics(self, metrics):
-        # Handle required key if missing
+        #Handle required key if missing
         required_keys = {"name", "config", "column_name", "schema_mapping"}
         for metric in metrics:
             missing_keys = required_keys - metric.keys()
@@ -363,18 +323,18 @@ class Evaluation:
                 raise ValueError(f"Column name '{column_name}' already exists.")
 
         headers = {
-            "Content-Type": "application/json",
+            'Content-Type': 'application/json',
             "Authorization": f"Bearer {os.getenv('RAGAAI_CATALYST_TOKEN')}",
-            "X-Project-Id": str(self.project_id),
+            'X-Project-Id': str(self.project_id),
         }
         metric_schema_mapping = self._update_base_json(metrics)
         try:
             response = requests.post(
-                f"{self.base_url}/playground/metric-evaluation",
-                headers=headers,
+                f'{self.base_url}/playground/metric-evaluation', 
+                headers=headers, 
                 json=metric_schema_mapping,
-                timeout=self.timeout,
-            )
+                timeout=self.timeout
+                )
             if response.status_code == 400:
                 raise ValueError(response.json()["message"])
             response.raise_for_status()
@@ -395,31 +355,24 @@ class Evaluation:
 
     def get_status(self):
         headers = {
-            "Content-Type": "application/json",
+            'Content-Type': 'application/json',
             "Authorization": f"Bearer {os.getenv('RAGAAI_CATALYST_TOKEN')}",
-            "X-Project-Id": str(self.project_id),
+            'X-Project-Id': str(self.project_id),
         }
         try:
             response = requests.get(
-                f"{self.base_url}/job/status", headers=headers, timeout=self.timeout
-            )
+                f'{self.base_url}/job/status', 
+                headers=headers, 
+                timeout=self.timeout)
             response.raise_for_status()
             if response.json()["success"]:
-                status_json = [
-                    item["status"]
-                    for item in response.json()["data"]["content"]
-                    if item["id"] == self.jobId
-                ][0]
+                status_json = [item["status"] for item in response.json()["data"]["content"] if item["id"]==self.jobId][0]
             if status_json == "Failed":
                 return print("Job failed. No results to fetch.")
             elif status_json == "In Progress":
-                return print(
-                    f"Job in progress. Please wait while the job completes.\nVisit Job Status: {self.base_url.removesuffix('/api')}/projects/job-status?projectId={self.project_id} to track"
-                )
+                return print(f"Job in progress. Please wait while the job completes.\nVisit Job Status: {self.base_url.removesuffix('/api')}/projects/job-status?projectId={self.project_id} to track")
             elif status_json == "Completed":
-                print(
-                    f"Job completed. Fetching results.\nVisit Job Status: {self.base_url.removesuffix('/api')}/projects/job-status?projectId={self.project_id} to check"
-                )
+                print(f"Job completed. Fetching results.\nVisit Job Status: {self.base_url.removesuffix('/api')}/projects/job-status?projectId={self.project_id} to check")
         except requests.exceptions.HTTPError as http_err:
             logger.error(f"HTTP error occurred: {http_err}")
         except requests.exceptions.ConnectionError as conn_err:
@@ -435,24 +388,25 @@ class Evaluation:
 
         def get_presignedUrl():
             headers = {
-                "Content-Type": "application/json",
+                'Content-Type': 'application/json',
                 "Authorization": f"Bearer {os.getenv('RAGAAI_CATALYST_TOKEN')}",
-                "X-Project-Id": str(self.project_id),
-            }
-
+                'X-Project-Id': str(self.project_id),
+                }
+            
             data = {
-                "fields": ["*"],
+                "fields": [
+                    "*"
+                ],
                 "datasetId": str(self.dataset_id),
                 "rowFilterList": [],
-                "export": True,
-            }
-            try:
+                "export": True
+                }
+            try:    
                 response = requests.post(
-                    f"{self.base_url}/v1/llm/docs",
-                    headers=headers,
+                    f'{self.base_url}/v1/llm/docs', 
+                    headers=headers, 
                     json=data,
-                    timeout=self.timeout,
-                )
+                    timeout=self.timeout)
                 response.raise_for_status()
                 return response.json()
             except requests.exceptions.HTTPError as http_err:
@@ -491,8 +445,8 @@ class Evaluation:
             df = pd.read_csv(io.StringIO(response_text))
 
             column_list = df.columns.to_list()
-            column_list = [col for col in column_list if not col.startswith("_")]
-            column_list = [col for col in column_list if "." not in col]
+            column_list = [col for col in column_list if not col.startswith('_')]
+            column_list = [col for col in column_list if '.' not in col]
             return df[column_list]
         else:
             return pd.DataFrame()