ragaai-catalyst 1.0.6.1b8__py3-none-any.whl → 1.0.7__py3-none-any.whl

ragaai_catalyst/dataset.py

@@ -18,7 +18,7 @@ class Dataset:
         Dataset.BASE_URL = (
             os.getenv("RAGAAI_CATALYST_BASE_URL")
             if os.getenv("RAGAAI_CATALYST_BASE_URL")
-            else "https://llm-platform.dev4.ragaai.ai/api"
+            else "https://llm-platform.prod5.ragaai.ai/api"
         )
 
     def list_datasets(self):
@@ -36,7 +36,7 @@ class Dataset:
             headers = {
                 "accept": "application/json, text/plain, */*",
                 "authorization": f"Bearer {os.getenv('RAGAAI_CATALYST_TOKEN')}",
-                "X-Project-Name": self.project_name
+                "X-Project-Name": self.project_name,
             }
             params = {
                 "projectName": self.project_name,

ragaai_catalyst/experiment.py

@@ -1,6 +1,7 @@
 import os
 import requests
 import logging
+import pandas as pd
 from .utils import response_checker
 from .ragaai_catalyst import RagaAICatalyst
 
@@ -32,7 +33,7 @@ class Experiment:
         Experiment.BASE_URL = (
             os.getenv("RAGAAI_CATALYST_BASE_URL")
             if os.getenv("RAGAAI_CATALYST_BASE_URL")
-            else "https://llm-platform.dev4.ragaai.ai/api"
+            else "https://llm-platform.prod5.ragaai.ai/api"
         )
         self.project_name = project_name
         self.experiment_name = experiment_name
@@ -66,7 +67,7 @@ class Experiment:
         def make_request():
             headers = {
                 "authorization": f"Bearer {os.getenv('RAGAAI_CATALYST_TOKEN')}",
-                "X-Project-Name": self.project_name
+                "X-Project-Name": self.project_name,
             }
             params = {
                 "name": self.project_name,
@@ -179,7 +180,7 @@ class Experiment:
                 response_checker(response, "Experiment.run_tests"),
             )
 
-    def get_status(self):
+    def get_status(self, job_id=None):
         """
         Retrieves the status of a job based on the provided job ID.
 
@@ -190,13 +191,18 @@ class Experiment:
             "Authorization": f'Bearer {os.getenv("RAGAAI_CATALYST_TOKEN")}',
             "X-Project-Name": self.project_name,
         }
-        if self.job_id is None:
+        if job_id is not None:
+            job_id_to_check = job_id
+        else:
+            job_id_to_check = self.job_id
+
+        if job_id_to_check is None:
             logger.warning("Attempt to fetch status without a valid job ID.")
             return "Please run an experiment test first"
         json_data = {
-            "jobId": self.job_id,
+            "jobId": job_id_to_check,
         }
-        logger.debug(f"Fetching status for Job ID: {self.job_id}")
+        logger.debug(f"Fetching status for Job ID: {job_id_to_check}")
         response = requests.get(
             f"{Experiment.BASE_URL}/job/status",
             headers=headers,
@@ -205,11 +211,10 @@ class Experiment:
         )
         status_code = response_checker(response, "Experiment.get_status")
         if status_code == 200:
-            # print(f"Status retrieved: Job ID {self.job_id} is active.")
             test_response = response.json()
             jobs = test_response["data"]["content"]
             for job in jobs:
-                if job["id"] == self.job_id:
+                if job["id"] == job_id_to_check:
                     return job["status"]
         elif status_code == 401:
             headers = {
@@ -239,7 +244,7 @@ class Experiment:
                 response_checker(response, "Experiment.get_status"),
             )
 
-    def get_results(self):
+    def get_results(self, job_id=None):
         """
         A function that retrieves results based on the experiment ID.
         It makes a POST request to the BASE_URL to fetch results using the provided JSON data.
@@ -247,15 +252,19 @@ class Experiment:
         If the status code is 401, it retries the request and returns the test response if successful.
         If the status is neither 200 nor 401, it logs an error and returns the response checker result.
         """
+        if job_id is not None:
+            job_id_to_use = job_id
+        else:
+            job_id_to_use = self.job_id
 
-        if self.job_id is None:
+        if job_id_to_use is None:
             logger.warning("Results fetch attempted without prior job execution.")
             return "Please run an experiment test first"
 
         headers = {
             "Content-Type": "application/json",
             "Authorization": f'Bearer {os.getenv("RAGAAI_CATALYST_TOKEN")}',
-            "X-Project-Id": self.project_id
+            "X-Project-Id": str(self.project_id),
         }
 
         json_data = {
@@ -266,7 +275,7 @@ class Experiment:
             "filterList": [],
         }
 
-        status_json = self.get_status()
+        status_json = self.get_status(job_id_to_use)
         if status_json == "Failed":
             return print("Job failed. No results to fetch.")
         elif status_json == "In Progress":
@@ -280,16 +289,32 @@ class Experiment:
             json=json_data,
             timeout=Experiment.TIMEOUT,
         )
-        # status_code = response_checker(response, "Experiment.get_test_results")
         if response.status_code == 200:
-            print(f"Results successfully retrieved.")
+            print("Results successfully retrieved.")
             test_response = response.json()
-            return test_response
+
+            if test_response["success"]:
+                parse_success, parsed_response = self.parse_response(test_response)
+                if parse_success:
+                    return parsed_response
+                else:
+                    logger.error(f"Failed to parse response: {test_response}")
+                    raise FailedToRetrieveResults(
+                        f"Failed to parse response: {test_response}"
+                    )
+
+            else:
+                logger.error(f"Failed to retrieve results for job: {job_id_to_use}")
+                raise FailedToRetrieveResults(
+                    f"Failed to retrieve results for job: {job_id_to_use}"
+                )
+
+            return parsed_response
         elif response.status_code == 401:
             headers = {
                 "Content-Type": "application/json",
                 "Authorization": f'Bearer {os.getenv("RAGAAI_CATALYST_TOKEN")}',
-                "X-Project-Id": self.project_id
+                "X-Project-Id": str(self.project_id),
             }
             response = requests.post(
                 f"{Experiment.BASE_URL}/v1/llm/docs",
@@ -297,7 +322,6 @@ class Experiment:
                 json=json_data,
                 timeout=Experiment.TIMEOUT,
             )
-            # status_code = response_checker(response, "Experiment.get_test_results")
             if response.status_code == 200:
                 test_response = response.json()
                 return test_response
@@ -305,8 +329,61 @@ class Experiment:
                 logger.error("Endpoint not responsive after retry attempts.")
                 return response_checker(response, "Experiment.get_test_results")
         else:
-
             return (
                 "Error in running tests",
                 response_checker(response, "Experiment.get_test_results"),
             )
+
+    def parse_response(self, response):
+        """
+        Parse the response to get the results
+        """
+        try:
+            x = pd.DataFrame(response["data"]["docs"])
+
+            column_names_to_replace = [
+                {item["columnName"]: item["displayName"]}
+                for item in response["data"]["columns"]
+            ]
+
+            if column_names_to_replace:
+                for item in column_names_to_replace:
+                    x = x.rename(columns=item)
+
+                dict_cols = [
+                    col
+                    for col in x.columns
+                    if x[col].dtype == "object"
+                    and x[col].apply(lambda y: isinstance(y, dict)).any()
+                ]
+
+                for dict_col in dict_cols:
+                    x[f"{dict_col}_reason"] = x[dict_col].apply(
+                        lambda y: y.get("reason") if isinstance(y, dict) else None
+                    )
+                    x[f"{dict_col}_metric_config"] = x[dict_col].apply(
+                        lambda y: (
+                            y.get("metric_config") if isinstance(y, dict) else None
+                        )
+                    )
+                    x[f"{dict_col}_status"] = x[dict_col].apply(
+                        lambda y: y.get("status") if isinstance(y, dict) else None
+                    )
+
+                    x = x.drop(columns=[dict_col])
+
+            x.columns = x.columns.str.replace("_reason_reason", "_reason")
+            x.columns = x.columns.str.replace("_reason_metric_config", "_metric_config")
+            x.columns = x.columns.str.replace("_reason_status", "_status")
+
+            x = x.drop(columns=["trace_uri"])
+
+            return True, x
+
+        except Exception as e:
+            logger.error(f"Failed to parse response: {e}", exc_info=True)
+            return False, pd.DataFrame()
+
+
+class FailedToRetrieveResults(Exception):
+    pass

{ragaai_catalyst-1.0.6.1b8.dist-info → ragaai_catalyst-1.0.7.dist-info}/METADATA

@@ -1,20 +1,26 @@
 Metadata-Version: 2.1
 Name: ragaai_catalyst
-Version: 1.0.6.1b8
+Version: 1.0.7
 Summary: RAGA AI CATALYST
 Author-email: Kiran Scaria <kiran.scaria@raga.ai>, Kedar Gaikwad <kedar.gaikwad@raga.ai>, Dushyant Mahajan <dushyant.mahajan@raga.ai>
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 Requires-Dist: aiohttp>=3.10.2
-Requires-Dist: opentelemetry-instrumentation-langchain~=0.28.0
-Requires-Dist: opentelemetry-instrumentation-openai~=0.28.0
-Requires-Dist: opentelemetry-semantic-conventions~=0.48b0
-Requires-Dist: opentelemetry-instrumentation~=0.48b0
-Requires-Dist: opentelemetry-api>=1.26.0
-Requires-Dist: opentelemetry-sdk>=1.26.0
+Requires-Dist: opentelemetry-api==1.25.0
+Requires-Dist: opentelemetry-sdk==1.25.0
+Requires-Dist: opentelemetry-exporter-otlp-proto-grpc==1.25.0
+Requires-Dist: opentelemetry-instrumentation==0.46b0
+Requires-Dist: opentelemetry-instrumentation-fastapi==0.46b0
+Requires-Dist: opentelemetry-instrumentation-asgi==0.46b0
+Requires-Dist: opentelemetry-semantic-conventions==0.46b0
+Requires-Dist: opentelemetry-util-http==0.46b0
+Requires-Dist: opentelemetry-instrumentation-langchain~=0.24.0
+Requires-Dist: opentelemetry-instrumentation-openai~=0.24.0
 Requires-Dist: langchain-core~=0.2.11
 Requires-Dist: langchain~=0.2.11
-Requires-Dist: openai>=1.36.0
+Requires-Dist: openai>=1.35.10
+Requires-Dist: pandas>=2.1.1
+Requires-Dist: tenacity==8.3.0
 Provides-Extra: dev
 Requires-Dist: pytest; extra == "dev"
 Requires-Dist: pytest-cov; extra == "dev"

{ragaai_catalyst-1.0.6.1b8.dist-info → ragaai_catalyst-1.0.7.dist-info}/RECORD

@@ -1,7 +1,7 @@
 ragaai_catalyst/__init__.py,sha256=0hf_H1xkhrpGnUWTbjvYOXXQta1S4JO2yud1Eqydt-I,226
 ragaai_catalyst/_version.py,sha256=JKt9KaVNOMVeGs8ojO6LvIZr7ZkMzNN-gCcvryy4x8E,460
-ragaai_catalyst/dataset.py,sha256=ok_EX6uCVbriOtWEWIcwoLG6oXe2muhwo6XkmGRUo7A,3558
-ragaai_catalyst/experiment.py,sha256=fHNH2RvyhR_MgOI3IwU5noCq-vXMXmVmgzmKkaZEY80,11964
+ragaai_catalyst/dataset.py,sha256=g0YH0QGDpCiJ4VULB3tukGFNLpWGw0ZRwLMP07ay2IE,3560
+ragaai_catalyst/experiment.py,sha256=dlp2S7kGHhz0IOjJ6vUnGGbZQ52U3PZbfFbWFiT26AM,14665
 ragaai_catalyst/ragaai_catalyst.py,sha256=DqEApmjxQufZZT4ap-IzoBmp3AX6RWMq7HXQ5kO0wsI,16468
 ragaai_catalyst/utils.py,sha256=TlhEFwLyRU690HvANbyoRycR3nQ67lxVUQoUOfTPYQ0,3772
 ragaai_catalyst/tracers/__init__.py,sha256=NppmJhD3sQ5R1q6teaZLS7rULj08Gb6JT8XiPRIe_B0,49
@@ -15,7 +15,7 @@ ragaai_catalyst/tracers/instrumentators/llamaindex.py,sha256=SMrRlR4xM7k9HK43hak
 ragaai_catalyst/tracers/instrumentators/openai.py,sha256=14R4KW9wQCR1xysLfsP_nxS7cqXrTPoD8En4MBAaZUU,379
 ragaai_catalyst/tracers/utils/__init__.py,sha256=KeMaZtYaTojilpLv65qH08QmpYclfpacDA0U3wg6Ybw,64
 ragaai_catalyst/tracers/utils/utils.py,sha256=ViygfJ7vZ7U0CTSA1lbxVloHp4NSlmfDzBRNCJuMhis,2374
-ragaai_catalyst-1.0.6.1b8.dist-info/METADATA,sha256=D4Evb1EwxmR3oEdhmsS5ZEGOYZLcSR7jGc2m87v8gR4,4834
-ragaai_catalyst-1.0.6.1b8.dist-info/WHEEL,sha256=UvcQYKBHoFqaQd6LKyqHw9fxEolWLQnlzP0h_LgJAfI,91
-ragaai_catalyst-1.0.6.1b8.dist-info/top_level.txt,sha256=HpgsdRgEJMk8nqrU6qdCYk3di7MJkDL0B19lkc7dLfM,16
-ragaai_catalyst-1.0.6.1b8.dist-info/RECORD,,
+ragaai_catalyst-1.0.7.dist-info/METADATA,sha256=fBubovTWPHax6eI9LinGIRuVz4k_AU4U8yAp4wilAjY,5119
+ragaai_catalyst-1.0.7.dist-info/WHEEL,sha256=UvcQYKBHoFqaQd6LKyqHw9fxEolWLQnlzP0h_LgJAfI,91
+ragaai_catalyst-1.0.7.dist-info/top_level.txt,sha256=HpgsdRgEJMk8nqrU6qdCYk3di7MJkDL0B19lkc7dLfM,16
+ragaai_catalyst-1.0.7.dist-info/RECORD,,