ragaai-catalyst 2.1.5b24__tar.gz → 2.1.5b26__tar.gz

{ragaai_catalyst-2.1.5b24/ragaai_catalyst.egg-info → ragaai_catalyst-2.1.5b26}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: ragaai_catalyst
-Version: 2.1.5b24
+Version: 2.1.5b26
 Summary: RAGA AI CATALYST
 Author-email: Kiran Scaria <kiran.scaria@raga.ai>, Kedar Gaikwad <kedar.gaikwad@raga.ai>, Dushyant Mahajan <dushyant.mahajan@raga.ai>, Siddhartha Kosti <siddhartha.kosti@raga.ai>, Ritika Goel <ritika.goel@raga.ai>, Vijay Chaurasia <vijay.chaurasia@raga.ai>
 Requires-Python: <3.13,>=3.9
@@ -28,7 +28,7 @@ Requires-Dist: Markdown>=3.7
 Requires-Dist: litellm==1.51.1
 Requires-Dist: tenacity==8.3.0
 Requires-Dist: tqdm>=4.66.5
-Requires-Dist: llama-index<0.11.0,>=0.10.0
+Requires-Dist: llama-index>=0.10.0
 Requires-Dist: pyopenssl>=24.2.1
 Requires-Dist: psutil~=6.0.0
 Requires-Dist: py-cpuinfo~=9.0.0
@@ -49,6 +49,8 @@ Requires-Dist: flake8; extra == "dev"
 
 RagaAI Catalyst is a comprehensive platform designed to enhance the management and optimization of LLM projects. It offers a wide range of features, including project management, dataset management, evaluation management, trace management, prompt management, synthetic data generation, and guardrail management. These functionalities enable you to efficiently evaluate, and safeguard your LLM applications.
 
+![RagaAI Catalyst](docs/img/main.png)
+
 ## Table of Contents
 
 - [RagaAI Catalyst](#ragaai-catalyst)
@@ -110,6 +112,7 @@ catalyst.project_use_cases()
 projects = catalyst.list_projects()
 print(projects)
 ```
+![Projects](docs/img/projects_new.png)
 
 ### Dataset Management
 Manage datasets efficiently for your projects:
@@ -135,6 +138,7 @@ dataset_manager.create_from_csv(
 dataset_manager.get_schema_mapping()
 
 ```
+![Dataset](docs/img/dataset.png)
 
 For more detailed information on Dataset Management, including CSV schema handling and advanced usage, please refer to the [Dataset Management documentation](docs/dataset_management.md).
 
@@ -189,7 +193,7 @@ print("Experiment Status:", status)
 results = evaluation.get_results()
 print("Experiment Results:", results)
 ```
-
+![Evaluation](docs/img/evaluation.png)
 
 
 ### Trace Management
@@ -221,6 +225,7 @@ tracer.stop()
 # Get upload status
 tracer.get_upload_status()
 ```
+![Trace](docs/img/trace_comp.png)
 
 
 ### Prompt Management
@@ -394,6 +399,7 @@ llm_caller = 'litellm'
 executor([message],prompt_params,model_params,llm_caller)
 
 ```
+![Guardrails](docs/img/guardrails.png)
 
 ### Agentic Tracing
 
@@ -432,6 +438,7 @@ with tracer:
     # Agent execution code
     pass
 ```
+![Tracing](docs/img/last_main.png)
 
 ### Red-teaming
 

{ragaai_catalyst-2.1.5b24 → ragaai_catalyst-2.1.5b26}/README.md

@@ -2,6 +2,8 @@
 
 RagaAI Catalyst is a comprehensive platform designed to enhance the management and optimization of LLM projects. It offers a wide range of features, including project management, dataset management, evaluation management, trace management, prompt management, synthetic data generation, and guardrail management. These functionalities enable you to efficiently evaluate, and safeguard your LLM applications.
 
+![RagaAI Catalyst](docs/img/main.png)
+
 ## Table of Contents
 
 - [RagaAI Catalyst](#ragaai-catalyst)
@@ -63,6 +65,7 @@ catalyst.project_use_cases()
 projects = catalyst.list_projects()
 print(projects)
 ```
+![Projects](docs/img/projects_new.png)
 
 ### Dataset Management
 Manage datasets efficiently for your projects:
@@ -88,6 +91,7 @@ dataset_manager.create_from_csv(
 dataset_manager.get_schema_mapping()
 
 ```
+![Dataset](docs/img/dataset.png)
 
 For more detailed information on Dataset Management, including CSV schema handling and advanced usage, please refer to the [Dataset Management documentation](docs/dataset_management.md).
 
@@ -142,7 +146,7 @@ print("Experiment Status:", status)
 results = evaluation.get_results()
 print("Experiment Results:", results)
 ```
-
+![Evaluation](docs/img/evaluation.png)
 
 
 ### Trace Management
@@ -174,6 +178,7 @@ tracer.stop()
 # Get upload status
 tracer.get_upload_status()
 ```
+![Trace](docs/img/trace_comp.png)
 
 
 ### Prompt Management
@@ -347,6 +352,7 @@ llm_caller = 'litellm'
 executor([message],prompt_params,model_params,llm_caller)
 
 ```
+![Guardrails](docs/img/guardrails.png)
 
 ### Agentic Tracing
 
@@ -385,6 +391,7 @@ with tracer:
     # Agent execution code
     pass
 ```
+![Tracing](docs/img/last_main.png)
 
 ### Red-teaming
 
@@ -417,4 +424,4 @@ def model(df: pd.DataFrame):
 
 
 scan_df = rt.run_scan(model=model, evaluators=["llm"], save_report=True)
-```
+```

ragaai_catalyst-2.1.5b26/examples/travel_agent/config.py

@@ -0,0 +1,27 @@
+
+
+import os
+from dotenv import load_dotenv
+
+from ragaai_catalyst import RagaAICatalyst, Tracer, init_tracing
+import uuid
+def initialize_tracing():
+    load_dotenv()
+    print(f"Access key: {os.getenv('RAGAAI_CATALYST_STAGING_ACCESS_KEY')}")
+    print(f"Secret key: {os.getenv('RAGAAI_CATALYST_STAGING_SECRET_KEY')}")
+    print(f"Base url: {os.getenv('RAGAAI_CATALYST_STAGING_BASE_URL')}")
+
+    catalyst = RagaAICatalyst(
+        access_key=os.getenv("RAGAAI_CATALYST_STAGING_ACCESS_KEY"),
+        secret_key=os.getenv("RAGAAI_CATALYST_STAGING_SECRET_KEY"),
+        base_url=os.getenv("RAGAAI_CATALYST_STAGING_BASE_URL"),
+    )
+
+    tracer = Tracer(
+        project_name="Execute_Metric_Test1",
+        dataset_name="travel_agent_dataset",
+        tracer_type="Agentic",  # langchain, llamaindex, Agentic
+    )
+
+    init_tracing(catalyst=catalyst, tracer=tracer)
+    return tracer

{ragaai_catalyst-2.1.5b24 → ragaai_catalyst-2.1.5b26}/examples/travel_agent/main.py

@@ -1,3 +1,6 @@
+import sys 
+sys.path.append('.')
+
 from dotenv import load_dotenv
 from tools import (
     llm_call,
@@ -112,4 +115,3 @@ def travel_agent():
 if __name__ == "__main__":
     with tracer:
         travel_agent()
-    

{ragaai_catalyst-2.1.5b24 → ragaai_catalyst-2.1.5b26}/examples/travel_agent/tools.py

@@ -26,6 +26,7 @@ def llm_call(prompt, max_tokens=512, model="gpt-4o-mini", name="default"):
         name="Hallucination",
         model="gpt-4o-mini",
         provider="openai",
+        lazy=True,
         display_name="Hallucination_display",
         mapping={
             'prompt': "goa to kashmir price",

{ragaai_catalyst-2.1.5b24 → ragaai_catalyst-2.1.5b26}/pyproject.toml

@@ -8,8 +8,7 @@ description = "RAGA AI CATALYST"
 readme = "README.md"
 requires-python = ">=3.9,<3.13"
 # license = {file = "LICENSE"}
-
-version = "2.1.5.b24"
+version = "2.1.5.b26"
 authors = [
     {name = "Kiran Scaria", email = "kiran.scaria@raga.ai"},
     {name = "Kedar Gaikwad", email = "kedar.gaikwad@raga.ai"},
@@ -42,7 +41,7 @@ dependencies = [
     "litellm==1.51.1",  
     "tenacity==8.3.0",
     "tqdm>=4.66.5",
-    "llama-index>=0.10.0,<0.11.0",
+    "llama-index>=0.10.0",
     "pyopenssl>=24.2.1",
     "psutil~=6.0.0",
     "py-cpuinfo~=9.0.0",

{ragaai_catalyst-2.1.5b24 → ragaai_catalyst-2.1.5b26}/ragaai_catalyst/evaluation.py

@@ -4,6 +4,7 @@ import pandas as pd
 import io
 from .ragaai_catalyst import RagaAICatalyst
 import logging
+import json
 
 logger = logging.getLogger(__name__)
 
@@ -18,7 +19,7 @@ class Evaluation:
         self.project_name = project_name
         self.dataset_name = dataset_name
         self.base_url = f"{RagaAICatalyst.BASE_URL}"
-        self.timeout = 10
+        self.timeout = 20
         self.jobId = None
         self.num_projects=99999
 
@@ -357,6 +358,52 @@ class Evaluation:
         except Exception as e:
             logger.error(f"An unexpected error occurred: {e}")
 
+    def append_metrics(self, display_name):
+        if not isinstance(display_name, str):
+            raise ValueError("display_name should be a string")
+        
+        headers = {
+            "Authorization": f"Bearer {os.getenv('RAGAAI_CATALYST_TOKEN')}",
+            'X-Project-Id': str(self.project_id),
+            'Content-Type': 'application/json',
+        }
+
+        payload = json.dumps({
+        "datasetId": self.dataset_id,
+        "metricParams": [
+            {
+            "metricSpec": {
+                "displayName": display_name
+            }
+            }
+        ]
+        })
+        
+        try:
+            response = requests.request(
+                "POST", 
+                f'{self.base_url}/v2/llm/metric-evaluation-rerun', 
+                headers=headers, 
+                data=payload, 
+                timeout=self.timeout)
+            if response.status_code == 400:
+                raise ValueError(response.json()["message"])
+            response.raise_for_status()
+            if response.json()["success"]:
+                print(response.json()["message"])
+                self.jobId = response.json()["data"]["jobId"]
+
+        except requests.exceptions.HTTPError as http_err:
+            logger.error(f"HTTP error occurred: {http_err}")
+        except requests.exceptions.ConnectionError as conn_err:
+            logger.error(f"Connection error occurred: {conn_err}")
+        except requests.exceptions.Timeout as timeout_err:
+            logger.error(f"Timeout error occurred: {timeout_err}")
+        except requests.exceptions.RequestException as req_err:
+            logger.error(f"An error occurred: {req_err}")
+        except Exception as e:
+            logger.error(f"An unexpected error occurred: {e}")
+
     def get_status(self):
         headers = {
             'Content-Type': 'application/json',
@@ -463,8 +510,11 @@ class Evaluation:
             df = pd.read_csv(io.StringIO(response_text))
 
             column_list = df.columns.to_list()
+            # Remove unwanted columns
             column_list = [col for col in column_list if not col.startswith('_')]
             column_list = [col for col in column_list if '.' not in col]
+            # Remove _claims_ columns
+            column_list = [col for col in column_list if '_claims_' not in col]
             return df[column_list]
         else:
             return pd.DataFrame()

{ragaai_catalyst-2.1.5b24 → ragaai_catalyst-2.1.5b26}/ragaai_catalyst/tracers/agentic_tracing/tracers/agent_tracer.py

@@ -106,48 +106,42 @@ class AgentTracerMixin:
                     if gt is not None:
                         span = self.span(name)
                         span.add_gt(gt)
-                    # Set agent context before initializing
-                    component_id = str(uuid.uuid4())
-                    hash_id = top_level_hash_id
-
-                    # Store the component ID in the instance
-                    self._agent_component_id = component_id
-
-                    # Get parent agent ID if exists
-                    parent_agent_id = tracer.current_agent_id.get()
-
-                    # Create agent component
-                    agent_component = tracer.create_agent_component(
-                        component_id=component_id,
-                        hash_id=hash_id,
-                        name=name,
-                        agent_type=agent_type,
-                        version=version,
-                        capabilities=capabilities or [],
-                        start_time=datetime.now().astimezone().isoformat(),
-                        memory_used=0,
-                        input_data=tracer._sanitize_input(args, kwargs),
-                        output_data=None,
-                        children=[],
-                        parent_id=parent_agent_id,
-                    )
-
-                    # Store component for later updates
-                    if not hasattr(tracer, "_agent_components"):
-                        tracer._agent_components = {}
-                    tracer._agent_components[component_id] = agent_component
 
-                    # If this is a nested agent, add it to parent's children
-                    if parent_agent_id:
-                        parent_children = tracer.agent_children.get()
-                        parent_children.append(agent_component)
-                        tracer.agent_children.set(parent_children)
-                    else:
-                        # Only add to root components if no parent
-                        tracer.add_component(agent_component)
-
-                    # Call original __init__ with this agent as current
-                    token = tracer.current_agent_id.set(component_id)
+                    if not hasattr(self, '_agent_component_id'):
+                        component_id = str(uuid.uuid4())
+                        self._agent_component_id = component_id
+                        
+                        # Get parent agent ID if exists
+                        parent_agent_id = tracer.current_agent_id.get()
+                        
+                        agent_component = tracer.create_agent_component(
+                            component_id=component_id,
+                            hash_id=top_level_hash_id,
+                            name=name,
+                            agent_type=agent_type,
+                            version=version,
+                            capabilities=capabilities or [],
+                            start_time=datetime.now().astimezone().isoformat(),
+                            memory_used=0,
+                            input_data=tracer._sanitize_input(args, kwargs),
+                            output_data=None,
+                            children=[],
+                            parent_id=parent_agent_id,
+                        )
+                        
+                        if not hasattr(tracer, "_agent_components"):
+                            tracer._agent_components = {}
+                        tracer._agent_components[component_id] = agent_component
+                        
+                        # For class agents, only add to parent's children if parent exists
+                        if parent_agent_id and parent_agent_id in tracer._agent_components:
+                            parent_component = tracer._agent_components[parent_agent_id]
+                            if not hasattr(parent_component, "children"):
+                                parent_component["children"] = []
+                            if component_id not in parent_component["children"]:
+                                parent_component["children"].append(component_id)
+                    
+                    token = tracer.current_agent_id.set(self._agent_component_id)
                     try:
                         original_init(self, *args, **kwargs)
                     finally:
@@ -160,7 +154,6 @@ class AgentTracerMixin:
                         if callable(attr_value):
 
                             def wrap_method(method):
-                                @self.file_tracker.trace_decorator
                                 @functools.wraps(method)
                                 def wrapped_method(self, *args, **kwargs):
                                     gt = kwargs.get("gt") if kwargs else None
@@ -225,15 +218,60 @@ class AgentTracerMixin:
                             setattr(target, attr_name, wrap_method(attr_value))
 
                 # Replace __init__ with wrapped version
+
                 target.__init__ = wrapped_init
+
+                # Wrap all methods to maintain parent-child relationship
+                for attr_name, attr_value in target.__dict__.items():
+                    if callable(attr_value) and not attr_name.startswith('__'):
+                        original_method = attr_value
+                        
+                        def create_wrapper(method):
+                            @self.file_tracker.trace_decorator
+                            @functools.wraps(method)
+                            def method_wrapper(self, *args, **kwargs):
+                                gt = kwargs.get("gt") if kwargs else None
+                                if gt is not None:
+                                    span = tracer.span(name)
+                                    span.add_gt(gt)
+                                # Use the class instance's agent ID as parent
+                                parent_id = getattr(self, '_agent_component_id', None)
+                                if parent_id:
+                                    if asyncio.iscoroutinefunction(method):
+                                        return tracer._trace_agent_execution(
+                                            method.__get__(self, type(self)),
+                                            name,
+                                            agent_type,
+                                            version,
+                                            capabilities,
+                                            top_level_hash_id,
+                                            *args,
+                                            **kwargs,
+                                        )
+                                    else:
+                                        return tracer._trace_sync_agent_execution(
+                                            method.__get__(self, type(self)),
+                                            name,
+                                            agent_type,
+                                            version,
+                                            capabilities,
+                                            top_level_hash_id,
+                                            *args,
+                                            **kwargs,
+                                        )
+                                else:
+                                    return method(self, *args, **kwargs)
+                            return method_wrapper
+                        
+                        setattr(target, attr_name, create_wrapper(original_method))
+
                 return target
             else:
-                # For function decorators, use existing sync/async tracing
-                is_async = asyncio.iscoroutinefunction(target)
-                if is_async:
-
-                    async def wrapper(*args, **kwargs):
-                        return await self._trace_agent_execution(
+                # For non-class targets (e.g., functions), use existing function wrapping logic
+                @functools.wraps(target)
+                def wrapper(*args, **kwargs):
+                    if asyncio.iscoroutinefunction(target):
+                        return tracer._trace_agent_execution(
                             target,
                             name,
                             agent_type,
@@ -243,12 +281,8 @@ class AgentTracerMixin:
                             *args,
                             **kwargs,
                         )
-
-                    return wrapper
-                else:
-
-                    def wrapper(*args, **kwargs):
-                        return self._trace_sync_agent_execution(
+                    else:
+                        return tracer._trace_sync_agent_execution(
                             target,
                             name,
                             agent_type,
@@ -258,16 +292,13 @@ class AgentTracerMixin:
                             *args,
                             **kwargs,
                         )
-
-                    return wrapper
+                return wrapper
 
         return decorator
 
     def _trace_sync_agent_execution(
-            self, func, name, agent_type, version, capabilities, top_level_hash_id, *args, **kwargs
+        self, func, name, agent_type, version, capabilities, top_level_hash_id, *args, **kwargs
     ):
-        hash_id = top_level_hash_id
-
         """Synchronous version of agent tracing"""
         if not self.is_active:
             return func(*args, **kwargs)
@@ -303,7 +334,7 @@ class AgentTracerMixin:
 
         try:
             # Execute the agent
-            result = self.file_tracker.trace_wrapper(func)(*args, **kwargs)
+            result = func(*args, **kwargs)
 
             # Calculate resource usage
             end_memory = psutil.Process().memory_info().rss
@@ -318,7 +349,7 @@ class AgentTracerMixin:
             # Create agent component with children and parent if exists
             agent_component = self.create_agent_component(
                 component_id=component_id,
-                hash_id=hash_id,
+                hash_id=top_level_hash_id,
                 name=name,
                 agent_type=agent_type,
                 version=version,
@@ -331,12 +362,17 @@ class AgentTracerMixin:
                 parent_id=parent_agent_id,
             )
 
-            # Add this component as a child to parent's children list
-            parent_children.append(agent_component)
-            self.agent_children.set(parent_children)
+            # Store component for updates
+            if not hasattr(self, "_agent_components"):
+                self._agent_components = {}
+            self._agent_components[component_id] = agent_component
 
-            # Only add to root components if no parent
-            if not parent_agent_id:
+            # Only add to hierarchy if this is a root component (no parent)
+            # or if the parent explicitly added it as a child
+            if parent_agent_id:
+                parent_children.append(agent_component)
+                self.agent_children.set(parent_children)
+            else:
                 self.add_component(agent_component)
 
             return result
@@ -351,16 +387,10 @@ class AgentTracerMixin:
             # Get children even in case of error
             children = self.agent_children.get()
 
-            # Set parent_id for all children
-            for child in children:
-                child["parent_id"] = component_id
-
-            # End tracking network calls for this component
-            self.end_component(component_id)
-
+            # Create error component
             agent_component = self.create_agent_component(
                 component_id=component_id,
-                hash_id=hash_id,
+                hash_id=top_level_hash_id,
                 name=name,
                 agent_type=agent_type,
                 version=version,
@@ -373,16 +403,20 @@ class AgentTracerMixin:
                 children=children,
                 parent_id=parent_agent_id,  # Add parent ID if exists
             )
-            # If this is a nested agent, add it to parent's children
+
+            # Store component for updates
+            if not hasattr(self, "_agent_components"):
+                self._agent_components = {}
+            self._agent_components[component_id] = agent_component
+
+            # Only add to hierarchy if this is a root component (no parent)
+            # or if the parent explicitly added it as a child
             if parent_agent_id:
-                parent_component = self._agent_components.get(parent_agent_id)
-                if parent_component:
-                    if "children" not in parent_component["data"]:
-                        parent_component["data"]["children"] = []
-                    parent_component["data"]["children"].append(agent_component)
+                parent_children.append(agent_component)
+                self.agent_children.set(parent_children)
             else:
                 # Only add to root components if no parent
-                self.add_component(agent_component)
+                self.add_component(agent_component, is_error=True)
 
             raise
         finally:
@@ -424,7 +458,7 @@ class AgentTracerMixin:
 
         try:
             # Execute the agent
-            result = await self.file_tracker.trace_wrapper(func)(*args, **kwargs)
+            result = await func(*args, **kwargs)
 
             # Calculate resource usage
             end_memory = psutil.Process().memory_info().rss
@@ -451,12 +485,17 @@ class AgentTracerMixin:
                 parent_id=parent_agent_id,
             )
 
-            # Add this component as a child to parent's children list
-            parent_children.append(agent_component)
-            self.agent_children.set(parent_children)
+            # Store component for updates
+            if not hasattr(self, "_agent_components"):
+                self._agent_components = {}
+            self._agent_components[component_id] = agent_component
 
-            # Only add to root components if no parent
-            if not parent_agent_id:
+            # Only add to hierarchy if this is a root component (no parent)
+            # or if the parent explicitly added it as a child
+            if parent_agent_id:
+                parent_children.append(agent_component)
+                self.agent_children.set(parent_children)
+            else:
                 self.add_component(agent_component)
 
             return result
@@ -471,13 +510,7 @@ class AgentTracerMixin:
             # Get children even in case of error
             children = self.agent_children.get()
 
-            # Set parent_id for all children
-            for child in children:
-                child["parent_id"] = component_id
-
-            # End tracking network calls for this component
-            self.end_component(component_id)
-
+            # Create error component
             agent_component = self.create_agent_component(
                 component_id=component_id,
                 hash_id=hash_id,
@@ -494,16 +527,19 @@ class AgentTracerMixin:
                 parent_id=parent_agent_id,  # Add parent ID if exists
             )
 
-            # If this is a nested agent, add it to parent's children
+            # Store component for updates
+            if not hasattr(self, "_agent_components"):
+                self._agent_components = {}
+            self._agent_components[component_id] = agent_component
+
+            # Only add to hierarchy if this is a root component (no parent)
+            # or if the parent explicitly added it as a child
             if parent_agent_id:
-                parent_component = self._agent_components.get(parent_agent_id)
-                if parent_component:
-                    if "children" not in parent_component["data"]:
-                        parent_component["data"]["children"] = []
-                    parent_component["data"]["children"].append(agent_component)
+                parent_children.append(agent_component)
+                self.agent_children.set(parent_children)
             else:
                 # Only add to root components if no parent
-                self.add_component(agent_component)
+                self.add_component(agent_component, is_error=True)
 
             raise
         finally:

{ragaai_catalyst-2.1.5b24 → ragaai_catalyst-2.1.5b26}/ragaai_catalyst/tracers/agentic_tracing/tracers/base.py

@@ -137,6 +137,7 @@ class BaseTracer:
         """Initialize a new trace"""
         self.tracking = True
         self.trace_id = str(uuid.uuid4())
+        self.file_tracker.trace_main_file()
         self.system_monitor = SystemMonitor(self.trace_id)
         threading.Thread(target=self._track_memory_usage).start()
         threading.Thread(target=self._track_cpu_usage).start()