ragaai-catalyst 2.0.7.2b1__py3-none-any.whl → 2.1b1__py3-none-any.whl

ragaai_catalyst/dataset.py

@@ -5,7 +5,6 @@ from typing import Union
 import logging
 from .ragaai_catalyst import RagaAICatalyst
 import pandas as pd
-import pdb
 logger = logging.getLogger(__name__)
 get_token = RagaAICatalyst.get_token
 
@@ -202,7 +201,6 @@ class Dataset:
 
         #### put csv to presigned URL
         def put_csv_to_presignedUrl(url):
-            # pdb.set_trace()
             headers = {
                 'Content-Type': 'text/csv',
                 'x-ms-blob-type': 'BlockBlob',
@@ -224,7 +222,6 @@ class Dataset:
         try:
 
             put_csv_response = put_csv_to_presignedUrl(url)
-            # pdb.set_trace()
             print(put_csv_response)
             if put_csv_response.status_code not in (200, 201):
                 raise ValueError('Unable to put csv to the presignedUrl')

ragaai_catalyst/evaluation.py

@@ -4,7 +4,6 @@ import pandas as pd
 import io
 from .ragaai_catalyst import RagaAICatalyst
 import logging
-import pdb
 
 logger = logging.getLogger(__name__)
 
@@ -330,7 +329,7 @@ class Evaluation:
         metric_schema_mapping = self._update_base_json(metrics)
         try:
             response = requests.post(
-                f'{self.base_url}/playground/metric-evaluation', 
+                f'{self.base_url}/v2/llm/metric-evaluation',
                 headers=headers, 
                 json=metric_schema_mapping,
                 timeout=self.timeout

ragaai_catalyst/tracers/__init__.py

@@ -1,3 +1,3 @@
 from .tracer import Tracer
 
-__all__ = ["Tracer"]
+__all__ = ["Tracer"] 

ragaai_catalyst/tracers/agentic_tracing/agent_tracer.py

@@ -4,43 +4,157 @@ from datetime import datetime
 import psutil
 from typing import Optional, Any, Dict, List
 from .unique_decorator import mydecorator
+from .unique_decorator import generate_unique_hash_simple
 
 import contextvars
 import asyncio
+from .file_name_tracker import TrackName
+
 
 class AgentTracerMixin:
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
+        self.file_tracker = TrackName()
         self.current_agent_id = contextvars.ContextVar("agent_id", default=None)
         self.current_agent_name = contextvars.ContextVar("agent_name", default=None)
         self.agent_children = contextvars.ContextVar("agent_children", default=[])
         self.component_network_calls = contextvars.ContextVar("component_network_calls", default={})
-        self._trace_sync_agent_execution = mydecorator(self._trace_sync_agent_execution)
-        self._trace_agent_execution = mydecorator(self._trace_agent_execution)
+        self.component_user_interaction = contextvars.ContextVar("component_user_interaction", default={})
+        self.gt = None
 
 
     def trace_agent(self, name: str, agent_type: str = "generic", version: str = "1.0.0", capabilities: List[str] = None):
-        def decorator(func):
-            # Check if the function is async
-            is_async = asyncio.iscoroutinefunction(func)
-
-            @functools.wraps(func)
-            async def async_wrapper(*args, **kwargs):
-                return await self._trace_agent_execution(
-                    func, name, agent_type, version, capabilities, *args, **kwargs
-                )
-
-            @functools.wraps(func)
-            def sync_wrapper(*args, **kwargs):
-                return self._trace_sync_agent_execution(
-                    func, name, agent_type, version, capabilities, *args, **kwargs
-                )
+        def decorator(target):
+            # Check if target is a class
+            is_class = isinstance(target, type)
+            tracer = self  # Store reference to tracer instance
+            top_level_hash_id = generate_unique_hash_simple(target)   # Generate hash based on the decorated target code
 
-            return async_wrapper if is_async else sync_wrapper
+            
+            if is_class:
+                # Store original __init__
+                original_init = target.__init__
+                
+                def wrapped_init(self, *args, **kwargs):
+                    self.gt = kwargs.get('gt', None) if kwargs else None
+                    # Set agent context before initializing
+                    component_id = str(uuid.uuid4())
+                    hash_id = top_level_hash_id
+                    
+                    # Store the component ID in the instance
+                    self._agent_component_id = component_id
+                    
+                    # Get parent agent ID if exists
+                    parent_agent_id = tracer.current_agent_id.get()
+                    
+                    # Create agent component
+                    agent_component = tracer.create_agent_component(
+                        component_id=component_id,
+                        hash_id=hash_id,
+                        name=name,
+                        agent_type=agent_type,
+                        version=version,
+                        capabilities=capabilities or [],
+                        start_time=datetime.now(),
+                        end_time=datetime.now(),
+                        memory_used=0,
+                        input_data=tracer._sanitize_input(args, kwargs),
+                        output_data=None,
+                        children=[],
+                        parent_id=parent_agent_id
+                    )
+                      
+                    # Store component for later updates
+                    if not hasattr(tracer, '_agent_components'):
+                        tracer._agent_components = {}
+                    tracer._agent_components[component_id] = agent_component
+                    
+                    # If this is a nested agent, add it to parent's children
+                    if parent_agent_id:
+                        parent_children = tracer.agent_children.get()
+                        parent_children.append(agent_component)
+                        tracer.agent_children.set(parent_children)
+                    else:
+                        # Only add to root components if no parent
+                        tracer.add_component(agent_component)
+                    
+                    # Call original __init__ with this agent as current
+                    token = tracer.current_agent_id.set(component_id)
+                    try:
+                        original_init(self, *args, **kwargs)
+                    finally:
+                        tracer.current_agent_id.reset(token)
+                
+                # Wrap all public methods to track execution
+                for attr_name in dir(target):
+                    if not attr_name.startswith('_'):
+                        attr_value = getattr(target, attr_name)
+                        if callable(attr_value):
+                            def wrap_method(method):
+                                @self.file_tracker.trace_decorator
+                                @functools.wraps(method)
+                                def wrapped_method(self, *args, **kwargs):
+                                    self.gt = kwargs.get('gt', None) if kwargs else None
+                                    # Set this agent as current during method execution
+                                    token = tracer.current_agent_id.set(self._agent_component_id)
+                                    
+                                    # Store parent's children before setting new empty list
+                                    parent_children = tracer.agent_children.get()
+                                    children_token = tracer.agent_children.set([])
+                                    
+                                    try:
+                                        start_time = datetime.now()
+                                        result = method(self, *args, **kwargs)
+                                        end_time = datetime.now()
+                                        
+                                        # Update agent component with method result
+                                        if hasattr(tracer, '_agent_components'):
+                                            component = tracer._agent_components.get(self._agent_component_id)
+                                            if component:
+                                                component['data']['output'] = tracer._sanitize_output(result)
+                                                component['data']['input'] = tracer._sanitize_input(args, kwargs)
+                                                component['start_time'] = start_time.isoformat()
+                                                component['end_time'] = end_time.isoformat()
+                                                
+                                                # Get children accumulated during method execution
+                                                children = tracer.agent_children.get()
+                                                if children:
+                                                    if 'children' not in component['data']:
+                                                        component['data']['children'] = []
+                                                    component['data']['children'].extend(children)
+                                                    
+                                                    # Add this component as a child to parent's children list
+                                                    parent_children.append(component)
+                                                    tracer.agent_children.set(parent_children)
+                                        return result
+                                    finally:
+                                        tracer.current_agent_id.reset(token)
+                                        tracer.agent_children.reset(children_token)
+                                return wrapped_method
+                            
+                            setattr(target, attr_name, wrap_method(attr_value))
+                
+                # Replace __init__ with wrapped version
+                target.__init__ = wrapped_init
+                return target
+            else:
+                # For function decorators, use existing sync/async tracing
+                is_async = asyncio.iscoroutinefunction(target)
+                if is_async:
+                    async def wrapper(*args, **kwargs):
+                        return await self._trace_agent_execution(target, name, agent_type, version, capabilities, top_level_hash_id, *args, **kwargs)
+                    return wrapper
+                else:
+                    def wrapper(*args, **kwargs):
+                        return self._trace_sync_agent_execution(target, name, agent_type, version, capabilities, *args, **kwargs)
+                    return wrapper
 
         return decorator
 
     def _trace_sync_agent_execution(self, func, name, agent_type, version, capabilities, *args, **kwargs):
+        # Generate a unique hash_id for this execution context
+        hash_id = str(uuid.uuid4())
+
         """Synchronous version of agent tracing"""
         if not self.is_active:
             return func(*args, **kwargs)
@@ -48,15 +162,21 @@ class AgentTracerMixin:
         start_time = datetime.now()
         start_memory = psutil.Process().memory_info().rss
         component_id = str(uuid.uuid4())
-        hash_id = self._trace_sync_agent_execution.hash_id
 
-        # Initialize empty children list for this agent
-        children_token = self.agent_children.set([])
+        # Extract ground truth if present
+        ground_truth = kwargs.pop('gt', None) if kwargs else None
+
+        # Get parent agent ID if exists
+        parent_agent_id = self.current_agent_id.get()
         
         # Set the current agent context
         agent_token = self.current_agent_id.set(component_id)
         agent_name_token = self.current_agent_name.set(name)
 
+        # Initialize empty children list for this agent
+        parent_children = self.agent_children.get()
+        children_token = self.agent_children.set([])
+        
         # Start tracking network calls for this component
         self.start_component(component_id)
 
@@ -71,15 +191,11 @@ class AgentTracerMixin:
 
             # Get children components collected during execution
             children = self.agent_children.get()
-            
-            # Set parent_id for all children
-            for child in children:
-                child["parent_id"] = component_id
 
             # End tracking network calls for this component
             self.end_component(component_id)
 
-            # Create agent component with children
+            # Create agent component with children and parent if exists
             agent_component = self.create_agent_component(
                 component_id=component_id,
                 hash_id=hash_id,
@@ -92,55 +208,29 @@ class AgentTracerMixin:
                 memory_used=memory_used,
                 input_data=self._sanitize_input(args, kwargs),
                 output_data=self._sanitize_output(result),
-                children=children
+                children=children,
+                parent_id=parent_agent_id
             )
 
-            self.add_component(agent_component)
-            return result
+            # Add ground truth to component data if present
+            if ground_truth is not None:
+                agent_component["data"]["gt"] = ground_truth
+                        
+            # Add this component as a child to parent's children list
+            parent_children.append(agent_component)
+            self.agent_children.set(parent_children)
 
-        except Exception as e:
-            error_component = {
-                "code": 500,
-                "type": type(e).__name__,
-                "message": str(e),
-                "details": {}
-            }
-            
-            # Get children even in case of error
-            children = self.agent_children.get()
-            
-            # Set parent_id for all children
-            for child in children:
-                child["parent_id"] = component_id
-            
-            # End tracking network calls for this component
-            self.end_component(component_id)
-            
-            agent_component = self.create_agent_component(
-                component_id=component_id,
-                hash_id=hash_id,
-                name=name,
-                agent_type=agent_type,
-                version=version,
-                capabilities=capabilities or [],
-                start_time=start_time,
-                end_time=datetime.now(),
-                memory_used=0,
-                input_data=self._sanitize_input(args, kwargs),
-                output_data=None,
-                error=error_component,
-                children=children
-            )
+            # Only add to root components if no parent
+            if not parent_agent_id:
+                self.add_component(agent_component)
 
-            self.add_component(agent_component)
-            raise
+            return result
         finally:
-            # Reset context variables
             self.current_agent_id.reset(agent_token)
             self.current_agent_name.reset(agent_name_token)
             self.agent_children.reset(children_token)
 
-    async def _trace_agent_execution(self, func, name, agent_type, version, capabilities, *args, **kwargs):
+    async def _trace_agent_execution(self, func, name, agent_type, version, capabilities, hash_id, *args, **kwargs):
         """Asynchronous version of agent tracing"""
         if not self.is_active:
             return await func(*args, **kwargs)
@@ -148,17 +238,20 @@ class AgentTracerMixin:
         start_time = datetime.now()
         start_memory = psutil.Process().memory_info().rss
         component_id = str(uuid.uuid4())
-        hash_id = self._trace_agent_execution.hash_id
 
-        # Initialize empty children list for this agent
-        children_token = self.agent_children.set([])
+        # Extract ground truth if present
+        ground_truth = kwargs.pop('gt', None) if kwargs else None
+
+        # Get parent agent ID if exists
+        parent_agent_id = self.current_agent_id.get()
         
         # Set the current agent context
         agent_token = self.current_agent_id.set(component_id)
         agent_name_token = self.current_agent_name.set(name)
 
-        # Start tracking network calls for this component
-        self.start_component(component_id)
+        # Initialize empty children list for this agent
+        parent_children = self.agent_children.get()
+        children_token = self.agent_children.set([])
 
         try:
             # Execute the agent
@@ -171,15 +264,8 @@ class AgentTracerMixin:
 
             # Get children components collected during execution
             children = self.agent_children.get()
-            
-            # Set parent_id for all children
-            for child in children:
-                child["parent_id"] = component_id
-
-            # End tracking network calls for this component
-            self.end_component(component_id)
 
-            # Create agent component with children
+            # Create agent component with children and parent if exists
             agent_component = self.create_agent_component(
                 component_id=component_id,
                 hash_id=hash_id,
@@ -192,12 +278,23 @@ class AgentTracerMixin:
                 memory_used=memory_used,
                 input_data=self._sanitize_input(args, kwargs),
                 output_data=self._sanitize_output(result),
-                children=children
+                children=children,
+                parent_id=parent_agent_id
             )
 
-            self.add_component(agent_component)
-            return result
+            # Add ground truth to component data if present
+            if ground_truth is not None:
+                agent_component["data"]["gt"] = ground_truth
+
+            # Add this component as a child to parent's children list
+            parent_children.append(agent_component)
+            self.agent_children.set(parent_children)
+
+            # Only add to root components if no parent
+            if not parent_agent_id:
+                self.add_component(agent_component)
 
+            return result
         except Exception as e:
             error_component = {
                 "code": 500,
@@ -229,10 +326,20 @@ class AgentTracerMixin:
                 input_data=self._sanitize_input(args, kwargs),
                 output_data=None,
                 error=error_component,
-                children=children
+                children=children,
+                parent_id=parent_agent_id  # Add parent ID if exists
             )
-
-            self.add_component(agent_component)
+            
+            # If this is a nested agent, add it to parent's children
+            if parent_agent_id:
+                parent_component = self._agent_components.get(parent_agent_id)
+                if parent_component:
+                    if 'children' not in parent_component['data']:
+                        parent_component['data']['children'] = []
+                    parent_component['data']['children'].append(agent_component)
+            else:
+                # Only add to root components if no parent
+                self.add_component(agent_component)
             raise
         finally:
             # Reset context variables
@@ -252,7 +359,7 @@ class AgentTracerMixin:
             "start_time": start_time.isoformat(),
             "end_time": kwargs["end_time"].isoformat(),
             "error": kwargs.get("error"),
-            "parent_id": None,
+            "parent_id": kwargs.get("parent_id"),
             "info": {
                 "agent_type": kwargs["agent_type"],
                 "version": kwargs["version"],
@@ -265,19 +372,12 @@ class AgentTracerMixin:
                 "children": kwargs.get("children", [])
             },
             "network_calls": self.component_network_calls.get(kwargs["component_id"], []),
-            "interactions": [{
-                "id": f"int_{uuid.uuid4()}",
-                "interaction_type": "input",
-                "timestamp": start_time.isoformat(),
-                "content": kwargs["input_data"]
-            }, {
-                "id": f"int_{uuid.uuid4()}",
-                "interaction_type": "output",
-                "timestamp": kwargs["end_time"].isoformat(),
-                "content": kwargs["output_data"]
-            }]
+            "interactions": self.component_user_interaction.get(kwargs["component_id"], [])
         }
 
+        if self.gt: 
+            component["data"]["gt"] = self.gt
+
         return component
 
     def start_component(self, component_id):
@@ -294,15 +394,26 @@ class AgentTracerMixin:
             component_network_calls[component_id] = []
         self.component_network_calls.set(component_network_calls)
 
-    def _sanitize_input(self, args: tuple, kwargs: dict) -> Dict:
-        """Sanitize and format input data"""
-        return {
-            "args": [str(arg) if not isinstance(arg, (int, float, bool, str, list, dict)) else arg for arg in args],
-            "kwargs": {
-                k: str(v) if not isinstance(v, (int, float, bool, str, list, dict)) else v 
-                for k, v in kwargs.items()
-            }
-        }
+    def _sanitize_input(self, args: tuple, kwargs: dict) -> str:
+        """Convert input arguments to text format.
+        
+        Args:
+            args: Input arguments tuple
+            kwargs: Input keyword arguments dict
+            
+        Returns:
+            str: Text representation of the input arguments
+        """
+        def _sanitize_value(value):
+            if isinstance(value, dict):
+                return str({k: _sanitize_value(v) for k, v in value.items()})
+            elif isinstance(value, (list, tuple)):
+                return str([_sanitize_value(item) for item in value])
+            return str(value)
+        
+        sanitized_args = [_sanitize_value(arg) for arg in args]
+        sanitized_kwargs = {k: _sanitize_value(v) for k, v in kwargs.items()}
+        return str({"args": sanitized_args, "kwargs": sanitized_kwargs})
 
     def _sanitize_output(self, output: Any) -> Any:
         """Sanitize and format output data"""

ragaai_catalyst/tracers/agentic_tracing/agentic_tracing.py

@@ -4,6 +4,7 @@ import json
 from datetime import datetime
 import uuid
 import os
+import builtins
 from pathlib import Path
 
 from .base import BaseTracer
@@ -11,6 +12,7 @@ from .llm_tracer import LLMTracerMixin
 from .tool_tracer import ToolTracerMixin
 from .agent_tracer import AgentTracerMixin
 from .network_tracer import NetworkTracer
+from .user_interaction_tracer import UserInteractionTracer
 
 from .data_structure import (
     Trace, Metadata, SystemInfo, OSInfo, EnvironmentInfo,
@@ -27,6 +29,7 @@ from ..upload_traces import UploadTraces
 class AgenticTracing(BaseTracer, LLMTracerMixin, ToolTracerMixin, AgentTracerMixin):
     def __init__(self, user_detail, auto_instrument_llm: bool = True):
         # Initialize all parent classes
+        self.user_interaction_tracer = UserInteractionTracer()
         LLMTracerMixin.__init__(self)
         ToolTracerMixin.__init__(self)
         AgentTracerMixin.__init__(self)
@@ -43,11 +46,13 @@ class AgenticTracing(BaseTracer, LLMTracerMixin, ToolTracerMixin, AgentTracerMix
         self.auto_instrument_llm = auto_instrument_llm
         self.tools: Dict[str, Tool] = {}
         self.call_depth = contextvars.ContextVar("call_depth", default=0)
+        self.current_component_id = contextvars.ContextVar("current_component_id", default=None)
         self.network_tracer = NetworkTracer()
         self.is_active = False
         self.current_agent_id = contextvars.ContextVar("current_agent_id", default=None)
         self.agent_children = contextvars.ContextVar("agent_children", default=[])
         self.component_network_calls = {}  # Store network calls per component
+        self.component_user_interaction = {}
         
         # Create output directory if it doesn't exist
         self.output_dir = Path("./traces")  # Using default traces directory
@@ -57,14 +62,27 @@ class AgenticTracing(BaseTracer, LLMTracerMixin, ToolTracerMixin, AgentTracerMix
         """Start tracking network calls for a component"""
         self.component_network_calls[component_id] = []
         self.network_tracer.network_calls = []  # Reset network calls
+        self.component_user_interaction[component_id] = []
+        self.current_component_id.set(component_id)
+        self.user_interaction_tracer.component_id.set(component_id)
 
     def end_component(self, component_id: str):
         """End tracking network calls for a component"""
         self.component_network_calls[component_id] = self.network_tracer.network_calls.copy()
         self.network_tracer.network_calls = []  # Reset for next component
+        self.component_user_interaction[component_id] = self.user_interaction_tracer.interactions.copy()
+        self.user_interaction_tracer.interactions = []
 
     def start(self):
         """Start tracing"""
+        # Setup user interaction tracing
+        self.user_interaction_tracer.project_id.set(self.project_id)
+        self.user_interaction_tracer.trace_id.set(self.trace_id)
+        self.user_interaction_tracer.tracer = self
+        self.user_interaction_tracer.component_id.set(self.current_component_id.get())
+        builtins.print = self.user_interaction_tracer.traced_print
+        builtins.input = self.user_interaction_tracer.traced_input
+        
         # Start base tracer (includes system info and resource monitoring)
         super().start()
         self.is_active = True
@@ -79,6 +97,10 @@ class AgenticTracing(BaseTracer, LLMTracerMixin, ToolTracerMixin, AgentTracerMix
     def stop(self):
         """Stop tracing and save results"""
         if self.is_active:
+            # Restore original print and input functions
+            builtins.print = self.user_interaction_tracer.original_print
+            builtins.input = self.user_interaction_tracer.original_input
+            
             # Calculate final metrics before stopping
             self._calculate_final_metrics()
             
@@ -143,51 +165,15 @@ class AgenticTracing(BaseTracer, LLMTracerMixin, ToolTracerMixin, AgentTracerMix
 
     def add_component(self, component_data: dict):
         """Add a component to the trace data"""
-        component_id = component_data["id"]
-        
         # Convert dict to appropriate Component type
+        filtered_data = {k: v for k, v in component_data.items() if k in ["id", "hash_id", "type", "name", "start_time", "end_time", "parent_id", "info", "data", "network_calls", "interactions"]}
+               
         if component_data["type"] == "llm":
-            component = LLMComponent(**component_data)
-            # Add network calls specific to this LLM component
-            component_data["network_calls"] = self.component_network_calls.get(component_id, [])
-            # Add user interaction for LLM calls
-            component_data["interactions"] = [{
-                "id": f"int_{uuid.uuid4()}",
-                "interaction_type": "input",
-                "content": component_data["data"]["input"]
-            }, {
-                "id": f"int_{uuid.uuid4()}",
-                "interaction_type": "output",
-                "content": component_data["data"]["output"]
-            }]
+            component = LLMComponent(**filtered_data)
         elif component_data["type"] == "agent":
-            component = AgentComponent(**component_data)
-            # Add network calls specific to this agent component
-            component_data["network_calls"] = self.component_network_calls.get(component_id, [])
-            # Add user interaction for agent
-            component_data["interactions"] = [{
-                "id": f"int_{uuid.uuid4()}",
-                "interaction_type": "input",
-                "content": component_data["data"]["input"]
-            }, {
-                "id": f"int_{uuid.uuid4()}",
-                "interaction_type": "output",
-                "content": component_data["data"]["output"]
-            }]
+            component = AgentComponent(**filtered_data)
         elif component_data["type"] == "tool":
-            component = ToolComponent(**component_data)
-            # Add network calls specific to this tool component
-            component_data["network_calls"] = self.component_network_calls.get(component_id, [])
-            # Add user interaction for tool
-            component_data["interactions"] = [{
-                "id": f"int_{uuid.uuid4()}",
-                "interaction_type": "input",
-                "content": component_data["data"]["input"]
-            }, {
-                "id": f"int_{uuid.uuid4()}",
-                "interaction_type": "output",
-                "content": component_data["data"]["output"]
-            }]
+            component = ToolComponent(**filtered_data)
         else:
             component = Component(**component_data)