ragaai-catalyst 2.1.5b24__py3-none-any.whl → 2.1.5b26__py3-none-any.whl

ragaai_catalyst/tracers/agentic_tracing/tracers/llm_tracer.py

@@ -13,7 +13,7 @@ import traceback
 import importlib
 import sys
 from litellm import model_cost
-from llama_index.core.base.llms.types import ChatResponse
+from llama_index.core.base.llms.types import ChatResponse,TextBlock, ChatMessage
 
 from .base import BaseTracer
 from ..utils.llm_utils import (
@@ -550,138 +550,132 @@ class LLMTracerMixin:
             error=None,
             parameters={},
     ):
-        # Update total metrics
-        self.total_tokens += usage.get("total_tokens", 0)
-        self.total_cost += cost.get("total_cost", 0)
-
-        network_calls = []
-        if self.auto_instrument_network:
-            network_calls = self.component_network_calls.get(component_id, [])
-
-        interactions = []
-        if self.auto_instrument_user_interaction:
-            input_output_interactions = []
-            for interaction in self.component_user_interaction.get(component_id, []):
-                if interaction["interaction_type"] in ["input", "output"]:
-                    input_output_interactions.append(interaction)
-            interactions.extend(input_output_interactions)
-        if self.auto_instrument_file_io:
-            file_io_interactions = []
-            for interaction in self.component_user_interaction.get(component_id, []):
-                if interaction["interaction_type"] in ["file_read", "file_write"]:
-                    file_io_interactions.append(interaction)
-            interactions.extend(file_io_interactions)
-
-        parameters_to_display = {}
-        if "run_manager" in parameters:
-            parameters_obj = parameters["run_manager"]
-            if hasattr(parameters_obj, "metadata"):
-                metadata = parameters_obj.metadata
-                # parameters = {'metadata': metadata}
-                parameters_to_display.update(metadata)
-
-        # Add only those keys in parameters that are single values and not objects, dict or list
-        for key, value in parameters.items():
-            if isinstance(value, (str, int, float, bool)):
-                parameters_to_display[key] = value
-
-        # Limit the number of parameters to display
-        parameters_to_display = dict(
-            list(parameters_to_display.items())[: self.MAX_PARAMETERS_TO_DISPLAY]
-        )
+        try:
+            # Update total metrics
+            self.total_tokens += usage.get("total_tokens", 0)
+            self.total_cost += cost.get("total_cost", 0)
+
+            network_calls = []
+            if self.auto_instrument_network:
+                network_calls = self.component_network_calls.get(component_id, [])
+
+            interactions = []
+            if self.auto_instrument_user_interaction:
+                input_output_interactions = []
+                for interaction in self.component_user_interaction.get(component_id, []):
+                    if interaction["interaction_type"] in ["input", "output"]:
+                        input_output_interactions.append(interaction)
+                interactions.extend(input_output_interactions)
+            if self.auto_instrument_file_io:
+                file_io_interactions = []
+                for interaction in self.component_user_interaction.get(component_id, []):
+                    if interaction["interaction_type"] in ["file_read", "file_write"]:
+                        file_io_interactions.append(interaction)
+                interactions.extend(file_io_interactions)
+
+            parameters_to_display = {}
+            if "run_manager" in parameters:
+                parameters_obj = parameters["run_manager"]
+                if hasattr(parameters_obj, "metadata"):
+                    metadata = parameters_obj.metadata
+                    # parameters = {'metadata': metadata}
+                    parameters_to_display.update(metadata)
+
+            # Add only those keys in parameters that are single values and not objects, dict or list
+            for key, value in parameters.items():
+                if isinstance(value, (str, int, float, bool)):
+                    parameters_to_display[key] = value
+
+            # Limit the number of parameters to display
+            parameters_to_display = dict(
+                list(parameters_to_display.items())[: self.MAX_PARAMETERS_TO_DISPLAY]
+            )
+
+            # Set the Context and GT
+            span_gt = None
+            span_context = None
+            if name in self.span_attributes_dict:
+                span_gt = self.span_attributes_dict[name].gt
+                span_context = self.span_attributes_dict[name].context
+
+                logger.debug(f"span context {span_context}, span_gt {span_gt}")
+
+            # Tags
+            tags = []
+            if name in self.span_attributes_dict:
+                tags = self.span_attributes_dict[name].tags or []
+
+            # Get End Time
+            end_time = datetime.now().astimezone().isoformat()
+
+            # Metrics
+            metrics = []
+            if name in self.span_attributes_dict:
+                raw_metrics = self.span_attributes_dict[name].metrics or []
+                for metric in raw_metrics:
+                    base_metric_name = metric["name"]
+                    counter = sum(1 for x in self.visited_metrics if x.startswith(base_metric_name))
+                    metric_name = f'{base_metric_name}_{counter}' if counter > 0 else base_metric_name
+                    self.visited_metrics.append(metric_name)
+                    metric["name"] = metric_name
+                    metrics.append(metric)
+
+            # TODO TO check i/p and o/p is according or not
+            input = input_data["args"] if hasattr(input_data, "args") else input_data
+            output = output_data.output_response if output_data else None
+            #print("Prompt input:",input)
+            prompt = self.convert_to_content(input)
+            #print("Prompt Output: ",prompt)
+            #print("Response input: ",output)
+            response = self.convert_to_content(output)
+            #print("Response output: ",response)
+
+            # TODO: Execute & Add the User requested metrics here
+            formatted_metrics = BaseTracer.get_formatted_metric(self.span_attributes_dict, self.project_id, name)
+            if formatted_metrics:
+                metrics.extend(formatted_metrics)
+
+            component = {
+                "id": component_id,
+                "hash_id": hash_id,
+                "source_hash_id": None,
+                "type": "llm",
+                "name": name,
+                "start_time": start_time,
+                "end_time": end_time,
+                "error": error,
+                "parent_id": self.current_agent_id.get(),
+                "info": {
+                    "model": llm_type,
+                    "version": version,
+                    "memory_used": memory_used,
+                    "cost": cost,
+                    "tokens": usage,
+                    "tags": tags,
+                    **parameters_to_display,
+                },
+                "extra_info": parameters,
+                "data": {
+                    "input": input,
+                    "output": output,
+                    "memory_used": memory_used,
+                },
+                "metrics": metrics,
+                "network_calls": network_calls,
+                "interactions": interactions,
+            }
+
+            # Assign context and gt if available
+            component["data"]["gt"] = span_gt
+            component["data"]["context"] = span_context
+
+            # Reset the SpanAttributes context variable
+            self.span_attributes_dict[name] = SpanAttributes(name)
+
+            return component
+        except Exception as e:
+            raise Exception("Failed to create LLM component")
 
-        # Set the Context and GT
-        span_gt = None
-        span_context = None
-        if name in self.span_attributes_dict:
-            span_gt = self.span_attributes_dict[name].gt
-            span_context = self.span_attributes_dict[name].context
-
-            logger.debug(f"span context {span_context}, span_gt {span_gt}")
-
-        # Tags
-        tags = []
-        if name in self.span_attributes_dict:
-            tags = self.span_attributes_dict[name].tags or []
-
-        # Get End Time
-        end_time = datetime.now().astimezone().isoformat()
-
-        # Metrics
-        metrics = []
-        if name in self.span_attributes_dict:
-            raw_metrics = self.span_attributes_dict[name].metrics or []
-            for metric in raw_metrics:
-                base_metric_name = metric["name"]
-                counter = sum(1 for x in self.visited_metrics if x.startswith(base_metric_name))
-                metric_name = f'{base_metric_name}_{counter}' if counter > 0 else base_metric_name
-                self.visited_metrics.append(metric_name)
-                metric["name"] = metric_name
-                metrics.append(metric)
-
-        # TODO TO check i/p and o/p is according or not
-        input = input_data["args"] if hasattr(input_data, "args") else input_data
-        output = output_data.output_response if output_data else None
-        #print("Prompt input:",input)
-        prompt = self.convert_to_content(input)
-        #print("Prompt Output: ",prompt)
-        #print("Response input: ",output)
-        response = self.convert_to_content(output)
-        #print("Response output: ",response)
-
-        # TODO: Execute & Add the User requested metrics here
-        formatted_metrics = BaseTracer.get_formatted_metric(self.span_attributes_dict, self.project_id, name)
-        if formatted_metrics:
-            metrics.extend(formatted_metrics)
-
-        component = {
-            "id": component_id,
-            "hash_id": hash_id,
-            "source_hash_id": None,
-            "type": "llm",
-            "name": name,
-            "start_time": start_time,
-            "end_time": end_time,
-            "error": error,
-            "parent_id": self.current_agent_id.get(),
-            "info": {
-                "model": llm_type,
-                "version": version,
-                "memory_used": memory_used,
-                "cost": cost,
-                "tokens": usage,
-                "tags": tags,
-                **parameters_to_display,
-            },
-            "extra_info": parameters,
-            "data": {
-                "input": input,
-                "output": output,
-                "memory_used": memory_used,
-            },
-            "metrics": metrics,
-            "network_calls": network_calls,
-            "interactions": interactions,
-        }
-
-        # Assign context and gt if available
-        component["data"]["gt"] = span_gt
-        component["data"]["context"] = span_context
-
-        # Reset the SpanAttributes context variable
-        self.span_attributes_dict[name] = SpanAttributes(name)
-
-        return component
-
-    # def convert_to_content(self, input_data):
-    #     if isinstance(input_data, dict):
-    #         messages = input_data.get("kwargs", {}).get("messages", [])
-    #     elif isinstance(input_data, list):
-    #         messages = input_data
-    #     else:
-    #         return ""
-        # return "\n".join(process_content(msg.get("content", "")) for msg in messages if msg.get("content"))
-  
     def convert_to_content(self, input_data):
         try:
             if isinstance(input_data, dict):
@@ -689,7 +683,6 @@ class LLMTracerMixin:
             elif isinstance(input_data, list):
                 if len(input_data)>0 and isinstance(input_data[0]['content'],ChatResponse):
                     extracted_messages = []
-
                     for item in input_data:
                         chat_response = item.get('content')
                         if hasattr(chat_response, 'message') and hasattr(chat_response.message, 'blocks'):
@@ -699,9 +692,10 @@ class LLMTracerMixin:
                     messages=extracted_messages
                     if isinstance(messages,list):
                         return "\n".join(messages)
-                    
-                    #messages=[msg["content"] for msg in input_data if isinstance(msg, dict) and "content" in msg]
-                    #messages = [msg["content"].message for msg in input_data if isinstance(msg, dict) and "content" in msg and isinstance(msg["content"], ChatResponse)]
+                elif len(input_data)>0 and isinstance(input_data[0]['content'],TextBlock):
+                    return " ".join(block.text for item in input_data for block in item['content'] if isinstance(block, TextBlock))
+                elif len(input_data)>0 and isinstance(input_data[0]['content'],ChatMessage):
+                    return " ".join(block.text for block in input_data[0]['content'].blocks if isinstance(block, TextBlock)) 
                 else:
                     messages = input_data
             elif isinstance(input_data,ChatResponse):
@@ -709,10 +703,9 @@ class LLMTracerMixin:
             else:
                 return ""
             res=""
-            # try:
             res="\n".join(msg.get("content", "").strip() for msg in messages if msg.get("content"))
         except Exception as e:
-            res=str(messages)
+            res=str(input_data)
         return res
 
     def process_content(content):
@@ -965,6 +958,10 @@ class LLMTracerMixin:
             metrics: List[Dict[str, Any]] = [],
             feedback: Optional[Any] = None,
     ):
+
+        start_memory = psutil.Process().memory_info().rss
+        start_time = datetime.now().astimezone().isoformat()
+
         if name not in self.span_attributes_dict:
             self.span_attributes_dict[name] = SpanAttributes(name)
         if tags:
@@ -996,7 +993,6 @@ class LLMTracerMixin:
         self.current_llm_call_name.set(name)
 
         def decorator(func):
-            @self.file_tracker.trace_decorator
             @functools.wraps(func)
             async def async_wrapper(*args, **kwargs):
                 gt = kwargs.get("gt") if kwargs else None
@@ -1018,14 +1014,34 @@ class LLMTracerMixin:
                     result = await func(*args, **kwargs)
                     return result
                 except Exception as e:
-                    error_info = {
-                        "error": {
-                            "type": type(e).__name__,
-                            "message": str(e),
-                            "traceback": traceback.format_exc(),
-                            "timestamp": datetime.now().astimezone().isoformat(),
-                        }
+                    error_component = {
+                        "type": type(e).__name__,
+                        "message": str(e),
+                        "traceback": traceback.format_exc(),
+                        "timestamp": datetime.now().astimezone().isoformat(),
                     }
+
+                    # End tracking network calls for this component
+                    self.end_component(component_id)
+
+                    end_memory = psutil.Process().memory_info().rss
+                    memory_used = max(0, end_memory - start_memory)
+
+                    llm_component = self.create_llm_component(
+                        component_id=component_id,
+                        hash_id=generate_unique_hash(func, args, kwargs),
+                        name=name,
+                        llm_type="unknown",
+                        version=None,
+                        memory_used=memory_used,
+                        start_time=start_time,
+                        input_data=extract_input_data(args, kwargs, None),
+                        output_data=None,
+                        error=error_component,
+                    )
+                    self.llm_data = llm_component
+                    self.add_component(llm_component, is_error=True)
+
                     raise
                 finally:
 
@@ -1070,7 +1086,6 @@ class LLMTracerMixin:
                     )
                     self.add_component(llm_component)
 
-            @self.file_tracker.trace_decorator
             @functools.wraps(func)
             def sync_wrapper(*args, **kwargs):
                 gt = kwargs.get("gt") if kwargs else None
@@ -1093,14 +1108,34 @@ class LLMTracerMixin:
                     result = func(*args, **kwargs)
                     return result
                 except Exception as e:
-                    error_info = {
-                        "error": {
-                            "type": type(e).__name__,
-                            "message": str(e),
-                            "traceback": traceback.format_exc(),
-                            "timestamp": datetime.now().astimezone().isoformat(),
-                        }
+                    error_component = {
+                        "type": type(e).__name__,
+                        "message": str(e),
+                        "traceback": traceback.format_exc(),
+                        "timestamp": datetime.now().astimezone().isoformat(),
                     }
+
+                    # End tracking network calls for this component
+                    self.end_component(component_id)
+
+                    end_memory = psutil.Process().memory_info().rss
+                    memory_used = max(0, end_memory - start_memory)
+
+                    llm_component = self.create_llm_component(
+                        component_id=component_id,
+                        hash_id=generate_unique_hash(func, args, kwargs),
+                        name=name,
+                        llm_type="unknown",
+                        version=None,
+                        memory_used=memory_used,
+                        start_time=start_time,
+                        input_data=extract_input_data(args, kwargs, None),
+                        output_data=None,
+                        error=error_component,
+                    )
+                    self.llm_data = llm_component
+                    self.add_component(llm_component, is_error=True)
+
                     raise
                 finally:
                     llm_component = self.llm_data

ragaai_catalyst/tracers/agentic_tracing/tracers/main_tracer.py

@@ -236,6 +236,8 @@ class AgenticTracing(
         total_cost = 0.0
         total_tokens = 0
 
+        processed_components = set()
+
         def process_component(component):
             nonlocal total_cost, total_tokens
             # Convert component to dict if it's an object
@@ -243,6 +245,11 @@ class AgenticTracing(
                 component.__dict__ if hasattr(component, "__dict__") else component
             )
 
+            comp_id = comp_dict.get("id") or comp_dict.get("component_id")
+            if comp_id in processed_components:
+                return  # Skip if already processed
+            processed_components.add(comp_id)
+
             if comp_dict.get("type") == "llm":
                 info = comp_dict.get("info", {})
                 if isinstance(info, dict):
@@ -372,66 +379,6 @@ class AgenticTracing(
 
         # Handle error case
         if is_error:
-            # Get the parent component if it exists
-            parent_id = component_data.get("parent_id")
-            children = self.agent_children.get()
-
-            # Set parent_id for all children
-            for child in children:
-                child["parent_id"] = parent_id
-
-            agent_tracer_mixin = AgentTracerMixin()
-            agent_tracer_mixin.component_network_calls = self.component_network_calls
-            agent_tracer_mixin.component_user_interaction = (
-                self.component_user_interaction
-            )
-
-            agent_tracer_mixin.span_attributes_dict[self.current_agent_name.get()] = (
-                SpanAttributes(self.current_agent_name.get())
-            )
-
-            # Create parent component with error info
-            parent_component = agent_tracer_mixin.create_agent_component(
-                component_id=parent_id,
-                hash_id=str(uuid.uuid4()),
-                source_hash_id=None,
-                type="agent",
-                name=self.current_agent_name.get(),
-                agent_type=self.agent_type.get(),
-                version=self.version.get(),
-                capabilities=self.capabilities.get(),
-                start_time=self.start_time,
-                end_time=datetime.now().astimezone().isoformat(),
-                memory_used=0,
-                input_data=self.input_data,
-                output_data=None,
-                children=children,
-                parent_id=None,  # Add parent ID if exists
-            )
-
-            filtered_data = {
-                k: v
-                for k, v in parent_component.items()
-                if k
-                in [
-                    "id",
-                    "hash_id",
-                    "source_hash_id",
-                    "type",
-                    "name",
-                    "start_time",
-                    "end_time",
-                    "parent_id",
-                    "info",
-                    "data",
-                    "network_calls",
-                    "interactions",
-                    "error",
-                ]
-            }
-            parent_agent_component = AgentComponent(**filtered_data)
-            # Add the parent component to trace and stop tracing
-            super().add_component(parent_agent_component)
             self.stop()
 
     def __enter__(self):

ragaai_catalyst/tracers/agentic_tracing/tracers/tool_tracer.py

@@ -255,7 +255,6 @@ class ToolTracerMixin:
             # Check if the function is async
             is_async = asyncio.iscoroutinefunction(func)
 
-            @self.file_tracker.trace_decorator
             @functools.wraps(func)
             async def async_wrapper(*args, **kwargs):
                 async_wrapper.metadata = metadata
@@ -267,7 +266,6 @@ class ToolTracerMixin:
                     func, name, tool_type, version, *args, **kwargs
                 )
 
-            @self.file_tracker.trace_decorator
             @functools.wraps(func)
             def sync_wrapper(*args, **kwargs):
                 sync_wrapper.metadata = metadata
@@ -309,7 +307,7 @@ class ToolTracerMixin:
 
         try:
             # Execute the tool
-            result = self.file_tracker.trace_wrapper(func)(*args, **kwargs)
+            result = func(*args, **kwargs)
 
             # Calculate resource usage
             end_memory = psutil.Process().memory_info().rss
@@ -359,7 +357,7 @@ class ToolTracerMixin:
                 error=error_component,
             )
 
-            self.add_component(tool_component)
+            self.add_component(tool_component, is_error=True)
 
             raise
         finally:
@@ -391,7 +389,7 @@ class ToolTracerMixin:
         self.start_component(component_id)
         try:
             # Execute the tool
-            result = await self.file_tracker.trace_wrapper(func)(*args, **kwargs)
+            result = await func(*args, **kwargs)
 
             # Calculate resource usage
             end_memory = psutil.Process().memory_info().rss
@@ -434,7 +432,7 @@ class ToolTracerMixin:
                 output_data=None,
                 error=error_component,
             )
-            self.add_component(tool_component)
+            self.add_component(tool_component, is_error=True)
 
             raise
         finally:

ragaai_catalyst/tracers/agentic_tracing/upload/upload_local_metric.py

@@ -11,13 +11,14 @@ logging_level = (
     else logger.setLevel(logging.INFO)
 )
 
+
 def calculate_metric(project_id, metric_name, model, provider, **kwargs):
     user_id = "1"
     org_domain = "raga"
 
     headers = {
         "Authorization": f"Bearer {os.getenv('RAGAAI_CATALYST_TOKEN')}",
-        "X-Project-Id": str(project_id),  # TODO to change it to project_id
+        "X-Project-Id": str(project_id),
         "Content-Type": "application/json"
     }
 
@@ -38,6 +39,7 @@ def calculate_metric(project_id, metric_name, model, provider, **kwargs):
                     "metric_name": metric_name,
                     "request_id": 1
                 },
+                "variable_mapping": kwargs,
                 "trace_object": {
                     "Data": {
                         "DocId": "doc-1",

ragaai_catalyst/tracers/agentic_tracing/utils/file_name_tracker.py

@@ -62,4 +62,8 @@ class TrackName:
         
     def reset(self):
         """Reset the file tracker by clearing all tracked files."""
-        self.files.clear()
+        self.files.clear()
+    
+    def trace_main_file(self):
+        frame = inspect.stack()[-1]
+        self.files.add(frame.filename)