ragaai-catalyst 2.1.5b1__py3-none-any.whl → 2.1.5b3__py3-none-any.whl

ragaai_catalyst/dataset.py

@@ -434,17 +434,16 @@ class Dataset:
             
             # Check response
             response_data = response.json()
-            if not response_data.get('success', False):
-                raise ValueError(response_data.get('message', 'Unknown error occurred'))
-            
-            print(f"Successfully added rows to dataset {dataset_name}")
-            return response_data
+            if response_data.get('success', False):
+                print(f"{response_data['message']}")
+            else:
+                raise ValueError(response_data.get('message', 'Failed to add rows'))
         
         except Exception as e:
             logger.error(f"Error in add_rows_to_dataset: {e}")
             raise
 
-    def add_columns(self,text_fields,dataset_name, column_name, provider, model,variables={}):
+    def add_columns(self, text_fields, dataset_name, column_name, provider, model, variables={}):
         """
         Add a column to a dataset with dynamically fetched model parameters
         
@@ -571,7 +570,7 @@ class Dataset:
                     variable_specs.append({
                         "name": key,
                         "type": "string",
-                        "schema": values
+                        "schema": "query"
                     })
                 add_column_payload["promptTemplate"]["variableSpecs"] = variable_specs
             
@@ -593,9 +592,10 @@ class Dataset:
             response.raise_for_status()
             response_data = response.json()
             
-            print("Column added successfully:")
-            print(json.dumps(response_data, indent=2))
-            return response_data
+            if response_data.get('success', False):
+                print(f"Column '{column_name}' added successfully to dataset '{dataset_name}'")
+            else:
+                raise ValueError(response_data.get('message', 'Failed to add column'))
         
         except requests.exceptions.RequestException as e:
             print(f"Error adding column: {e}")

ragaai_catalyst/ragaai_catalyst.py

@@ -2,7 +2,7 @@ import os
 import logging
 import requests
 from typing import Dict, Optional, Union
-
+import re
 logger = logging.getLogger("RagaAICatalyst")
 
 
@@ -55,10 +55,11 @@ class RagaAICatalyst:
         self.api_keys = api_keys or {}
 
         if base_url:
-            RagaAICatalyst.BASE_URL = base_url
+            RagaAICatalyst.BASE_URL = self._normalize_base_url(base_url)
             try:
+                #set the os.environ["RAGAAI_CATALYST_BASE_URL"] before getting the token as it is used in the get_token method
+                os.environ["RAGAAI_CATALYST_BASE_URL"] = RagaAICatalyst.BASE_URL
                 self.get_token()
-                os.environ["RAGAAI_CATALYST_BASE_URL"] = base_url
             except requests.exceptions.RequestException:
                 raise ConnectionError(
                     "The provided base_url is not accessible. Please re-check the base_url."
@@ -71,6 +72,14 @@ class RagaAICatalyst:
         if self.api_keys:
             self._upload_keys()
 
+    @staticmethod
+    def _normalize_base_url(url):
+        url = re.sub(r'(?<!:)//+', '/', url)  # Ignore the `://` part of URLs and remove extra // if any
+        url = url.rstrip("/") # To remove trailing slashes
+        if not url.endswith("/api"): # To ensure it ends with /api
+            url = f"{url}/api"
+        return url
+
     def _set_access_key_secret_key(self, access_key, secret_key):
         os.environ["RAGAAI_CATALYST_ACCESS_KEY"] = access_key
         os.environ["RAGAAI_CATALYST_SECRET_KEY"] = secret_key

ragaai_catalyst/tracers/agentic_tracing/tracers/base.py

@@ -1,8 +1,5 @@
 import json
 import os
-import platform
-import psutil
-import pkg_resources
 from datetime import datetime
 from pathlib import Path
 from typing import List, Any, Dict
@@ -16,20 +13,9 @@ from ..data.data_structure import (
     Trace,
     Metadata,
     SystemInfo,
-    OSInfo,
-    EnvironmentInfo,
     Resources,
-    CPUResource,
-    MemoryResource,
-    DiskResource,
-    NetworkResource,
-    ResourceInfo,
-    MemoryInfo,
-    DiskInfo,
-    NetworkInfo,
     Component,
 )
-
 from ..upload.upload_agentic_traces import UploadAgenticTraces
 from ..upload.upload_code import upload_code
 from ..upload.upload_trace_metric import upload_trace_metric
@@ -37,9 +23,8 @@ from ..utils.file_name_tracker import TrackName
 from ..utils.zip_list_of_unique_files import zip_list_of_unique_files
 from ..utils.span_attributes import SpanAttributes
 from ..utils.create_dataset_schema import create_dataset_schema_with_trace
+from ..utils.system_monitor import SystemMonitor
 
-
-# Configure logging to show debug messages (which includes info messages as well)
 import logging
 
 logger = logging.getLogger(__name__)
@@ -76,12 +61,12 @@ class TracerJSONEncoder(json.JSONEncoder):
 class BaseTracer:
     def __init__(self, user_details):
         self.user_details = user_details
-        self.project_name = self.user_details["project_name"]  # Access the project_name
-        self.dataset_name = self.user_details["dataset_name"]  # Access the dataset_name
-        self.project_id = self.user_details["project_id"]  # Access the project_id
-        self.trace_name = self.user_details["trace_name"]  # Access the trace_name
+        self.project_name = self.user_details["project_name"]
+        self.dataset_name = self.user_details["dataset_name"]
+        self.project_id = self.user_details["project_id"]
+        self.trace_name = self.user_details["trace_name"]
         self.visited_metrics = []
-        self.trace_metrics = []  # Store metrics here
+        self.trace_metrics = []
 
         # Initialize trace data
         self.trace_id = None
@@ -97,117 +82,60 @@ class BaseTracer:
         self.network_usage_list = []
         self.tracking_thread = None
         self.tracking = False
+        self.system_monitor = None
 
     def _get_system_info(self) -> SystemInfo:
-        # Get OS info
-        os_info = OSInfo(
-            name=platform.system(),
-            version=platform.version(),
-            platform=platform.machine(),
-            kernel_version=platform.release(),
-        )
-
-        # Get Python environment info
-        installed_packages = [
-            f"{pkg.key}=={pkg.version}" for pkg in pkg_resources.working_set
-        ]
-        env_info = EnvironmentInfo(
-            name="Python",
-            version=platform.python_version(),
-            packages=installed_packages,
-            env_path=sys.prefix,
-            command_to_run=f"python {sys.argv[0]}",
-        )
-
-        return SystemInfo(
-            id=f"sys_{self.trace_id}",
-            os=os_info,
-            environment=env_info,
-            source_code="Path to the source code .zip file in format hashid.zip",  # TODO: Implement source code archiving
-        )
+        return self.system_monitor.get_system_info()
 
     def _get_resources(self) -> Resources:
-        # CPU info
-        cpu_info = ResourceInfo(
-            name=platform.processor(),
-            cores=psutil.cpu_count(logical=False),
-            threads=psutil.cpu_count(logical=True),
-        )
-        cpu = CPUResource(info=cpu_info, interval="5s", values=[psutil.cpu_percent()])
-
-        # Memory info
-        memory = psutil.virtual_memory()
-        mem_info = MemoryInfo(
-            total=memory.total / (1024**3),  # Convert to GB
-            free=memory.available / (1024**3),
-        )
-        mem = MemoryResource(info=mem_info, interval="5s", values=[memory.percent])
-
-        # Disk info
-        disk = psutil.disk_usage("/")
-        disk_info = DiskInfo(total=disk.total / (1024**3), free=disk.free / (1024**3))
-        disk_io = psutil.disk_io_counters()
-        disk_resource = DiskResource(
-            info=disk_info,
-            interval="5s",
-            read=[disk_io.read_bytes / (1024**2)],  # MB
-            write=[disk_io.write_bytes / (1024**2)],
-        )
-
-        # Network info
-        net_io = psutil.net_io_counters()
-        net_info = NetworkInfo(
-            upload_speed=net_io.bytes_sent / (1024**2),  # MB
-            download_speed=net_io.bytes_recv / (1024**2),
-        )
-        net = NetworkResource(
-            info=net_info,
-            interval="5s",
-            uploads=[net_io.bytes_sent / (1024**2)],
-            downloads=[net_io.bytes_recv / (1024**2)],
-        )
-
-        return Resources(cpu=cpu, memory=mem, disk=disk_resource, network=net)
+        return self.system_monitor.get_resources()
 
     def _track_memory_usage(self):
         self.memory_usage_list = []
         while self.tracking:
-            memory_usage = psutil.Process().memory_info().rss
-            self.memory_usage_list.append(memory_usage / (1024 * 1024))  # Convert to MB and append to the list
-            time.sleep(self.interval_time)
+            usage = self.system_monitor.track_memory_usage()
+            self.memory_usage_list.append(usage)
+            try:
+                time.sleep(self.interval_time)
+            except Exception as e:
+                logger.warning(f"Sleep interrupted in memory tracking: {str(e)}")
 
     def _track_cpu_usage(self):
         self.cpu_usage_list = []
         while self.tracking:
-            cpu_usage = psutil.cpu_percent(interval=self.interval_time)
-            self.cpu_usage_list.append(cpu_usage)
-            time.sleep(self.interval_time)
+            usage = self.system_monitor.track_cpu_usage(self.interval_time)
+            self.cpu_usage_list.append(usage)
+            try:
+                time.sleep(self.interval_time)
+            except Exception as e:
+                logger.warning(f"Sleep interrupted in CPU tracking: {str(e)}")
 
     def _track_disk_usage(self):
         self.disk_usage_list = []
         while self.tracking:
-            disk_io = psutil.disk_io_counters()
-            self.disk_usage_list.append({
-                'disk_read': disk_io.read_bytes / (1024 * 1024),  # Convert to MB
-                'disk_write': disk_io.write_bytes / (1024 * 1024)  # Convert to MB
-            })
-            time.sleep(self.interval_time)
+            usage = self.system_monitor.track_disk_usage()
+            self.disk_usage_list.append(usage)
+            try:
+                time.sleep(self.interval_time)
+            except Exception as e:
+                logger.warning(f"Sleep interrupted in disk tracking: {str(e)}")
 
     def _track_network_usage(self):
         self.network_usage_list = []
         while self.tracking:
-            net_io = psutil.net_io_counters()
-            self.network_usage_list.append({
-                'uploads': net_io.bytes_sent / (1024 * 1024),  # Convert to MB
-                'downloads': net_io.bytes_recv / (1024 * 1024)  # Convert to MB
-            })
-            time.sleep(self.interval_time)
+            usage = self.system_monitor.track_network_usage()
+            self.network_usage_list.append(usage)
+            try:
+                time.sleep(self.interval_time)
+            except Exception as e:
+                logger.warning(f"Sleep interrupted in network tracking: {str(e)}")
 
     def start(self):
         """Initialize a new trace"""
         self.tracking = True
-        self.tracking_thread = threading.Thread(target=self._track_memory_usage)
-        self.tracking_thread.start()
+        self.trace_id = str(uuid.uuid4())
+        self.system_monitor = SystemMonitor(self.trace_id)
+        threading.Thread(target=self._track_memory_usage).start()
         threading.Thread(target=self._track_cpu_usage).start()
         threading.Thread(target=self._track_disk_usage).start()
         threading.Thread(target=self._track_network_usage).start()
@@ -223,9 +151,6 @@ class BaseTracer:
             resources=self._get_resources(),
         )
 
-        # Generate a unique trace ID, when trace starts
-        self.trace_id = str(uuid.uuid4())
-
         # Get the start time
         self.start_time = datetime.now().astimezone().isoformat()
 
@@ -257,8 +182,6 @@ class BaseTracer:
 
             #track memory usage
             self.tracking = False
-            if self.tracking_thread is not None:
-                self.tracking_thread.join()
             self.trace.metadata.resources.memory.values = self.memory_usage_list
 
             #track cpu usage
@@ -695,12 +618,11 @@ class BaseTracer:
         # Process additional interactions and network calls
         if "interactions" in child:
             for interaction in child["interactions"]:
-                if interaction!=[]:
-                    interaction["id"] = str(interaction_id)
-                    interaction["span_id"] = child.get("id")
-                    interaction["error"] = None
-                    interactions.append(interaction)
-                    interaction_id += 1
+                interaction["id"] = str(interaction_id)
+                interaction["span_id"] = child.get("id")
+                interaction["error"] = None
+                interactions.append(interaction)
+                interaction_id += 1
 
         if "network_calls" in child:
             for child_network_call in child["network_calls"]:
@@ -877,16 +799,15 @@ class BaseTracer:
             # Process interactions from span.data if they exist
             if span.interactions:
                 for span_interaction in span.interactions:
-                    if span_interaction != []:
-                        interaction = {}
-                        interaction["id"] = str(interaction_id)
-                        interaction["span_id"] = span.id
-                        interaction["interaction_type"] = span_interaction.type
-                        interaction["content"] = span_interaction.content
-                        interaction["timestamp"] = span_interaction.timestamp
-                        interaction["error"] = span.error
-                        interactions.append(interaction)
-                        interaction_id += 1
+                    interaction = {}
+                    interaction["id"] = str(interaction_id)
+                    interaction["span_id"] = span.id
+                    interaction["interaction_type"] = span_interaction.type
+                    interaction["content"] = span_interaction.content
+                    interaction["timestamp"] = span_interaction.timestamp
+                    interaction["error"] = span.error
+                    interactions.append(interaction)
+                    interaction_id += 1
 
             if span.network_calls:
                 for span_network_call in span.network_calls:

ragaai_catalyst/tracers/agentic_tracing/tracers/llm_tracer.py

@@ -78,6 +78,7 @@ class LLMTracerMixin:
 
         if "openai" in sys.modules:
             self.patch_openai_methods(sys.modules["openai"])
+            self.patch_openai_beta_methods(sys.modules["openai"])
         if "litellm" in sys.modules:
             self.patch_litellm_methods(sys.modules["litellm"])
         if "anthropic" in sys.modules:
@@ -97,6 +98,7 @@ class LLMTracerMixin:
             self.patch_vertex_ai_methods, "vertexai.generative_models"
         )
         wrapt.register_post_import_hook(self.patch_openai_methods, "openai")
+        wrapt.register_post_import_hook(self.patch_openai_beta_methods, "openai")
         wrapt.register_post_import_hook(self.patch_litellm_methods, "litellm")
         wrapt.register_post_import_hook(self.patch_anthropic_methods, "anthropic")
         wrapt.register_post_import_hook(
@@ -135,6 +137,42 @@ class LLMTracerMixin:
             # Log the error but continue execution
             print(f"Warning: Failed to patch OpenAI methods: {str(e)}")
 
+    def patch_openai_beta_methods(self, openai_module):
+        """
+        Patch the new openai.beta endpoints (threads, runs, messages, etc.)
+        so that calls like openai.beta.threads.create(...) or
+        openai.beta.threads.runs.create(...) are automatically traced.
+        """
+        # Make sure openai_module has a 'beta' attribute
+        if not hasattr(openai_module, "beta"):
+            return
+
+        beta_module = openai_module.beta
+
+        # Patch openai.beta.threads
+        import openai
+        openai.api_type = "openai"
+        if hasattr(beta_module, "threads"):
+            threads_obj = beta_module.threads
+            # Patch top-level methods on openai.beta.threads
+            for method_name in ["create", "list"]:
+                if hasattr(threads_obj, method_name):
+                    self.wrap_method(threads_obj, method_name)
+
+            # Patch the nested objects: messages, runs
+            if hasattr(threads_obj, "messages"):
+                messages_obj = threads_obj.messages
+                for method_name in ["create", "list"]:
+                    if hasattr(messages_obj, method_name):
+                        self.wrap_method(messages_obj, method_name)
+
+            if hasattr(threads_obj, "runs"):
+                runs_obj = threads_obj.runs
+                for method_name in ["create", "retrieve", "list"]:
+                    if hasattr(runs_obj, method_name):
+                        self.wrap_method(runs_obj, method_name)
+
+
     def patch_anthropic_methods(self, module):
         if hasattr(module, "Anthropic"):
             client_class = getattr(module, "Anthropic")

ragaai_catalyst/tracers/agentic_tracing/utils/system_monitor.py

@@ -0,0 +1,215 @@
+import platform
+import psutil
+import sys
+import pkg_resources
+import logging
+from typing import Dict, List, Optional
+from ..data.data_structure import (
+    SystemInfo,
+    OSInfo,
+    EnvironmentInfo,
+    Resources,
+    CPUResource,
+    MemoryResource,
+    DiskResource,
+    NetworkResource,
+    ResourceInfo,
+    MemoryInfo,
+    DiskInfo,
+    NetworkInfo,
+)
+
+logger = logging.getLogger(__name__)
+
+class SystemMonitor:
+    def __init__(self, trace_id: str):
+        self.trace_id = trace_id
+
+    def get_system_info(self) -> SystemInfo:
+        # Initialize with None values
+        os_info = OSInfo(
+            name=None,
+            version=None,
+            platform=None,
+            kernel_version=None,
+        )
+        env_info = EnvironmentInfo(
+            name=None,
+            version=None,
+            packages=[],
+            env_path=None,
+            command_to_run=None,
+        )
+
+        try:
+            # Get OS info
+            os_info = OSInfo(
+                name=platform.system(),
+                version=platform.version(),
+                platform=platform.machine(),
+                kernel_version=platform.release(),
+            )
+        except Exception as e:
+            logger.warning(f"Failed to get OS info: {str(e)}")
+             
+        try:
+            # Get Python environment info
+            installed_packages = [
+                f"{pkg.key}=={pkg.version}" for pkg in pkg_resources.working_set
+            ]
+            env_info = EnvironmentInfo(
+                name="Python",
+                version=platform.python_version(),
+                packages=installed_packages,
+                env_path=sys.prefix,
+                command_to_run=f"python {sys.argv[0]}",
+            )
+        except Exception as e:
+            logger.warning(f"Failed to get environment info: {str(e)}")
+             
+
+        # Always return a valid SystemInfo object
+        return SystemInfo(
+            id=f"sys_{self.trace_id}",
+            os=os_info,
+            environment=env_info,
+            source_code="",
+        )
+
+    def get_resources(self) -> Resources:
+        # Initialize with None values
+        cpu_info = ResourceInfo(
+            name=None,
+            cores=None,
+            threads=None,
+        )
+        cpu = CPUResource(info=cpu_info, interval="5s", values=[])
+
+        mem_info = MemoryInfo(
+            total=None,
+            free=None,
+        )
+        mem = MemoryResource(info=mem_info, interval="5s", values=[])
+
+        disk_info = DiskInfo(
+            total=None,
+            free=None,
+        )
+        disk_resource = DiskResource(
+            info=disk_info,
+            interval="5s",
+            read=[],
+            write=[],
+        )
+
+        net_info = NetworkInfo(
+            upload_speed=None,
+            download_speed=None,
+        )
+        net = NetworkResource(
+            info=net_info,
+            interval="5s",
+            uploads=[],
+            downloads=[],
+        )
+
+        try:
+            # CPU info
+            cpu_info = ResourceInfo(
+                name=platform.processor(),
+                cores=psutil.cpu_count(logical=False),
+                threads=psutil.cpu_count(logical=True),
+            )
+            cpu = CPUResource(info=cpu_info, interval="5s", values=[psutil.cpu_percent()])
+        except Exception as e:
+            logger.warning(f"Failed to get CPU info: {str(e)}")
+             
+
+        try:
+            # Memory info
+            memory = psutil.virtual_memory()
+            mem_info = MemoryInfo(
+                total=memory.total / (1024**3),  # Convert to GB
+                free=memory.available / (1024**3),
+            )
+            mem = MemoryResource(info=mem_info, interval="5s", values=[memory.percent])
+        except Exception as e:
+            logger.warning(f"Failed to get memory info: {str(e)}")
+             
+
+        try:
+            # Disk info
+            disk = psutil.disk_usage("/")
+            disk_info = DiskInfo(total=disk.total / (1024**3), free=disk.free / (1024**3))
+            disk_io = psutil.disk_io_counters()
+            disk_resource = DiskResource(
+                info=disk_info,
+                interval="5s",
+                read=[disk_io.read_bytes / (1024**2)],  # MB
+                write=[disk_io.write_bytes / (1024**2)],
+            )
+        except Exception as e:
+            logger.warning(f"Failed to get disk info: {str(e)}")
+             
+        try:
+            # Network info
+            net_io = psutil.net_io_counters()
+            net_info = NetworkInfo(
+                upload_speed=net_io.bytes_sent / (1024**2),  # MB
+                download_speed=net_io.bytes_recv / (1024**2),
+            )
+            net = NetworkResource(
+                info=net_info,
+                interval="5s",
+                uploads=[net_io.bytes_sent / (1024**2)],
+                downloads=[net_io.bytes_recv / (1024**2)],
+            )
+        except Exception as e:
+            logger.warning(f"Failed to get network info: {str(e)}")
+            
+
+        # Always return a valid Resources object
+        return Resources(cpu=cpu, memory=mem, disk=disk_resource, network=net)
+
+    def track_memory_usage(self) -> Optional[float]:
+        """Track memory usage in MB"""
+        try:
+            memory_usage = psutil.Process().memory_info().rss
+            return memory_usage / (1024 * 1024)  # Convert to MB
+        except Exception as e:
+            logger.warning(f"Failed to track memory usage: {str(e)}")
+            return None  
+
+    def track_cpu_usage(self, interval: float) -> Optional[float]:
+        """Track CPU usage percentage"""
+        try:
+            return psutil.cpu_percent(interval=interval)
+        except Exception as e:
+            logger.warning(f"Failed to track CPU usage: {str(e)}")
+            return None  
+
+    def track_disk_usage(self) -> Dict[str, Optional[float]]:
+        """Track disk I/O in MB"""
+        default_response = {'disk_read': None, 'disk_write': None}
+        try:
+            disk_io = psutil.disk_io_counters()
+            return {
+                'disk_read': disk_io.read_bytes / (1024 * 1024),  # Convert to MB
+                'disk_write': disk_io.write_bytes / (1024 * 1024)  # Convert to MB
+            }
+        except Exception as e:
+            logger.warning(f"Failed to track disk usage: {str(e)}")
+            return default_response 
+
+    def track_network_usage(self) -> Dict[str, Optional[float]]:
+        """Track network I/O in MB"""
+        default_response = {'uploads': None, 'downloads': None}
+        try:
+            net_io = psutil.net_io_counters()
+            return {
+                'uploads': net_io.bytes_sent / (1024 * 1024),  # Convert to MB
+                'downloads': net_io.bytes_recv / (1024 * 1024)  # Convert to MB
+            }
+        except Exception as e:
+            logger.warning(f"Failed to track network usage: {str(e)}")
+            return default_response 

ragaai_catalyst/tracers/tracer.py

@@ -1,4 +1,6 @@
+from audioop import add
 import os
+import uuid
 import datetime
 import logging
 import asyncio
@@ -6,6 +8,13 @@ import aiohttp
 import requests
 from contextlib import contextmanager
 from concurrent.futures import ThreadPoolExecutor
+from ragaai_catalyst.tracers.langchain_callback import LangchainTracer
+from ragaai_catalyst.tracers.utils.convert_langchain_callbacks_output import convert_langchain_callbacks_output
+
+from ragaai_catalyst.tracers.utils.langchain_tracer_extraction_logic import langchain_tracer_extraction
+from ragaai_catalyst.tracers.upload_traces import UploadTraces
+import tempfile
+import json
 
 from opentelemetry.sdk import trace as trace_sdk
 from opentelemetry.sdk.trace.export import SimpleSpanProcessor
@@ -118,6 +127,7 @@ class Tracer(AgenticTracing):
         self.timeout = 30
         self.num_projects = 100
         self.start_time = datetime.datetime.now().astimezone().isoformat()
+        self.model_cost_dict = load_model_costs()
 
         if update_llm_cost:
             # First update the model costs file from GitHub
@@ -152,11 +162,12 @@ class Tracer(AgenticTracing):
             raise
 
         if tracer_type == "langchain":
-            self.raga_client = RagaExporter(project_name=self.project_name, dataset_name=self.dataset_name)
+            # self.raga_client = RagaExporter(project_name=self.project_name, dataset_name=self.dataset_name)
 
-            self._tracer_provider = self._setup_provider()
-            self._instrumentor = self._setup_instrumentor(tracer_type)
-            self.is_instrumented = False
+            # self._tracer_provider = self._setup_provider()
+            # self._instrumentor = self._setup_instrumentor(tracer_type)
+            # self.is_instrumented = False
+            # self._upload_task = None
             self._upload_task = None
         elif tracer_type == "llamaindex":
             self._upload_task = None
@@ -239,11 +250,12 @@ class Tracer(AgenticTracing):
     def start(self):
         """Start the tracer."""
         if self.tracer_type == "langchain":
-            if not self.is_instrumented:
-                self._instrumentor().instrument(tracer_provider=self._tracer_provider)
-                self.is_instrumented = True
-            print(f"Tracer started for project: {self.project_name}")
-            return self
+            # if not self.is_instrumented:
+            #     self._instrumentor().instrument(tracer_provider=self._tracer_provider)
+            #     self.is_instrumented = True
+            # print(f"Tracer started for project: {self.project_name}")
+            self.langchain_tracer = LangchainTracer()
+            return self.langchain_tracer.start()
         elif self.tracer_type == "llamaindex":
             from ragaai_catalyst.tracers.llamaindex_callback import LlamaIndexTracer
             return LlamaIndexTracer(self._pass_user_data()).start()
@@ -254,17 +266,74 @@ class Tracer(AgenticTracing):
     def stop(self):
         """Stop the tracer and initiate trace upload."""
         if self.tracer_type == "langchain":
-            if not self.is_instrumented:
-                logger.warning("Tracer was not started. No traces to upload.")
-                return "No traces to upload"
-
-            print("Stopping tracer and initiating trace upload...")
-            self._cleanup()
-            self._upload_task = self._run_async(self._upload_traces())
-            self.is_active = False
-            self.dataset_name = None
+            # if not self.is_instrumented:
+            #     logger.warning("Tracer was not started. No traces to upload.")
+            #     return "No traces to upload"
+
+            # print("Stopping tracer and initiating trace upload...")
+            # self._cleanup()
+            # self._upload_task = self._run_async(self._upload_traces())
+            # self.is_active = False
+            # self.dataset_name = None
+
+            # filename = f"langchain_callback_traces.json"
+            # filepath = os.path.join(tempfile.gettempdir(), filename) 
+            
+            user_detail = self._pass_user_data()
+            data, additional_metadata = self.langchain_tracer.stop()
+
+            # Add cost if possible
+            # import pdb; pdb.set_trace()
+            if additional_metadata['model_name']:
+                try:
+                    model_cost_data = self.model_cost_dict[additional_metadata['model_name']]
+                    prompt_cost = additional_metadata["tokens"]["prompt"]*model_cost_data["input_cost_per_token"]
+                    completion_cost = additional_metadata["tokens"]["completion"]*model_cost_data["output_cost_per_token"]
+                    # additional_metadata.setdefault('cost', {})["prompt_cost"] = prompt_cost
+                    # additional_metadata.setdefault('cost', {})["completion_cost"] = completion_cost
+                    additional_metadata.setdefault('cost', {})["total_cost"] = prompt_cost + completion_cost 
+                except Exception as e:
+                    logger.warning(f"Error adding cost: {e}")
+
+            # with open(filepath, 'r') as f:
+            #     data = json.load(f)
+            additional_metadata["total_tokens"] = additional_metadata["tokens"]["total"]
+            additional_metadata["total_cost"] = additional_metadata["cost"]["total_cost"]
+
+            del additional_metadata["tokens"]
+            del additional_metadata["cost"]
+            
+            combined_metadata = user_detail['trace_user_detail']['metadata'].copy()
+            combined_metadata.update(additional_metadata)
+            combined_metadata
+
+            langchain_traces = langchain_tracer_extraction(data)
+            final_result = convert_langchain_callbacks_output(langchain_traces)
+            final_result[0]['project_name'] = user_detail['project_name']
+            final_result[0]['trace_id'] = str(uuid.uuid4())
+            final_result[0]['session_id'] = None
+            final_result[0]['metadata'] = combined_metadata
+            final_result[0]['pipeline'] = user_detail['trace_user_detail']['pipeline']
+
+            filepath_3 = os.path.join(os.getcwd(), "final_result.json")
+            with open(filepath_3, 'w') as f:
+                json.dump(final_result, f, indent=2)
             
-            return "Trace upload initiated. Use get_upload_status() to check the status."
+            
+            print(filepath_3)
+
+            additional_metadata_keys = additional_metadata.keys() if additional_metadata else None
+
+            UploadTraces(json_file_path=filepath_3,
+                         project_name=self.project_name,
+                         project_id=self.project_id,
+                         dataset_name=self.dataset_name,
+                         user_detail=user_detail,
+                         base_url=self.base_url
+                         ).upload_traces(additional_metadata_keys=additional_metadata_keys)
+            
+            return 
+
         elif self.tracer_type == "llamaindex":
             from ragaai_catalyst.tracers.llamaindex_callback import LlamaIndexTracer
             return LlamaIndexTracer(self._pass_user_data()).stop()

ragaai_catalyst/tracers/upload_traces.py

@@ -20,7 +20,7 @@ class UploadTraces:
         self.base_url = base_url
         self.timeout = 10
 
-    def _create_dataset_schema_with_trace(self):
+    def _create_dataset_schema_with_trace(self, additional_metadata_keys=None, additional_pipeline_keys=None):
         SCHEMA_MAPPING_NEW = {
             "trace_id": {"columnType": "traceId"},
             "trace_uri": {"columnType": "traceUri"},
@@ -34,6 +34,15 @@ class UploadTraces:
             "vector_store":{"columnType":"pipeline"},
             "feedback": {"columnType":"feedBack"}
         }
+
+        if additional_metadata_keys:
+            for key in additional_metadata_keys:
+                SCHEMA_MAPPING_NEW[key] = {"columnType": "metadata"}
+
+        if additional_pipeline_keys:
+            for key in additional_pipeline_keys:
+                SCHEMA_MAPPING_NEW[key] = {"columnType": "pipeline"}
+                
         def make_request():
             headers = {
                 "Content-Type": "application/json",
@@ -119,9 +128,14 @@ class UploadTraces:
                                     data=payload,
                                     timeout=self.timeout)
 
-    def upload_traces(self):
-        self._create_dataset_schema_with_trace()
-        presignedUrl = self._get_presigned_url()
-        self._put_presigned_url(presignedUrl, self.json_file_path)
-        self._insert_traces(presignedUrl)
-        print("Traces uploaded")
+    def upload_traces(self, additional_metadata_keys=None, additional_pipeline_keys=None):
+        try:
+            self._create_dataset_schema_with_trace(additional_metadata_keys, additional_pipeline_keys)
+            presignedUrl = self._get_presigned_url()
+            if presignedUrl is None:
+                return
+            self._put_presigned_url(presignedUrl, self.json_file_path)
+            self._insert_traces(presignedUrl)
+            print("Traces uploaded")
+        except Exception as e:
+            print(f"Error while uploading agentic traces: {e}")

ragaai_catalyst/tracers/utils/convert_langchain_callbacks_output.py

@@ -0,0 +1,61 @@
+import json
+
+def convert_langchain_callbacks_output(result, project_name="", metadata="", pipeline=""):
+    initial_struc = [{
+        "project_name": project_name,
+        "trace_id": "NA",
+        "session_id": "NA",
+        "metadata" : metadata,
+        "pipeline" : pipeline,
+        "traces" : []
+    }]
+    traces_data = []
+
+    prompt = result["data"]["prompt"]
+    response = result["data"]["response"]
+    context = result["data"]["context"]
+    final_prompt = ""
+
+    prompt_structured_data = {
+        "traceloop.entity.input": json.dumps({
+            "kwargs": {
+                "input": prompt,
+            }
+        })
+    }    
+    prompt_data = {
+        "name": "retrieve_documents.langchain.workflow",
+        "attributes": prompt_structured_data,
+    }
+
+    traces_data.append(prompt_data)
+
+    context_structured_data = {
+        "traceloop.entity.input": json.dumps({
+            "kwargs": {
+                "context": context
+            }
+        }),
+        "traceloop.entity.output": json.dumps({
+            "kwargs": {
+                "text": prompt
+            }
+        })
+    }
+    context_data = {
+        "name": "PromptTemplate.langchain.task",
+        "attributes": context_structured_data,
+    }
+    traces_data.append(context_data)
+
+    response_structured_data = {"gen_ai.completion.0.content": response,
+                                "gen_ai.prompt.0.content": prompt}
+    response_data = {
+        "name": "ChatOpenAI.langchain.task",
+        "attributes" : response_structured_data
+    }
+    traces_data.append(response_data)
+
+    initial_struc[0]["traces"] = traces_data
+
+    return initial_struc

ragaai_catalyst/tracers/utils/langchain_tracer_extraction_logic.py

@@ -0,0 +1,81 @@
+import json
+import uuid
+
+def langchain_tracer_extraction(data):
+    trace_aggregate = {}
+    import uuid
+
+    def generate_trace_id():
+        """
+        Generate a random trace ID using UUID4.
+        Returns a string representation of the UUID with no hyphens.
+        """
+        return '0x'+str(uuid.uuid4()).replace('-', '')
+
+    trace_aggregate["tracer_type"] = "langchain"
+    trace_aggregate['trace_id'] = generate_trace_id()
+    trace_aggregate['session_id'] = None
+    trace_aggregate["pipeline"] = {
+        'llm_model': 'gpt-3.5-turbo', 
+        'vector_store': 'faiss',
+        'embed_model': 'text-embedding-ada-002'
+        }
+    trace_aggregate["metadata"] = {
+        'key1': 'value1',
+        'key2': 'value2',
+        'log_source': 'langchain_tracer',
+        'recorded_on': '2024-06-14 08:57:27.324410'
+        }
+    trace_aggregate["prompt_length"] = 0
+    trace_aggregate["data"] = {}
+
+    def get_prompt(data):
+        # if "chain_starts" in data and data["chain_starts"] != []:
+        #     for item in data["chain_starts"]:
+
+        if "chat_model_calls" in data and data["chat_model_calls"] != []:
+            for item in data["chat_model_calls"]:
+                messages = item["messages"][0]
+                for message in messages:
+                    if message["type"]=="human":
+                        human_messages = message["content"].strip()
+                        return human_messages
+        if  "llm_calls" in data and data["llm_calls"] != []:
+            if "llm_start" in data["llm_calls"][0]["event"]:
+                for item in data["llm_calls"]:
+                    prompt = item["prompts"]
+                    return prompt[0].strip()
+
+    def get_response(data):
+        for item in data["llm_calls"]:
+            if item["event"] == "llm_end":
+                # import pdb; pdb.set_trace()
+                llm_end_responses = item["response"]["generations"][0]
+                for llm_end_response in llm_end_responses:
+                    response = llm_end_response["text"]
+                return response.strip()
+
+    def get_context(data):
+        if "retriever_actions" in data and data["retriever_actions"] != []:
+            for item in data["retriever_actions"]:
+                if item["event"] == "retriever_end":
+                    context = item["documents"][0]["page_content"].replace('\n', ' ')
+                    return context
+        if "chat_model_calls" in data and data["chat_model_calls"] != []:
+            for item in data["chat_model_calls"]:
+                messages = item["messages"][0]
+                for message in messages:
+                    if message["type"]=="system":
+                        content = message["content"].strip().replace('\n', ' ')
+                        return content
+
+
+    prompt = get_prompt(data)
+    response = get_response(data)
+    context = get_context(data)
+
+    trace_aggregate["data"]["prompt"]=prompt
+    trace_aggregate["data"]["response"]=response
+    trace_aggregate["data"]["context"]=context
+
+    return trace_aggregate

{ragaai_catalyst-2.1.5b1.dist-info → ragaai_catalyst-2.1.5b3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: ragaai_catalyst
-Version: 2.1.5b1
+Version: 2.1.5b3
 Summary: RAGA AI CATALYST
 Author-email: Kiran Scaria <kiran.scaria@raga.ai>, Kedar Gaikwad <kedar.gaikwad@raga.ai>, Dushyant Mahajan <dushyant.mahajan@raga.ai>, Siddhartha Kosti <siddhartha.kosti@raga.ai>, Ritika Goel <ritika.goel@raga.ai>, Vijay Chaurasia <vijay.chaurasia@raga.ai>
 Requires-Python: <3.13,>=3.9

{ragaai_catalyst-2.1.5b1.dist-info → ragaai_catalyst-2.1.5b3.dist-info}/RECORD

@@ -1,6 +1,6 @@
 ragaai_catalyst/__init__.py,sha256=tvESV8UuVtth14E89wQxgf0WvQZSApWfDeLiIdmMhkE,791
 ragaai_catalyst/_version.py,sha256=JKt9KaVNOMVeGs8ojO6LvIZr7ZkMzNN-gCcvryy4x8E,460
-ragaai_catalyst/dataset.py,sha256=Mc0L28lO4f7zD8DraQMHWzuizJf8Q0Z-DxNx3pS4OFo,23409
+ragaai_catalyst/dataset.py,sha256=8EbZEpcV4Fb8oO0vhYIK_WIwGyYvJ4VzwGeIZDtu2E8,23475
 ragaai_catalyst/evaluation.py,sha256=34H2bYZNSrcu0jMQgDZw1OLVbQU80PaVLo2avju8POM,20311
 ragaai_catalyst/experiment.py,sha256=8yQo1phCHlpnJ-4CqCaIbLXg_1ZlAuLGI9kqGBl-OTE,18859
 ragaai_catalyst/guard_executor.py,sha256=llPbE3DyVtrybojXknzBZj8-dtUrGBQwi9-ZiPJxGRo,3762
@@ -8,14 +8,14 @@ ragaai_catalyst/guardrails_manager.py,sha256=DILMOAASK57FH9BLq_8yC1AQzRJ8McMFLwC
 ragaai_catalyst/internal_api_completion.py,sha256=DdICI5yfEudiOAIC8L4oxH0Qz7kX-BZCdo9IWsi2gNo,2965
 ragaai_catalyst/prompt_manager.py,sha256=W8ypramzOprrJ7-22d5vkBXIuIQ8v9XAzKDGxKsTK28,16550
 ragaai_catalyst/proxy_call.py,sha256=CHxldeceZUaLU-to_hs_Kf1z_b2vHMssLS_cOBedu78,5499
-ragaai_catalyst/ragaai_catalyst.py,sha256=FdqMzwuQLqS2-3JJDsTQ8uh2itllOxfPrRUjb8Kwmn0,17428
+ragaai_catalyst/ragaai_catalyst.py,sha256=5nVg3_-lcvhrXjNkPTeGhe3tdUjm_4ZIctOcqWXBkRA,17939
 ragaai_catalyst/synthetic_data_generation.py,sha256=uDV9tNwto2xSkWg5XHXUvjErW-4P34CTrxaJpRfezyA,19250
 ragaai_catalyst/utils.py,sha256=TlhEFwLyRU690HvANbyoRycR3nQ67lxVUQoUOfTPYQ0,3772
 ragaai_catalyst/tracers/__init__.py,sha256=LfgTes-nHpazssbGKnn8kyLZNr49kIPrlkrqqoTFTfc,301
 ragaai_catalyst/tracers/distributed.py,sha256=AIRvS5Ur4jbFDXsUkYuCTmtGoHHx3LOG4n5tWOh610U,10330
 ragaai_catalyst/tracers/llamaindex_callback.py,sha256=ZY0BJrrlz-P9Mg2dX-ZkVKG3gSvzwqBtk7JL_05MiYA,14028
-ragaai_catalyst/tracers/tracer.py,sha256=S_ANRm5zSMvQiUyQTRwyUepFci_T3AN26wAOXoURfyc,15648
-ragaai_catalyst/tracers/upload_traces.py,sha256=mT5rverNUL5Rcal9VR5_c75wHBAUrm2pvYetTZqP3ok,4796
+ragaai_catalyst/tracers/tracer.py,sha256=k2HjH6ONaabbPvoX6xJRck-A2l-9GVW7Nueimuu-Ua8,19096
+ragaai_catalyst/tracers/upload_traces.py,sha256=2TWdRTN6FMaX-dqDv8BJWQS0xrCGYKkXEYOi2kK3Z3Y,5487
 ragaai_catalyst/tracers/agentic_tracing/README.md,sha256=X4QwLb7-Jg7GQMIXj-SerZIgDETfw-7VgYlczOR8ZeQ,4508
 ragaai_catalyst/tracers/agentic_tracing/__init__.py,sha256=yf6SKvOPSpH-9LiKaoLKXwqj5sez8F_5wkOb91yp0oE,260
 ragaai_catalyst/tracers/agentic_tracing/data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -28,10 +28,10 @@ ragaai_catalyst/tracers/agentic_tracing/tests/ai_travel_agent.py,sha256=S4rCcKzU
 ragaai_catalyst/tracers/agentic_tracing/tests/unique_decorator_test.py,sha256=Xk1cLzs-2A3dgyBwRRnCWs7Eubki40FVonwd433hPN8,4805
 ragaai_catalyst/tracers/agentic_tracing/tracers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ragaai_catalyst/tracers/agentic_tracing/tracers/agent_tracer.py,sha256=--wvhOJ-J2433WPatIS3wx6VFeCUIcgRT5_ZjGQDv2c,26364
-ragaai_catalyst/tracers/agentic_tracing/tracers/base.py,sha256=RFHbmzFdkl1zV4ZnM6VOAnCeTMqSKmgdH8VuWsrBwf4,40120
+ragaai_catalyst/tracers/agentic_tracing/tracers/base.py,sha256=88rX7OkOGEyVNECUrc4bYqODyulXve_-99d9ku5hBeQ,37373
 ragaai_catalyst/tracers/agentic_tracing/tracers/custom_tracer.py,sha256=l3x3uFO5ov93I7UUrUX1M06WVGy2ug2jEZ1G7o315z4,13075
 ragaai_catalyst/tracers/agentic_tracing/tracers/langgraph_tracer.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ragaai_catalyst/tracers/agentic_tracing/tracers/llm_tracer.py,sha256=fuSVks5-OchQLXqN6T18CtDjE8ufrCcksckcC3WW-_k,30150
+ragaai_catalyst/tracers/agentic_tracing/tracers/llm_tracer.py,sha256=s6BRoBteCRF8XrXGnmZ98ZWPrSONC5RObPXNaq-im3w,31782
 ragaai_catalyst/tracers/agentic_tracing/tracers/main_tracer.py,sha256=6hsg-Yw11v4qeELI1CWrdX8BXf-wJrTF5smBI5prgoo,15873
 ragaai_catalyst/tracers/agentic_tracing/tracers/network_tracer.py,sha256=m8CxYkl7iMiFya_lNwN1ykBc3Pmo-2pR_2HmpptwHWQ,10352
 ragaai_catalyst/tracers/agentic_tracing/tracers/tool_tracer.py,sha256=4rWL7fIJE5wN0nwh6fMWyh3OrrenZHJkNzyQXikyzQI,13771
@@ -49,6 +49,7 @@ ragaai_catalyst/tracers/agentic_tracing/utils/get_user_trace_metrics.py,sha256=v
 ragaai_catalyst/tracers/agentic_tracing/utils/llm_utils.py,sha256=wlXCuaRe81s-7FWdJ_MquXFGRZZfNrZxLIIxl-Ohbqk,15541
 ragaai_catalyst/tracers/agentic_tracing/utils/model_costs.json,sha256=GXV1s349reRMpYF_EkK-b6peSb4SY-17WnlkvpuQ4sM,294430
 ragaai_catalyst/tracers/agentic_tracing/utils/span_attributes.py,sha256=MqeRNGxzeuh9qTK0NbYMftl9V9Z0V7gMgBoHkrXP56k,1592
+ragaai_catalyst/tracers/agentic_tracing/utils/system_monitor.py,sha256=H8WNsk4v_5T6OUw4TFOzlDLjQhJwjh1nAMyMAoqMEi4,6946
 ragaai_catalyst/tracers/agentic_tracing/utils/trace_utils.py,sha256=RciiDdo2riibEoM8X0FKHaXi78y3bWwNkV8U0leqigk,3508
 ragaai_catalyst/tracers/agentic_tracing/utils/unique_decorator.py,sha256=DQHjcEuqEKsNSWaNs7SoOaq50yK4Jsl966S7mBnV-zA,5723
 ragaai_catalyst/tracers/agentic_tracing/utils/zip_list_of_unique_files.py,sha256=8SAN4pjA3XKBWlUZiDGXGh26ve1qRH_fv0VwewsfCFc,18818
@@ -60,9 +61,11 @@ ragaai_catalyst/tracers/instrumentators/langchain.py,sha256=yMN0qVF0pUVk6R5M1vJo
 ragaai_catalyst/tracers/instrumentators/llamaindex.py,sha256=SMrRlR4xM7k9HK43hakE8rkrWHxMlmtmWD-AX6TeByc,416
 ragaai_catalyst/tracers/instrumentators/openai.py,sha256=14R4KW9wQCR1xysLfsP_nxS7cqXrTPoD8En4MBAaZUU,379
 ragaai_catalyst/tracers/utils/__init__.py,sha256=KeMaZtYaTojilpLv65qH08QmpYclfpacDA0U3wg6Ybw,64
+ragaai_catalyst/tracers/utils/convert_langchain_callbacks_output.py,sha256=ofrNrxf2b1hpjDh_zeaxiYq86azn1MF3kW8-ViYPEg0,1641
+ragaai_catalyst/tracers/utils/langchain_tracer_extraction_logic.py,sha256=cghjCuUe8w-2MZdh9xgtRGe3y219u26GGzpnuY4Wt6Q,3047
 ragaai_catalyst/tracers/utils/utils.py,sha256=ViygfJ7vZ7U0CTSA1lbxVloHp4NSlmfDzBRNCJuMhis,2374
-ragaai_catalyst-2.1.5b1.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-ragaai_catalyst-2.1.5b1.dist-info/METADATA,sha256=raEM7cStptlPTu7ph3OyOU7UanQ3_7-T9dQbFXlFMbs,12764
-ragaai_catalyst-2.1.5b1.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-ragaai_catalyst-2.1.5b1.dist-info/top_level.txt,sha256=HpgsdRgEJMk8nqrU6qdCYk3di7MJkDL0B19lkc7dLfM,16
-ragaai_catalyst-2.1.5b1.dist-info/RECORD,,
+ragaai_catalyst-2.1.5b3.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+ragaai_catalyst-2.1.5b3.dist-info/METADATA,sha256=i-IVw7tVuDCXGNCIBH8Lsovatn4x67VrhV-hf-HYWYQ,12764
+ragaai_catalyst-2.1.5b3.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+ragaai_catalyst-2.1.5b3.dist-info/top_level.txt,sha256=HpgsdRgEJMk8nqrU6qdCYk3di7MJkDL0B19lkc7dLfM,16
+ragaai_catalyst-2.1.5b3.dist-info/RECORD,,