ragaai-catalyst 2.0.7.2__py3-none-any.whl → 2.0.7.2b0__py3-none-any.whl

ragaai_catalyst/tracers/agentic_tracing/utils/trace_utils.py

@@ -0,0 +1,74 @@
+import json
+import os
+from importlib import resources
+from dataclasses import asdict
+
+
+def convert_usage_to_dict(usage):
+    # Initialize the token_usage dictionary with default values
+    token_usage = {
+        "input": 0,
+        "completion": 0,
+        "reasoning": 0,  # Default reasoning tokens to 0 unless specified
+    }
+
+    if usage:
+        if isinstance(usage, dict):
+            # Access usage data as dictionary keys
+            token_usage["input"] = usage.get("prompt_tokens", 0)
+            token_usage["completion"] = usage.get("completion_tokens", 0)
+            # If reasoning tokens are provided, adjust accordingly
+            token_usage["reasoning"] = usage.get("reasoning_tokens", 0)
+        else:
+            # Handle the case where usage is not a dictionary
+            # This could be an object with attributes, or something else
+            try:
+                token_usage["input"] = getattr(usage, "prompt_tokens", 0)
+                token_usage["completion"] = getattr(usage, "completion_tokens", 0)
+                token_usage["reasoning"] = getattr(usage, "reasoning_tokens", 0)
+            except AttributeError:
+                # If attributes are not found, log or handle the error as needed
+                print(f"Warning: Unexpected usage type: {type(usage)}")
+
+    return token_usage
+
+
+def calculate_cost(
+    token_usage,
+    input_cost_per_token=0.0,
+    output_cost_per_token=0.0,
+    reasoning_cost_per_token=0.0,
+):
+    input_tokens = token_usage.get("prompt_tokens", 0)
+    output_tokens = token_usage.get("completion_tokens", 0)
+    reasoning_tokens = token_usage.get("reasoning_tokens", 0)
+
+    input_cost = input_tokens * input_cost_per_token
+    output_cost = output_tokens * output_cost_per_token
+    reasoning_cost = reasoning_tokens * reasoning_cost_per_token
+
+    total_cost = input_cost + output_cost + reasoning_cost
+
+    return {
+        "input": input_cost,
+        "completion": output_cost,
+        "reasoning": reasoning_cost,
+        "total": total_cost,
+    }
+
+
+def load_model_costs():
+    try:
+        current_dir = os.path.dirname(os.path.abspath(__file__))
+        model_costs_path = os.path.join(current_dir, "model_costs.json")
+        with open(model_costs_path, "r") as file:
+            return json.load(file)
+    except FileNotFoundError:
+        with resources.open_text("utils", "model_costs.json") as file:
+            return json.load(file)
+
+
+def log_event(event_data, log_file_path):
+    event_data = asdict(event_data)
+    with open(log_file_path, "a") as f:
+        f.write(json.dumps(event_data) + "\n")

ragaai_catalyst/tracers/tracer.py

@@ -19,11 +19,13 @@ from .instrumentators import (
 from .utils import get_unique_key
 # from .llamaindex_callback import LlamaIndexTracer
 from ..ragaai_catalyst import RagaAICatalyst
+from .agentic_tracing.agentic_tracing import AgenticTracing
+from .agentic_tracing.llm_tracer import LLMTracerMixin
 
 logger = logging.getLogger(__name__)
 
 
-class Tracer:
+class Tracer(AgenticTracing):
     NUM_PROJECTS = 100
     TIMEOUT = 10
     def __init__(
@@ -41,6 +43,7 @@ class Tracer:
 
         Args:
             project_name (str): The name of the project.
+            dataset_name (str): The name of the dataset.
             tracer_type (str, optional): The type of tracer. Defaults to None.
             pipeline (dict, optional): The pipeline configuration. Defaults to None.
             metadata (dict, optional): The metadata. Defaults to None.
@@ -50,16 +53,28 @@ class Tracer:
         Returns:
             None
         """
+        # Set auto_instrument_llm to True to enable automatic LLM tracing
+        user_detail = {
+            "project_name": project_name,
+            "project_id": None,  # Will be set after project validation
+            "dataset_name": dataset_name,
+            "trace_user_detail": {"metadata": metadata} if metadata else {}
+        }
+        super().__init__(user_detail=user_detail, auto_instrument_llm=True)
+        self.is_active = True
         self.project_name = project_name
         self.dataset_name = dataset_name
         self.tracer_type = tracer_type
         self.metadata = self._improve_metadata(metadata, tracer_type)
+        self.metadata["total_cost"] = 0.0
+        self.metadata["total_tokens"] = 0
         self.pipeline = pipeline
         self.description = description
         self.upload_timeout = upload_timeout
         self.base_url = f"{RagaAICatalyst.BASE_URL}"
         self.timeout = 10
         self.num_projects = 100
+        self.start_time = datetime.datetime.now(datetime.timezone.utc)
 
         try:
             response = requests.get(
@@ -81,6 +96,7 @@ class Tracer:
             self.project_id = [
                 project["id"] for project in response.json()["data"]["content"] if project["name"] == project_name
             ][0]
+            self._pass_user_data()
 
         except requests.exceptions.RequestException as e:
             logger.error(f"Failed to retrieve projects list: {e}")
@@ -98,7 +114,8 @@ class Tracer:
             from .llamaindex_callback import LlamaIndexTracer
 
         else:
-            raise ValueError (f"Currently supported tracer types are 'langchain' and 'llamaindex'.")
+            self._upload_task = None
+            # raise ValueError (f"Currently supported tracer types are 'langchain' and 'llamaindex'.")
 
     def _improve_metadata(self, metadata, tracer_type):
         if metadata is None:
@@ -157,7 +174,9 @@ class Tracer:
         elif self.tracer_type == "llamaindex":
             from .llamaindex_callback import LlamaIndexTracer
             return LlamaIndexTracer(self._pass_user_data()).start()
-            
+        else:
+            super().start()
+            return self
 
     def stop(self):
         """Stop the tracer and initiate trace upload."""
@@ -172,7 +191,9 @@ class Tracer:
             return "Trace upload initiated. Use get_upload_status() to check the status."
         elif self.tracer_type == "llamaindex":
             from .llamaindex_callback import LlamaIndexTracer
-            return LlamaIndexTracer().stop()
+            return LlamaIndexTracer(self._pass_user_data()).stop()
+        else:
+            super().stop()
 
     def get_upload_status(self):
         """Check the status of the trace upload."""
@@ -262,6 +283,7 @@ class Tracer:
         # Reset instrumentation flag
         self.is_instrumented = False
         # Note: We're not resetting all attributes here to allow for upload status checking
+
     def _pass_user_data(self):
         return {"project_name":self.project_name, 
                 "project_id": self.project_id,

ragaai_catalyst/tracers/upload_traces.py

@@ -0,0 +1,127 @@
+import requests
+import json
+import os
+from datetime import datetime
+
+
+class UploadTraces:
+    def __init__(self, 
+                 json_file_path,
+                 project_name,
+                 project_id,
+                 dataset_name,
+                 user_detail,
+                 base_url):
+        self.json_file_path = json_file_path
+        self.project_name = project_name
+        self.project_id = project_id
+        self.dataset_name = dataset_name
+        self.user_detail = user_detail
+        self.base_url = base_url
+        self.timeout = 10
+
+    def _create_dataset_schema_with_trace(self):
+        SCHEMA_MAPPING_NEW = {
+            "trace_id": {"columnType": "traceId"},
+            "trace_uri": {"columnType": "traceUri"},
+            "prompt": {"columnType": "prompt"},
+            "response":{"columnType": "response"},
+            "context": {"columnType": "context"},
+            "llm_model": {"columnType":"pipeline"},
+            "recorded_on": {"columnType": "metadata"},
+            "embed_model": {"columnType":"pipeline"},
+            "log_source": {"columnType": "metadata"},
+            "vector_store":{"columnType":"pipeline"},
+            "feedback": {"columnType":"feedBack"}
+        }
+        def make_request():
+            headers = {
+                "Content-Type": "application/json",
+                "Authorization": f"Bearer {os.getenv('RAGAAI_CATALYST_TOKEN')}",
+                "X-Project-Name": self.project_name,
+            }
+            payload = json.dumps({
+                "datasetName": self.dataset_name,
+                "schemaMapping": SCHEMA_MAPPING_NEW,
+                "traceFolderUrl": None,
+            })
+            response = requests.request("POST",
+                f"{self.base_url}/v1/llm/dataset/logs",
+                headers=headers,
+                data=payload,
+                timeout=self.timeout
+            )
+
+            return response
+
+        response = make_request()
+
+        if response.status_code == 401:
+            # get_token()  # Fetch a new token and set it in the environment
+            response = make_request()  # Retry the request
+        if response.status_code != 200:
+            return response.status_code
+        return response.status_code
+
+    def _get_presigned_url(self):
+        payload = json.dumps({
+                "datasetName": self.dataset_name,
+                "numFiles": 1,
+            })
+        headers = {
+            "Content-Type": "application/json",
+            "Authorization": f"Bearer {os.getenv('RAGAAI_CATALYST_TOKEN')}",
+            "X-Project-Name": self.project_name,
+        }
+
+        response = requests.request("GET", 
+                                    f"{self.base_url}/v1/llm/presigned-url", 
+                                    headers=headers, 
+                                    data=payload,
+                                    timeout=self.timeout)
+        if response.status_code == 200:
+            presignedUrls = response.json()["data"]["presignedUrls"][0]
+            return presignedUrls
+
+    def _put_presigned_url(self, presignedUrl, filename):
+        headers = {
+                "Content-Type": "application/json",
+            }
+
+        if "blob.core.windows.net" in presignedUrl:  # Azure
+            headers["x-ms-blob-type"] = "BlockBlob"
+        print(f"Uploading traces...")
+        with open(filename) as f:
+            payload = f.read().replace("\n", "").replace("\r", "").encode()
+            
+
+        response = requests.request("PUT", 
+                                    presignedUrl, 
+                                    headers=headers, 
+                                    data=payload,
+                                    timeout=self.timeout)
+        if response.status_code != 200 or response.status_code != 201:
+            return response, response.status_code
+
+    def _insert_traces(self, presignedUrl):
+        headers = {
+                "Authorization": f"Bearer {os.getenv('RAGAAI_CATALYST_TOKEN')}",
+                "Content-Type": "application/json",
+                "X-Project-Name": self.project_name,
+            }
+        payload = json.dumps({
+                "datasetName": self.dataset_name,
+                "presignedUrl": presignedUrl,
+            })
+        response = requests.request("POST", 
+                                    f"{self.base_url}/v1/llm/insert/trace", 
+                                    headers=headers, 
+                                    data=payload,
+                                    timeout=self.timeout)
+
+    def upload_traces(self):
+        self._create_dataset_schema_with_trace()
+        presignedUrl = self._get_presigned_url()
+        self._put_presigned_url(presignedUrl, self.json_file_path)
+        self._insert_traces(presignedUrl)
+        print("Traces uplaoded")

ragaai_catalyst-2.0.7.2b0.dist-info/METADATA

@@ -0,0 +1,39 @@
+Metadata-Version: 2.1
+Name: ragaai_catalyst
+Version: 2.0.7.2b0
+Summary: RAGA AI CATALYST
+Author-email: Kiran Scaria <kiran.scaria@raga.ai>, Kedar Gaikwad <kedar.gaikwad@raga.ai>, Dushyant Mahajan <dushyant.mahajan@raga.ai>, Siddhartha Kosti <siddhartha.kosti@raga.ai>, Ritika Goel <ritika.goel@raga.ai>, Vijay Chaurasia <vijay.chaurasia@raga.ai>
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Requires-Dist: aiohttp>=3.10.2
+Requires-Dist: opentelemetry-api==1.25.0
+Requires-Dist: opentelemetry-sdk==1.25.0
+Requires-Dist: opentelemetry-exporter-otlp-proto-grpc==1.25.0
+Requires-Dist: opentelemetry-instrumentation==0.46b0
+Requires-Dist: opentelemetry-instrumentation-fastapi==0.46b0
+Requires-Dist: opentelemetry-instrumentation-asgi==0.46b0
+Requires-Dist: opentelemetry-semantic-conventions==0.46b0
+Requires-Dist: opentelemetry-util-http==0.46b0
+Requires-Dist: opentelemetry-instrumentation-langchain~=0.24.0
+Requires-Dist: opentelemetry-instrumentation-openai~=0.24.0
+Requires-Dist: langchain-core>=0.2.11
+Requires-Dist: langchain>=0.2.11
+Requires-Dist: openai>=1.57.0
+Requires-Dist: pandas>=2.1.1
+Requires-Dist: groq>=0.11.0
+Requires-Dist: PyPDF2>=3.0.1
+Requires-Dist: google-generativeai>=0.8.2
+Requires-Dist: Markdown>=3.7
+Requires-Dist: litellm==1.51.1
+Requires-Dist: tenacity==8.3.0
+Requires-Dist: tqdm>=4.66.5
+Requires-Dist: llama-index==0.10.0
+Requires-Dist: pyopenssl==24.2.1
+Requires-Dist: psutil>=5.9.5
+Provides-Extra: dev
+Requires-Dist: pytest; extra == "dev"
+Requires-Dist: pytest-cov; extra == "dev"
+Requires-Dist: black; extra == "dev"
+Requires-Dist: isort; extra == "dev"
+Requires-Dist: mypy; extra == "dev"
+Requires-Dist: flake8; extra == "dev"

ragaai_catalyst-2.0.7.2b0.dist-info/RECORD

@@ -0,0 +1,50 @@
+ragaai_catalyst/__init__.py,sha256=BdIJ_UUre0uEnRTsLw_hE0C0muWk6XWNZqdVOel22R4,537
+ragaai_catalyst/_version.py,sha256=JKt9KaVNOMVeGs8ojO6LvIZr7ZkMzNN-gCcvryy4x8E,460
+ragaai_catalyst/dataset.py,sha256=j_vu3Xkp_8qYW0J9Qnn53Uyh98MsugqYl5zxhOv9EOg,10731
+ragaai_catalyst/evaluation.py,sha256=xrJ8kxw-fGNJgHsBTJCdH4jrxwzI6jpzUIQ9po-V4kc,20327
+ragaai_catalyst/experiment.py,sha256=8KvqgJg5JVnt9ghhGDJvdb4mN7ETBX_E5gNxBT0Nsn8,19010
+ragaai_catalyst/guard_executor.py,sha256=llPbE3DyVtrybojXknzBZj8-dtUrGBQwi9-ZiPJxGRo,3762
+ragaai_catalyst/guardrails_manager.py,sha256=DILMOAASK57FH9BLq_8yC1AQzRJ8McMFLwCXgYwNAd4,11904
+ragaai_catalyst/internal_api_completion.py,sha256=DdICI5yfEudiOAIC8L4oxH0Qz7kX-BZCdo9IWsi2gNo,2965
+ragaai_catalyst/prompt_manager.py,sha256=W8ypramzOprrJ7-22d5vkBXIuIQ8v9XAzKDGxKsTK28,16550
+ragaai_catalyst/proxy_call.py,sha256=CHxldeceZUaLU-to_hs_Kf1z_b2vHMssLS_cOBedu78,5499
+ragaai_catalyst/ragaai_catalyst.py,sha256=FdqMzwuQLqS2-3JJDsTQ8uh2itllOxfPrRUjb8Kwmn0,17428
+ragaai_catalyst/synthetic_data_generation.py,sha256=uDV9tNwto2xSkWg5XHXUvjErW-4P34CTrxaJpRfezyA,19250
+ragaai_catalyst/utils.py,sha256=TlhEFwLyRU690HvANbyoRycR3nQ67lxVUQoUOfTPYQ0,3772
+ragaai_catalyst/tracers/__init__.py,sha256=NppmJhD3sQ5R1q6teaZLS7rULj08Gb6JT8XiPRIe_B0,49
+ragaai_catalyst/tracers/llamaindex_callback.py,sha256=vPE7MieKjfwLrLUnnPs20Df0xNYqoCCj-Mt2NbiuiKU,14023
+ragaai_catalyst/tracers/tracer.py,sha256=F7LiIUKj1mCl9tl_xwJOWLP-80ICuw5KVBsismLpOM0,12280
+ragaai_catalyst/tracers/upload_traces.py,sha256=hs0PEmit3n3_uUqrdbwcBdyK5Nbkik3JQVwJMEwYTd4,4796
+ragaai_catalyst/tracers/agentic_tracing/Untitled-1.json,sha256=kqwtLzflk342U7Tvjf5Bari9m3I0dVMVU7wNShDwuW4,43639
+ragaai_catalyst/tracers/agentic_tracing/__init__.py,sha256=6QyQI8P7aNFHTantNJOP1ZdSNrDKBLhlg_gGNj7tm1c,73
+ragaai_catalyst/tracers/agentic_tracing/agent_tracer.py,sha256=n0fuZOguYScuRoRijz-16okfh54HWlpbTmt133msi64,11815
+ragaai_catalyst/tracers/agentic_tracing/agentic_tracing.py,sha256=XvoOqEJjqzhE5Xa8mJ2knik6qrecxd1-kRZAFl7P0FI,8680
+ragaai_catalyst/tracers/agentic_tracing/base.py,sha256=4YWt9Sdrs4IZoJldBN_tl0DCu4zLLEYyDoLvs36oFiw,9955
+ragaai_catalyst/tracers/agentic_tracing/data_structure.py,sha256=YRZFTzenIIPJ-eDblnUUjY1QeECu6DozfhCgmj38n9s,4696
+ragaai_catalyst/tracers/agentic_tracing/llm_tracer.py,sha256=hxvUc4CYeVncgzJThykyg8Dd98lgogSGbMch8pjF5C8,36766
+ragaai_catalyst/tracers/agentic_tracing/network_tracer.py,sha256=6FTA15xMnum9omM_0Jd9cMIuWdKu1gR5Tc8fOXAkP8E,10068
+ragaai_catalyst/tracers/agentic_tracing/sample.py,sha256=S4rCcKzU_5SB62BYEbNn_1VbbTdG4396N8rdZ3ZNGcE,5654
+ragaai_catalyst/tracers/agentic_tracing/tool_tracer.py,sha256=O9HJ-lOTK5WYyv6u27W0pvQp4vfsuZGYDua6Ev2zzFc,8407
+ragaai_catalyst/tracers/agentic_tracing/unique_decorator.py,sha256=WtcALkwS8dsJhqc1641toyH1lVwWyzFwRLBKY56XKJI,7448
+ragaai_catalyst/tracers/agentic_tracing/unique_decorator_test.py,sha256=Xk1cLzs-2A3dgyBwRRnCWs7Eubki40FVonwd433hPN8,4805
+ragaai_catalyst/tracers/agentic_tracing/user_interaction_tracer.py,sha256=7Y-31rcoJ4YeZn9gSnqJgujGpRX6rKAkq6Y0KDIMTOo,2085
+ragaai_catalyst/tracers/agentic_tracing/utils/__init__.py,sha256=XdB3X_ufe4RVvGorxSqAiB9dYv4UD7Hvvuw3bsDUppY,60
+ragaai_catalyst/tracers/agentic_tracing/utils/api_utils.py,sha256=JyNCbfpW-w4O9CjtemTqmor2Rh1WGpQwhRaDSRmBxw8,689
+ragaai_catalyst/tracers/agentic_tracing/utils/data_classes.py,sha256=gFOadWzUORETaH3Y_HerHXs_aVFEt5ry2NQ36domW58,1267
+ragaai_catalyst/tracers/agentic_tracing/utils/generic.py,sha256=WwXT01xmp8MSr7KinuDCSK9a1ifpLcT7ajFkvYviG_A,1190
+ragaai_catalyst/tracers/agentic_tracing/utils/llm_utils.py,sha256=noCDrF1tG-n1LSdooq_-kkBw8z35YvsiGGhcy1-yL4M,5810
+ragaai_catalyst/tracers/agentic_tracing/utils/model_costs.json,sha256=5jvhriJJkWT-nz8S-KsgsjOn9g7D2hKYob8URl9FHOk,215176
+ragaai_catalyst/tracers/agentic_tracing/utils/trace_utils.py,sha256=9cFzfFqIA968bUG7LNTjdN7zbdEXUtcvRKg883ade2c,2586
+ragaai_catalyst/tracers/exporters/__init__.py,sha256=kVA8zp05h3phu4e-iHSlnznp_PzMRczB7LphSsZgUjg,138
+ragaai_catalyst/tracers/exporters/file_span_exporter.py,sha256=RgGteu-NVGprXKkynvyIO5yOjpbtA41R3W_NzCjnkwE,6445
+ragaai_catalyst/tracers/exporters/raga_exporter.py,sha256=rQ5Wj71f2Ke3qLlV8KiWCskbGBR-ia_hlzDx86rPrEo,18188
+ragaai_catalyst/tracers/instrumentators/__init__.py,sha256=FgnMQupoRTzmVsG9YKsLQera2Pfs-AluZv8CxwavoyQ,253
+ragaai_catalyst/tracers/instrumentators/langchain.py,sha256=yMN0qVF0pUVk6R5M1vJoUXezDo1ejs4klCFRlE8x4vE,574
+ragaai_catalyst/tracers/instrumentators/llamaindex.py,sha256=SMrRlR4xM7k9HK43hakE8rkrWHxMlmtmWD-AX6TeByc,416
+ragaai_catalyst/tracers/instrumentators/openai.py,sha256=14R4KW9wQCR1xysLfsP_nxS7cqXrTPoD8En4MBAaZUU,379
+ragaai_catalyst/tracers/utils/__init__.py,sha256=KeMaZtYaTojilpLv65qH08QmpYclfpacDA0U3wg6Ybw,64
+ragaai_catalyst/tracers/utils/utils.py,sha256=ViygfJ7vZ7U0CTSA1lbxVloHp4NSlmfDzBRNCJuMhis,2374
+ragaai_catalyst-2.0.7.2b0.dist-info/METADATA,sha256=dBXbhc2TWIyfwewQ2tmlnEcu8DU6hOyZVIPX22Gl-6w,1677
+ragaai_catalyst-2.0.7.2b0.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+ragaai_catalyst-2.0.7.2b0.dist-info/top_level.txt,sha256=HpgsdRgEJMk8nqrU6qdCYk3di7MJkDL0B19lkc7dLfM,16
+ragaai_catalyst-2.0.7.2b0.dist-info/RECORD,,