ragaai-catalyst 2.1.4.1b0__py3-none-any.whl → 2.1.4.1b1__py3-none-any.whl

ragaai_catalyst/tracers/agentic_tracing/data/data_structure.py

@@ -271,7 +271,7 @@ class ComponentInfo:
     cost: Optional[Dict[str, float]] = None
 
 class Trace:
-    def __init__(self, id: str, trace_name: str, project_name: str, start_time: str, end_time: str, metadata: Optional[Metadata] = None, data: Optional[List[Dict[str, Any]]] = None, replays: Optional[Dict[str, Any]] = None):
+    def __init__(self, id: str, trace_name: str, project_name: str, start_time: str, end_time: str, metadata: Optional[Metadata] = None, data: Optional[List[Dict[str, Any]]] = None, replays: Optional[Dict[str, Any]] = None, metrics: Optional[List[Dict[str, Any]]] = None):
         self.id = id
         self.trace_name = trace_name
         self.project_name = project_name
@@ -280,6 +280,7 @@ class Trace:
         self.metadata = metadata or Metadata()
         self.data = data or []
         self.replays = replays
+        self.metrics = metrics or []
 
     def to_dict(self):
         return {
@@ -288,7 +289,8 @@ class Trace:
             "project_name": self.project_name,
             "start_time": self.start_time,
             "end_time": self.end_time,
-            "metadata": self.metadata.to_dict() if self.metadata else None,
+            "metadata": self.metadata,
             "data": self.data,
             "replays": self.replays,
+            "metrics": self.metrics
         }

ragaai_catalyst/tracers/agentic_tracing/tracers/base.py

@@ -5,7 +5,7 @@ import psutil
 import pkg_resources
 from datetime import datetime
 from pathlib import Path
-from typing import List, Any
+from typing import List, Any, Dict
 import uuid
 import sys
 import tempfile
@@ -81,6 +81,7 @@ class BaseTracer:
         self.project_id = self.user_details["project_id"]  # Access the project_id
         self.trace_name = self.user_details["trace_name"]  # Access the trace_name
         self.visited_metrics = []
+        self.trace_metrics = []  # Store metrics here
 
         # Initialize trace data
         self.trace_id = None
@@ -211,6 +212,10 @@ class BaseTracer:
         threading.Thread(target=self._track_disk_usage).start()
         threading.Thread(target=self._track_network_usage).start()
 
+        # Reset metrics
+        self.visited_metrics = []
+        self.trace_metrics = []
+
         metadata = Metadata(
             cost={},
             tokens={},
@@ -241,6 +246,7 @@ class BaseTracer:
             metadata=metadata,
             data=self.data_key,
             replays={"source": None},
+            metrics=[]  # Initialize empty metrics list
         )
 
     def stop(self):
@@ -300,8 +306,12 @@ class BaseTracer:
             # replace source code with zip_path
             self.trace.metadata.system_info.source_code = hash_id
 
+            # Add metrics to trace before saving
+            trace_data = self.trace.to_dict()
+
+            trace_data["metrics"] = self.trace_metrics
+
             # Clean up trace_data before saving
-            trace_data = self.trace.__dict__
             cleaned_trace_data = self._clean_trace(trace_data)
 
             # Format interactions and add to trace
@@ -609,9 +619,7 @@ class BaseTracer:
                     "span_id": child.get("id"),
                     "interaction_type": "llm_call_end",
                     "name": child.get("name"),
-                    "content": {
-                        "response": child.get("data", {}).get("output")
-                    },
+                    "content": {"response": child.get("data", {}).get("output")},
                     "timestamp": child.get("end_time"),
                     "error": child.get("error"),
                 }
@@ -882,8 +890,90 @@ class BaseTracer:
 
         return {"workflow": sorted_interactions}
 
+    def add_metrics(
+        self,
+        name: str | List[Dict[str, Any]] | Dict[str, Any] = None,
+        score: float | int = None,
+        reasoning: str = "",
+        cost: float = None,
+        latency: float = None,
+        metadata: Dict[str, Any] = None,
+        config: Dict[str, Any] = None,
+    ):
+        """Add metrics at the trace level.
+
+        Can be called in two ways:
+        1. With individual parameters:
+           tracer.add_metrics(name="metric_name", score=0.9, reasoning="Good performance")
+           
+        2. With a dictionary or list of dictionaries:
+           tracer.add_metrics({"name": "metric_name", "score": 0.9})
+           tracer.add_metrics([{"name": "metric1", "score": 0.9}, {"name": "metric2", "score": 0.8}])
+
+        Args:
+            name: Either the metric name (str) or a metric dictionary/list of dictionaries
+            score: Score value (float or int) when using individual parameters
+            reasoning: Optional explanation for the score
+            cost: Optional cost associated with the metric
+            latency: Optional latency measurement
+            metadata: Optional additional metadata as key-value pairs
+            config: Optional configuration parameters
+        """
+        if not hasattr(self, 'trace'):
+            logger.warning("Cannot add metrics before trace is initialized. Call start() first.")
+            return
+
+        # Convert individual parameters to metric dict if needed
+        if isinstance(name, str):
+            metrics = [{
+                "name": name,
+                "score": score,
+                "reasoning": reasoning,
+                "cost": cost,
+                "latency": latency,
+                "metadata": metadata or {},
+                "config": config or {}
+            }]
+        else:
+            # Handle dict or list input
+            metrics = name if isinstance(name, list) else [name] if isinstance(name, dict) else []
+
+        try:
+            for metric in metrics:
+                if not isinstance(metric, dict):
+                    raise ValueError(f"Expected dict, got {type(metric)}")
+                
+                if "name" not in metric or "score" not in metric:
+                    raise ValueError("Metric must contain 'name' and 'score' fields")
+
+                # Handle duplicate metric names
+                metric_name = metric["name"]
+                if metric_name in self.visited_metrics:
+                    count = sum(1 for m in self.visited_metrics if m.startswith(metric_name))
+                    metric_name = f"{metric_name}_{count + 1}"
+                self.visited_metrics.append(metric_name)
+
+                formatted_metric = {
+                    "name": metric_name,  # Use potentially modified name
+                    "score": metric["score"],
+                    "reason": metric.get("reasoning", ""),
+                    "source": "user",
+                    "cost": metric.get("cost"),
+                    "latency": metric.get("latency"),
+                    "metadata": metric.get("metadata", {}),
+                    "mappings": [],
+                    "config": metric.get("config", {})
+                }
+                
+                self.trace_metrics.append(formatted_metric)
+                logger.debug(f"Added trace-level metric: {formatted_metric}")
+
+        except ValueError as e:
+            logger.error(f"Validation Error: {e}")
+        except Exception as e:
+            logger.error(f"Error adding metric: {e}")
+    
     def span(self, span_name):
         if span_name not in self.span_attributes_dict:
             self.span_attributes_dict[span_name] = SpanAttributes(span_name)
-        return self.span_attributes_dict[span_name]
-    
+        return self.span_attributes_dict[span_name]

ragaai_catalyst/tracers/agentic_tracing/upload/upload_trace_metric.py

@@ -59,6 +59,13 @@ def _get_children_metrics_of_agent(children_traces):
 
 def get_trace_metrics_from_trace(traces):
     metrics = []
+
+    # get trace level metrics
+    if "metrics" in traces.keys():
+        if len(traces["metrics"]) > 0:
+            metrics.extend(traces["metrics"])
+
+    # get span level metrics
     for span in traces["data"][0]["spans"]:
         if span["type"] == "agent":
             children_metric = _get_children_metrics_of_agent(span["data"]["children"])

{ragaai_catalyst-2.1.4.1b0.dist-info → ragaai_catalyst-2.1.4.1b1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: ragaai_catalyst
-Version: 2.1.4.1b0
+Version: 2.1.4.1b1
 Summary: RAGA AI CATALYST
 Author-email: Kiran Scaria <kiran.scaria@raga.ai>, Kedar Gaikwad <kedar.gaikwad@raga.ai>, Dushyant Mahajan <dushyant.mahajan@raga.ai>, Siddhartha Kosti <siddhartha.kosti@raga.ai>, Ritika Goel <ritika.goel@raga.ai>, Vijay Chaurasia <vijay.chaurasia@raga.ai>
 Requires-Python: <3.13,>=3.9

{ragaai_catalyst-2.1.4.1b0.dist-info → ragaai_catalyst-2.1.4.1b1.dist-info}/RECORD

@@ -18,7 +18,7 @@ ragaai_catalyst/tracers/upload_traces.py,sha256=mT5rverNUL5Rcal9VR5_c75wHBAUrm2p
 ragaai_catalyst/tracers/agentic_tracing/README.md,sha256=X4QwLb7-Jg7GQMIXj-SerZIgDETfw-7VgYlczOR8ZeQ,4508
 ragaai_catalyst/tracers/agentic_tracing/__init__.py,sha256=yf6SKvOPSpH-9LiKaoLKXwqj5sez8F_5wkOb91yp0oE,260
 ragaai_catalyst/tracers/agentic_tracing/data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ragaai_catalyst/tracers/agentic_tracing/data/data_structure.py,sha256=nFnwqL1-Uznwndi2ugDnhziUbIASlcBYnM6Dyq7pPt8,9243
+ragaai_catalyst/tracers/agentic_tracing/data/data_structure.py,sha256=icAtNzKN_I0YtfuJ3RF8BdZJK3ohqxkVZIdvM5_YugY,9327
 ragaai_catalyst/tracers/agentic_tracing/tests/FinancialAnalysisSystem.ipynb,sha256=0qZxjWqYCTAVvdo3Tsp544D8Am48wfeMQ9RKpKgAL8g,34291
 ragaai_catalyst/tracers/agentic_tracing/tests/GameActivityEventPlanner.ipynb,sha256=QCMFJYbGX0fd9eMW4PqyQLZjyWuTXo7n1nqO_hMLf0s,4225
 ragaai_catalyst/tracers/agentic_tracing/tests/TravelPlanner.ipynb,sha256=fU3inXoemJbdTkGAQl_N1UwVEZ10LrKv4gCEpbQ4ISg,43481
@@ -27,7 +27,7 @@ ragaai_catalyst/tracers/agentic_tracing/tests/ai_travel_agent.py,sha256=S4rCcKzU
 ragaai_catalyst/tracers/agentic_tracing/tests/unique_decorator_test.py,sha256=Xk1cLzs-2A3dgyBwRRnCWs7Eubki40FVonwd433hPN8,4805
 ragaai_catalyst/tracers/agentic_tracing/tracers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ragaai_catalyst/tracers/agentic_tracing/tracers/agent_tracer.py,sha256=aLiq5nPie5TT61QYtvAtvErsxjPFYiUxjayn5aCX1_k,25543
-ragaai_catalyst/tracers/agentic_tracing/tracers/base.py,sha256=u2DX_BsMGsuJkWqN6ucxboMO8GnvOcArKJe5L08y-YI,35014
+ragaai_catalyst/tracers/agentic_tracing/tracers/base.py,sha256=13AkXbO6NE1a2rlimsROW3527vUnmin8_bRMNZfXarg,38783
 ragaai_catalyst/tracers/agentic_tracing/tracers/custom_tracer.py,sha256=uay8lU7T-CKsVu8KvWX31qfMqufK9S3Ive7XKo2Ksmk,12252
 ragaai_catalyst/tracers/agentic_tracing/tracers/langgraph_tracer.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ragaai_catalyst/tracers/agentic_tracing/tracers/llm_tracer.py,sha256=fn5qxb365GmQkJy_yZAY5TiiWRMFKPNJdYk8KFr8uWA,29343
@@ -38,7 +38,7 @@ ragaai_catalyst/tracers/agentic_tracing/tracers/user_interaction_tracer.py,sha25
 ragaai_catalyst/tracers/agentic_tracing/upload/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ragaai_catalyst/tracers/agentic_tracing/upload/upload_agentic_traces.py,sha256=1MDKXAAPzOEdxFKWWQrRgrmM3kz--DGXSywGXQmR3lQ,6041
 ragaai_catalyst/tracers/agentic_tracing/upload/upload_code.py,sha256=HgpMgI-JTWZrizcM7GGUIaAgaZF4aRT3D0dJXVEkblY,4271
-ragaai_catalyst/tracers/agentic_tracing/upload/upload_trace_metric.py,sha256=iFagpPltlg6aKvdyAFvXsuxyjUUjcHAMmvXlevL-uYM,3312
+ragaai_catalyst/tracers/agentic_tracing/upload/upload_trace_metric.py,sha256=id66gfx-XYj_zsAmicBKojBOqJQ__FJLSoZ0db56aes,3493
 ragaai_catalyst/tracers/agentic_tracing/utils/__init__.py,sha256=XdB3X_ufe4RVvGorxSqAiB9dYv4UD7Hvvuw3bsDUppY,60
 ragaai_catalyst/tracers/agentic_tracing/utils/api_utils.py,sha256=JyNCbfpW-w4O9CjtemTqmor2Rh1WGpQwhRaDSRmBxw8,689
 ragaai_catalyst/tracers/agentic_tracing/utils/create_dataset_schema.py,sha256=lgvJL0cakJrX8WGsnU05YGvotequSj6HgSohyR4OJNE,804
@@ -60,8 +60,8 @@ ragaai_catalyst/tracers/instrumentators/llamaindex.py,sha256=SMrRlR4xM7k9HK43hak
 ragaai_catalyst/tracers/instrumentators/openai.py,sha256=14R4KW9wQCR1xysLfsP_nxS7cqXrTPoD8En4MBAaZUU,379
 ragaai_catalyst/tracers/utils/__init__.py,sha256=KeMaZtYaTojilpLv65qH08QmpYclfpacDA0U3wg6Ybw,64
 ragaai_catalyst/tracers/utils/utils.py,sha256=ViygfJ7vZ7U0CTSA1lbxVloHp4NSlmfDzBRNCJuMhis,2374
-ragaai_catalyst-2.1.4.1b0.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-ragaai_catalyst-2.1.4.1b0.dist-info/METADATA,sha256=V5fmS2GcC5Ykd7yakB0_gDvqRPKHxzTtkp7TCt6T3O4,12794
-ragaai_catalyst-2.1.4.1b0.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-ragaai_catalyst-2.1.4.1b0.dist-info/top_level.txt,sha256=HpgsdRgEJMk8nqrU6qdCYk3di7MJkDL0B19lkc7dLfM,16
-ragaai_catalyst-2.1.4.1b0.dist-info/RECORD,,
+ragaai_catalyst-2.1.4.1b1.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+ragaai_catalyst-2.1.4.1b1.dist-info/METADATA,sha256=w1o5CYUrQ3M0x_vn1GBYsOGXSTPYUxToxp4RIRjI1jI,12794
+ragaai_catalyst-2.1.4.1b1.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+ragaai_catalyst-2.1.4.1b1.dist-info/top_level.txt,sha256=HpgsdRgEJMk8nqrU6qdCYk3di7MJkDL0B19lkc7dLfM,16
+ragaai_catalyst-2.1.4.1b1.dist-info/RECORD,,