ragaai-catalyst 2.1.3b0__py3-none-any.whl → 2.1.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (27) hide show
  1. ragaai_catalyst/tracers/agentic_tracing/data/data_structure.py +37 -11
  2. ragaai_catalyst/tracers/agentic_tracing/tracers/agent_tracer.py +240 -81
  3. ragaai_catalyst/tracers/agentic_tracing/tracers/base.py +632 -114
  4. ragaai_catalyst/tracers/agentic_tracing/tracers/custom_tracer.py +316 -0
  5. ragaai_catalyst/tracers/agentic_tracing/tracers/langgraph_tracer.py +0 -0
  6. ragaai_catalyst/tracers/agentic_tracing/tracers/llm_tracer.py +229 -82
  7. ragaai_catalyst/tracers/agentic_tracing/tracers/main_tracer.py +214 -59
  8. ragaai_catalyst/tracers/agentic_tracing/tracers/network_tracer.py +16 -14
  9. ragaai_catalyst/tracers/agentic_tracing/tracers/tool_tracer.py +147 -28
  10. ragaai_catalyst/tracers/agentic_tracing/tracers/user_interaction_tracer.py +88 -2
  11. ragaai_catalyst/tracers/agentic_tracing/upload/upload_agentic_traces.py +9 -51
  12. ragaai_catalyst/tracers/agentic_tracing/upload/upload_trace_metric.py +83 -0
  13. ragaai_catalyst/tracers/agentic_tracing/utils/create_dataset_schema.py +26 -0
  14. ragaai_catalyst/tracers/agentic_tracing/utils/get_user_trace_metrics.py +28 -0
  15. ragaai_catalyst/tracers/agentic_tracing/utils/llm_utils.py +45 -15
  16. ragaai_catalyst/tracers/agentic_tracing/utils/model_costs.json +2520 -2152
  17. ragaai_catalyst/tracers/agentic_tracing/utils/span_attributes.py +59 -0
  18. ragaai_catalyst/tracers/agentic_tracing/utils/trace_utils.py +23 -0
  19. ragaai_catalyst/tracers/agentic_tracing/utils/zip_list_of_unique_files.py +284 -15
  20. ragaai_catalyst/tracers/llamaindex_callback.py +5 -5
  21. ragaai_catalyst/tracers/tracer.py +83 -10
  22. ragaai_catalyst/tracers/upload_traces.py +1 -1
  23. ragaai_catalyst-2.1.4.dist-info/METADATA +431 -0
  24. {ragaai_catalyst-2.1.3b0.dist-info → ragaai_catalyst-2.1.4.dist-info}/RECORD +26 -20
  25. ragaai_catalyst-2.1.3b0.dist-info/METADATA +0 -43
  26. {ragaai_catalyst-2.1.3b0.dist-info → ragaai_catalyst-2.1.4.dist-info}/WHEEL +0 -0
  27. {ragaai_catalyst-2.1.3b0.dist-info → ragaai_catalyst-2.1.4.dist-info}/top_level.txt +0 -0
@@ -1,12 +1,22 @@
1
+ import os
1
2
  import uuid
2
3
  from datetime import datetime
3
4
  import psutil
4
5
  import functools
5
6
  from typing import Optional, Any, Dict, List
6
- from ..utils.unique_decorator import generate_unique_hash_simple, mydecorator
7
+ from ..utils.unique_decorator import generate_unique_hash_simple
7
8
  import contextvars
8
9
  import asyncio
9
10
  from ..utils.file_name_tracker import TrackName
11
+ from ..utils.span_attributes import SpanAttributes
12
+ import logging
13
+
14
+ logger = logging.getLogger(__name__)
15
+ logging_level = (
16
+ logger.setLevel(logging.DEBUG)
17
+ if os.getenv("DEBUG")
18
+ else logger.setLevel(logging.INFO)
19
+ )
10
20
 
11
21
 
12
22
  class ToolTracerMixin:
@@ -19,17 +29,68 @@ class ToolTracerMixin:
19
29
  self.component_user_interaction = {}
20
30
  self.gt = None
21
31
 
32
+ # add auto_instrument option
33
+ self.auto_instrument_tool = False
34
+ self.auto_instrument_user_interaction = False
35
+ self.auto_instrument_network = False
36
+
37
+ # take care of auto_instrument
38
+ def instrument_tool_calls(self):
39
+ self.auto_instrument_tool = True
40
+
41
+ def instrument_user_interaction_calls(self):
42
+ self.auto_instrument_user_interaction = True
43
+
44
+ def instrument_network_calls(self):
45
+ self.auto_instrument_network = True
46
+
47
+ def trace_tool(
48
+ self,
49
+ name: str,
50
+ tool_type: str = "generic",
51
+ version: str = "1.0.0",
52
+ tags: List[str] = [],
53
+ metadata: Dict[str, Any] = {},
54
+ metrics: List[Dict[str, Any]] = [],
55
+ feedback: Optional[Any] = None,
56
+ ):
57
+ if name not in self.span_attributes_dict:
58
+ self.span_attributes_dict[name] = SpanAttributes(name)
59
+ if tags:
60
+ self.span(name).add_tags(tags)
61
+ if metadata:
62
+ self.span(name).add_metadata(metadata)
63
+ if metrics:
64
+ if isinstance(metrics, dict):
65
+ metrics = [metrics]
66
+ try:
67
+ for metric in metrics:
68
+ self.span(name).add_metrics(
69
+ name=metric["name"],
70
+ score=metric["score"],
71
+ reasoning=metric.get("reasoning", ""),
72
+ cost=metric.get("cost", None),
73
+ latency=metric.get("latency", None),
74
+ metadata=metric.get("metadata", {}),
75
+ config=metric.get("config", {}),
76
+ )
77
+ except ValueError as e:
78
+ logger.error(f"Validation Error: {e}")
79
+ except Exception as e:
80
+ logger.error(f"Error adding metric: {e}")
81
+
82
+ if feedback:
83
+ self.span(name).add_feedback(feedback)
22
84
 
23
- def trace_tool(self, name: str, tool_type: str = "generic", version: str = "1.0.0"):
24
85
  def decorator(func):
25
86
  # Add metadata attribute to the function
26
87
  metadata = {
27
88
  "name": name,
28
89
  "tool_type": tool_type,
29
90
  "version": version,
30
- "is_active": True
91
+ "is_active": self.is_active,
31
92
  }
32
-
93
+
33
94
  # Check if the function is async
34
95
  is_async = asyncio.iscoroutinefunction(func)
35
96
 
@@ -37,7 +98,7 @@ class ToolTracerMixin:
37
98
  @functools.wraps(func)
38
99
  async def async_wrapper(*args, **kwargs):
39
100
  async_wrapper.metadata = metadata
40
- self.gt = kwargs.get('gt', None) if kwargs else None
101
+ self.gt = kwargs.get("gt", None) if kwargs else None
41
102
  return await self._trace_tool_execution(
42
103
  func, name, tool_type, version, *args, **kwargs
43
104
  )
@@ -46,7 +107,7 @@ class ToolTracerMixin:
46
107
  @functools.wraps(func)
47
108
  def sync_wrapper(*args, **kwargs):
48
109
  sync_wrapper.metadata = metadata
49
- self.gt = kwargs.get('gt', None) if kwargs else None
110
+ self.gt = kwargs.get("gt", None) if kwargs else None
50
111
  return self._trace_sync_tool_execution(
51
112
  func, name, tool_type, version, *args, **kwargs
52
113
  )
@@ -57,11 +118,16 @@ class ToolTracerMixin:
57
118
 
58
119
  return decorator
59
120
 
60
- def _trace_sync_tool_execution(self, func, name, tool_type, version, *args, **kwargs):
121
+ def _trace_sync_tool_execution(
122
+ self, func, name, tool_type, version, *args, **kwargs
123
+ ):
61
124
  """Synchronous version of tool tracing"""
62
125
  if not self.is_active:
63
126
  return func(*args, **kwargs)
64
127
 
128
+ if not self.auto_instrument_tool:
129
+ return func(*args, **kwargs)
130
+
65
131
  start_time = datetime.now().astimezone()
66
132
  start_memory = psutil.Process().memory_info().rss
67
133
  component_id = str(uuid.uuid4())
@@ -91,10 +157,11 @@ class ToolTracerMixin:
91
157
  memory_used=memory_used,
92
158
  start_time=start_time,
93
159
  input_data=self._sanitize_input(args, kwargs),
94
- output_data=self._sanitize_output(result)
160
+ output_data=self._sanitize_output(result),
95
161
  )
96
162
 
97
163
  self.add_component(tool_component)
164
+
98
165
  return result
99
166
 
100
167
  except Exception as e:
@@ -102,12 +169,12 @@ class ToolTracerMixin:
102
169
  "code": 500,
103
170
  "type": type(e).__name__,
104
171
  "message": str(e),
105
- "details": {}
172
+ "details": {},
106
173
  }
107
-
174
+
108
175
  # End tracking network calls for this component
109
176
  self.end_component(component_id)
110
-
177
+
111
178
  tool_component = self.create_tool_component(
112
179
  component_id=component_id,
113
180
  hash_id=hash_id,
@@ -118,22 +185,29 @@ class ToolTracerMixin:
118
185
  start_time=start_time,
119
186
  input_data=self._sanitize_input(args, kwargs),
120
187
  output_data=None,
121
- error=error_component
188
+ error=error_component,
122
189
  )
123
190
 
124
191
  self.add_component(tool_component)
192
+
125
193
  raise
126
194
 
127
- async def _trace_tool_execution(self, func, name, tool_type, version, *args, **kwargs):
195
+ async def _trace_tool_execution(
196
+ self, func, name, tool_type, version, *args, **kwargs
197
+ ):
128
198
  """Asynchronous version of tool tracing"""
129
199
  if not self.is_active:
130
200
  return await func(*args, **kwargs)
131
201
 
202
+ if not self.auto_instrument_tool:
203
+ return await func(*args, **kwargs)
204
+
132
205
  start_time = datetime.now().astimezone()
133
206
  start_memory = psutil.Process().memory_info().rss
134
207
  component_id = str(uuid.uuid4())
135
208
  hash_id = generate_unique_hash_simple(func)
136
209
 
210
+ self.start_component(component_id)
137
211
  try:
138
212
  # Execute the tool
139
213
  result = await func(*args, **kwargs)
@@ -141,6 +215,7 @@ class ToolTracerMixin:
141
215
  # Calculate resource usage
142
216
  end_memory = psutil.Process().memory_info().rss
143
217
  memory_used = max(0, end_memory - start_memory)
218
+ self.end_component(component_id)
144
219
 
145
220
  # Create tool component
146
221
  tool_component = self.create_tool_component(
@@ -152,9 +227,10 @@ class ToolTracerMixin:
152
227
  start_time=start_time,
153
228
  memory_used=memory_used,
154
229
  input_data=self._sanitize_input(args, kwargs),
155
- output_data=self._sanitize_output(result)
230
+ output_data=self._sanitize_output(result),
156
231
  )
157
232
  self.add_component(tool_component)
233
+
158
234
  return result
159
235
 
160
236
  except Exception as e:
@@ -162,9 +238,9 @@ class ToolTracerMixin:
162
238
  "code": 500,
163
239
  "type": type(e).__name__,
164
240
  "message": str(e),
165
- "details": {}
241
+ "details": {},
166
242
  }
167
-
243
+
168
244
  tool_component = self.create_tool_component(
169
245
  component_id=component_id,
170
246
  hash_id=hash_id,
@@ -175,15 +251,42 @@ class ToolTracerMixin:
175
251
  memory_used=0,
176
252
  input_data=self._sanitize_input(args, kwargs),
177
253
  output_data=None,
178
- error=error_component
254
+ error=error_component,
179
255
  )
180
256
  self.add_component(tool_component)
257
+
181
258
  raise
182
259
 
183
260
  def create_tool_component(self, **kwargs):
184
-
185
-
186
261
  """Create a tool component according to the data structure"""
262
+ network_calls = []
263
+ if self.auto_instrument_network:
264
+ network_calls = self.component_network_calls.get(kwargs["component_id"], [])
265
+ interactions = []
266
+ if self.auto_instrument_user_interaction:
267
+ interactions = self.component_user_interaction.get(
268
+ kwargs["component_id"], []
269
+ )
270
+
271
+ # Get tags, metrics
272
+ name = kwargs["name"]
273
+ # tags
274
+ tags = []
275
+ if name in self.span_attributes_dict:
276
+ tags = self.span_attributes_dict[name].tags or []
277
+
278
+ # metrics
279
+ metrics = []
280
+ if name in self.span_attributes_dict:
281
+ raw_metrics = self.span_attributes_dict[name].metrics or []
282
+ for metric in raw_metrics:
283
+ base_metric_name = metric["name"]
284
+ counter = sum(1 for x in self.visited_metrics if x.startswith(base_metric_name))
285
+ metric_name = f'{base_metric_name}_{counter}' if counter > 0 else base_metric_name
286
+ self.visited_metrics.append(metric_name)
287
+ metric["name"] = metric_name
288
+ metrics.append(metric)
289
+
187
290
  start_time = kwargs["start_time"]
188
291
  component = {
189
292
  "id": kwargs["component_id"],
@@ -198,20 +301,25 @@ class ToolTracerMixin:
198
301
  "info": {
199
302
  "tool_type": kwargs["tool_type"],
200
303
  "version": kwargs["version"],
201
- "memory_used": kwargs["memory_used"]
304
+ "memory_used": kwargs["memory_used"],
305
+ "tags": tags,
202
306
  },
203
307
  "data": {
204
308
  "input": kwargs["input_data"],
205
309
  "output": kwargs["output_data"],
206
- "memory_used": kwargs["memory_used"]
310
+ "memory_used": kwargs["memory_used"],
207
311
  },
208
- "network_calls": self.component_network_calls.get(kwargs["component_id"], []),
209
- "interactions": self.component_user_interaction.get(kwargs["component_id"], [])
312
+ "metrics": metrics,
313
+ "network_calls": network_calls,
314
+ "interactions": interactions,
210
315
  }
211
316
 
212
- if self.gt:
317
+ if self.gt:
213
318
  component["data"]["gt"] = self.gt
214
319
 
320
+ # Reset the SpanAttributes context variable
321
+ self.span_attributes_dict[kwargs["name"]] = SpanAttributes(kwargs["name"])
322
+
215
323
  return component
216
324
 
217
325
  def start_component(self, component_id):
@@ -223,15 +331,26 @@ class ToolTracerMixin:
223
331
  def _sanitize_input(self, args: tuple, kwargs: dict) -> Dict:
224
332
  """Sanitize and format input data"""
225
333
  return {
226
- "args": [str(arg) if not isinstance(arg, (int, float, bool, str, list, dict)) else arg for arg in args],
334
+ "args": [
335
+ (
336
+ str(arg)
337
+ if not isinstance(arg, (int, float, bool, str, list, dict))
338
+ else arg
339
+ )
340
+ for arg in args
341
+ ],
227
342
  "kwargs": {
228
- k: str(v) if not isinstance(v, (int, float, bool, str, list, dict)) else v
343
+ k: (
344
+ str(v)
345
+ if not isinstance(v, (int, float, bool, str, list, dict))
346
+ else v
347
+ )
229
348
  for k, v in kwargs.items()
230
- }
349
+ },
231
350
  }
232
351
 
233
352
  def _sanitize_output(self, output: Any) -> Any:
234
353
  """Sanitize and format output data"""
235
354
  if isinstance(output, (int, float, bool, str, list, dict)):
236
355
  return output
237
- return str(output)
356
+ return str(output)
@@ -3,6 +3,35 @@ from datetime import datetime
3
3
  import contextvars
4
4
  import inspect
5
5
  import uuid
6
+ from typing import Optional, Any
7
+
8
+ class TracedFile:
9
+ def __init__(self, file_obj, file_path: str, tracer):
10
+ self._file = file_obj
11
+ self._file_path = file_path
12
+ self._tracer = tracer
13
+
14
+ def write(self, content: str) -> int:
15
+ self._tracer.trace_file_operation("write", self._file_path, content=content)
16
+ return self._file.write(content)
17
+
18
+ def read(self, size: Optional[int] = None) -> str:
19
+ content = self._file.read() if size is None else self._file.read(size)
20
+ self._tracer.trace_file_operation("read", self._file_path, content=content)
21
+ return content
22
+
23
+ def close(self) -> None:
24
+ return self._file.close()
25
+
26
+ def __enter__(self):
27
+ return self
28
+
29
+ def __exit__(self, exc_type, exc_val, exc_tb):
30
+ self.close()
31
+ return None
32
+
33
+ def __getattr__(self, name: str) -> Any:
34
+ return getattr(self._file, name)
6
35
 
7
36
  class UserInteractionTracer:
8
37
  def __init__(self, *args, **kwargs):
@@ -12,6 +41,7 @@ class UserInteractionTracer:
12
41
  self.component_id = contextvars.ContextVar("component_id", default=None)
13
42
  self.original_input = builtins.input
14
43
  self.original_print = builtins.print
44
+ self.original_open = builtins.open
15
45
  self.interactions = []
16
46
 
17
47
  def traced_input(self, prompt=""):
@@ -25,9 +55,10 @@ class UserInteractionTracer:
25
55
 
26
56
  self.interactions.append({
27
57
  "id": str(uuid.uuid4()),
58
+ "component_id": self.component_id.get(),
28
59
  "interaction_type": "input",
29
60
  "content": content,
30
- "timestamp": datetime.now().isoformat()
61
+ "timestamp": datetime.now().astimezone().isoformat()
31
62
  })
32
63
  return content
33
64
 
@@ -36,8 +67,63 @@ class UserInteractionTracer:
36
67
 
37
68
  self.interactions.append({
38
69
  "id": str(uuid.uuid4()),
70
+ "component_id": self.component_id.get(),
39
71
  "interaction_type": "output",
40
72
  "content": content,
41
- "timestamp": datetime.now().isoformat()
73
+ "timestamp": datetime.now().astimezone().isoformat()
42
74
  })
43
75
  return self.original_print(*args, **kwargs)
76
+
77
+ def traced_open(self, file: str, mode: str = 'r', *args, **kwargs):
78
+ # Skip tracing for system and virtual environment paths
79
+ system_paths = [
80
+ 'site-packages',
81
+ 'dist-packages',
82
+ '/proc/',
83
+ '/sys/',
84
+ '/var/lib/',
85
+ '/usr/lib/',
86
+ '/System/Library'
87
+ ]
88
+
89
+ file_str = str(file)
90
+ if any(path in file_str for path in system_paths):
91
+ return self.original_open(file, mode, *args, **kwargs)
92
+
93
+ file_obj = self.original_open(file, mode, *args, **kwargs)
94
+ return TracedFile(file_obj, file, self)
95
+
96
+ def trace_file_operation(self, operation: str, file_path: str, **kwargs):
97
+ interaction_type = f"file_{operation}"
98
+
99
+ # Check for existing interaction with same file_path and operation
100
+ for existing in reversed(self.interactions):
101
+ if (existing.get("file_path") == file_path and
102
+ existing.get("interaction_type") == interaction_type):
103
+ # Merge content if it exists
104
+ if "content" in kwargs and "content" in existing:
105
+ existing["content"] += kwargs["content"]
106
+ return
107
+ break
108
+
109
+ # If no matching interaction found or couldn't merge, create new one
110
+ interaction = {
111
+ "id": str(uuid.uuid4()),
112
+ "component_id": self.component_id.get(),
113
+ "interaction_type": interaction_type,
114
+ "file_path": file_path,
115
+ "timestamp": datetime.now().astimezone().isoformat()
116
+ }
117
+ interaction.update(kwargs)
118
+ self.interactions.append(interaction)
119
+
120
+ def __enter__(self):
121
+ builtins.input = self.traced_input
122
+ builtins.print = self.traced_print
123
+ builtins.open = self.traced_open
124
+ return self
125
+
126
+ def __exit__(self, exc_type, exc_val, exc_tb):
127
+ builtins.input = self.original_input
128
+ builtins.print = self.original_print
129
+ builtins.open = self.original_open
@@ -18,50 +18,7 @@ class UploadAgenticTraces:
18
18
  self.dataset_name = dataset_name
19
19
  self.user_detail = user_detail
20
20
  self.base_url = base_url
21
- self.timeout = 99999
22
-
23
- def _create_dataset_schema_with_trace(self):
24
- SCHEMA_MAPPING_NEW = {
25
- "trace_id": {"columnType": "traceId"},
26
- "trace_uri": {"columnType": "traceUri"},
27
- "prompt": {"columnType": "prompt"},
28
- "response":{"columnType": "response"},
29
- "context": {"columnType": "context"},
30
- "llm_model": {"columnType":"pipeline"},
31
- "recorded_on": {"columnType": "metadata"},
32
- "embed_model": {"columnType":"pipeline"},
33
- "log_source": {"columnType": "metadata"},
34
- "vector_store":{"columnType":"pipeline"},
35
- "feedback": {"columnType":"feedBack"}
36
- }
37
- def make_request():
38
- headers = {
39
- "Content-Type": "application/json",
40
- "Authorization": f"Bearer {os.getenv('RAGAAI_CATALYST_TOKEN')}",
41
- "X-Project-Name": self.project_name,
42
- }
43
- payload = json.dumps({
44
- "datasetName": self.dataset_name,
45
- # "schemaMapping": SCHEMA_MAPPING_NEW,
46
- "traceFolderUrl": None,
47
- })
48
- response = requests.request("POST",
49
- f"{self.base_url}/v1/llm/dataset/logs",
50
- headers=headers,
51
- data=payload,
52
- timeout=self.timeout
53
- )
54
-
55
- return response
56
-
57
- response = make_request()
58
-
59
- if response.status_code == 401:
60
- # get_token() # Fetch a new token and set it in the environment
61
- response = make_request() # Retry the request
62
- if response.status_code != 200:
63
- return response.status_code
64
- return response.status_code
21
+ self.timeout = 30
65
22
 
66
23
 
67
24
  def _get_presigned_url(self):
@@ -181,10 +138,11 @@ class UploadAgenticTraces:
181
138
  return None
182
139
 
183
140
  def upload_agentic_traces(self):
184
- self._create_dataset_schema_with_trace()
185
- presignedUrl = self._get_presigned_url()
186
- if presignedUrl is None:
187
- return
188
- self._put_presigned_url(presignedUrl, self.json_file_path)
189
- self.insert_traces(presignedUrl)
190
- print("Agentic Traces uploaded")
141
+ try:
142
+ presignedUrl = self._get_presigned_url()
143
+ if presignedUrl is None:
144
+ return
145
+ self._put_presigned_url(presignedUrl, self.json_file_path)
146
+ self.insert_traces(presignedUrl)
147
+ except Exception as e:
148
+ print(f"Error while uploading agentic traces: {e}")
@@ -0,0 +1,83 @@
1
+ import requests
2
+ import os
3
+ import json
4
+ from ....ragaai_catalyst import RagaAICatalyst
5
+ from ..utils.get_user_trace_metrics import get_user_trace_metrics
6
+
7
+ def upload_trace_metric(json_file_path, dataset_name, project_name):
8
+ try:
9
+ with open(json_file_path, "r") as f:
10
+ traces = json.load(f)
11
+ metrics = get_trace_metrics_from_trace(traces)
12
+ metrics = _change_metrics_format_for_payload(metrics)
13
+
14
+ user_trace_metrics = get_user_trace_metrics(project_name, dataset_name)
15
+ if user_trace_metrics:
16
+ user_trace_metrics_list = [metric["displayName"] for metric in user_trace_metrics]
17
+
18
+ if user_trace_metrics:
19
+ for metric in metrics:
20
+ if metric["displayName"] in user_trace_metrics_list:
21
+ metricConfig = next((user_metric["metricConfig"] for user_metric in user_trace_metrics if user_metric["displayName"] == metric["displayName"]), None)
22
+ if not metricConfig or metricConfig.get("Metric Source", {}).get("value") != "user":
23
+ raise ValueError(f"Metrics {metric['displayName']} already exist in dataset {dataset_name} of project {project_name}.")
24
+
25
+ headers = {
26
+ "Content-Type": "application/json",
27
+ "Authorization": f"Bearer {os.getenv('RAGAAI_CATALYST_TOKEN')}",
28
+ "X-Project-Name": project_name,
29
+ }
30
+ payload = json.dumps({
31
+ "datasetName": dataset_name,
32
+ "metrics": metrics
33
+ })
34
+ response = requests.request("POST",
35
+ f"{RagaAICatalyst.BASE_URL}/v1/llm/trace/metrics",
36
+ headers=headers,
37
+ data=payload,
38
+ timeout=10)
39
+ if response.status_code != 200:
40
+ raise ValueError(f"Error inserting agentic trace metrics")
41
+ except requests.exceptions.RequestException as e:
42
+ raise ValueError(f"Error submitting traces: {e}")
43
+ return None
44
+
45
+ return response
46
+
47
+
48
+ def _get_children_metrics_of_agent(children_traces):
49
+ metrics = []
50
+ for span in children_traces:
51
+ metrics.extend(span.get("metrics", []))
52
+ if span["type"] != "agent":
53
+ metric = span.get("metrics", [])
54
+ if metric:
55
+ metrics.extend(metric)
56
+ else:
57
+ metrics.extend(_get_children_metrics_of_agent(span["data"]["children"]))
58
+ return metrics
59
+
60
+ def get_trace_metrics_from_trace(traces):
61
+ metrics = []
62
+ for span in traces["data"][0]["spans"]:
63
+ if span["type"] == "agent":
64
+ children_metric = _get_children_metrics_of_agent(span["data"]["children"])
65
+ if children_metric:
66
+ metrics.extend(children_metric)
67
+ else:
68
+ metric = span.get("metrics", [])
69
+ if metric:
70
+ metrics.extend(metric)
71
+ return metrics
72
+
73
+ def _change_metrics_format_for_payload(metrics):
74
+ formatted_metrics = []
75
+ for metric in metrics:
76
+ if any(m["name"] == metric["name"] for m in formatted_metrics):
77
+ continue
78
+ formatted_metrics.append({
79
+ "name": metric["name"],
80
+ "displayName": metric["name"],
81
+ "config": {"source": "user"},
82
+ })
83
+ return formatted_metrics
@@ -0,0 +1,26 @@
1
+ import os
2
+ import json
3
+ import re
4
+ import requests
5
+ from ragaai_catalyst.tracers.agentic_tracing.tracers.base import RagaAICatalyst
6
+
7
+ def create_dataset_schema_with_trace(project_name, dataset_name):
8
+ def make_request():
9
+ headers = {
10
+ "Content-Type": "application/json",
11
+ "Authorization": f"Bearer {os.getenv('RAGAAI_CATALYST_TOKEN')}",
12
+ "X-Project-Name": project_name,
13
+ }
14
+ payload = json.dumps({
15
+ "datasetName": dataset_name,
16
+ "traceFolderUrl": None,
17
+ })
18
+ response = requests.request("POST",
19
+ f"{RagaAICatalyst.BASE_URL}/v1/llm/dataset/logs",
20
+ headers=headers,
21
+ data=payload,
22
+ timeout=10
23
+ )
24
+ return response
25
+ response = make_request()
26
+ return response
@@ -0,0 +1,28 @@
1
+ import requests
2
+ import os
3
+ from ....ragaai_catalyst import RagaAICatalyst
4
+ from ....dataset import Dataset
5
+
6
+ def get_user_trace_metrics(project_name, dataset_name):
7
+ try:
8
+ list_datasets = Dataset(project_name=project_name).list_datasets()
9
+ if not list_datasets:
10
+ return []
11
+ elif dataset_name not in list_datasets:
12
+ return []
13
+ else:
14
+ headers = {
15
+ "Authorization": f"Bearer {os.getenv('RAGAAI_CATALYST_TOKEN')}",
16
+ "X-Project-Name": project_name,
17
+ }
18
+ response = requests.request("GET",
19
+ f"{RagaAICatalyst.BASE_URL}/v1/llm/trace/metrics?datasetName={dataset_name}",
20
+ headers=headers, timeout=10)
21
+ if response.status_code != 200:
22
+ print(f"Error fetching traces metrics: {response.json()['message']}")
23
+ return None
24
+
25
+ return response.json()["data"]["columns"]
26
+ except Exception as e:
27
+ print(f"Error fetching traces metrics: {e}")
28
+ return None