ragaai-catalyst 2.1.3__py3-none-any.whl → 2.1.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ragaai_catalyst/tracers/agentic_tracing/data/data_structure.py +37 -11
- ragaai_catalyst/tracers/agentic_tracing/tracers/agent_tracer.py +240 -81
- ragaai_catalyst/tracers/agentic_tracing/tracers/base.py +632 -114
- ragaai_catalyst/tracers/agentic_tracing/tracers/custom_tracer.py +316 -0
- ragaai_catalyst/tracers/agentic_tracing/tracers/langgraph_tracer.py +0 -0
- ragaai_catalyst/tracers/agentic_tracing/tracers/llm_tracer.py +229 -82
- ragaai_catalyst/tracers/agentic_tracing/tracers/main_tracer.py +214 -59
- ragaai_catalyst/tracers/agentic_tracing/tracers/network_tracer.py +16 -14
- ragaai_catalyst/tracers/agentic_tracing/tracers/tool_tracer.py +147 -28
- ragaai_catalyst/tracers/agentic_tracing/tracers/user_interaction_tracer.py +88 -2
- ragaai_catalyst/tracers/agentic_tracing/upload/upload_agentic_traces.py +9 -51
- ragaai_catalyst/tracers/agentic_tracing/upload/upload_trace_metric.py +83 -0
- ragaai_catalyst/tracers/agentic_tracing/utils/create_dataset_schema.py +26 -0
- ragaai_catalyst/tracers/agentic_tracing/utils/get_user_trace_metrics.py +28 -0
- ragaai_catalyst/tracers/agentic_tracing/utils/llm_utils.py +45 -15
- ragaai_catalyst/tracers/agentic_tracing/utils/model_costs.json +2520 -2152
- ragaai_catalyst/tracers/agentic_tracing/utils/span_attributes.py +59 -0
- ragaai_catalyst/tracers/agentic_tracing/utils/trace_utils.py +23 -0
- ragaai_catalyst/tracers/agentic_tracing/utils/zip_list_of_unique_files.py +284 -15
- ragaai_catalyst/tracers/llamaindex_callback.py +5 -5
- ragaai_catalyst/tracers/tracer.py +83 -10
- ragaai_catalyst/tracers/upload_traces.py +1 -1
- ragaai_catalyst-2.1.4.dist-info/METADATA +431 -0
- {ragaai_catalyst-2.1.3.dist-info → ragaai_catalyst-2.1.4.dist-info}/RECORD +26 -20
- ragaai_catalyst-2.1.3.dist-info/METADATA +0 -43
- {ragaai_catalyst-2.1.3.dist-info → ragaai_catalyst-2.1.4.dist-info}/WHEEL +0 -0
- {ragaai_catalyst-2.1.3.dist-info → ragaai_catalyst-2.1.4.dist-info}/top_level.txt +0 -0
@@ -1,12 +1,22 @@
|
|
1
|
+
import os
|
1
2
|
import uuid
|
2
3
|
from datetime import datetime
|
3
4
|
import psutil
|
4
5
|
import functools
|
5
6
|
from typing import Optional, Any, Dict, List
|
6
|
-
from ..utils.unique_decorator import generate_unique_hash_simple
|
7
|
+
from ..utils.unique_decorator import generate_unique_hash_simple
|
7
8
|
import contextvars
|
8
9
|
import asyncio
|
9
10
|
from ..utils.file_name_tracker import TrackName
|
11
|
+
from ..utils.span_attributes import SpanAttributes
|
12
|
+
import logging
|
13
|
+
|
14
|
+
logger = logging.getLogger(__name__)
|
15
|
+
logging_level = (
|
16
|
+
logger.setLevel(logging.DEBUG)
|
17
|
+
if os.getenv("DEBUG")
|
18
|
+
else logger.setLevel(logging.INFO)
|
19
|
+
)
|
10
20
|
|
11
21
|
|
12
22
|
class ToolTracerMixin:
|
@@ -19,17 +29,68 @@ class ToolTracerMixin:
|
|
19
29
|
self.component_user_interaction = {}
|
20
30
|
self.gt = None
|
21
31
|
|
32
|
+
# add auto_instrument option
|
33
|
+
self.auto_instrument_tool = False
|
34
|
+
self.auto_instrument_user_interaction = False
|
35
|
+
self.auto_instrument_network = False
|
36
|
+
|
37
|
+
# take care of auto_instrument
|
38
|
+
def instrument_tool_calls(self):
|
39
|
+
self.auto_instrument_tool = True
|
40
|
+
|
41
|
+
def instrument_user_interaction_calls(self):
|
42
|
+
self.auto_instrument_user_interaction = True
|
43
|
+
|
44
|
+
def instrument_network_calls(self):
|
45
|
+
self.auto_instrument_network = True
|
46
|
+
|
47
|
+
def trace_tool(
|
48
|
+
self,
|
49
|
+
name: str,
|
50
|
+
tool_type: str = "generic",
|
51
|
+
version: str = "1.0.0",
|
52
|
+
tags: List[str] = [],
|
53
|
+
metadata: Dict[str, Any] = {},
|
54
|
+
metrics: List[Dict[str, Any]] = [],
|
55
|
+
feedback: Optional[Any] = None,
|
56
|
+
):
|
57
|
+
if name not in self.span_attributes_dict:
|
58
|
+
self.span_attributes_dict[name] = SpanAttributes(name)
|
59
|
+
if tags:
|
60
|
+
self.span(name).add_tags(tags)
|
61
|
+
if metadata:
|
62
|
+
self.span(name).add_metadata(metadata)
|
63
|
+
if metrics:
|
64
|
+
if isinstance(metrics, dict):
|
65
|
+
metrics = [metrics]
|
66
|
+
try:
|
67
|
+
for metric in metrics:
|
68
|
+
self.span(name).add_metrics(
|
69
|
+
name=metric["name"],
|
70
|
+
score=metric["score"],
|
71
|
+
reasoning=metric.get("reasoning", ""),
|
72
|
+
cost=metric.get("cost", None),
|
73
|
+
latency=metric.get("latency", None),
|
74
|
+
metadata=metric.get("metadata", {}),
|
75
|
+
config=metric.get("config", {}),
|
76
|
+
)
|
77
|
+
except ValueError as e:
|
78
|
+
logger.error(f"Validation Error: {e}")
|
79
|
+
except Exception as e:
|
80
|
+
logger.error(f"Error adding metric: {e}")
|
81
|
+
|
82
|
+
if feedback:
|
83
|
+
self.span(name).add_feedback(feedback)
|
22
84
|
|
23
|
-
def trace_tool(self, name: str, tool_type: str = "generic", version: str = "1.0.0"):
|
24
85
|
def decorator(func):
|
25
86
|
# Add metadata attribute to the function
|
26
87
|
metadata = {
|
27
88
|
"name": name,
|
28
89
|
"tool_type": tool_type,
|
29
90
|
"version": version,
|
30
|
-
"is_active":
|
91
|
+
"is_active": self.is_active,
|
31
92
|
}
|
32
|
-
|
93
|
+
|
33
94
|
# Check if the function is async
|
34
95
|
is_async = asyncio.iscoroutinefunction(func)
|
35
96
|
|
@@ -37,7 +98,7 @@ class ToolTracerMixin:
|
|
37
98
|
@functools.wraps(func)
|
38
99
|
async def async_wrapper(*args, **kwargs):
|
39
100
|
async_wrapper.metadata = metadata
|
40
|
-
self.gt = kwargs.get(
|
101
|
+
self.gt = kwargs.get("gt", None) if kwargs else None
|
41
102
|
return await self._trace_tool_execution(
|
42
103
|
func, name, tool_type, version, *args, **kwargs
|
43
104
|
)
|
@@ -46,7 +107,7 @@ class ToolTracerMixin:
|
|
46
107
|
@functools.wraps(func)
|
47
108
|
def sync_wrapper(*args, **kwargs):
|
48
109
|
sync_wrapper.metadata = metadata
|
49
|
-
self.gt = kwargs.get(
|
110
|
+
self.gt = kwargs.get("gt", None) if kwargs else None
|
50
111
|
return self._trace_sync_tool_execution(
|
51
112
|
func, name, tool_type, version, *args, **kwargs
|
52
113
|
)
|
@@ -57,11 +118,16 @@ class ToolTracerMixin:
|
|
57
118
|
|
58
119
|
return decorator
|
59
120
|
|
60
|
-
def _trace_sync_tool_execution(
|
121
|
+
def _trace_sync_tool_execution(
|
122
|
+
self, func, name, tool_type, version, *args, **kwargs
|
123
|
+
):
|
61
124
|
"""Synchronous version of tool tracing"""
|
62
125
|
if not self.is_active:
|
63
126
|
return func(*args, **kwargs)
|
64
127
|
|
128
|
+
if not self.auto_instrument_tool:
|
129
|
+
return func(*args, **kwargs)
|
130
|
+
|
65
131
|
start_time = datetime.now().astimezone()
|
66
132
|
start_memory = psutil.Process().memory_info().rss
|
67
133
|
component_id = str(uuid.uuid4())
|
@@ -91,10 +157,11 @@ class ToolTracerMixin:
|
|
91
157
|
memory_used=memory_used,
|
92
158
|
start_time=start_time,
|
93
159
|
input_data=self._sanitize_input(args, kwargs),
|
94
|
-
output_data=self._sanitize_output(result)
|
160
|
+
output_data=self._sanitize_output(result),
|
95
161
|
)
|
96
162
|
|
97
163
|
self.add_component(tool_component)
|
164
|
+
|
98
165
|
return result
|
99
166
|
|
100
167
|
except Exception as e:
|
@@ -102,12 +169,12 @@ class ToolTracerMixin:
|
|
102
169
|
"code": 500,
|
103
170
|
"type": type(e).__name__,
|
104
171
|
"message": str(e),
|
105
|
-
"details": {}
|
172
|
+
"details": {},
|
106
173
|
}
|
107
|
-
|
174
|
+
|
108
175
|
# End tracking network calls for this component
|
109
176
|
self.end_component(component_id)
|
110
|
-
|
177
|
+
|
111
178
|
tool_component = self.create_tool_component(
|
112
179
|
component_id=component_id,
|
113
180
|
hash_id=hash_id,
|
@@ -118,22 +185,29 @@ class ToolTracerMixin:
|
|
118
185
|
start_time=start_time,
|
119
186
|
input_data=self._sanitize_input(args, kwargs),
|
120
187
|
output_data=None,
|
121
|
-
error=error_component
|
188
|
+
error=error_component,
|
122
189
|
)
|
123
190
|
|
124
191
|
self.add_component(tool_component)
|
192
|
+
|
125
193
|
raise
|
126
194
|
|
127
|
-
async def _trace_tool_execution(
|
195
|
+
async def _trace_tool_execution(
|
196
|
+
self, func, name, tool_type, version, *args, **kwargs
|
197
|
+
):
|
128
198
|
"""Asynchronous version of tool tracing"""
|
129
199
|
if not self.is_active:
|
130
200
|
return await func(*args, **kwargs)
|
131
201
|
|
202
|
+
if not self.auto_instrument_tool:
|
203
|
+
return await func(*args, **kwargs)
|
204
|
+
|
132
205
|
start_time = datetime.now().astimezone()
|
133
206
|
start_memory = psutil.Process().memory_info().rss
|
134
207
|
component_id = str(uuid.uuid4())
|
135
208
|
hash_id = generate_unique_hash_simple(func)
|
136
209
|
|
210
|
+
self.start_component(component_id)
|
137
211
|
try:
|
138
212
|
# Execute the tool
|
139
213
|
result = await func(*args, **kwargs)
|
@@ -141,6 +215,7 @@ class ToolTracerMixin:
|
|
141
215
|
# Calculate resource usage
|
142
216
|
end_memory = psutil.Process().memory_info().rss
|
143
217
|
memory_used = max(0, end_memory - start_memory)
|
218
|
+
self.end_component(component_id)
|
144
219
|
|
145
220
|
# Create tool component
|
146
221
|
tool_component = self.create_tool_component(
|
@@ -152,9 +227,10 @@ class ToolTracerMixin:
|
|
152
227
|
start_time=start_time,
|
153
228
|
memory_used=memory_used,
|
154
229
|
input_data=self._sanitize_input(args, kwargs),
|
155
|
-
output_data=self._sanitize_output(result)
|
230
|
+
output_data=self._sanitize_output(result),
|
156
231
|
)
|
157
232
|
self.add_component(tool_component)
|
233
|
+
|
158
234
|
return result
|
159
235
|
|
160
236
|
except Exception as e:
|
@@ -162,9 +238,9 @@ class ToolTracerMixin:
|
|
162
238
|
"code": 500,
|
163
239
|
"type": type(e).__name__,
|
164
240
|
"message": str(e),
|
165
|
-
"details": {}
|
241
|
+
"details": {},
|
166
242
|
}
|
167
|
-
|
243
|
+
|
168
244
|
tool_component = self.create_tool_component(
|
169
245
|
component_id=component_id,
|
170
246
|
hash_id=hash_id,
|
@@ -175,15 +251,42 @@ class ToolTracerMixin:
|
|
175
251
|
memory_used=0,
|
176
252
|
input_data=self._sanitize_input(args, kwargs),
|
177
253
|
output_data=None,
|
178
|
-
error=error_component
|
254
|
+
error=error_component,
|
179
255
|
)
|
180
256
|
self.add_component(tool_component)
|
257
|
+
|
181
258
|
raise
|
182
259
|
|
183
260
|
def create_tool_component(self, **kwargs):
|
184
|
-
|
185
|
-
|
186
261
|
"""Create a tool component according to the data structure"""
|
262
|
+
network_calls = []
|
263
|
+
if self.auto_instrument_network:
|
264
|
+
network_calls = self.component_network_calls.get(kwargs["component_id"], [])
|
265
|
+
interactions = []
|
266
|
+
if self.auto_instrument_user_interaction:
|
267
|
+
interactions = self.component_user_interaction.get(
|
268
|
+
kwargs["component_id"], []
|
269
|
+
)
|
270
|
+
|
271
|
+
# Get tags, metrics
|
272
|
+
name = kwargs["name"]
|
273
|
+
# tags
|
274
|
+
tags = []
|
275
|
+
if name in self.span_attributes_dict:
|
276
|
+
tags = self.span_attributes_dict[name].tags or []
|
277
|
+
|
278
|
+
# metrics
|
279
|
+
metrics = []
|
280
|
+
if name in self.span_attributes_dict:
|
281
|
+
raw_metrics = self.span_attributes_dict[name].metrics or []
|
282
|
+
for metric in raw_metrics:
|
283
|
+
base_metric_name = metric["name"]
|
284
|
+
counter = sum(1 for x in self.visited_metrics if x.startswith(base_metric_name))
|
285
|
+
metric_name = f'{base_metric_name}_{counter}' if counter > 0 else base_metric_name
|
286
|
+
self.visited_metrics.append(metric_name)
|
287
|
+
metric["name"] = metric_name
|
288
|
+
metrics.append(metric)
|
289
|
+
|
187
290
|
start_time = kwargs["start_time"]
|
188
291
|
component = {
|
189
292
|
"id": kwargs["component_id"],
|
@@ -198,20 +301,25 @@ class ToolTracerMixin:
|
|
198
301
|
"info": {
|
199
302
|
"tool_type": kwargs["tool_type"],
|
200
303
|
"version": kwargs["version"],
|
201
|
-
"memory_used": kwargs["memory_used"]
|
304
|
+
"memory_used": kwargs["memory_used"],
|
305
|
+
"tags": tags,
|
202
306
|
},
|
203
307
|
"data": {
|
204
308
|
"input": kwargs["input_data"],
|
205
309
|
"output": kwargs["output_data"],
|
206
|
-
"memory_used": kwargs["memory_used"]
|
310
|
+
"memory_used": kwargs["memory_used"],
|
207
311
|
},
|
208
|
-
"
|
209
|
-
"
|
312
|
+
"metrics": metrics,
|
313
|
+
"network_calls": network_calls,
|
314
|
+
"interactions": interactions,
|
210
315
|
}
|
211
316
|
|
212
|
-
if self.gt:
|
317
|
+
if self.gt:
|
213
318
|
component["data"]["gt"] = self.gt
|
214
319
|
|
320
|
+
# Reset the SpanAttributes context variable
|
321
|
+
self.span_attributes_dict[kwargs["name"]] = SpanAttributes(kwargs["name"])
|
322
|
+
|
215
323
|
return component
|
216
324
|
|
217
325
|
def start_component(self, component_id):
|
@@ -223,15 +331,26 @@ class ToolTracerMixin:
|
|
223
331
|
def _sanitize_input(self, args: tuple, kwargs: dict) -> Dict:
|
224
332
|
"""Sanitize and format input data"""
|
225
333
|
return {
|
226
|
-
"args": [
|
334
|
+
"args": [
|
335
|
+
(
|
336
|
+
str(arg)
|
337
|
+
if not isinstance(arg, (int, float, bool, str, list, dict))
|
338
|
+
else arg
|
339
|
+
)
|
340
|
+
for arg in args
|
341
|
+
],
|
227
342
|
"kwargs": {
|
228
|
-
k:
|
343
|
+
k: (
|
344
|
+
str(v)
|
345
|
+
if not isinstance(v, (int, float, bool, str, list, dict))
|
346
|
+
else v
|
347
|
+
)
|
229
348
|
for k, v in kwargs.items()
|
230
|
-
}
|
349
|
+
},
|
231
350
|
}
|
232
351
|
|
233
352
|
def _sanitize_output(self, output: Any) -> Any:
|
234
353
|
"""Sanitize and format output data"""
|
235
354
|
if isinstance(output, (int, float, bool, str, list, dict)):
|
236
355
|
return output
|
237
|
-
return str(output)
|
356
|
+
return str(output)
|
@@ -3,6 +3,35 @@ from datetime import datetime
|
|
3
3
|
import contextvars
|
4
4
|
import inspect
|
5
5
|
import uuid
|
6
|
+
from typing import Optional, Any
|
7
|
+
|
8
|
+
class TracedFile:
|
9
|
+
def __init__(self, file_obj, file_path: str, tracer):
|
10
|
+
self._file = file_obj
|
11
|
+
self._file_path = file_path
|
12
|
+
self._tracer = tracer
|
13
|
+
|
14
|
+
def write(self, content: str) -> int:
|
15
|
+
self._tracer.trace_file_operation("write", self._file_path, content=content)
|
16
|
+
return self._file.write(content)
|
17
|
+
|
18
|
+
def read(self, size: Optional[int] = None) -> str:
|
19
|
+
content = self._file.read() if size is None else self._file.read(size)
|
20
|
+
self._tracer.trace_file_operation("read", self._file_path, content=content)
|
21
|
+
return content
|
22
|
+
|
23
|
+
def close(self) -> None:
|
24
|
+
return self._file.close()
|
25
|
+
|
26
|
+
def __enter__(self):
|
27
|
+
return self
|
28
|
+
|
29
|
+
def __exit__(self, exc_type, exc_val, exc_tb):
|
30
|
+
self.close()
|
31
|
+
return None
|
32
|
+
|
33
|
+
def __getattr__(self, name: str) -> Any:
|
34
|
+
return getattr(self._file, name)
|
6
35
|
|
7
36
|
class UserInteractionTracer:
|
8
37
|
def __init__(self, *args, **kwargs):
|
@@ -12,6 +41,7 @@ class UserInteractionTracer:
|
|
12
41
|
self.component_id = contextvars.ContextVar("component_id", default=None)
|
13
42
|
self.original_input = builtins.input
|
14
43
|
self.original_print = builtins.print
|
44
|
+
self.original_open = builtins.open
|
15
45
|
self.interactions = []
|
16
46
|
|
17
47
|
def traced_input(self, prompt=""):
|
@@ -25,9 +55,10 @@ class UserInteractionTracer:
|
|
25
55
|
|
26
56
|
self.interactions.append({
|
27
57
|
"id": str(uuid.uuid4()),
|
58
|
+
"component_id": self.component_id.get(),
|
28
59
|
"interaction_type": "input",
|
29
60
|
"content": content,
|
30
|
-
"timestamp": datetime.now().isoformat()
|
61
|
+
"timestamp": datetime.now().astimezone().isoformat()
|
31
62
|
})
|
32
63
|
return content
|
33
64
|
|
@@ -36,8 +67,63 @@ class UserInteractionTracer:
|
|
36
67
|
|
37
68
|
self.interactions.append({
|
38
69
|
"id": str(uuid.uuid4()),
|
70
|
+
"component_id": self.component_id.get(),
|
39
71
|
"interaction_type": "output",
|
40
72
|
"content": content,
|
41
|
-
"timestamp": datetime.now().isoformat()
|
73
|
+
"timestamp": datetime.now().astimezone().isoformat()
|
42
74
|
})
|
43
75
|
return self.original_print(*args, **kwargs)
|
76
|
+
|
77
|
+
def traced_open(self, file: str, mode: str = 'r', *args, **kwargs):
|
78
|
+
# Skip tracing for system and virtual environment paths
|
79
|
+
system_paths = [
|
80
|
+
'site-packages',
|
81
|
+
'dist-packages',
|
82
|
+
'/proc/',
|
83
|
+
'/sys/',
|
84
|
+
'/var/lib/',
|
85
|
+
'/usr/lib/',
|
86
|
+
'/System/Library'
|
87
|
+
]
|
88
|
+
|
89
|
+
file_str = str(file)
|
90
|
+
if any(path in file_str for path in system_paths):
|
91
|
+
return self.original_open(file, mode, *args, **kwargs)
|
92
|
+
|
93
|
+
file_obj = self.original_open(file, mode, *args, **kwargs)
|
94
|
+
return TracedFile(file_obj, file, self)
|
95
|
+
|
96
|
+
def trace_file_operation(self, operation: str, file_path: str, **kwargs):
|
97
|
+
interaction_type = f"file_{operation}"
|
98
|
+
|
99
|
+
# Check for existing interaction with same file_path and operation
|
100
|
+
for existing in reversed(self.interactions):
|
101
|
+
if (existing.get("file_path") == file_path and
|
102
|
+
existing.get("interaction_type") == interaction_type):
|
103
|
+
# Merge content if it exists
|
104
|
+
if "content" in kwargs and "content" in existing:
|
105
|
+
existing["content"] += kwargs["content"]
|
106
|
+
return
|
107
|
+
break
|
108
|
+
|
109
|
+
# If no matching interaction found or couldn't merge, create new one
|
110
|
+
interaction = {
|
111
|
+
"id": str(uuid.uuid4()),
|
112
|
+
"component_id": self.component_id.get(),
|
113
|
+
"interaction_type": interaction_type,
|
114
|
+
"file_path": file_path,
|
115
|
+
"timestamp": datetime.now().astimezone().isoformat()
|
116
|
+
}
|
117
|
+
interaction.update(kwargs)
|
118
|
+
self.interactions.append(interaction)
|
119
|
+
|
120
|
+
def __enter__(self):
|
121
|
+
builtins.input = self.traced_input
|
122
|
+
builtins.print = self.traced_print
|
123
|
+
builtins.open = self.traced_open
|
124
|
+
return self
|
125
|
+
|
126
|
+
def __exit__(self, exc_type, exc_val, exc_tb):
|
127
|
+
builtins.input = self.original_input
|
128
|
+
builtins.print = self.original_print
|
129
|
+
builtins.open = self.original_open
|
@@ -18,50 +18,7 @@ class UploadAgenticTraces:
|
|
18
18
|
self.dataset_name = dataset_name
|
19
19
|
self.user_detail = user_detail
|
20
20
|
self.base_url = base_url
|
21
|
-
self.timeout =
|
22
|
-
|
23
|
-
def _create_dataset_schema_with_trace(self):
|
24
|
-
SCHEMA_MAPPING_NEW = {
|
25
|
-
"trace_id": {"columnType": "traceId"},
|
26
|
-
"trace_uri": {"columnType": "traceUri"},
|
27
|
-
"prompt": {"columnType": "prompt"},
|
28
|
-
"response":{"columnType": "response"},
|
29
|
-
"context": {"columnType": "context"},
|
30
|
-
"llm_model": {"columnType":"pipeline"},
|
31
|
-
"recorded_on": {"columnType": "metadata"},
|
32
|
-
"embed_model": {"columnType":"pipeline"},
|
33
|
-
"log_source": {"columnType": "metadata"},
|
34
|
-
"vector_store":{"columnType":"pipeline"},
|
35
|
-
"feedback": {"columnType":"feedBack"}
|
36
|
-
}
|
37
|
-
def make_request():
|
38
|
-
headers = {
|
39
|
-
"Content-Type": "application/json",
|
40
|
-
"Authorization": f"Bearer {os.getenv('RAGAAI_CATALYST_TOKEN')}",
|
41
|
-
"X-Project-Name": self.project_name,
|
42
|
-
}
|
43
|
-
payload = json.dumps({
|
44
|
-
"datasetName": self.dataset_name,
|
45
|
-
# "schemaMapping": SCHEMA_MAPPING_NEW,
|
46
|
-
"traceFolderUrl": None,
|
47
|
-
})
|
48
|
-
response = requests.request("POST",
|
49
|
-
f"{self.base_url}/v1/llm/dataset/logs",
|
50
|
-
headers=headers,
|
51
|
-
data=payload,
|
52
|
-
timeout=self.timeout
|
53
|
-
)
|
54
|
-
|
55
|
-
return response
|
56
|
-
|
57
|
-
response = make_request()
|
58
|
-
|
59
|
-
if response.status_code == 401:
|
60
|
-
# get_token() # Fetch a new token and set it in the environment
|
61
|
-
response = make_request() # Retry the request
|
62
|
-
if response.status_code != 200:
|
63
|
-
return response.status_code
|
64
|
-
return response.status_code
|
21
|
+
self.timeout = 30
|
65
22
|
|
66
23
|
|
67
24
|
def _get_presigned_url(self):
|
@@ -181,10 +138,11 @@ class UploadAgenticTraces:
|
|
181
138
|
return None
|
182
139
|
|
183
140
|
def upload_agentic_traces(self):
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
|
188
|
-
|
189
|
-
|
190
|
-
|
141
|
+
try:
|
142
|
+
presignedUrl = self._get_presigned_url()
|
143
|
+
if presignedUrl is None:
|
144
|
+
return
|
145
|
+
self._put_presigned_url(presignedUrl, self.json_file_path)
|
146
|
+
self.insert_traces(presignedUrl)
|
147
|
+
except Exception as e:
|
148
|
+
print(f"Error while uploading agentic traces: {e}")
|
@@ -0,0 +1,83 @@
|
|
1
|
+
import requests
|
2
|
+
import os
|
3
|
+
import json
|
4
|
+
from ....ragaai_catalyst import RagaAICatalyst
|
5
|
+
from ..utils.get_user_trace_metrics import get_user_trace_metrics
|
6
|
+
|
7
|
+
def upload_trace_metric(json_file_path, dataset_name, project_name):
|
8
|
+
try:
|
9
|
+
with open(json_file_path, "r") as f:
|
10
|
+
traces = json.load(f)
|
11
|
+
metrics = get_trace_metrics_from_trace(traces)
|
12
|
+
metrics = _change_metrics_format_for_payload(metrics)
|
13
|
+
|
14
|
+
user_trace_metrics = get_user_trace_metrics(project_name, dataset_name)
|
15
|
+
if user_trace_metrics:
|
16
|
+
user_trace_metrics_list = [metric["displayName"] for metric in user_trace_metrics]
|
17
|
+
|
18
|
+
if user_trace_metrics:
|
19
|
+
for metric in metrics:
|
20
|
+
if metric["displayName"] in user_trace_metrics_list:
|
21
|
+
metricConfig = next((user_metric["metricConfig"] for user_metric in user_trace_metrics if user_metric["displayName"] == metric["displayName"]), None)
|
22
|
+
if not metricConfig or metricConfig.get("Metric Source", {}).get("value") != "user":
|
23
|
+
raise ValueError(f"Metrics {metric['displayName']} already exist in dataset {dataset_name} of project {project_name}.")
|
24
|
+
|
25
|
+
headers = {
|
26
|
+
"Content-Type": "application/json",
|
27
|
+
"Authorization": f"Bearer {os.getenv('RAGAAI_CATALYST_TOKEN')}",
|
28
|
+
"X-Project-Name": project_name,
|
29
|
+
}
|
30
|
+
payload = json.dumps({
|
31
|
+
"datasetName": dataset_name,
|
32
|
+
"metrics": metrics
|
33
|
+
})
|
34
|
+
response = requests.request("POST",
|
35
|
+
f"{RagaAICatalyst.BASE_URL}/v1/llm/trace/metrics",
|
36
|
+
headers=headers,
|
37
|
+
data=payload,
|
38
|
+
timeout=10)
|
39
|
+
if response.status_code != 200:
|
40
|
+
raise ValueError(f"Error inserting agentic trace metrics")
|
41
|
+
except requests.exceptions.RequestException as e:
|
42
|
+
raise ValueError(f"Error submitting traces: {e}")
|
43
|
+
return None
|
44
|
+
|
45
|
+
return response
|
46
|
+
|
47
|
+
|
48
|
+
def _get_children_metrics_of_agent(children_traces):
|
49
|
+
metrics = []
|
50
|
+
for span in children_traces:
|
51
|
+
metrics.extend(span.get("metrics", []))
|
52
|
+
if span["type"] != "agent":
|
53
|
+
metric = span.get("metrics", [])
|
54
|
+
if metric:
|
55
|
+
metrics.extend(metric)
|
56
|
+
else:
|
57
|
+
metrics.extend(_get_children_metrics_of_agent(span["data"]["children"]))
|
58
|
+
return metrics
|
59
|
+
|
60
|
+
def get_trace_metrics_from_trace(traces):
|
61
|
+
metrics = []
|
62
|
+
for span in traces["data"][0]["spans"]:
|
63
|
+
if span["type"] == "agent":
|
64
|
+
children_metric = _get_children_metrics_of_agent(span["data"]["children"])
|
65
|
+
if children_metric:
|
66
|
+
metrics.extend(children_metric)
|
67
|
+
else:
|
68
|
+
metric = span.get("metrics", [])
|
69
|
+
if metric:
|
70
|
+
metrics.extend(metric)
|
71
|
+
return metrics
|
72
|
+
|
73
|
+
def _change_metrics_format_for_payload(metrics):
|
74
|
+
formatted_metrics = []
|
75
|
+
for metric in metrics:
|
76
|
+
if any(m["name"] == metric["name"] for m in formatted_metrics):
|
77
|
+
continue
|
78
|
+
formatted_metrics.append({
|
79
|
+
"name": metric["name"],
|
80
|
+
"displayName": metric["name"],
|
81
|
+
"config": {"source": "user"},
|
82
|
+
})
|
83
|
+
return formatted_metrics
|
@@ -0,0 +1,26 @@
|
|
1
|
+
import os
|
2
|
+
import json
|
3
|
+
import re
|
4
|
+
import requests
|
5
|
+
from ragaai_catalyst.tracers.agentic_tracing.tracers.base import RagaAICatalyst
|
6
|
+
|
7
|
+
def create_dataset_schema_with_trace(project_name, dataset_name):
|
8
|
+
def make_request():
|
9
|
+
headers = {
|
10
|
+
"Content-Type": "application/json",
|
11
|
+
"Authorization": f"Bearer {os.getenv('RAGAAI_CATALYST_TOKEN')}",
|
12
|
+
"X-Project-Name": project_name,
|
13
|
+
}
|
14
|
+
payload = json.dumps({
|
15
|
+
"datasetName": dataset_name,
|
16
|
+
"traceFolderUrl": None,
|
17
|
+
})
|
18
|
+
response = requests.request("POST",
|
19
|
+
f"{RagaAICatalyst.BASE_URL}/v1/llm/dataset/logs",
|
20
|
+
headers=headers,
|
21
|
+
data=payload,
|
22
|
+
timeout=10
|
23
|
+
)
|
24
|
+
return response
|
25
|
+
response = make_request()
|
26
|
+
return response
|
@@ -0,0 +1,28 @@
|
|
1
|
+
import requests
|
2
|
+
import os
|
3
|
+
from ....ragaai_catalyst import RagaAICatalyst
|
4
|
+
from ....dataset import Dataset
|
5
|
+
|
6
|
+
def get_user_trace_metrics(project_name, dataset_name):
|
7
|
+
try:
|
8
|
+
list_datasets = Dataset(project_name=project_name).list_datasets()
|
9
|
+
if not list_datasets:
|
10
|
+
return []
|
11
|
+
elif dataset_name not in list_datasets:
|
12
|
+
return []
|
13
|
+
else:
|
14
|
+
headers = {
|
15
|
+
"Authorization": f"Bearer {os.getenv('RAGAAI_CATALYST_TOKEN')}",
|
16
|
+
"X-Project-Name": project_name,
|
17
|
+
}
|
18
|
+
response = requests.request("GET",
|
19
|
+
f"{RagaAICatalyst.BASE_URL}/v1/llm/trace/metrics?datasetName={dataset_name}",
|
20
|
+
headers=headers, timeout=10)
|
21
|
+
if response.status_code != 200:
|
22
|
+
print(f"Error fetching traces metrics: {response.json()['message']}")
|
23
|
+
return None
|
24
|
+
|
25
|
+
return response.json()["data"]["columns"]
|
26
|
+
except Exception as e:
|
27
|
+
print(f"Error fetching traces metrics: {e}")
|
28
|
+
return None
|