ragaai-catalyst 2.1.3__py3-none-any.whl → 2.1.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ragaai_catalyst/tracers/agentic_tracing/data/data_structure.py +37 -11
- ragaai_catalyst/tracers/agentic_tracing/tracers/agent_tracer.py +240 -81
- ragaai_catalyst/tracers/agentic_tracing/tracers/base.py +632 -114
- ragaai_catalyst/tracers/agentic_tracing/tracers/custom_tracer.py +316 -0
- ragaai_catalyst/tracers/agentic_tracing/tracers/langgraph_tracer.py +0 -0
- ragaai_catalyst/tracers/agentic_tracing/tracers/llm_tracer.py +229 -82
- ragaai_catalyst/tracers/agentic_tracing/tracers/main_tracer.py +214 -59
- ragaai_catalyst/tracers/agentic_tracing/tracers/network_tracer.py +16 -14
- ragaai_catalyst/tracers/agentic_tracing/tracers/tool_tracer.py +147 -28
- ragaai_catalyst/tracers/agentic_tracing/tracers/user_interaction_tracer.py +88 -2
- ragaai_catalyst/tracers/agentic_tracing/upload/upload_agentic_traces.py +9 -51
- ragaai_catalyst/tracers/agentic_tracing/upload/upload_trace_metric.py +83 -0
- ragaai_catalyst/tracers/agentic_tracing/utils/create_dataset_schema.py +26 -0
- ragaai_catalyst/tracers/agentic_tracing/utils/get_user_trace_metrics.py +28 -0
- ragaai_catalyst/tracers/agentic_tracing/utils/llm_utils.py +45 -15
- ragaai_catalyst/tracers/agentic_tracing/utils/model_costs.json +2520 -2152
- ragaai_catalyst/tracers/agentic_tracing/utils/span_attributes.py +59 -0
- ragaai_catalyst/tracers/agentic_tracing/utils/trace_utils.py +23 -0
- ragaai_catalyst/tracers/agentic_tracing/utils/zip_list_of_unique_files.py +284 -15
- ragaai_catalyst/tracers/llamaindex_callback.py +5 -5
- ragaai_catalyst/tracers/tracer.py +83 -10
- ragaai_catalyst/tracers/upload_traces.py +1 -1
- ragaai_catalyst-2.1.4.dist-info/METADATA +431 -0
- {ragaai_catalyst-2.1.3.dist-info → ragaai_catalyst-2.1.4.dist-info}/RECORD +26 -20
- ragaai_catalyst-2.1.3.dist-info/METADATA +0 -43
- {ragaai_catalyst-2.1.3.dist-info → ragaai_catalyst-2.1.4.dist-info}/WHEEL +0 -0
- {ragaai_catalyst-2.1.3.dist-info → ragaai_catalyst-2.1.4.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,316 @@
|
|
1
|
+
import sys
|
2
|
+
import uuid
|
3
|
+
import psutil
|
4
|
+
import threading
|
5
|
+
from datetime import datetime
|
6
|
+
import functools
|
7
|
+
from typing import Optional, Any, Dict, List
|
8
|
+
from ..utils.unique_decorator import generate_unique_hash_simple, mydecorator
|
9
|
+
import contextvars
|
10
|
+
import asyncio
|
11
|
+
from ..utils.file_name_tracker import TrackName
|
12
|
+
|
13
|
+
|
14
|
+
class CustomTracerMixin:
|
15
|
+
def __init__(self, *args, **kwargs):
|
16
|
+
super().__init__(*args, **kwargs)
|
17
|
+
self.file_tracker = TrackName()
|
18
|
+
self.current_custom_name = contextvars.ContextVar("custom_name", default=None)
|
19
|
+
self.current_custom_id = contextvars.ContextVar("custom_id", default=None)
|
20
|
+
self.component_network_calls = {}
|
21
|
+
self.component_user_interaction = {}
|
22
|
+
self.gt = None
|
23
|
+
|
24
|
+
# Add auto instrument flags
|
25
|
+
self.auto_instrument_custom = False
|
26
|
+
self.auto_instrument_user_interaction = False
|
27
|
+
self.auto_instrument_network = False
|
28
|
+
|
29
|
+
def trace_custom(self, name: str = None, custom_type: str = "generic", version: str = "1.0.0", trace_variables: bool = True):
|
30
|
+
def decorator(func):
|
31
|
+
# Add metadata attribute to the function
|
32
|
+
metadata = {
|
33
|
+
"name": name or func.__name__,
|
34
|
+
"custom_type": custom_type,
|
35
|
+
"version": version,
|
36
|
+
"trace_variables": trace_variables,
|
37
|
+
"is_active": True
|
38
|
+
}
|
39
|
+
|
40
|
+
# Check if the function is async
|
41
|
+
is_async = asyncio.iscoroutinefunction(func)
|
42
|
+
|
43
|
+
@self.file_tracker.trace_decorator
|
44
|
+
@functools.wraps(func)
|
45
|
+
async def async_wrapper(*args, **kwargs):
|
46
|
+
async_wrapper.metadata = metadata
|
47
|
+
self.gt = kwargs.get('gt', None) if kwargs else None
|
48
|
+
return await self._trace_custom_execution(
|
49
|
+
func, name or func.__name__, custom_type, version, trace_variables, *args, **kwargs
|
50
|
+
)
|
51
|
+
|
52
|
+
@self.file_tracker.trace_decorator
|
53
|
+
@functools.wraps(func)
|
54
|
+
def sync_wrapper(*args, **kwargs):
|
55
|
+
sync_wrapper.metadata = metadata
|
56
|
+
self.gt = kwargs.get('gt', None) if kwargs else None
|
57
|
+
return self._trace_sync_custom_execution(
|
58
|
+
func, name or func.__name__, custom_type, version, trace_variables, *args, **kwargs
|
59
|
+
)
|
60
|
+
|
61
|
+
wrapper = async_wrapper if is_async else sync_wrapper
|
62
|
+
wrapper.metadata = metadata
|
63
|
+
return wrapper
|
64
|
+
|
65
|
+
return decorator
|
66
|
+
|
67
|
+
def _trace_sync_custom_execution(self, func, name, custom_type, version, trace_variables, *args, **kwargs):
|
68
|
+
"""Synchronous version of custom tracing"""
|
69
|
+
if not self.is_active or not self.auto_instrument_custom:
|
70
|
+
return func(*args, **kwargs)
|
71
|
+
|
72
|
+
start_time = datetime.now().astimezone().isoformat()
|
73
|
+
start_memory = psutil.Process().memory_info().rss
|
74
|
+
component_id = str(uuid.uuid4())
|
75
|
+
hash_id = generate_unique_hash_simple(func)
|
76
|
+
variable_traces = []
|
77
|
+
|
78
|
+
# Set up variable tracing if enabled
|
79
|
+
if trace_variables:
|
80
|
+
def trace_variables_func(frame, event, arg):
|
81
|
+
if event == 'line' and frame.f_code == func.__code__:
|
82
|
+
try:
|
83
|
+
locals_dict = {k: v for k, v in frame.f_locals.items()
|
84
|
+
if not k.startswith('__') and isinstance(v, (int, float, bool, str, list, dict, tuple, set))}
|
85
|
+
if locals_dict:
|
86
|
+
variable_traces.append({
|
87
|
+
'variables': locals_dict,
|
88
|
+
'timestamp': datetime.now().astimezone().isoformat()
|
89
|
+
})
|
90
|
+
except:
|
91
|
+
pass
|
92
|
+
return trace_variables_func
|
93
|
+
|
94
|
+
|
95
|
+
# Start tracking network calls for this component
|
96
|
+
self.start_component(component_id)
|
97
|
+
|
98
|
+
try:
|
99
|
+
# Execute the function
|
100
|
+
result = func(*args, **kwargs)
|
101
|
+
|
102
|
+
# Calculate resource usage
|
103
|
+
end_time = datetime.now().astimezone().isoformat()
|
104
|
+
end_memory = psutil.Process().memory_info().rss
|
105
|
+
memory_used = max(0, end_memory - start_memory)
|
106
|
+
|
107
|
+
# End tracking network calls for this component
|
108
|
+
self.end_component(component_id)
|
109
|
+
|
110
|
+
# Create custom component
|
111
|
+
custom_component = self.create_custom_component(
|
112
|
+
component_id=component_id,
|
113
|
+
hash_id=hash_id,
|
114
|
+
name=name,
|
115
|
+
custom_type=custom_type,
|
116
|
+
version=version,
|
117
|
+
memory_used=memory_used,
|
118
|
+
start_time=start_time,
|
119
|
+
end_time=end_time,
|
120
|
+
variable_traces=variable_traces,
|
121
|
+
input_data=self._sanitize_input(args, kwargs),
|
122
|
+
output_data=self._sanitize_output(result)
|
123
|
+
)
|
124
|
+
|
125
|
+
self.add_component(custom_component)
|
126
|
+
return result
|
127
|
+
|
128
|
+
except Exception as e:
|
129
|
+
error_component = {
|
130
|
+
"code": 500,
|
131
|
+
"type": type(e).__name__,
|
132
|
+
"message": str(e),
|
133
|
+
"details": {}
|
134
|
+
}
|
135
|
+
|
136
|
+
# End tracking network calls for this component
|
137
|
+
self.end_component(component_id)
|
138
|
+
|
139
|
+
end_time = datetime.now().astimezone().isoformat()
|
140
|
+
|
141
|
+
custom_component = self.create_custom_component(
|
142
|
+
component_id=component_id,
|
143
|
+
hash_id=hash_id,
|
144
|
+
name=name,
|
145
|
+
custom_type=custom_type,
|
146
|
+
version=version,
|
147
|
+
memory_used=0,
|
148
|
+
start_time=start_time,
|
149
|
+
end_time=end_time,
|
150
|
+
variable_traces=variable_traces,
|
151
|
+
input_data=self._sanitize_input(args, kwargs),
|
152
|
+
output_data=None,
|
153
|
+
error=error_component
|
154
|
+
)
|
155
|
+
|
156
|
+
self.add_component(custom_component)
|
157
|
+
raise
|
158
|
+
|
159
|
+
async def _trace_custom_execution(self, func, name, custom_type, version, trace_variables, *args, **kwargs):
|
160
|
+
"""Asynchronous version of custom tracing"""
|
161
|
+
if not self.is_active or not self.auto_instrument_custom:
|
162
|
+
return await func(*args, **kwargs)
|
163
|
+
|
164
|
+
start_time = datetime.now().astimezone().isoformat()
|
165
|
+
start_memory = psutil.Process().memory_info().rss
|
166
|
+
component_id = str(uuid.uuid4())
|
167
|
+
hash_id = generate_unique_hash_simple(func)
|
168
|
+
variable_traces = []
|
169
|
+
|
170
|
+
# Set up variable tracing if enabled
|
171
|
+
if trace_variables:
|
172
|
+
def trace_variables_func(frame, event, arg):
|
173
|
+
if event == 'line' and frame.f_code == func.__code__:
|
174
|
+
try:
|
175
|
+
locals_dict = {k: v for k, v in frame.f_locals.items()
|
176
|
+
if not k.startswith('__') and isinstance(v, (int, float, bool, str, list, dict, tuple, set))}
|
177
|
+
if locals_dict:
|
178
|
+
variable_traces.append({
|
179
|
+
'variables': locals_dict,
|
180
|
+
'timestamp': datetime.now().astimezone().isoformat()
|
181
|
+
})
|
182
|
+
except:
|
183
|
+
pass
|
184
|
+
return trace_variables_func
|
185
|
+
|
186
|
+
try:
|
187
|
+
# Execute the function
|
188
|
+
result = await func(*args, **kwargs)
|
189
|
+
|
190
|
+
# Calculate resource usage
|
191
|
+
end_time = datetime.now().astimezone().isoformat()
|
192
|
+
end_memory = psutil.Process().memory_info().rss
|
193
|
+
memory_used = max(0, end_memory - start_memory)
|
194
|
+
|
195
|
+
# Create custom component
|
196
|
+
custom_component = self.create_custom_component(
|
197
|
+
component_id=component_id,
|
198
|
+
hash_id=hash_id,
|
199
|
+
name=name,
|
200
|
+
custom_type=custom_type,
|
201
|
+
version=version,
|
202
|
+
start_time=start_time,
|
203
|
+
end_time=end_time,
|
204
|
+
memory_used=memory_used,
|
205
|
+
variable_traces=variable_traces,
|
206
|
+
input_data=self._sanitize_input(args, kwargs),
|
207
|
+
output_data=self._sanitize_output(result)
|
208
|
+
)
|
209
|
+
self.add_component(custom_component)
|
210
|
+
return result
|
211
|
+
|
212
|
+
except Exception as e:
|
213
|
+
error_component = {
|
214
|
+
"code": 500,
|
215
|
+
"type": type(e).__name__,
|
216
|
+
"message": str(e),
|
217
|
+
"details": {}
|
218
|
+
}
|
219
|
+
|
220
|
+
end_time = datetime.now().astimezone().isoformat()
|
221
|
+
|
222
|
+
custom_component = self.create_custom_component(
|
223
|
+
component_id=component_id,
|
224
|
+
hash_id=hash_id,
|
225
|
+
name=name,
|
226
|
+
custom_type=custom_type,
|
227
|
+
version=version,
|
228
|
+
start_time=start_time,
|
229
|
+
end_time=end_time,
|
230
|
+
memory_used=0,
|
231
|
+
variable_traces=variable_traces,
|
232
|
+
input_data=self._sanitize_input(args, kwargs),
|
233
|
+
output_data=None,
|
234
|
+
error=error_component
|
235
|
+
)
|
236
|
+
self.add_component(custom_component)
|
237
|
+
raise
|
238
|
+
|
239
|
+
def create_custom_component(self, **kwargs):
|
240
|
+
"""Create a custom component according to the data structure"""
|
241
|
+
start_time = kwargs["start_time"]
|
242
|
+
|
243
|
+
network_calls = []
|
244
|
+
if self.auto_instrument_network:
|
245
|
+
network_calls = self.component_network_calls.get(kwargs["component_id"], [])
|
246
|
+
|
247
|
+
interactions = []
|
248
|
+
if self.auto_instrument_user_interaction:
|
249
|
+
interactions = self.component_user_interaction.get(kwargs["component_id"], [])
|
250
|
+
|
251
|
+
component = {
|
252
|
+
"id": kwargs["component_id"],
|
253
|
+
"hash_id": kwargs["hash_id"],
|
254
|
+
"source_hash_id": None,
|
255
|
+
"type": "custom",
|
256
|
+
"name": kwargs["name"],
|
257
|
+
"start_time": start_time,
|
258
|
+
"end_time": kwargs["end_time"],
|
259
|
+
"error": kwargs.get("error"),
|
260
|
+
"parent_id": self.current_agent_id.get() if hasattr(self, 'current_agent_id') else None,
|
261
|
+
"info": {
|
262
|
+
"custom_type": kwargs["custom_type"],
|
263
|
+
"version": kwargs["version"],
|
264
|
+
"memory_used": kwargs["memory_used"]
|
265
|
+
},
|
266
|
+
"data": {
|
267
|
+
"input": kwargs["input_data"],
|
268
|
+
"output": kwargs["output_data"],
|
269
|
+
"memory_used": kwargs["memory_used"],
|
270
|
+
"variable_traces": kwargs.get("variable_traces", [])
|
271
|
+
},
|
272
|
+
"network_calls": network_calls,
|
273
|
+
"interactions": interactions
|
274
|
+
}
|
275
|
+
|
276
|
+
if self.gt:
|
277
|
+
component["data"]["gt"] = self.gt
|
278
|
+
|
279
|
+
return component
|
280
|
+
|
281
|
+
def start_component(self, component_id):
|
282
|
+
"""Start tracking network calls for a component"""
|
283
|
+
self.component_network_calls[component_id] = []
|
284
|
+
|
285
|
+
def end_component(self, component_id):
|
286
|
+
"""End tracking network calls for a component"""
|
287
|
+
pass
|
288
|
+
|
289
|
+
def _sanitize_input(self, args: tuple, kwargs: dict) -> Dict:
|
290
|
+
"""Sanitize and format input data"""
|
291
|
+
return {
|
292
|
+
"args": [str(arg) if not isinstance(arg, (int, float, bool, str, list, dict)) else arg for arg in args],
|
293
|
+
"kwargs": {
|
294
|
+
k: str(v) if not isinstance(v, (int, float, bool, str, list, dict)) else v
|
295
|
+
for k, v in kwargs.items()
|
296
|
+
}
|
297
|
+
}
|
298
|
+
|
299
|
+
def _sanitize_output(self, output: Any) -> Any:
|
300
|
+
"""Sanitize and format output data"""
|
301
|
+
if isinstance(output, (int, float, bool, str, list, dict)):
|
302
|
+
return output
|
303
|
+
return str(output)
|
304
|
+
|
305
|
+
# Auto instrumentation methods
|
306
|
+
def instrument_custom_calls(self):
|
307
|
+
"""Enable auto-instrumentation for custom calls"""
|
308
|
+
self.auto_instrument_custom = True
|
309
|
+
|
310
|
+
def instrument_user_interaction_calls(self):
|
311
|
+
"""Enable auto-instrumentation for user interaction calls"""
|
312
|
+
self.auto_instrument_user_interaction = True
|
313
|
+
|
314
|
+
def instrument_network_calls(self):
|
315
|
+
"""Enable auto-instrumentation for network calls"""
|
316
|
+
self.auto_instrument_network = True
|
File without changes
|