ragaai-catalyst 2.1.3__py3-none-any.whl → 2.1.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (27) hide show
  1. ragaai_catalyst/tracers/agentic_tracing/data/data_structure.py +37 -11
  2. ragaai_catalyst/tracers/agentic_tracing/tracers/agent_tracer.py +240 -81
  3. ragaai_catalyst/tracers/agentic_tracing/tracers/base.py +632 -114
  4. ragaai_catalyst/tracers/agentic_tracing/tracers/custom_tracer.py +316 -0
  5. ragaai_catalyst/tracers/agentic_tracing/tracers/langgraph_tracer.py +0 -0
  6. ragaai_catalyst/tracers/agentic_tracing/tracers/llm_tracer.py +229 -82
  7. ragaai_catalyst/tracers/agentic_tracing/tracers/main_tracer.py +214 -59
  8. ragaai_catalyst/tracers/agentic_tracing/tracers/network_tracer.py +16 -14
  9. ragaai_catalyst/tracers/agentic_tracing/tracers/tool_tracer.py +147 -28
  10. ragaai_catalyst/tracers/agentic_tracing/tracers/user_interaction_tracer.py +88 -2
  11. ragaai_catalyst/tracers/agentic_tracing/upload/upload_agentic_traces.py +9 -51
  12. ragaai_catalyst/tracers/agentic_tracing/upload/upload_trace_metric.py +83 -0
  13. ragaai_catalyst/tracers/agentic_tracing/utils/create_dataset_schema.py +26 -0
  14. ragaai_catalyst/tracers/agentic_tracing/utils/get_user_trace_metrics.py +28 -0
  15. ragaai_catalyst/tracers/agentic_tracing/utils/llm_utils.py +45 -15
  16. ragaai_catalyst/tracers/agentic_tracing/utils/model_costs.json +2520 -2152
  17. ragaai_catalyst/tracers/agentic_tracing/utils/span_attributes.py +59 -0
  18. ragaai_catalyst/tracers/agentic_tracing/utils/trace_utils.py +23 -0
  19. ragaai_catalyst/tracers/agentic_tracing/utils/zip_list_of_unique_files.py +284 -15
  20. ragaai_catalyst/tracers/llamaindex_callback.py +5 -5
  21. ragaai_catalyst/tracers/tracer.py +83 -10
  22. ragaai_catalyst/tracers/upload_traces.py +1 -1
  23. ragaai_catalyst-2.1.4.dist-info/METADATA +431 -0
  24. {ragaai_catalyst-2.1.3.dist-info → ragaai_catalyst-2.1.4.dist-info}/RECORD +26 -20
  25. ragaai_catalyst-2.1.3.dist-info/METADATA +0 -43
  26. {ragaai_catalyst-2.1.3.dist-info → ragaai_catalyst-2.1.4.dist-info}/WHEEL +0 -0
  27. {ragaai_catalyst-2.1.3.dist-info → ragaai_catalyst-2.1.4.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,316 @@
1
+ import sys
2
+ import uuid
3
+ import psutil
4
+ import threading
5
+ from datetime import datetime
6
+ import functools
7
+ from typing import Optional, Any, Dict, List
8
+ from ..utils.unique_decorator import generate_unique_hash_simple, mydecorator
9
+ import contextvars
10
+ import asyncio
11
+ from ..utils.file_name_tracker import TrackName
12
+
13
+
14
+ class CustomTracerMixin:
15
+ def __init__(self, *args, **kwargs):
16
+ super().__init__(*args, **kwargs)
17
+ self.file_tracker = TrackName()
18
+ self.current_custom_name = contextvars.ContextVar("custom_name", default=None)
19
+ self.current_custom_id = contextvars.ContextVar("custom_id", default=None)
20
+ self.component_network_calls = {}
21
+ self.component_user_interaction = {}
22
+ self.gt = None
23
+
24
+ # Add auto instrument flags
25
+ self.auto_instrument_custom = False
26
+ self.auto_instrument_user_interaction = False
27
+ self.auto_instrument_network = False
28
+
29
+ def trace_custom(self, name: str = None, custom_type: str = "generic", version: str = "1.0.0", trace_variables: bool = True):
30
+ def decorator(func):
31
+ # Add metadata attribute to the function
32
+ metadata = {
33
+ "name": name or func.__name__,
34
+ "custom_type": custom_type,
35
+ "version": version,
36
+ "trace_variables": trace_variables,
37
+ "is_active": True
38
+ }
39
+
40
+ # Check if the function is async
41
+ is_async = asyncio.iscoroutinefunction(func)
42
+
43
+ @self.file_tracker.trace_decorator
44
+ @functools.wraps(func)
45
+ async def async_wrapper(*args, **kwargs):
46
+ async_wrapper.metadata = metadata
47
+ self.gt = kwargs.get('gt', None) if kwargs else None
48
+ return await self._trace_custom_execution(
49
+ func, name or func.__name__, custom_type, version, trace_variables, *args, **kwargs
50
+ )
51
+
52
+ @self.file_tracker.trace_decorator
53
+ @functools.wraps(func)
54
+ def sync_wrapper(*args, **kwargs):
55
+ sync_wrapper.metadata = metadata
56
+ self.gt = kwargs.get('gt', None) if kwargs else None
57
+ return self._trace_sync_custom_execution(
58
+ func, name or func.__name__, custom_type, version, trace_variables, *args, **kwargs
59
+ )
60
+
61
+ wrapper = async_wrapper if is_async else sync_wrapper
62
+ wrapper.metadata = metadata
63
+ return wrapper
64
+
65
+ return decorator
66
+
67
+ def _trace_sync_custom_execution(self, func, name, custom_type, version, trace_variables, *args, **kwargs):
68
+ """Synchronous version of custom tracing"""
69
+ if not self.is_active or not self.auto_instrument_custom:
70
+ return func(*args, **kwargs)
71
+
72
+ start_time = datetime.now().astimezone().isoformat()
73
+ start_memory = psutil.Process().memory_info().rss
74
+ component_id = str(uuid.uuid4())
75
+ hash_id = generate_unique_hash_simple(func)
76
+ variable_traces = []
77
+
78
+ # Set up variable tracing if enabled
79
+ if trace_variables:
80
+ def trace_variables_func(frame, event, arg):
81
+ if event == 'line' and frame.f_code == func.__code__:
82
+ try:
83
+ locals_dict = {k: v for k, v in frame.f_locals.items()
84
+ if not k.startswith('__') and isinstance(v, (int, float, bool, str, list, dict, tuple, set))}
85
+ if locals_dict:
86
+ variable_traces.append({
87
+ 'variables': locals_dict,
88
+ 'timestamp': datetime.now().astimezone().isoformat()
89
+ })
90
+ except:
91
+ pass
92
+ return trace_variables_func
93
+
94
+
95
+ # Start tracking network calls for this component
96
+ self.start_component(component_id)
97
+
98
+ try:
99
+ # Execute the function
100
+ result = func(*args, **kwargs)
101
+
102
+ # Calculate resource usage
103
+ end_time = datetime.now().astimezone().isoformat()
104
+ end_memory = psutil.Process().memory_info().rss
105
+ memory_used = max(0, end_memory - start_memory)
106
+
107
+ # End tracking network calls for this component
108
+ self.end_component(component_id)
109
+
110
+ # Create custom component
111
+ custom_component = self.create_custom_component(
112
+ component_id=component_id,
113
+ hash_id=hash_id,
114
+ name=name,
115
+ custom_type=custom_type,
116
+ version=version,
117
+ memory_used=memory_used,
118
+ start_time=start_time,
119
+ end_time=end_time,
120
+ variable_traces=variable_traces,
121
+ input_data=self._sanitize_input(args, kwargs),
122
+ output_data=self._sanitize_output(result)
123
+ )
124
+
125
+ self.add_component(custom_component)
126
+ return result
127
+
128
+ except Exception as e:
129
+ error_component = {
130
+ "code": 500,
131
+ "type": type(e).__name__,
132
+ "message": str(e),
133
+ "details": {}
134
+ }
135
+
136
+ # End tracking network calls for this component
137
+ self.end_component(component_id)
138
+
139
+ end_time = datetime.now().astimezone().isoformat()
140
+
141
+ custom_component = self.create_custom_component(
142
+ component_id=component_id,
143
+ hash_id=hash_id,
144
+ name=name,
145
+ custom_type=custom_type,
146
+ version=version,
147
+ memory_used=0,
148
+ start_time=start_time,
149
+ end_time=end_time,
150
+ variable_traces=variable_traces,
151
+ input_data=self._sanitize_input(args, kwargs),
152
+ output_data=None,
153
+ error=error_component
154
+ )
155
+
156
+ self.add_component(custom_component)
157
+ raise
158
+
159
+ async def _trace_custom_execution(self, func, name, custom_type, version, trace_variables, *args, **kwargs):
160
+ """Asynchronous version of custom tracing"""
161
+ if not self.is_active or not self.auto_instrument_custom:
162
+ return await func(*args, **kwargs)
163
+
164
+ start_time = datetime.now().astimezone().isoformat()
165
+ start_memory = psutil.Process().memory_info().rss
166
+ component_id = str(uuid.uuid4())
167
+ hash_id = generate_unique_hash_simple(func)
168
+ variable_traces = []
169
+
170
+ # Set up variable tracing if enabled
171
+ if trace_variables:
172
+ def trace_variables_func(frame, event, arg):
173
+ if event == 'line' and frame.f_code == func.__code__:
174
+ try:
175
+ locals_dict = {k: v for k, v in frame.f_locals.items()
176
+ if not k.startswith('__') and isinstance(v, (int, float, bool, str, list, dict, tuple, set))}
177
+ if locals_dict:
178
+ variable_traces.append({
179
+ 'variables': locals_dict,
180
+ 'timestamp': datetime.now().astimezone().isoformat()
181
+ })
182
+ except:
183
+ pass
184
+ return trace_variables_func
185
+
186
+ try:
187
+ # Execute the function
188
+ result = await func(*args, **kwargs)
189
+
190
+ # Calculate resource usage
191
+ end_time = datetime.now().astimezone().isoformat()
192
+ end_memory = psutil.Process().memory_info().rss
193
+ memory_used = max(0, end_memory - start_memory)
194
+
195
+ # Create custom component
196
+ custom_component = self.create_custom_component(
197
+ component_id=component_id,
198
+ hash_id=hash_id,
199
+ name=name,
200
+ custom_type=custom_type,
201
+ version=version,
202
+ start_time=start_time,
203
+ end_time=end_time,
204
+ memory_used=memory_used,
205
+ variable_traces=variable_traces,
206
+ input_data=self._sanitize_input(args, kwargs),
207
+ output_data=self._sanitize_output(result)
208
+ )
209
+ self.add_component(custom_component)
210
+ return result
211
+
212
+ except Exception as e:
213
+ error_component = {
214
+ "code": 500,
215
+ "type": type(e).__name__,
216
+ "message": str(e),
217
+ "details": {}
218
+ }
219
+
220
+ end_time = datetime.now().astimezone().isoformat()
221
+
222
+ custom_component = self.create_custom_component(
223
+ component_id=component_id,
224
+ hash_id=hash_id,
225
+ name=name,
226
+ custom_type=custom_type,
227
+ version=version,
228
+ start_time=start_time,
229
+ end_time=end_time,
230
+ memory_used=0,
231
+ variable_traces=variable_traces,
232
+ input_data=self._sanitize_input(args, kwargs),
233
+ output_data=None,
234
+ error=error_component
235
+ )
236
+ self.add_component(custom_component)
237
+ raise
238
+
239
+ def create_custom_component(self, **kwargs):
240
+ """Create a custom component according to the data structure"""
241
+ start_time = kwargs["start_time"]
242
+
243
+ network_calls = []
244
+ if self.auto_instrument_network:
245
+ network_calls = self.component_network_calls.get(kwargs["component_id"], [])
246
+
247
+ interactions = []
248
+ if self.auto_instrument_user_interaction:
249
+ interactions = self.component_user_interaction.get(kwargs["component_id"], [])
250
+
251
+ component = {
252
+ "id": kwargs["component_id"],
253
+ "hash_id": kwargs["hash_id"],
254
+ "source_hash_id": None,
255
+ "type": "custom",
256
+ "name": kwargs["name"],
257
+ "start_time": start_time,
258
+ "end_time": kwargs["end_time"],
259
+ "error": kwargs.get("error"),
260
+ "parent_id": self.current_agent_id.get() if hasattr(self, 'current_agent_id') else None,
261
+ "info": {
262
+ "custom_type": kwargs["custom_type"],
263
+ "version": kwargs["version"],
264
+ "memory_used": kwargs["memory_used"]
265
+ },
266
+ "data": {
267
+ "input": kwargs["input_data"],
268
+ "output": kwargs["output_data"],
269
+ "memory_used": kwargs["memory_used"],
270
+ "variable_traces": kwargs.get("variable_traces", [])
271
+ },
272
+ "network_calls": network_calls,
273
+ "interactions": interactions
274
+ }
275
+
276
+ if self.gt:
277
+ component["data"]["gt"] = self.gt
278
+
279
+ return component
280
+
281
+ def start_component(self, component_id):
282
+ """Start tracking network calls for a component"""
283
+ self.component_network_calls[component_id] = []
284
+
285
+ def end_component(self, component_id):
286
+ """End tracking network calls for a component"""
287
+ pass
288
+
289
+ def _sanitize_input(self, args: tuple, kwargs: dict) -> Dict:
290
+ """Sanitize and format input data"""
291
+ return {
292
+ "args": [str(arg) if not isinstance(arg, (int, float, bool, str, list, dict)) else arg for arg in args],
293
+ "kwargs": {
294
+ k: str(v) if not isinstance(v, (int, float, bool, str, list, dict)) else v
295
+ for k, v in kwargs.items()
296
+ }
297
+ }
298
+
299
+ def _sanitize_output(self, output: Any) -> Any:
300
+ """Sanitize and format output data"""
301
+ if isinstance(output, (int, float, bool, str, list, dict)):
302
+ return output
303
+ return str(output)
304
+
305
+ # Auto instrumentation methods
306
+ def instrument_custom_calls(self):
307
+ """Enable auto-instrumentation for custom calls"""
308
+ self.auto_instrument_custom = True
309
+
310
+ def instrument_user_interaction_calls(self):
311
+ """Enable auto-instrumentation for user interaction calls"""
312
+ self.auto_instrument_user_interaction = True
313
+
314
+ def instrument_network_calls(self):
315
+ """Enable auto-instrumentation for network calls"""
316
+ self.auto_instrument_network = True