ragaai-catalyst 2.1.4.1b0__py3-none-any.whl → 2.1.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ragaai_catalyst/__init__.py +23 -2
- ragaai_catalyst/dataset.py +462 -1
- ragaai_catalyst/evaluation.py +76 -7
- ragaai_catalyst/ragaai_catalyst.py +52 -10
- ragaai_catalyst/redteaming/__init__.py +7 -0
- ragaai_catalyst/redteaming/config/detectors.toml +13 -0
- ragaai_catalyst/redteaming/data_generator/scenario_generator.py +95 -0
- ragaai_catalyst/redteaming/data_generator/test_case_generator.py +120 -0
- ragaai_catalyst/redteaming/evaluator.py +125 -0
- ragaai_catalyst/redteaming/llm_generator.py +136 -0
- ragaai_catalyst/redteaming/llm_generator_old.py +83 -0
- ragaai_catalyst/redteaming/red_teaming.py +331 -0
- ragaai_catalyst/redteaming/requirements.txt +4 -0
- ragaai_catalyst/redteaming/tests/grok.ipynb +97 -0
- ragaai_catalyst/redteaming/tests/stereotype.ipynb +2258 -0
- ragaai_catalyst/redteaming/upload_result.py +38 -0
- ragaai_catalyst/redteaming/utils/issue_description.py +114 -0
- ragaai_catalyst/redteaming/utils/rt.png +0 -0
- ragaai_catalyst/redteaming_old.py +171 -0
- ragaai_catalyst/synthetic_data_generation.py +400 -22
- ragaai_catalyst/tracers/__init__.py +17 -1
- ragaai_catalyst/tracers/agentic_tracing/data/data_structure.py +4 -2
- ragaai_catalyst/tracers/agentic_tracing/tracers/agent_tracer.py +212 -148
- ragaai_catalyst/tracers/agentic_tracing/tracers/base.py +657 -247
- ragaai_catalyst/tracers/agentic_tracing/tracers/custom_tracer.py +50 -19
- ragaai_catalyst/tracers/agentic_tracing/tracers/llm_tracer.py +588 -177
- ragaai_catalyst/tracers/agentic_tracing/tracers/main_tracer.py +99 -100
- ragaai_catalyst/tracers/agentic_tracing/tracers/network_tracer.py +3 -3
- ragaai_catalyst/tracers/agentic_tracing/tracers/tool_tracer.py +230 -29
- ragaai_catalyst/tracers/agentic_tracing/upload/trace_uploader.py +358 -0
- ragaai_catalyst/tracers/agentic_tracing/upload/upload_agentic_traces.py +75 -20
- ragaai_catalyst/tracers/agentic_tracing/upload/upload_code.py +55 -11
- ragaai_catalyst/tracers/agentic_tracing/upload/upload_local_metric.py +74 -0
- ragaai_catalyst/tracers/agentic_tracing/upload/upload_trace_metric.py +47 -16
- ragaai_catalyst/tracers/agentic_tracing/utils/create_dataset_schema.py +4 -2
- ragaai_catalyst/tracers/agentic_tracing/utils/file_name_tracker.py +26 -3
- ragaai_catalyst/tracers/agentic_tracing/utils/llm_utils.py +182 -17
- ragaai_catalyst/tracers/agentic_tracing/utils/model_costs.json +1233 -497
- ragaai_catalyst/tracers/agentic_tracing/utils/span_attributes.py +81 -10
- ragaai_catalyst/tracers/agentic_tracing/utils/supported_llm_provider.toml +34 -0
- ragaai_catalyst/tracers/agentic_tracing/utils/system_monitor.py +215 -0
- ragaai_catalyst/tracers/agentic_tracing/utils/trace_utils.py +0 -32
- ragaai_catalyst/tracers/agentic_tracing/utils/unique_decorator.py +3 -1
- ragaai_catalyst/tracers/agentic_tracing/utils/zip_list_of_unique_files.py +73 -47
- ragaai_catalyst/tracers/distributed.py +300 -0
- ragaai_catalyst/tracers/exporters/__init__.py +3 -1
- ragaai_catalyst/tracers/exporters/dynamic_trace_exporter.py +160 -0
- ragaai_catalyst/tracers/exporters/ragaai_trace_exporter.py +129 -0
- ragaai_catalyst/tracers/langchain_callback.py +809 -0
- ragaai_catalyst/tracers/llamaindex_instrumentation.py +424 -0
- ragaai_catalyst/tracers/tracer.py +301 -55
- ragaai_catalyst/tracers/upload_traces.py +24 -7
- ragaai_catalyst/tracers/utils/convert_langchain_callbacks_output.py +61 -0
- ragaai_catalyst/tracers/utils/convert_llama_instru_callback.py +69 -0
- ragaai_catalyst/tracers/utils/extraction_logic_llama_index.py +74 -0
- ragaai_catalyst/tracers/utils/langchain_tracer_extraction_logic.py +82 -0
- ragaai_catalyst/tracers/utils/model_prices_and_context_window_backup.json +9365 -0
- ragaai_catalyst/tracers/utils/trace_json_converter.py +269 -0
- {ragaai_catalyst-2.1.4.1b0.dist-info → ragaai_catalyst-2.1.5.dist-info}/METADATA +367 -45
- ragaai_catalyst-2.1.5.dist-info/RECORD +97 -0
- {ragaai_catalyst-2.1.4.1b0.dist-info → ragaai_catalyst-2.1.5.dist-info}/WHEEL +1 -1
- ragaai_catalyst-2.1.4.1b0.dist-info/RECORD +0 -67
- {ragaai_catalyst-2.1.4.1b0.dist-info → ragaai_catalyst-2.1.5.dist-info}/LICENSE +0 -0
- {ragaai_catalyst-2.1.4.1b0.dist-info → ragaai_catalyst-2.1.5.dist-info}/top_level.txt +0 -0
@@ -25,6 +25,7 @@ class CustomTracerMixin:
|
|
25
25
|
self.auto_instrument_custom = False
|
26
26
|
self.auto_instrument_user_interaction = False
|
27
27
|
self.auto_instrument_network = False
|
28
|
+
self.auto_instrument_file_io = False
|
28
29
|
|
29
30
|
def trace_custom(self, name: str = None, custom_type: str = "generic", version: str = "1.0.0", trace_variables: bool = True):
|
30
31
|
def decorator(func):
|
@@ -40,20 +41,24 @@ class CustomTracerMixin:
|
|
40
41
|
# Check if the function is async
|
41
42
|
is_async = asyncio.iscoroutinefunction(func)
|
42
43
|
|
43
|
-
@self.file_tracker.trace_decorator
|
44
44
|
@functools.wraps(func)
|
45
45
|
async def async_wrapper(*args, **kwargs):
|
46
46
|
async_wrapper.metadata = metadata
|
47
|
-
|
47
|
+
gt = kwargs.get('gt') if kwargs else None
|
48
|
+
if gt is not None:
|
49
|
+
span = self.span(name)
|
50
|
+
span.add_gt(gt)
|
48
51
|
return await self._trace_custom_execution(
|
49
52
|
func, name or func.__name__, custom_type, version, trace_variables, *args, **kwargs
|
50
53
|
)
|
51
54
|
|
52
|
-
@self.file_tracker.trace_decorator
|
53
55
|
@functools.wraps(func)
|
54
56
|
def sync_wrapper(*args, **kwargs):
|
55
57
|
sync_wrapper.metadata = metadata
|
56
|
-
|
58
|
+
gt = kwargs.get('gt') if kwargs else None
|
59
|
+
if gt is not None:
|
60
|
+
span = self.span(name)
|
61
|
+
span.add_gt(gt)
|
57
62
|
return self._trace_sync_custom_execution(
|
58
63
|
func, name or func.__name__, custom_type, version, trace_variables, *args, **kwargs
|
59
64
|
)
|
@@ -153,7 +158,7 @@ class CustomTracerMixin:
|
|
153
158
|
error=error_component
|
154
159
|
)
|
155
160
|
|
156
|
-
self.add_component(custom_component)
|
161
|
+
self.add_component(custom_component, is_error=True)
|
157
162
|
raise
|
158
163
|
|
159
164
|
async def _trace_custom_execution(self, func, name, custom_type, version, trace_variables, *args, **kwargs):
|
@@ -233,7 +238,7 @@ class CustomTracerMixin:
|
|
233
238
|
output_data=None,
|
234
239
|
error=error_component
|
235
240
|
)
|
236
|
-
self.add_component(custom_component)
|
241
|
+
self.add_component(custom_component, is_error=True)
|
237
242
|
raise
|
238
243
|
|
239
244
|
def create_custom_component(self, **kwargs):
|
@@ -246,8 +251,18 @@ class CustomTracerMixin:
|
|
246
251
|
|
247
252
|
interactions = []
|
248
253
|
if self.auto_instrument_user_interaction:
|
249
|
-
|
250
|
-
|
254
|
+
input_output_interactions = []
|
255
|
+
for interaction in self.component_user_interaction.get(kwargs["component_id"], []):
|
256
|
+
if interaction["interaction_type"] in ["input", "output"]:
|
257
|
+
input_output_interactions.append(interaction)
|
258
|
+
interactions.extend(input_output_interactions)
|
259
|
+
if self.auto_instrument_file_io:
|
260
|
+
file_io_interactions = []
|
261
|
+
for interaction in self.component_user_interaction.get(kwargs["component_id"], []):
|
262
|
+
if interaction["interaction_type"] in ["file_read", "file_write"]:
|
263
|
+
file_io_interactions.append(interaction)
|
264
|
+
interactions.extend(file_io_interactions)
|
265
|
+
|
251
266
|
component = {
|
252
267
|
"id": kwargs["component_id"],
|
253
268
|
"hash_id": kwargs["hash_id"],
|
@@ -273,9 +288,13 @@ class CustomTracerMixin:
|
|
273
288
|
"interactions": interactions
|
274
289
|
}
|
275
290
|
|
276
|
-
if self.
|
277
|
-
|
278
|
-
|
291
|
+
if kwargs["name"] in self.span_attributes_dict:
|
292
|
+
span_gt = self.span_attributes_dict[kwargs["name"]].gt
|
293
|
+
if span_gt is not None:
|
294
|
+
component["data"]["gt"] = span_gt
|
295
|
+
span_context = self.span_attributes_dict[kwargs["name"]].context
|
296
|
+
if span_context:
|
297
|
+
component["data"]["context"] = span_context
|
279
298
|
return component
|
280
299
|
|
281
300
|
def start_component(self, component_id):
|
@@ -286,15 +305,23 @@ class CustomTracerMixin:
|
|
286
305
|
"""End tracking network calls for a component"""
|
287
306
|
pass
|
288
307
|
|
289
|
-
def _sanitize_input(self, args: tuple, kwargs: dict) ->
|
290
|
-
|
291
|
-
|
292
|
-
|
293
|
-
|
294
|
-
|
295
|
-
|
308
|
+
def _sanitize_input(self, args: tuple, kwargs: dict) -> dict:
|
309
|
+
"""Sanitize and format input data, including handling of nested lists and dictionaries."""
|
310
|
+
|
311
|
+
def sanitize_value(value):
|
312
|
+
if isinstance(value, (int, float, bool, str)):
|
313
|
+
return value
|
314
|
+
elif isinstance(value, list):
|
315
|
+
return [sanitize_value(item) for item in value]
|
316
|
+
elif isinstance(value, dict):
|
317
|
+
return {key: sanitize_value(val) for key, val in value.items()}
|
318
|
+
else:
|
319
|
+
return str(value) # Convert non-standard types to string
|
320
|
+
|
321
|
+
return {
|
322
|
+
"args": [sanitize_value(arg) for arg in args],
|
323
|
+
"kwargs": {key: sanitize_value(val) for key, val in kwargs.items()},
|
296
324
|
}
|
297
|
-
}
|
298
325
|
|
299
326
|
def _sanitize_output(self, output: Any) -> Any:
|
300
327
|
"""Sanitize and format output data"""
|
@@ -314,3 +341,7 @@ class CustomTracerMixin:
|
|
314
341
|
def instrument_network_calls(self):
|
315
342
|
"""Enable auto-instrumentation for network calls"""
|
316
343
|
self.auto_instrument_network = True
|
344
|
+
|
345
|
+
def instrument_file_io_calls(self):
|
346
|
+
"""Enable auto-instrumentation for file IO calls"""
|
347
|
+
self.auto_instrument_file_io = True
|