ragaai-catalyst 2.1.4.1b0__py3-none-any.whl → 2.1.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. ragaai_catalyst/__init__.py +23 -2
  2. ragaai_catalyst/dataset.py +462 -1
  3. ragaai_catalyst/evaluation.py +76 -7
  4. ragaai_catalyst/ragaai_catalyst.py +52 -10
  5. ragaai_catalyst/redteaming/__init__.py +7 -0
  6. ragaai_catalyst/redteaming/config/detectors.toml +13 -0
  7. ragaai_catalyst/redteaming/data_generator/scenario_generator.py +95 -0
  8. ragaai_catalyst/redteaming/data_generator/test_case_generator.py +120 -0
  9. ragaai_catalyst/redteaming/evaluator.py +125 -0
  10. ragaai_catalyst/redteaming/llm_generator.py +136 -0
  11. ragaai_catalyst/redteaming/llm_generator_old.py +83 -0
  12. ragaai_catalyst/redteaming/red_teaming.py +331 -0
  13. ragaai_catalyst/redteaming/requirements.txt +4 -0
  14. ragaai_catalyst/redteaming/tests/grok.ipynb +97 -0
  15. ragaai_catalyst/redteaming/tests/stereotype.ipynb +2258 -0
  16. ragaai_catalyst/redteaming/upload_result.py +38 -0
  17. ragaai_catalyst/redteaming/utils/issue_description.py +114 -0
  18. ragaai_catalyst/redteaming/utils/rt.png +0 -0
  19. ragaai_catalyst/redteaming_old.py +171 -0
  20. ragaai_catalyst/synthetic_data_generation.py +400 -22
  21. ragaai_catalyst/tracers/__init__.py +17 -1
  22. ragaai_catalyst/tracers/agentic_tracing/data/data_structure.py +4 -2
  23. ragaai_catalyst/tracers/agentic_tracing/tracers/agent_tracer.py +212 -148
  24. ragaai_catalyst/tracers/agentic_tracing/tracers/base.py +657 -247
  25. ragaai_catalyst/tracers/agentic_tracing/tracers/custom_tracer.py +50 -19
  26. ragaai_catalyst/tracers/agentic_tracing/tracers/llm_tracer.py +588 -177
  27. ragaai_catalyst/tracers/agentic_tracing/tracers/main_tracer.py +99 -100
  28. ragaai_catalyst/tracers/agentic_tracing/tracers/network_tracer.py +3 -3
  29. ragaai_catalyst/tracers/agentic_tracing/tracers/tool_tracer.py +230 -29
  30. ragaai_catalyst/tracers/agentic_tracing/upload/trace_uploader.py +358 -0
  31. ragaai_catalyst/tracers/agentic_tracing/upload/upload_agentic_traces.py +75 -20
  32. ragaai_catalyst/tracers/agentic_tracing/upload/upload_code.py +55 -11
  33. ragaai_catalyst/tracers/agentic_tracing/upload/upload_local_metric.py +74 -0
  34. ragaai_catalyst/tracers/agentic_tracing/upload/upload_trace_metric.py +47 -16
  35. ragaai_catalyst/tracers/agentic_tracing/utils/create_dataset_schema.py +4 -2
  36. ragaai_catalyst/tracers/agentic_tracing/utils/file_name_tracker.py +26 -3
  37. ragaai_catalyst/tracers/agentic_tracing/utils/llm_utils.py +182 -17
  38. ragaai_catalyst/tracers/agentic_tracing/utils/model_costs.json +1233 -497
  39. ragaai_catalyst/tracers/agentic_tracing/utils/span_attributes.py +81 -10
  40. ragaai_catalyst/tracers/agentic_tracing/utils/supported_llm_provider.toml +34 -0
  41. ragaai_catalyst/tracers/agentic_tracing/utils/system_monitor.py +215 -0
  42. ragaai_catalyst/tracers/agentic_tracing/utils/trace_utils.py +0 -32
  43. ragaai_catalyst/tracers/agentic_tracing/utils/unique_decorator.py +3 -1
  44. ragaai_catalyst/tracers/agentic_tracing/utils/zip_list_of_unique_files.py +73 -47
  45. ragaai_catalyst/tracers/distributed.py +300 -0
  46. ragaai_catalyst/tracers/exporters/__init__.py +3 -1
  47. ragaai_catalyst/tracers/exporters/dynamic_trace_exporter.py +160 -0
  48. ragaai_catalyst/tracers/exporters/ragaai_trace_exporter.py +129 -0
  49. ragaai_catalyst/tracers/langchain_callback.py +809 -0
  50. ragaai_catalyst/tracers/llamaindex_instrumentation.py +424 -0
  51. ragaai_catalyst/tracers/tracer.py +301 -55
  52. ragaai_catalyst/tracers/upload_traces.py +24 -7
  53. ragaai_catalyst/tracers/utils/convert_langchain_callbacks_output.py +61 -0
  54. ragaai_catalyst/tracers/utils/convert_llama_instru_callback.py +69 -0
  55. ragaai_catalyst/tracers/utils/extraction_logic_llama_index.py +74 -0
  56. ragaai_catalyst/tracers/utils/langchain_tracer_extraction_logic.py +82 -0
  57. ragaai_catalyst/tracers/utils/model_prices_and_context_window_backup.json +9365 -0
  58. ragaai_catalyst/tracers/utils/trace_json_converter.py +269 -0
  59. {ragaai_catalyst-2.1.4.1b0.dist-info → ragaai_catalyst-2.1.5.dist-info}/METADATA +367 -45
  60. ragaai_catalyst-2.1.5.dist-info/RECORD +97 -0
  61. {ragaai_catalyst-2.1.4.1b0.dist-info → ragaai_catalyst-2.1.5.dist-info}/WHEEL +1 -1
  62. ragaai_catalyst-2.1.4.1b0.dist-info/RECORD +0 -67
  63. {ragaai_catalyst-2.1.4.1b0.dist-info → ragaai_catalyst-2.1.5.dist-info}/LICENSE +0 -0
  64. {ragaai_catalyst-2.1.4.1b0.dist-info → ragaai_catalyst-2.1.5.dist-info}/top_level.txt +0 -0
@@ -25,6 +25,7 @@ class CustomTracerMixin:
25
25
  self.auto_instrument_custom = False
26
26
  self.auto_instrument_user_interaction = False
27
27
  self.auto_instrument_network = False
28
+ self.auto_instrument_file_io = False
28
29
 
29
30
  def trace_custom(self, name: str = None, custom_type: str = "generic", version: str = "1.0.0", trace_variables: bool = True):
30
31
  def decorator(func):
@@ -40,20 +41,24 @@ class CustomTracerMixin:
40
41
  # Check if the function is async
41
42
  is_async = asyncio.iscoroutinefunction(func)
42
43
 
43
- @self.file_tracker.trace_decorator
44
44
  @functools.wraps(func)
45
45
  async def async_wrapper(*args, **kwargs):
46
46
  async_wrapper.metadata = metadata
47
- self.gt = kwargs.get('gt', None) if kwargs else None
47
+ gt = kwargs.get('gt') if kwargs else None
48
+ if gt is not None:
49
+ span = self.span(name)
50
+ span.add_gt(gt)
48
51
  return await self._trace_custom_execution(
49
52
  func, name or func.__name__, custom_type, version, trace_variables, *args, **kwargs
50
53
  )
51
54
 
52
- @self.file_tracker.trace_decorator
53
55
  @functools.wraps(func)
54
56
  def sync_wrapper(*args, **kwargs):
55
57
  sync_wrapper.metadata = metadata
56
- self.gt = kwargs.get('gt', None) if kwargs else None
58
+ gt = kwargs.get('gt') if kwargs else None
59
+ if gt is not None:
60
+ span = self.span(name)
61
+ span.add_gt(gt)
57
62
  return self._trace_sync_custom_execution(
58
63
  func, name or func.__name__, custom_type, version, trace_variables, *args, **kwargs
59
64
  )
@@ -153,7 +158,7 @@ class CustomTracerMixin:
153
158
  error=error_component
154
159
  )
155
160
 
156
- self.add_component(custom_component)
161
+ self.add_component(custom_component, is_error=True)
157
162
  raise
158
163
 
159
164
  async def _trace_custom_execution(self, func, name, custom_type, version, trace_variables, *args, **kwargs):
@@ -233,7 +238,7 @@ class CustomTracerMixin:
233
238
  output_data=None,
234
239
  error=error_component
235
240
  )
236
- self.add_component(custom_component)
241
+ self.add_component(custom_component, is_error=True)
237
242
  raise
238
243
 
239
244
  def create_custom_component(self, **kwargs):
@@ -246,8 +251,18 @@ class CustomTracerMixin:
246
251
 
247
252
  interactions = []
248
253
  if self.auto_instrument_user_interaction:
249
- interactions = self.component_user_interaction.get(kwargs["component_id"], [])
250
-
254
+ input_output_interactions = []
255
+ for interaction in self.component_user_interaction.get(kwargs["component_id"], []):
256
+ if interaction["interaction_type"] in ["input", "output"]:
257
+ input_output_interactions.append(interaction)
258
+ interactions.extend(input_output_interactions)
259
+ if self.auto_instrument_file_io:
260
+ file_io_interactions = []
261
+ for interaction in self.component_user_interaction.get(kwargs["component_id"], []):
262
+ if interaction["interaction_type"] in ["file_read", "file_write"]:
263
+ file_io_interactions.append(interaction)
264
+ interactions.extend(file_io_interactions)
265
+
251
266
  component = {
252
267
  "id": kwargs["component_id"],
253
268
  "hash_id": kwargs["hash_id"],
@@ -273,9 +288,13 @@ class CustomTracerMixin:
273
288
  "interactions": interactions
274
289
  }
275
290
 
276
- if self.gt:
277
- component["data"]["gt"] = self.gt
278
-
291
+ if kwargs["name"] in self.span_attributes_dict:
292
+ span_gt = self.span_attributes_dict[kwargs["name"]].gt
293
+ if span_gt is not None:
294
+ component["data"]["gt"] = span_gt
295
+ span_context = self.span_attributes_dict[kwargs["name"]].context
296
+ if span_context:
297
+ component["data"]["context"] = span_context
279
298
  return component
280
299
 
281
300
  def start_component(self, component_id):
@@ -286,15 +305,23 @@ class CustomTracerMixin:
286
305
  """End tracking network calls for a component"""
287
306
  pass
288
307
 
289
- def _sanitize_input(self, args: tuple, kwargs: dict) -> Dict:
290
- """Sanitize and format input data"""
291
- return {
292
- "args": [str(arg) if not isinstance(arg, (int, float, bool, str, list, dict)) else arg for arg in args],
293
- "kwargs": {
294
- k: str(v) if not isinstance(v, (int, float, bool, str, list, dict)) else v
295
- for k, v in kwargs.items()
308
+ def _sanitize_input(self, args: tuple, kwargs: dict) -> dict:
309
+ """Sanitize and format input data, including handling of nested lists and dictionaries."""
310
+
311
+ def sanitize_value(value):
312
+ if isinstance(value, (int, float, bool, str)):
313
+ return value
314
+ elif isinstance(value, list):
315
+ return [sanitize_value(item) for item in value]
316
+ elif isinstance(value, dict):
317
+ return {key: sanitize_value(val) for key, val in value.items()}
318
+ else:
319
+ return str(value) # Convert non-standard types to string
320
+
321
+ return {
322
+ "args": [sanitize_value(arg) for arg in args],
323
+ "kwargs": {key: sanitize_value(val) for key, val in kwargs.items()},
296
324
  }
297
- }
298
325
 
299
326
  def _sanitize_output(self, output: Any) -> Any:
300
327
  """Sanitize and format output data"""
@@ -314,3 +341,7 @@ class CustomTracerMixin:
314
341
  def instrument_network_calls(self):
315
342
  """Enable auto-instrumentation for network calls"""
316
343
  self.auto_instrument_network = True
344
+
345
+ def instrument_file_io_calls(self):
346
+ """Enable auto-instrumentation for file IO calls"""
347
+ self.auto_instrument_file_io = True