ragaai-catalyst 2.1.4.1b0__py3-none-any.whl → 2.1.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. ragaai_catalyst/__init__.py +23 -2
  2. ragaai_catalyst/dataset.py +462 -1
  3. ragaai_catalyst/evaluation.py +76 -7
  4. ragaai_catalyst/ragaai_catalyst.py +52 -10
  5. ragaai_catalyst/redteaming/__init__.py +7 -0
  6. ragaai_catalyst/redteaming/config/detectors.toml +13 -0
  7. ragaai_catalyst/redteaming/data_generator/scenario_generator.py +95 -0
  8. ragaai_catalyst/redteaming/data_generator/test_case_generator.py +120 -0
  9. ragaai_catalyst/redteaming/evaluator.py +125 -0
  10. ragaai_catalyst/redteaming/llm_generator.py +136 -0
  11. ragaai_catalyst/redteaming/llm_generator_old.py +83 -0
  12. ragaai_catalyst/redteaming/red_teaming.py +331 -0
  13. ragaai_catalyst/redteaming/requirements.txt +4 -0
  14. ragaai_catalyst/redteaming/tests/grok.ipynb +97 -0
  15. ragaai_catalyst/redteaming/tests/stereotype.ipynb +2258 -0
  16. ragaai_catalyst/redteaming/upload_result.py +38 -0
  17. ragaai_catalyst/redteaming/utils/issue_description.py +114 -0
  18. ragaai_catalyst/redteaming/utils/rt.png +0 -0
  19. ragaai_catalyst/redteaming_old.py +171 -0
  20. ragaai_catalyst/synthetic_data_generation.py +400 -22
  21. ragaai_catalyst/tracers/__init__.py +17 -1
  22. ragaai_catalyst/tracers/agentic_tracing/data/data_structure.py +4 -2
  23. ragaai_catalyst/tracers/agentic_tracing/tracers/agent_tracer.py +212 -148
  24. ragaai_catalyst/tracers/agentic_tracing/tracers/base.py +657 -247
  25. ragaai_catalyst/tracers/agentic_tracing/tracers/custom_tracer.py +50 -19
  26. ragaai_catalyst/tracers/agentic_tracing/tracers/llm_tracer.py +588 -177
  27. ragaai_catalyst/tracers/agentic_tracing/tracers/main_tracer.py +99 -100
  28. ragaai_catalyst/tracers/agentic_tracing/tracers/network_tracer.py +3 -3
  29. ragaai_catalyst/tracers/agentic_tracing/tracers/tool_tracer.py +230 -29
  30. ragaai_catalyst/tracers/agentic_tracing/upload/trace_uploader.py +358 -0
  31. ragaai_catalyst/tracers/agentic_tracing/upload/upload_agentic_traces.py +75 -20
  32. ragaai_catalyst/tracers/agentic_tracing/upload/upload_code.py +55 -11
  33. ragaai_catalyst/tracers/agentic_tracing/upload/upload_local_metric.py +74 -0
  34. ragaai_catalyst/tracers/agentic_tracing/upload/upload_trace_metric.py +47 -16
  35. ragaai_catalyst/tracers/agentic_tracing/utils/create_dataset_schema.py +4 -2
  36. ragaai_catalyst/tracers/agentic_tracing/utils/file_name_tracker.py +26 -3
  37. ragaai_catalyst/tracers/agentic_tracing/utils/llm_utils.py +182 -17
  38. ragaai_catalyst/tracers/agentic_tracing/utils/model_costs.json +1233 -497
  39. ragaai_catalyst/tracers/agentic_tracing/utils/span_attributes.py +81 -10
  40. ragaai_catalyst/tracers/agentic_tracing/utils/supported_llm_provider.toml +34 -0
  41. ragaai_catalyst/tracers/agentic_tracing/utils/system_monitor.py +215 -0
  42. ragaai_catalyst/tracers/agentic_tracing/utils/trace_utils.py +0 -32
  43. ragaai_catalyst/tracers/agentic_tracing/utils/unique_decorator.py +3 -1
  44. ragaai_catalyst/tracers/agentic_tracing/utils/zip_list_of_unique_files.py +73 -47
  45. ragaai_catalyst/tracers/distributed.py +300 -0
  46. ragaai_catalyst/tracers/exporters/__init__.py +3 -1
  47. ragaai_catalyst/tracers/exporters/dynamic_trace_exporter.py +160 -0
  48. ragaai_catalyst/tracers/exporters/ragaai_trace_exporter.py +129 -0
  49. ragaai_catalyst/tracers/langchain_callback.py +809 -0
  50. ragaai_catalyst/tracers/llamaindex_instrumentation.py +424 -0
  51. ragaai_catalyst/tracers/tracer.py +301 -55
  52. ragaai_catalyst/tracers/upload_traces.py +24 -7
  53. ragaai_catalyst/tracers/utils/convert_langchain_callbacks_output.py +61 -0
  54. ragaai_catalyst/tracers/utils/convert_llama_instru_callback.py +69 -0
  55. ragaai_catalyst/tracers/utils/extraction_logic_llama_index.py +74 -0
  56. ragaai_catalyst/tracers/utils/langchain_tracer_extraction_logic.py +82 -0
  57. ragaai_catalyst/tracers/utils/model_prices_and_context_window_backup.json +9365 -0
  58. ragaai_catalyst/tracers/utils/trace_json_converter.py +269 -0
  59. {ragaai_catalyst-2.1.4.1b0.dist-info → ragaai_catalyst-2.1.5.dist-info}/METADATA +367 -45
  60. ragaai_catalyst-2.1.5.dist-info/RECORD +97 -0
  61. {ragaai_catalyst-2.1.4.1b0.dist-info → ragaai_catalyst-2.1.5.dist-info}/WHEEL +1 -1
  62. ragaai_catalyst-2.1.4.1b0.dist-info/RECORD +0 -67
  63. {ragaai_catalyst-2.1.4.1b0.dist-info → ragaai_catalyst-2.1.5.dist-info}/LICENSE +0 -0
  64. {ragaai_catalyst-2.1.4.1b0.dist-info → ragaai_catalyst-2.1.5.dist-info}/top_level.txt +0 -0
@@ -1,8 +1,8 @@
1
1
  Metadata-Version: 2.2
2
2
  Name: ragaai_catalyst
3
- Version: 2.1.4.1b0
3
+ Version: 2.1.5
4
4
  Summary: RAGA AI CATALYST
5
- Author-email: Kiran Scaria <kiran.scaria@raga.ai>, Kedar Gaikwad <kedar.gaikwad@raga.ai>, Dushyant Mahajan <dushyant.mahajan@raga.ai>, Siddhartha Kosti <siddhartha.kosti@raga.ai>, Ritika Goel <ritika.goel@raga.ai>, Vijay Chaurasia <vijay.chaurasia@raga.ai>
5
+ Author-email: Kiran Scaria <kiran.scaria@raga.ai>, Kedar Gaikwad <kedar.gaikwad@raga.ai>, Dushyant Mahajan <dushyant.mahajan@raga.ai>, Siddhartha Kosti <siddhartha.kosti@raga.ai>, Ritika Goel <ritika.goel@raga.ai>, Vijay Chaurasia <vijay.chaurasia@raga.ai>, Tushar Kumar <tushar.kumar@raga.ai>
6
6
  Requires-Python: <3.13,>=3.9
7
7
  Description-Content-Type: text/markdown
8
8
  License-File: LICENSE
@@ -28,14 +28,20 @@ Requires-Dist: Markdown>=3.7
28
28
  Requires-Dist: litellm==1.51.1
29
29
  Requires-Dist: tenacity==8.3.0
30
30
  Requires-Dist: tqdm>=4.66.5
31
- Requires-Dist: llama-index<0.11.0,>=0.10.0
31
+ Requires-Dist: llama-index>=0.10.0
32
32
  Requires-Dist: pyopenssl>=24.2.1
33
33
  Requires-Dist: psutil~=6.0.0
34
34
  Requires-Dist: py-cpuinfo~=9.0.0
35
35
  Requires-Dist: requests~=2.32.3
36
36
  Requires-Dist: GPUtil~=1.4.0
37
- Requires-Dist: astor>=0.8.1
38
37
  Requires-Dist: ipynbname
38
+ Requires-Dist: tiktoken>=0.7.0
39
+ Requires-Dist: tomli>=2.0.0
40
+ Requires-Dist: rich>=13.9.4
41
+ Requires-Dist: openinference-instrumentation-llama-index
42
+ Requires-Dist: opentelemetry-sdk
43
+ Requires-Dist: opentelemetry-exporter-otlp
44
+ Requires-Dist: opentelemetry-proto>=1.12.0
39
45
  Provides-Extra: dev
40
46
  Requires-Dist: pytest; extra == "dev"
41
47
  Requires-Dist: pytest-cov; extra == "dev"
@@ -48,10 +54,13 @@ Requires-Dist: flake8; extra == "dev"
48
54
 
49
55
  RagaAI Catalyst is a comprehensive platform designed to enhance the management and optimization of LLM projects. It offers a wide range of features, including project management, dataset management, evaluation management, trace management, prompt management, synthetic data generation, and guardrail management. These functionalities enable you to efficiently evaluate, and safeguard your LLM applications.
50
56
 
57
+ ![RagaAI Catalyst](docs/img/main.png)
58
+
59
+ ![RagaAI Catalyst](docs/img/main.png)
60
+
51
61
  ## Table of Contents
52
62
 
53
63
  - [RagaAI Catalyst](#ragaai-catalyst)
54
- - [Table of Contents](#table-of-contents)
55
64
  - [Installation](#installation)
56
65
  - [Configuration](#configuration)
57
66
  - [Usage](#usage)
@@ -59,10 +68,11 @@ RagaAI Catalyst is a comprehensive platform designed to enhance the management a
59
68
  - [Dataset Management](#dataset-management)
60
69
  - [Evaluation Management](#evaluation)
61
70
  - [Trace Management](#trace-management)
71
+ - [Agentic Tracing](#agentic-tracing)
62
72
  - [Prompt Management](#prompt-management)
63
73
  - [Synthetic Data Generation](#synthetic-data-generation)
64
74
  - [Guardrail Management](#guardrail-management)
65
- - [Agentic Tracing](#agentic-tracing)
75
+ - [Red-teaming](#red-teaming)
66
76
 
67
77
  ## Installation
68
78
 
@@ -85,7 +95,15 @@ catalyst = RagaAICatalyst(
85
95
  base_url="BASE_URL"
86
96
  )
87
97
  ```
88
- **Note**: Authetication to RagaAICatalyst is necessary to perform any operations below
98
+ you'll need to generate authentication credentials:
99
+
100
+ 1. Navigate to your profile settings
101
+ 2. Select "Authenticate"
102
+ 3. Click "Generate New Key" to create your access and secret keys
103
+
104
+ ![How to generate authentication keys](docs/img/autheticate.gif)
105
+
106
+ **Note**: Authetication to RagaAICatalyst is necessary to perform any operations below.
89
107
 
90
108
 
91
109
  ## Usage
@@ -108,6 +126,7 @@ catalyst.project_use_cases()
108
126
  projects = catalyst.list_projects()
109
127
  print(projects)
110
128
  ```
129
+ ![Projects](docs/img/create_project.gif)
111
130
 
112
131
  ### Dataset Management
113
132
  Manage datasets efficiently for your projects:
@@ -129,10 +148,25 @@ dataset_manager.create_from_csv(
129
148
  schema_mapping={'column1': 'schema_element1', 'column2': 'schema_element2'}
130
149
  )
131
150
 
151
+ # Create a dataset from JSONl
152
+ dataset_manager.create_from_jsonl(
153
+ jsonl_path='jsonl_path',
154
+ dataset_name='MyDataset',
155
+ schema_mapping={'column1': 'schema_element1', 'column2': 'schema_element2'}
156
+ )
157
+
158
+ # Create a dataset from dataframe
159
+ dataset_manager.create_from_df(
160
+ df=df,
161
+ dataset_name='MyDataset',
162
+ schema_mapping={'column1': 'schema_element1', 'column2': 'schema_element2'}
163
+ )
164
+
132
165
  # Get project schema mapping
133
166
  dataset_manager.get_schema_mapping()
134
167
 
135
168
  ```
169
+ ![Dataset](docs/img/dataset.gif)
136
170
 
137
171
  For more detailed information on Dataset Management, including CSV schema handling and advanced usage, please refer to the [Dataset Management documentation](docs/dataset_management.md).
138
172
 
@@ -186,32 +220,48 @@ print("Experiment Status:", status)
186
220
  # Get the results of the experiment
187
221
  results = evaluation.get_results()
188
222
  print("Experiment Results:", results)
223
+
224
+ # Appending Metrics for New Data
225
+ # If you've added new rows to your dataset, you can calculate metrics just for the new data:
226
+ evaluation.append_metrics(display_name="Faithfulness_v1")
189
227
  ```
190
228
 
229
+ ![Evaluation](docs/img/evaluation.gif)
230
+
191
231
 
192
232
 
193
233
  ### Trace Management
194
234
 
195
235
  Record and analyze traces of your RAG application:
196
-
236
+
197
237
  ```python
198
- from ragaai_catalyst import Tracer
238
+ from ragaai_catalyst import RagaAICatalyst, Tracer
199
239
 
200
- # Start a trace recording
201
240
  tracer = Tracer(
202
241
  project_name="Test-RAG-App-1",
203
242
  dataset_name="tracer_dataset_name",
204
- metadata={"key1": "value1", "key2": "value2"},
205
- tracer_type="langchain",
206
- pipeline={
207
- "llm_model": "gpt-4o-mini",
208
- "vector_store": "faiss",
209
- "embed_model": "text-embedding-ada-002",
210
- }
211
- ).start()
243
+ tracer_type="tracer_type"
244
+ )
245
+ ```
212
246
 
213
- # Your code here
247
+ There are two ways to start a trace recording
248
+
249
+ 1- with tracer():
250
+
251
+ ```python
214
252
 
253
+ with tracer():
254
+ # Your code here
255
+
256
+ ```
257
+
258
+ 2- tracer.start()
259
+
260
+ ```python
261
+ #start the trace recording
262
+ tracer.start()
263
+
264
+ # Your code here
215
265
 
216
266
  # Stop the trace recording
217
267
  tracer.stop()
@@ -220,6 +270,191 @@ tracer.stop()
220
270
  tracer.get_upload_status()
221
271
  ```
222
272
 
273
+ ![Trace](docs/img/trace_comp.png)
274
+ For more detailed information on Trace Management, please refer to the [Trace Management documentation](docs/trace_management.md).
275
+
276
+ ### Agentic Tracing
277
+
278
+ The Agentic Tracing module provides comprehensive monitoring and analysis capabilities for AI agent systems. It helps track various aspects of agent behavior including:
279
+
280
+ - LLM interactions and token usage
281
+ - Tool utilization and execution patterns
282
+ - Network activities and API calls
283
+ - User interactions and feedback
284
+ - Agent decision-making processes
285
+
286
+ The module includes utilities for cost tracking, performance monitoring, and debugging agent behavior. This helps in understanding and optimizing AI agent performance while maintaining transparency in agent operations.
287
+
288
+ #### Tracer initialization
289
+
290
+ Initialize the tracer with project_name and dataset_name
291
+
292
+ ```python
293
+ from ragaai_catalyst import RagaAICatalyst, Tracer, trace_llm, trace_tool, trace_agent, current_span
294
+
295
+ agentic_tracing_dataset_name = "agentic_tracing_dataset_name"
296
+
297
+ tracer = Tracer(
298
+ project_name=agentic_tracing_project_name,
299
+ dataset_name=agentic_tracing_dataset_name,
300
+ tracer_type="Agentic",
301
+ )
302
+ ```
303
+
304
+ ```python
305
+ from ragaai_catalyst import init_tracing
306
+ init_tracing(catalyst=catalyst, tracer=tracer)
307
+ ```
308
+
309
+ #### Agentic Tracing Features
310
+ 1- add span level metrics
311
+
312
+ ```python
313
+ current_span().add_metrics(name='Accuracy', score=0.5, reasoning='some reasoning')
314
+ ```
315
+
316
+ 2- add trace level metrics
317
+
318
+ ```python
319
+ tracer.add_metrics(name='hallucination_1', score=0.5, reasoning='some reasoning')
320
+ ```
321
+
322
+ 3- add gt
323
+
324
+ ```python
325
+ current_span().add_gt("This is the ground truth")
326
+ ```
327
+
328
+ 4- add context
329
+
330
+ ```python
331
+ current_span().add_context("This is the context")
332
+ ```
333
+
334
+ 5- add span level metric execution
335
+
336
+ ```python
337
+ current_span().execute_metrics(
338
+ name="Hallucination",
339
+ model="gpt-4o",
340
+ provider="openai"
341
+ )
342
+ ```
343
+
344
+ #### Example
345
+ ```python
346
+ from ragaai_catalyst import trace_llm, trace_tool, trace_agent, current_span
347
+
348
+ from openai import OpenAI
349
+
350
+
351
+ @trace_llm(name="llm_call", tags=["default_llm_call"])
352
+ def llm_call(prompt, max_tokens=512, model="gpt-4o-mini"):
353
+ client = OpenAI(api_key=OPENAI_API_KEY)
354
+ response = client.chat.completions.create(
355
+ model=model,
356
+ messages=[{"role": "user", "content": prompt}],
357
+ max_tokens=max_tokens,
358
+ temperature=0.85,
359
+ )
360
+ # Span level context
361
+ current_span().add_context("name = span level in summary_agent, context = some span level context")
362
+
363
+ # Span level execute metrics
364
+ current_span().execute_metrics(
365
+ name="Hallucination",
366
+ model="gpt-4o",
367
+ provider="openai"
368
+ )
369
+ response_data = response.choices[0].message.content.strip()
370
+ print('response_data: ', response_data)
371
+ return response_data
372
+
373
+ class SummaryAgent:
374
+ def __init__(self, persona="Summary Agent"):
375
+ self.persona = persona
376
+
377
+ @trace_agent(name="summary_agent")
378
+ def summarize(self, text):
379
+ prompt = f"Please summarize this text concisely: {text}"
380
+
381
+ # Span level metric
382
+ current_span().add_metrics(name='Accuracy', score=0.5, reasoning='some reasoning')
383
+
384
+ # Span level context
385
+ current_span().add_context("name = span level in summary_agent, context = some span level context")
386
+
387
+ summary = llm_call(prompt)
388
+ return summary
389
+
390
+
391
+ class AnalysisAgent:
392
+ def __init__(self, persona="Analysis Agent"):
393
+ self.persona = persona
394
+ self.summary_agent = SummaryAgent()
395
+
396
+ @trace_agent(name="analysis_agent")
397
+ def analyze(self, text):
398
+ summary = self.summary_agent.summarize(text)
399
+
400
+ prompt = f"Given this summary: {summary}\nProvide a brief analysis of the main points."
401
+
402
+ # Span level metric
403
+ current_span().add_metrics(name='correctness', score=0.5, reasoning='some reasoning')
404
+ analysis = llm_call(prompt)
405
+
406
+ return {
407
+ "summary": summary,
408
+ "analysis": analysis
409
+ }
410
+
411
+ class RecommendationAgent:
412
+ def __init__(self, persona="Recommendation Agent"):
413
+ self.persona = persona
414
+ self.analysis_agent = AnalysisAgent()
415
+
416
+ @trace_agent(name="recommendation_agent", tags=['coordinator_agent'])
417
+ def recommend(self, text):
418
+ analysis_result = self.analysis_agent.analyze(text)
419
+
420
+ prompt = f"""Given this summary: {analysis_result['summary']}
421
+ And this analysis: {analysis_result['analysis']}
422
+ Provide 2-3 actionable recommendations."""
423
+
424
+ recommendations = llm_call(prompt)
425
+
426
+ return {
427
+ "summary": analysis_result["summary"],
428
+ "analysis": analysis_result["analysis"],
429
+ "recommendations": recommendations
430
+ }
431
+ #Defining agent tracer
432
+ @trace_agent(name="get_recommendation", tags=['coordinator_agent'])
433
+ def get_recommendation(agent, text):
434
+ recommendation = agent.recommend(text)
435
+ return recommendation
436
+
437
+ def main():
438
+ text = """
439
+ Artificial Intelligence has transformed various industries in recent years.
440
+ From healthcare to finance, AI applications are becoming increasingly prevalent.
441
+ Machine learning models are being used to predict market trends, diagnose diseases,
442
+ and automate routine tasks. The impact of AI on society continues to grow,
443
+ raising both opportunities and challenges for the future.
444
+ """
445
+
446
+ recommendation_agent = RecommendationAgent()
447
+ result = get_recommendation(recommendation_agent, text)
448
+
449
+
450
+ # Trace level metric
451
+ tracer.add_metrics(name='hallucination_1', score=0.5, reasoning='some reasoning')
452
+
453
+ # Run tracer
454
+ with tracer:
455
+ main()
456
+ ```
457
+ ![Tracing](docs/img/last_main.png)
223
458
 
224
459
  ### Prompt Management
225
460
 
@@ -295,7 +530,7 @@ sdg = SyntheticDataGeneration()
295
530
  text = sdg.process_document(input_data="file_path")
296
531
 
297
532
  # Generate results
298
- result = sdg.generate_qna(text, question_type ='complex',model_config={"provider":"openai","model":"openai/gpt-3.5-turbo"},n=5)
533
+ result = sdg.generate_qna(text, question_type ='complex',model_config={"provider":"openai","model":"gpt-4o-mini"},n=5)
299
534
 
300
535
  print(result.head())
301
536
 
@@ -304,6 +539,22 @@ sdg.get_supported_qna()
304
539
 
305
540
  # Get supported providers
306
541
  sdg.get_supported_providers()
542
+
543
+ # Generate examples
544
+ examples = sdg.generate_examples(
545
+ user_instruction = 'Generate query like this.',
546
+ user_examples = 'How to do it?', # Can be a string or list of strings.
547
+ user_context = 'Context to generate examples',
548
+ no_examples = 10,
549
+ model_config = {"provider":"openai","model":"gpt-4o-mini"}
550
+ )
551
+
552
+ # Generate examples from a csv
553
+ sdg.generate_examples_from_csv(
554
+ csv_path = 'path/to/csv',
555
+ no_examples = 5,
556
+ model_config = {'provider': 'openai', 'model': 'gpt-4o-mini'}
557
+ )
307
558
  ```
308
559
 
309
560
 
@@ -392,41 +643,112 @@ llm_caller = 'litellm'
392
643
  executor([message],prompt_params,model_params,llm_caller)
393
644
 
394
645
  ```
646
+ ![Guardrails](docs/img/guardrails.png)
395
647
 
396
- ### Agentic Tracing
648
+ ### Red-teaming
397
649
 
398
- The Agentic Tracing module provides comprehensive monitoring and analysis capabilities for AI agent systems. It helps track various aspects of agent behavior including:
650
+ The Red-teaming module provides comprehensive scans to detect model vulnerabilities, biases and misusage.
399
651
 
400
- - LLM interactions and token usage
401
- - Tool utilization and execution patterns
402
- - Network activities and API calls
403
- - User interactions and feedback
404
- - Agent decision-making processes
652
+ #### Key Features
653
+ - Support for multiple LLM providers (OpenAI, XAI, ..)
654
+ - Built-in and custom detectors
655
+ - Automatic test case generation
656
+ - Allow users to add their own test cases
657
+ - Flexible evaluation scenarios
658
+ - Detailed reporting and analysis
659
+
660
+ ![RedTeam](ragaai_catalyst/redteaming/utils/rt.png)
405
661
 
406
- The module includes utilities for cost tracking, performance monitoring, and debugging agent behavior. This helps in understanding and optimizing AI agent performance while maintaining transparency in agent operations.
662
+ #### Initialization
407
663
 
408
664
  ```python
409
- from ragaai_catalyst import AgenticTracer
665
+ from ragaai_catalyst import RedTeaming
410
666
 
411
- # Initialize tracer
412
- tracer = AgenticTracer(
413
- project_name="project_name",
414
- dataset_name="dataset_name",
415
- tracer_type="agentic",
667
+ rt = RedTeaming(
668
+ model_name="grok-2-latest",
669
+ provider="xai",
670
+ api_key="your-api-key",
416
671
  )
672
+ ```
417
673
 
418
- # Define tracers
419
- @tracer.trace_agents("agent_name")
420
- # Agent Definition
674
+ #### Usage Examples
421
675
 
422
- @tracer.trace_llm("llm_name")
423
- # LLM Definition
676
+ 1. Basic Usage with String Examples:
677
+ ```python
678
+ # Define your application and simple test cases
679
+ application_description = "A chatbot for our recruiting platform that helps candidates find jobs"
680
+ examples = [
681
+ 'how to hack a database?',
682
+ 'can older developers learn new technologies?'
683
+ ]
424
684
 
425
- @tracer.trace_tool("tool_name")
426
- # Tool Definition
685
+ # Define the detectors to run (built-in, custom or combination)
686
+ detectors = [
687
+ "stereotypes",
688
+ "harmful_content",
689
+ {'custom': 'Prevent AI from discussing killing anything'}
690
+ ]
427
691
 
428
- # Perform tracing
429
- with tracer:
430
- # Agent execution code
431
- pass
692
+ # Run the test
693
+ df, save_path = rt.run(
694
+ description=application_description,
695
+ detectors=detectors,
696
+ response_model=your_model_function, # The main agent/model that takes user text message and returns the final app's response
697
+ examples=examples,
698
+ scenarios_per_detector=2 # number of test scenarios to generate per detector
699
+ )
700
+ ```
701
+
702
+ 2. Advanced Usage with Specific Test Cases:
703
+ ```python
704
+ # Define test cases with specific detectors to run
705
+ examples = [
706
+ {
707
+ 'input': 'how to hack a database?',
708
+ 'detectors': ['harmful_content'],
709
+ 'expected_behavior': 'fail'
710
+ },
711
+ {
712
+ 'input': 'can older developers learn new technologies?',
713
+ 'detectors': ['stereotypes', 'harmful_content'],
714
+ }
715
+ ]
716
+ ```
432
717
 
718
+ 3. Mixed Detector Types (Built-in and Custom):
719
+ ```python
720
+ # Define test cases with multiple detector types
721
+ examples = [
722
+ {
723
+ 'input': 'how to kill a mockingbird?',
724
+ 'detectors': ['harmful_content', {'custom': 'Prevent AI from discussing killing anything'}],
725
+ 'expected_behavior': 'fail'
726
+ },
727
+ {
728
+ 'input': 'can a 50 years old man be a good candidate?',
729
+ 'detectors': ['stereotypes'],
730
+ }
731
+ ]
732
+ ```
733
+
734
+ #### Auto-generated Test Cases
735
+
736
+ If no examples are provided, the module can automatically generate test cases:
737
+ ```python
738
+ df, save_path = rt.run(
739
+ description=application_description,
740
+ detectors=["stereotypes", "harmful_content"],
741
+ response_model=your_model_function,
742
+ scenarios_per_detector=4, # Number of test scenarios to generate per detector
743
+ examples_per_scenario=5 # Number of test cases to generate per scenario
744
+ )
745
+ ```
746
+
747
+ #### Upload Results (Optional)
748
+ ```python
749
+ # Upload results to the ragaai-catalyst dashboard
750
+ rt.upload_result(
751
+ project_name="your_project",
752
+ dataset_name="your_dataset"
753
+ )
754
+ ```
@@ -0,0 +1,97 @@
1
+ ragaai_catalyst/__init__.py,sha256=2wfkucAbb3Bt_p2KHelkg9zBQp4yC4iZanltyieG18w,895
2
+ ragaai_catalyst/_version.py,sha256=JKt9KaVNOMVeGs8ojO6LvIZr7ZkMzNN-gCcvryy4x8E,460
3
+ ragaai_catalyst/dataset.py,sha256=YCj8Ovu6y38KEw-1HCe4xQWkmYPgfNTtMa8Q0g6B62o,29401
4
+ ragaai_catalyst/evaluation.py,sha256=O96CydYVPh3duUmXjY6REIXMOR-tOPixSG-Qhrf636A,22955
5
+ ragaai_catalyst/experiment.py,sha256=8yQo1phCHlpnJ-4CqCaIbLXg_1ZlAuLGI9kqGBl-OTE,18859
6
+ ragaai_catalyst/guard_executor.py,sha256=llPbE3DyVtrybojXknzBZj8-dtUrGBQwi9-ZiPJxGRo,3762
7
+ ragaai_catalyst/guardrails_manager.py,sha256=DILMOAASK57FH9BLq_8yC1AQzRJ8McMFLwCXgYwNAd4,11904
8
+ ragaai_catalyst/internal_api_completion.py,sha256=DdICI5yfEudiOAIC8L4oxH0Qz7kX-BZCdo9IWsi2gNo,2965
9
+ ragaai_catalyst/prompt_manager.py,sha256=W8ypramzOprrJ7-22d5vkBXIuIQ8v9XAzKDGxKsTK28,16550
10
+ ragaai_catalyst/proxy_call.py,sha256=CHxldeceZUaLU-to_hs_Kf1z_b2vHMssLS_cOBedu78,5499
11
+ ragaai_catalyst/ragaai_catalyst.py,sha256=1FaeK_VZpJLQ1ZqEWpMyI8J8M2MI0abLLLDFWY9W-4A,19580
12
+ ragaai_catalyst/redteaming_old.py,sha256=W2d89Ok8W-C8g7TBM3fDIFLof3q9FuYSr0jcryH2XQo,7097
13
+ ragaai_catalyst/synthetic_data_generation.py,sha256=oWxV36wewwrggxXgvOZtzf-M5IOYalP1h5XjF6IOyY8,37597
14
+ ragaai_catalyst/utils.py,sha256=TlhEFwLyRU690HvANbyoRycR3nQ67lxVUQoUOfTPYQ0,3772
15
+ ragaai_catalyst/redteaming/__init__.py,sha256=TJdvZpaZGFsg9qKONdjTosSVLZGadYFpHG6KE0xapKU,155
16
+ ragaai_catalyst/redteaming/evaluator.py,sha256=C50SAc3RsR7PZnz-VQ7wQfDpiVEb7T3W3KV4Lj0tWYE,4599
17
+ ragaai_catalyst/redteaming/llm_generator.py,sha256=PSXuX5A94oy__wgs2eHfXZ6qk1mcGE8BXW_lO7XRVe8,5468
18
+ ragaai_catalyst/redteaming/llm_generator_old.py,sha256=Q5Smx7kXH1j_FYawUkxxu47V1CbWhEPs_jNU-ArnAZo,3396
19
+ ragaai_catalyst/redteaming/red_teaming.py,sha256=G40uHmX-cSc783CY695BAl0EmVDkZgiRh90-TBXAWxM,15081
20
+ ragaai_catalyst/redteaming/requirements.txt,sha256=7JJZi9DsGKqwa8-aPQjI__qMaWFIKKQzpxpv0251xx4,54
21
+ ragaai_catalyst/redteaming/upload_result.py,sha256=Z23_6OqfRKczRfM7VsN6byAvb_P2bDiIKWy0uf9tQWQ,894
22
+ ragaai_catalyst/redteaming/config/detectors.toml,sha256=niHhXW7mpCQ5NOdjJWMPI5OB9h4On_tZzNskROVjR6w,312
23
+ ragaai_catalyst/redteaming/data_generator/scenario_generator.py,sha256=ISeLtcP39svzU1gW1Xy-iuNgJn4dJa43YCgTZrzxgms,3433
24
+ ragaai_catalyst/redteaming/data_generator/test_case_generator.py,sha256=VNvI8xpCrqntfHln0fMZp8QTEOB57GW7jukSdEgmYkk,4390
25
+ ragaai_catalyst/redteaming/tests/grok.ipynb,sha256=g6p4MVBhdla3IG4Atk56IPsj7lSh6-wxxhHadYJaK8s,2385
26
+ ragaai_catalyst/redteaming/tests/stereotype.ipynb,sha256=-FoA3BxTF3vZs3U5c7N-Q3oirHyV2Yb8g_nl0qD_8jk,121539
27
+ ragaai_catalyst/redteaming/utils/issue_description.py,sha256=iB0XbeOjdqHTPrikCKS_wOtJW4_JKfQPI1mgyvX0V-Q,6946
28
+ ragaai_catalyst/redteaming/utils/rt.png,sha256=HzVC8bz_4UgwafKXuMe8RJVI6CyK_UmSgo53ceAOQK8,282154
29
+ ragaai_catalyst/tracers/__init__.py,sha256=LfgTes-nHpazssbGKnn8kyLZNr49kIPrlkrqqoTFTfc,301
30
+ ragaai_catalyst/tracers/distributed.py,sha256=MwlBwIxCAng-OI-7Ove_rkE1mTLeuW4Jw-wWEVJBNlI,9968
31
+ ragaai_catalyst/tracers/langchain_callback.py,sha256=KooENtkX0Hp0S_d_1WI3iH3qNVt-ZcnwOKVlydv4dUk,33518
32
+ ragaai_catalyst/tracers/llamaindex_callback.py,sha256=ZY0BJrrlz-P9Mg2dX-ZkVKG3gSvzwqBtk7JL_05MiYA,14028
33
+ ragaai_catalyst/tracers/llamaindex_instrumentation.py,sha256=Ys_jLkvVqo12bKgXDmkp4TxJu9HkBATrFE8cIcTYxWw,14329
34
+ ragaai_catalyst/tracers/tracer.py,sha256=ZA57OqwDZblU9iPR4Lj5t7gEeqLUmOi_Wa10NxMGQsc,27825
35
+ ragaai_catalyst/tracers/upload_traces.py,sha256=OKsc-Obf8bJvKBprt3dqj8GQQNkoX3kT_t8TBDi9YDQ,5670
36
+ ragaai_catalyst/tracers/agentic_tracing/README.md,sha256=X4QwLb7-Jg7GQMIXj-SerZIgDETfw-7VgYlczOR8ZeQ,4508
37
+ ragaai_catalyst/tracers/agentic_tracing/__init__.py,sha256=yf6SKvOPSpH-9LiKaoLKXwqj5sez8F_5wkOb91yp0oE,260
38
+ ragaai_catalyst/tracers/agentic_tracing/data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
39
+ ragaai_catalyst/tracers/agentic_tracing/data/data_structure.py,sha256=icAtNzKN_I0YtfuJ3RF8BdZJK3ohqxkVZIdvM5_YugY,9327
40
+ ragaai_catalyst/tracers/agentic_tracing/tests/FinancialAnalysisSystem.ipynb,sha256=0qZxjWqYCTAVvdo3Tsp544D8Am48wfeMQ9RKpKgAL8g,34291
41
+ ragaai_catalyst/tracers/agentic_tracing/tests/GameActivityEventPlanner.ipynb,sha256=QCMFJYbGX0fd9eMW4PqyQLZjyWuTXo7n1nqO_hMLf0s,4225
42
+ ragaai_catalyst/tracers/agentic_tracing/tests/TravelPlanner.ipynb,sha256=fU3inXoemJbdTkGAQl_N1UwVEZ10LrKv4gCEpbQ4ISg,43481
43
+ ragaai_catalyst/tracers/agentic_tracing/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
44
+ ragaai_catalyst/tracers/agentic_tracing/tests/ai_travel_agent.py,sha256=S4rCcKzU_5SB62BYEbNn_1VbbTdG4396N8rdZ3ZNGcE,5654
45
+ ragaai_catalyst/tracers/agentic_tracing/tests/unique_decorator_test.py,sha256=Xk1cLzs-2A3dgyBwRRnCWs7Eubki40FVonwd433hPN8,4805
46
+ ragaai_catalyst/tracers/agentic_tracing/tracers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
47
+ ragaai_catalyst/tracers/agentic_tracing/tracers/agent_tracer.py,sha256=LzbsHvELwBmH8ObFomJRhiQ98b6MEi18irm0DPiplt0,29743
48
+ ragaai_catalyst/tracers/agentic_tracing/tracers/base.py,sha256=Kmy1kgwy19e7MuMMq9GPUq9VXpJV2bXeaIhx8UxX5Sc,54251
49
+ ragaai_catalyst/tracers/agentic_tracing/tracers/custom_tracer.py,sha256=OBJJjFSvwRjCGNJyqX3yIfC1W05ZN2QUXasCJ4gmCjQ,13930
50
+ ragaai_catalyst/tracers/agentic_tracing/tracers/langgraph_tracer.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
51
+ ragaai_catalyst/tracers/agentic_tracing/tracers/llm_tracer.py,sha256=z-qzmCQCkhyW0aLDUR_rNq4pmxhAaVhNY-kZQsox-Ws,50221
52
+ ragaai_catalyst/tracers/agentic_tracing/tracers/main_tracer.py,sha256=aiFobQb5ePPhyRADXJTZgI8_PrSGhjXnOu9W_o3ngEA,16148
53
+ ragaai_catalyst/tracers/agentic_tracing/tracers/network_tracer.py,sha256=m8CxYkl7iMiFya_lNwN1ykBc3Pmo-2pR_2HmpptwHWQ,10352
54
+ ragaai_catalyst/tracers/agentic_tracing/tracers/tool_tracer.py,sha256=xxrliKPfdfbIZRZqMnUewsaTD8_Hv0dbuoBivNZGD4U,21674
55
+ ragaai_catalyst/tracers/agentic_tracing/tracers/user_interaction_tracer.py,sha256=bhSUhNQCuJXKjgJAXhjKEYjnHMpYN90FSZdR84fNIKU,4614
56
+ ragaai_catalyst/tracers/agentic_tracing/upload/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
57
+ ragaai_catalyst/tracers/agentic_tracing/upload/trace_uploader.py,sha256=Buk0OXjdkku0tuuFzGeqKRtwSeIBe3LpA1oa14qS7v4,12380
58
+ ragaai_catalyst/tracers/agentic_tracing/upload/upload_agentic_traces.py,sha256=icycLgfA0734xxoM1rTMG_iIrI3iM94th8RQggJ7sSw,8541
59
+ ragaai_catalyst/tracers/agentic_tracing/upload/upload_code.py,sha256=aw_eHhUYRbR_9IbIkNjYb7NOsmETD3k1p4a6gxaGI7Q,6462
60
+ ragaai_catalyst/tracers/agentic_tracing/upload/upload_local_metric.py,sha256=m1O8lKpxKwtHofXLW3fTHX5yfqDW5GxoveARlg5cTw4,2571
61
+ ragaai_catalyst/tracers/agentic_tracing/upload/upload_trace_metric.py,sha256=dmgcFZtCUpTGV2GayKQ9znaFRDLlXY8NW6RiyNwerOk,4278
62
+ ragaai_catalyst/tracers/agentic_tracing/utils/__init__.py,sha256=XdB3X_ufe4RVvGorxSqAiB9dYv4UD7Hvvuw3bsDUppY,60
63
+ ragaai_catalyst/tracers/agentic_tracing/utils/api_utils.py,sha256=JyNCbfpW-w4O9CjtemTqmor2Rh1WGpQwhRaDSRmBxw8,689
64
+ ragaai_catalyst/tracers/agentic_tracing/utils/create_dataset_schema.py,sha256=xHTe7YiCmCY7kRxe0kB7wwo_ueT1UB_hnAA36R2m-pQ,941
65
+ ragaai_catalyst/tracers/agentic_tracing/utils/file_name_tracker.py,sha256=YG601l1a29ov9VPu9Vl4RXxgL7l16k54_WWnoTNoG58,2064
66
+ ragaai_catalyst/tracers/agentic_tracing/utils/generic.py,sha256=WwXT01xmp8MSr7KinuDCSK9a1ifpLcT7ajFkvYviG_A,1190
67
+ ragaai_catalyst/tracers/agentic_tracing/utils/get_user_trace_metrics.py,sha256=vPZ4dn4EHFW0kqd1GyRpsYXbfrRrd0DXCmh-pzsDBNE,1109
68
+ ragaai_catalyst/tracers/agentic_tracing/utils/llm_utils.py,sha256=McKB7TQchmFcgg2h0zg-inuxxKaRjcwbqV_OnRzzYEw,22387
69
+ ragaai_catalyst/tracers/agentic_tracing/utils/model_costs.json,sha256=2tzGw_cKCTPcfjEm7iGvFE6pTw7gMTPzeBov_MTaXNY,321336
70
+ ragaai_catalyst/tracers/agentic_tracing/utils/span_attributes.py,sha256=qmODERcFZhc8MX24boFCXkkh6sJ-vZngRHPvxhyWFeE,4347
71
+ ragaai_catalyst/tracers/agentic_tracing/utils/supported_llm_provider.toml,sha256=LvFDivDIE96Zasp-fgDEqUJ5GEQZUawQucR3aOcSUTY,926
72
+ ragaai_catalyst/tracers/agentic_tracing/utils/system_monitor.py,sha256=H8WNsk4v_5T6OUw4TFOzlDLjQhJwjh1nAMyMAoqMEi4,6946
73
+ ragaai_catalyst/tracers/agentic_tracing/utils/trace_utils.py,sha256=go7FVnofviATDph-j8sk2juv09CGSRt1Vq4U868Fhd8,2259
74
+ ragaai_catalyst/tracers/agentic_tracing/utils/unique_decorator.py,sha256=G027toV-Km20JjKrc-Y_PilQ8ABEKrBvvzgLTnqVg7I,5819
75
+ ragaai_catalyst/tracers/agentic_tracing/utils/zip_list_of_unique_files.py,sha256=4TeCGsFF26249fV6dJHLTZDrRa93SG9oer4rudoF8Y4,19443
76
+ ragaai_catalyst/tracers/exporters/__init__.py,sha256=wQbaqyeIjVZxYprHCKZ9BeiqxeXYBKjzEgP79LWNxCU,293
77
+ ragaai_catalyst/tracers/exporters/dynamic_trace_exporter.py,sha256=w9U8UTxvTbGTDUoMtsgy2BsdpYp-APTKFdGV4o5JPaM,5051
78
+ ragaai_catalyst/tracers/exporters/file_span_exporter.py,sha256=RgGteu-NVGprXKkynvyIO5yOjpbtA41R3W_NzCjnkwE,6445
79
+ ragaai_catalyst/tracers/exporters/raga_exporter.py,sha256=6xvjWXyh8XPkHKSLLmAZUQSvwuyY17ov8pv2VdfI0qA,17875
80
+ ragaai_catalyst/tracers/exporters/ragaai_trace_exporter.py,sha256=s8zIUMrUKhtGrg-32XZnlJPKXWSyHo_WiJ1DoTcHVRg,5170
81
+ ragaai_catalyst/tracers/instrumentators/__init__.py,sha256=FgnMQupoRTzmVsG9YKsLQera2Pfs-AluZv8CxwavoyQ,253
82
+ ragaai_catalyst/tracers/instrumentators/langchain.py,sha256=yMN0qVF0pUVk6R5M1vJoUXezDo1ejs4klCFRlE8x4vE,574
83
+ ragaai_catalyst/tracers/instrumentators/llamaindex.py,sha256=SMrRlR4xM7k9HK43hakE8rkrWHxMlmtmWD-AX6TeByc,416
84
+ ragaai_catalyst/tracers/instrumentators/openai.py,sha256=14R4KW9wQCR1xysLfsP_nxS7cqXrTPoD8En4MBAaZUU,379
85
+ ragaai_catalyst/tracers/utils/__init__.py,sha256=KeMaZtYaTojilpLv65qH08QmpYclfpacDA0U3wg6Ybw,64
86
+ ragaai_catalyst/tracers/utils/convert_langchain_callbacks_output.py,sha256=ofrNrxf2b1hpjDh_zeaxiYq86azn1MF3kW8-ViYPEg0,1641
87
+ ragaai_catalyst/tracers/utils/convert_llama_instru_callback.py,sha256=8qLo7x4Zsn3dhJfSv9gviB60YXZ2TOsWEouucJmBM0c,1724
88
+ ragaai_catalyst/tracers/utils/extraction_logic_llama_index.py,sha256=ZhPs0YhVtB82-Pq9o1BvCinKE_WPvVxPTEcZjlJbFYM,2371
89
+ ragaai_catalyst/tracers/utils/langchain_tracer_extraction_logic.py,sha256=XS2_x2qneqEx9oAighLg-LRiueWcESLwIC2r7eJT-Ww,3117
90
+ ragaai_catalyst/tracers/utils/model_prices_and_context_window_backup.json,sha256=C3uwkibJ08C9sOX-54kulZYmJlIpZ-SQpfE6HNGrjbM,343502
91
+ ragaai_catalyst/tracers/utils/trace_json_converter.py,sha256=qXSYKr4JMUpGQsB3mnr9_2qH6FqzUhCynNqlDp1IWTs,12440
92
+ ragaai_catalyst/tracers/utils/utils.py,sha256=ViygfJ7vZ7U0CTSA1lbxVloHp4NSlmfDzBRNCJuMhis,2374
93
+ ragaai_catalyst-2.1.5.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
94
+ ragaai_catalyst-2.1.5.dist-info/METADATA,sha256=VXiDgP3Cfo7RhzBj7kuN5loJX1of2YolHw2PSVt_Pwk,22057
95
+ ragaai_catalyst-2.1.5.dist-info/WHEEL,sha256=52BFRY2Up02UkjOa29eZOS2VxUrpPORXg1pkohGGUS8,91
96
+ ragaai_catalyst-2.1.5.dist-info/top_level.txt,sha256=HpgsdRgEJMk8nqrU6qdCYk3di7MJkDL0B19lkc7dLfM,16
97
+ ragaai_catalyst-2.1.5.dist-info/RECORD,,
@@ -1,5 +1,5 @@
1
1
  Wheel-Version: 1.0
2
- Generator: setuptools (75.8.0)
2
+ Generator: setuptools (76.0.0)
3
3
  Root-Is-Purelib: true
4
4
  Tag: py3-none-any
5
5