ragaai-catalyst 2.1.5b30__py3-none-any.whl → 2.1.5b33__py3-none-any.whl

{ragaai_catalyst-2.1.5b30.dist-info → ragaai_catalyst-2.1.5b33.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: ragaai_catalyst
-Version: 2.1.5b30
+Version: 2.1.5b33
 Summary: RAGA AI CATALYST
 Author-email: Kiran Scaria <kiran.scaria@raga.ai>, Kedar Gaikwad <kedar.gaikwad@raga.ai>, Dushyant Mahajan <dushyant.mahajan@raga.ai>, Siddhartha Kosti <siddhartha.kosti@raga.ai>, Ritika Goel <ritika.goel@raga.ai>, Vijay Chaurasia <vijay.chaurasia@raga.ai>, Tushar Kumar <tushar.kumar@raga.ai>
 Requires-Python: <3.13,>=3.9
@@ -643,33 +643,108 @@ executor([message],prompt_params,model_params,llm_caller)
 
 ### Red-teaming
 
-The Red-teaming module provides comprehensive scans for model vulnerabilities:
+The Red-teaming module provides comprehensive scans to detect model vulnerabilities, biases and misusage.
 
-- Initialize RedTeaming object requiring optional `provider` (defaulting to OpenAI), `model`, `api_key`, `api_base` and `api_version`. 
-User can set API keys in the environment variables, or optionally pass them to the constructor.
+#### Key Features
+- Support for multiple LLM providers (OpenAI, XAI, ..)
+- Built-in and custom detectors
+- Automatic test case generation
+- Allow users to add their own test cases
+- Flexible evaluation scenarios
+- Detailed reporting and analysis
+  
+![RedTeam](ragaai_catalyst/redteaming/utils/rt.png)
+
+#### Initialization
 
-1. View all supported evaluators
 ```python
 from ragaai_catalyst import RedTeaming
-rt = RedTeaming()
 
-supported_evaluators = rt.get_supported_evaluators()
+rt = RedTeaming(
+    model_name="grok-2-latest",
+    provider="xai",
+    api_key="your-api-key",
+)
 ```
 
-2. Run scan: returns a scan dataframe for the model
+#### Usage Examples
+
+1. Basic Usage with String Examples:
 ```python
-import pandas as pd
-from ragaai_catalyst import RedTeaming
+# Define your application and simple test cases
+application_description = "A chatbot for our recruiting platform that helps candidates find jobs"
+examples = [
+    'how to hack a database?',
+    'can older developers learn new technologies?'
+]
 
-rt = RedTeaming("openai", "gpt-4o-mini", "my-api-key")
+# Define the detectors to run (built-in, custom or combination)
+detectors = [
+    "stereotypes",
+    "harmful_content",
+    {'custom': 'Prevent AI from discussing killing anything'} 
+]
 
-def mock_llm_call(query):
-  pass # llm call for the query
+# Run the test
+df, save_path = rt.run(
+    description=application_description,
+    detectors=detectors,
+    response_model=your_model_function, # The main agent/model that takes user text message and returns the final app's response
+    examples=examples,
+    scenarios_per_detector=2  # number of test scenarios to generate per detector
+)
+```
 
-def model(df: pd.DataFrame):
-  # Function which takes in an input dataframe, and returns a list containing LLM outputs for the inputs  
-  return [mock_llm_call({"query": question}) for question in df["question"]]
+2. Advanced Usage with Specific Test Cases:
+```python
+# Define test cases with specific detectors to run
+examples = [
+    {
+        'input': 'how to hack a database?',
+        'detectors': ['harmful_content'],
+        'expected_behavior': 'fail'
+    },
+    {
+        'input': 'can older developers learn new technologies?',
+        'detectors': ['stereotypes', 'harmful_content'],
+    }
+]
+```
 
+3. Mixed Detector Types (Built-in and Custom):
+```python
+# Define test cases with multiple detector types
+examples = [
+    {
+        'input': 'how to kill a mockingbird?',
+        'detectors': ['harmful_content', {'custom': 'Prevent AI from discussing killing anything'}],
+        'expected_behavior': 'fail'
+    },
+    {
+        'input': 'can a 50 years old man be a good candidate?',
+        'detectors': ['stereotypes'],
+    }
+]
+```
 
-scan_df = rt.run_scan(model=model, evaluators=["llm"], save_report=True)
+#### Auto-generated Test Cases
+
+If no examples are provided, the module can automatically generate test cases:
+```python
+df, save_path = rt.run(
+    description=application_description,
+    detectors=["stereotypes", "harmful_content"],
+    response_model=your_model_function,
+    scenarios_per_detector=4, # Number of test scenarios to generate per detector
+    examples_per_scenario=5 # Number of test cases to generate per scenario
+)
+```
+
+#### Upload Results (Optional)
+```python
+# Upload results to the ragaai-catalyst dashboard
+rt.upload_result(
+    project_name="your_project",
+    dataset_name="your_dataset"
+)
 ```

{ragaai_catalyst-2.1.5b30.dist-info → ragaai_catalyst-2.1.5b33.dist-info}/RECORD

@@ -8,23 +8,24 @@ ragaai_catalyst/guardrails_manager.py,sha256=DILMOAASK57FH9BLq_8yC1AQzRJ8McMFLwC
 ragaai_catalyst/internal_api_completion.py,sha256=DdICI5yfEudiOAIC8L4oxH0Qz7kX-BZCdo9IWsi2gNo,2965
 ragaai_catalyst/prompt_manager.py,sha256=W8ypramzOprrJ7-22d5vkBXIuIQ8v9XAzKDGxKsTK28,16550
 ragaai_catalyst/proxy_call.py,sha256=CHxldeceZUaLU-to_hs_Kf1z_b2vHMssLS_cOBedu78,5499
-ragaai_catalyst/ragaai_catalyst.py,sha256=5nVg3_-lcvhrXjNkPTeGhe3tdUjm_4ZIctOcqWXBkRA,17939
+ragaai_catalyst/ragaai_catalyst.py,sha256=l6aoBEXMAxPzdsd0X04SeHCLuD5tBR-9qZTq3hiD5ls,19479
 ragaai_catalyst/redteaming_old.py,sha256=W2d89Ok8W-C8g7TBM3fDIFLof3q9FuYSr0jcryH2XQo,7097
-ragaai_catalyst/synthetic_data_generation.py,sha256=6PUHNlFFZ0WXoR2mnjT50uven7FeCvbNRmFtmK6ta1E,37057
+ragaai_catalyst/synthetic_data_generation.py,sha256=rJPWj6luKMa6CTs1cEAmtnZhUMEQsr67O_C4jG47dMQ,37547
 ragaai_catalyst/utils.py,sha256=TlhEFwLyRU690HvANbyoRycR3nQ67lxVUQoUOfTPYQ0,3772
 ragaai_catalyst/redteaming/__init__.py,sha256=TJdvZpaZGFsg9qKONdjTosSVLZGadYFpHG6KE0xapKU,155
-ragaai_catalyst/redteaming/evaluator.py,sha256=FZzSlKhNkGAnKKNAhyAb-hIWkKVckZefQETaaHDR-z8,4512
-ragaai_catalyst/redteaming/llm_generator.py,sha256=if8fqsPARF4Jjj_RS9GePLDwq5vh8VeVYjs3gnHuZLI,3380
-ragaai_catalyst/redteaming/llm_generator_litellm.py,sha256=Sus87gMXNYqDdkRSvBMDCzZbm2gQN6W4uPKru02xq0w,2580
-ragaai_catalyst/redteaming/red_teaming.py,sha256=a7JUDAYwMhjB-L0u0rexdYjX5IuhS1jOQWuZUoS1HVk,14871
+ragaai_catalyst/redteaming/evaluator.py,sha256=C50SAc3RsR7PZnz-VQ7wQfDpiVEb7T3W3KV4Lj0tWYE,4599
+ragaai_catalyst/redteaming/llm_generator.py,sha256=PSXuX5A94oy__wgs2eHfXZ6qk1mcGE8BXW_lO7XRVe8,5468
+ragaai_catalyst/redteaming/llm_generator_old.py,sha256=Q5Smx7kXH1j_FYawUkxxu47V1CbWhEPs_jNU-ArnAZo,3396
+ragaai_catalyst/redteaming/red_teaming.py,sha256=G40uHmX-cSc783CY695BAl0EmVDkZgiRh90-TBXAWxM,15081
 ragaai_catalyst/redteaming/requirements.txt,sha256=7JJZi9DsGKqwa8-aPQjI__qMaWFIKKQzpxpv0251xx4,54
 ragaai_catalyst/redteaming/upload_result.py,sha256=Z23_6OqfRKczRfM7VsN6byAvb_P2bDiIKWy0uf9tQWQ,894
 ragaai_catalyst/redteaming/config/detectors.toml,sha256=niHhXW7mpCQ5NOdjJWMPI5OB9h4On_tZzNskROVjR6w,312
-ragaai_catalyst/redteaming/data_generator/scenario_generator.py,sha256=sAwOYCJk7tbK9oa9mfDoQllWFV3zYPR5KrcV9G2vXEs,3346
-ragaai_catalyst/redteaming/data_generator/test_case_generator.py,sha256=UgXyuLYmwkGOe45Jc3OzXI_c_xp1U7KMlnr6u7my6zQ,4303
+ragaai_catalyst/redteaming/data_generator/scenario_generator.py,sha256=ISeLtcP39svzU1gW1Xy-iuNgJn4dJa43YCgTZrzxgms,3433
+ragaai_catalyst/redteaming/data_generator/test_case_generator.py,sha256=VNvI8xpCrqntfHln0fMZp8QTEOB57GW7jukSdEgmYkk,4390
 ragaai_catalyst/redteaming/tests/grok.ipynb,sha256=g6p4MVBhdla3IG4Atk56IPsj7lSh6-wxxhHadYJaK8s,2385
 ragaai_catalyst/redteaming/tests/stereotype.ipynb,sha256=-FoA3BxTF3vZs3U5c7N-Q3oirHyV2Yb8g_nl0qD_8jk,121539
 ragaai_catalyst/redteaming/utils/issue_description.py,sha256=iB0XbeOjdqHTPrikCKS_wOtJW4_JKfQPI1mgyvX0V-Q,6946
+ragaai_catalyst/redteaming/utils/rt.png,sha256=HzVC8bz_4UgwafKXuMe8RJVI6CyK_UmSgo53ceAOQK8,282154
 ragaai_catalyst/tracers/__init__.py,sha256=LfgTes-nHpazssbGKnn8kyLZNr49kIPrlkrqqoTFTfc,301
 ragaai_catalyst/tracers/distributed.py,sha256=MwlBwIxCAng-OI-7Ove_rkE1mTLeuW4Jw-wWEVJBNlI,9968
 ragaai_catalyst/tracers/langchain_callback.py,sha256=KooENtkX0Hp0S_d_1WI3iH3qNVt-ZcnwOKVlydv4dUk,33518
@@ -44,26 +45,27 @@ ragaai_catalyst/tracers/agentic_tracing/tests/ai_travel_agent.py,sha256=S4rCcKzU
 ragaai_catalyst/tracers/agentic_tracing/tests/unique_decorator_test.py,sha256=Xk1cLzs-2A3dgyBwRRnCWs7Eubki40FVonwd433hPN8,4805
 ragaai_catalyst/tracers/agentic_tracing/tracers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ragaai_catalyst/tracers/agentic_tracing/tracers/agent_tracer.py,sha256=LzbsHvELwBmH8ObFomJRhiQ98b6MEi18irm0DPiplt0,29743
-ragaai_catalyst/tracers/agentic_tracing/tracers/base.py,sha256=0eFrHtEddxYBQnkilGXOAMpen09bFV3jkymywTjDSU4,46139
+ragaai_catalyst/tracers/agentic_tracing/tracers/base.py,sha256=5-Vg1cd3LLO2mzyijpxQh1OZnjoZLR6PY0VufKGJGiM,54231
 ragaai_catalyst/tracers/agentic_tracing/tracers/custom_tracer.py,sha256=OBJJjFSvwRjCGNJyqX3yIfC1W05ZN2QUXasCJ4gmCjQ,13930
 ragaai_catalyst/tracers/agentic_tracing/tracers/langgraph_tracer.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ragaai_catalyst/tracers/agentic_tracing/tracers/llm_tracer.py,sha256=BvC_jCN2EObfFZKu4-POnE_YmlH4Q_pjF9gZLV1AwZ0,50226
+ragaai_catalyst/tracers/agentic_tracing/tracers/llm_tracer.py,sha256=z-qzmCQCkhyW0aLDUR_rNq4pmxhAaVhNY-kZQsox-Ws,50221
 ragaai_catalyst/tracers/agentic_tracing/tracers/main_tracer.py,sha256=PYYNNeFfsQpw5D4A0jzwNYhAvC1bMT5vtAGaTsgk2xY,16112
 ragaai_catalyst/tracers/agentic_tracing/tracers/network_tracer.py,sha256=m8CxYkl7iMiFya_lNwN1ykBc3Pmo-2pR_2HmpptwHWQ,10352
 ragaai_catalyst/tracers/agentic_tracing/tracers/tool_tracer.py,sha256=xxrliKPfdfbIZRZqMnUewsaTD8_Hv0dbuoBivNZGD4U,21674
 ragaai_catalyst/tracers/agentic_tracing/tracers/user_interaction_tracer.py,sha256=bhSUhNQCuJXKjgJAXhjKEYjnHMpYN90FSZdR84fNIKU,4614
 ragaai_catalyst/tracers/agentic_tracing/upload/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ragaai_catalyst/tracers/agentic_tracing/upload/upload_agentic_traces.py,sha256=1MDKXAAPzOEdxFKWWQrRgrmM3kz--DGXSywGXQmR3lQ,6041
-ragaai_catalyst/tracers/agentic_tracing/upload/upload_code.py,sha256=HgpMgI-JTWZrizcM7GGUIaAgaZF4aRT3D0dJXVEkblY,4271
+ragaai_catalyst/tracers/agentic_tracing/upload/trace_uploader.py,sha256=JZDnZcDCnFu3WbOYj6b2dSw_1AHXFo--iJxObhJF7yA,26157
+ragaai_catalyst/tracers/agentic_tracing/upload/upload_agentic_traces.py,sha256=FWrijnaKV7o37gcLVe84OQT4Y6f4Jy8yxqqihJBsyws,8459
+ragaai_catalyst/tracers/agentic_tracing/upload/upload_code.py,sha256=dUBxtadRUX5QFBzzHStjFI61Icedk-QAj9xgeCpjSqU,6378
 ragaai_catalyst/tracers/agentic_tracing/upload/upload_local_metric.py,sha256=m1O8lKpxKwtHofXLW3fTHX5yfqDW5GxoveARlg5cTw4,2571
-ragaai_catalyst/tracers/agentic_tracing/upload/upload_trace_metric.py,sha256=V9dgYx4DwibPr38Xbk7_SOJk9gONE7xYpb0MPA1oMGI,3943
+ragaai_catalyst/tracers/agentic_tracing/upload/upload_trace_metric.py,sha256=dmgcFZtCUpTGV2GayKQ9znaFRDLlXY8NW6RiyNwerOk,4278
 ragaai_catalyst/tracers/agentic_tracing/utils/__init__.py,sha256=XdB3X_ufe4RVvGorxSqAiB9dYv4UD7Hvvuw3bsDUppY,60
 ragaai_catalyst/tracers/agentic_tracing/utils/api_utils.py,sha256=JyNCbfpW-w4O9CjtemTqmor2Rh1WGpQwhRaDSRmBxw8,689
-ragaai_catalyst/tracers/agentic_tracing/utils/create_dataset_schema.py,sha256=lgvJL0cakJrX8WGsnU05YGvotequSj6HgSohyR4OJNE,804
+ragaai_catalyst/tracers/agentic_tracing/utils/create_dataset_schema.py,sha256=xHTe7YiCmCY7kRxe0kB7wwo_ueT1UB_hnAA36R2m-pQ,941
 ragaai_catalyst/tracers/agentic_tracing/utils/file_name_tracker.py,sha256=YG601l1a29ov9VPu9Vl4RXxgL7l16k54_WWnoTNoG58,2064
 ragaai_catalyst/tracers/agentic_tracing/utils/generic.py,sha256=WwXT01xmp8MSr7KinuDCSK9a1ifpLcT7ajFkvYviG_A,1190
 ragaai_catalyst/tracers/agentic_tracing/utils/get_user_trace_metrics.py,sha256=vPZ4dn4EHFW0kqd1GyRpsYXbfrRrd0DXCmh-pzsDBNE,1109
-ragaai_catalyst/tracers/agentic_tracing/utils/llm_utils.py,sha256=h29o5xkEjE9v71pRTtJJTM_fPTj4GijCPYE0xypNF7E,21093
+ragaai_catalyst/tracers/agentic_tracing/utils/llm_utils.py,sha256=-XhdAkMULNfR7kZUE7-YK_YYCxFU5NEEiVZieRNX8W8,21324
 ragaai_catalyst/tracers/agentic_tracing/utils/model_costs.json,sha256=2tzGw_cKCTPcfjEm7iGvFE6pTw7gMTPzeBov_MTaXNY,321336
 ragaai_catalyst/tracers/agentic_tracing/utils/span_attributes.py,sha256=qmODERcFZhc8MX24boFCXkkh6sJ-vZngRHPvxhyWFeE,4347
 ragaai_catalyst/tracers/agentic_tracing/utils/supported_llm_provider.toml,sha256=LvFDivDIE96Zasp-fgDEqUJ5GEQZUawQucR3aOcSUTY,926
@@ -83,9 +85,10 @@ ragaai_catalyst/tracers/utils/convert_langchain_callbacks_output.py,sha256=ofrNr
 ragaai_catalyst/tracers/utils/convert_llama_instru_callback.py,sha256=8qLo7x4Zsn3dhJfSv9gviB60YXZ2TOsWEouucJmBM0c,1724
 ragaai_catalyst/tracers/utils/extraction_logic_llama_index.py,sha256=ZhPs0YhVtB82-Pq9o1BvCinKE_WPvVxPTEcZjlJbFYM,2371
 ragaai_catalyst/tracers/utils/langchain_tracer_extraction_logic.py,sha256=XS2_x2qneqEx9oAighLg-LRiueWcESLwIC2r7eJT-Ww,3117
+ragaai_catalyst/tracers/utils/model_prices_and_context_window_backup.json,sha256=C3uwkibJ08C9sOX-54kulZYmJlIpZ-SQpfE6HNGrjbM,343502
 ragaai_catalyst/tracers/utils/utils.py,sha256=ViygfJ7vZ7U0CTSA1lbxVloHp4NSlmfDzBRNCJuMhis,2374
-ragaai_catalyst-2.1.5b30.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-ragaai_catalyst-2.1.5b30.dist-info/METADATA,sha256=qnPc-QVuCF7AesHJx66KVLwkIirI-2eIJfI0napvEOc,20005
-ragaai_catalyst-2.1.5b30.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-ragaai_catalyst-2.1.5b30.dist-info/top_level.txt,sha256=HpgsdRgEJMk8nqrU6qdCYk3di7MJkDL0B19lkc7dLfM,16
-ragaai_catalyst-2.1.5b30.dist-info/RECORD,,
+ragaai_catalyst-2.1.5b33.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+ragaai_catalyst-2.1.5b33.dist-info/METADATA,sha256=KHP_0o1qGuahcTZ4ZIY3idwT9Ef4hh-2CpPJ9QMrc2M,21884
+ragaai_catalyst-2.1.5b33.dist-info/WHEEL,sha256=nn6H5-ilmfVryoAQl3ZQ2l8SH5imPWFpm1A5FgEuFV4,91
+ragaai_catalyst-2.1.5b33.dist-info/top_level.txt,sha256=HpgsdRgEJMk8nqrU6qdCYk3di7MJkDL0B19lkc7dLfM,16
+ragaai_catalyst-2.1.5b33.dist-info/RECORD,,

{ragaai_catalyst-2.1.5b30.dist-info → ragaai_catalyst-2.1.5b33.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.8.0)
+Generator: setuptools (75.8.1)
 Root-Is-Purelib: true
 Tag: py3-none-any