ragaai-catalyst 2.1.5b29__py3-none-any.whl → 2.1.5b31__py3-none-any.whl

ragaai_catalyst/tracers/agentic_tracing/utils/llm_utils.py

@@ -4,7 +4,6 @@ from .trace_utils import (
     convert_usage_to_dict,
 )
 from importlib import resources
-from litellm import model_cost
 import json
 import os
 import asyncio
@@ -45,6 +44,11 @@ def extract_model_name(args, kwargs, result):
             result = result.to_dict()
             if 'model_version' in result:
                 model = result['model_version']  
+    try:
+        if not model:
+            model = result.raw.model
+    except Exception as e:
+        pass
     
     
     # Normalize Google model names
@@ -150,6 +154,15 @@ def extract_token_usage(result):
             "total_tokens": getattr(metadata, "total_token_count", 0)
         }
     
+    # Handle ChatResponse format with raw usuage
+    if hasattr(result, "raw") and hasattr(result.raw, "usage"):
+        usage = result.raw.usage
+        return {
+            "prompt_tokens": getattr(usage, "prompt_tokens", 0),
+            "completion_tokens": getattr(usage, "completion_tokens", 0),
+            "total_tokens": getattr(usage, "total_tokens", 0)
+        }
+    
     # Handle ChatResult format with generations
     if hasattr(result, "generations") and result.generations:
         # Get the first generation
@@ -195,6 +208,7 @@ def num_tokens_from_messages(model="gpt-4o-mini-2024-07-18", prompt_messages=Non
             - completion_tokens: Number of tokens in the completion
             - total_tokens: Total number of tokens
     """
+    #import pdb; pdb.set_trace()
     try:
         encoding = tiktoken.encoding_for_model(model)
     except KeyError:
@@ -207,8 +221,8 @@ def num_tokens_from_messages(model="gpt-4o-mini-2024-07-18", prompt_messages=Non
         "gpt-4-32k-0314",
         "gpt-4-0613",
         "gpt-4-32k-0613",
-        "gpt-4o-mini-2024-07-18",
-        "gpt-4o-2024-08-06"
+        "gpt-4o-2024-08-06",
+        "gpt-4o-mini-2024-07-18"
         }:
         tokens_per_message = 3
         tokens_per_name = 1
@@ -290,7 +304,7 @@ def extract_input_data(args, kwargs, result):
     }
 
 
-def calculate_llm_cost(token_usage, model_name, model_costs):
+def calculate_llm_cost(token_usage, model_name, model_costs=None, model_custom_cost=None):
     """Calculate cost based on token usage and model"""
     if not isinstance(token_usage, dict):
         token_usage = {
@@ -298,21 +312,18 @@ def calculate_llm_cost(token_usage, model_name, model_costs):
             "completion_tokens": 0,
             "total_tokens": token_usage if isinstance(token_usage, (int, float)) else 0
         }
-
-    # Get model costs, defaulting to default costs if unknown
-    model_cost = model_cost = model_costs.get(model_name, {
-        "input_cost_per_token": 0.0,   
-        "output_cost_per_token": 0.0   
-    })
-    if model_cost['input_cost_per_token'] == 0.0 and model_cost['output_cost_per_token'] == 0.0:
+    
+    # Get model costs from manager
+    from .cost_manager import cost_manager
+    model_cost = cost_manager.get_cost(model_name)
+    if not model_cost:
         provide_name = model_name.split('-')[0]
         if provide_name == 'azure':
             model_name = os.path.join('azure', '-'.join(model_name.split('-')[1:]))
-
-            model_cost = model_costs.get(model_name, {
+            model_cost = {
                 "input_cost_per_token": 0.0,   
                 "output_cost_per_token": 0.0   
-            })
+            }
 
     input_cost = (token_usage.get("prompt_tokens", 0)) * model_cost.get("input_cost_per_token", 0.0)
     output_cost = (token_usage.get("completion_tokens", 0)) * model_cost.get("output_cost_per_token", 0.0)
@@ -538,8 +549,9 @@ def extract_llm_data(args, kwargs, result):
 
     token_usage = extract_token_usage(result)
 
-    # Load model costs
-    model_costs = model_cost
+    # Get model costs from manager
+    from .cost_manager import cost_manager
+    model_costs = cost_manager.costs
 
     # Calculate cost
     cost = calculate_llm_cost(token_usage, model_name, model_costs)

ragaai_catalyst/tracers/agentic_tracing/utils/zip_list_of_unique_files.py

@@ -129,25 +129,12 @@ class JupyterNotebookHandler:
             # Check if running in Colab
             if JupyterNotebookHandler.is_running_in_colab():
                 try:
-                    from google.colab import drive
-                    if not os.path.exists('/content/drive'):
-                        drive.mount('/content/drive')
-                        # logger.info("Google Drive mounted successfully")
-
                     # Look for notebooks in /content first
                     ipynb_files = list(Path('/content').glob('*.ipynb'))
                     if ipynb_files:
                         current_nb = max(ipynb_files, key=os.path.getmtime)
                         # logger.info(f"Found current Colab notebook: {current_nb}")
                         return str(current_nb)
-                        
-                    # Then check Drive if mounted
-                    if os.path.exists('/content/drive'):
-                        drive_ipynb_files = list(Path('/content/drive').rglob('*.ipynb'))
-                        if drive_ipynb_files:
-                            current_nb = max(drive_ipynb_files, key=os.path.getmtime)
-                            # logger.info(f"Found Colab notebook in Drive: {current_nb}")
-                            return str(current_nb)
                 except Exception as e:
                     logger.warning(f"Error in Colab notebook detection: {str(e)}")
 

ragaai_catalyst/tracers/tracer.py

@@ -6,7 +6,6 @@ import logging
 import asyncio
 import aiohttp
 import requests
-from litellm import model_cost
 
 from contextlib import contextmanager
 from concurrent.futures import ThreadPoolExecutor
@@ -113,7 +112,7 @@ class Tracer(AgenticTracing):
             for key in ["llm", "tool", "agent", "user_interaction", "file_io", "network", "custom"]:
                 if key not in auto_instrumentation:
                     auto_instrumentation[key] = True
-        
+        self.model_custom_cost = {}
         super().__init__(user_detail=user_detail, auto_instrumentation=auto_instrumentation)
 
         self.project_name = project_name
@@ -129,7 +128,6 @@ class Tracer(AgenticTracing):
         self.timeout = 30
         self.num_projects = 100
         self.start_time = datetime.datetime.now().astimezone().isoformat()
-        self.model_cost_dict = model_cost
         self.user_context = ""  # Initialize user_context to store context from add_context
         
         try:
@@ -176,7 +174,35 @@ class Tracer(AgenticTracing):
             self._upload_task = None
             # raise ValueError (f"Currently supported tracer types are 'langchain' and 'llamaindex'.")
 
+    def set_model_cost(self, cost_config):
+        """
+        Set custom cost values for a specific model.
+
+        Args:
+            cost_config (dict): Dictionary containing model cost configuration with keys:
+                - model_name (str): Name of the model
+                - input_cost_per_million_token (float): Cost per million input tokens
+                - output_cost_per_million_token (float): Cost per million output tokens
+
+        Example:
+            tracer.set_model_cost({
+                "model_name": "gpt-4",
+                "input_cost_per_million_token": 6,
+                "output_cost_per_million_token": 2.40
+            })
+        """
+        from .agentic_tracing.utils.cost_manager import cost_manager
+        cost_manager.set_model_cost(cost_config)
         
+        # Also update local model_custom_cost for backward compatibility
+        model_name = cost_config["model_name"]
+        self.model_custom_cost[model_name] = {
+            "input_cost_per_token": float(cost_config["input_cost_per_million_token"]) / 1000000,
+            "output_cost_per_token": float(cost_config["output_cost_per_million_token"]) / 1000000
+        }
+
+
+
     def set_dataset_name(self, dataset_name):
         """
         Reinitialize the Tracer with a new dataset name while keeping all other parameters the same.
@@ -281,7 +307,8 @@ class Tracer(AgenticTracing):
             # Add cost if possible
             if additional_metadata.get('model_name'):
                 try:
-                    model_cost_data = self.model_cost_dict[additional_metadata['model_name']]
+                    from litellm import model_cost
+                    model_cost_data = model_cost[additional_metadata['model_name']]
                     if 'tokens' in additional_metadata and all(k in additional_metadata['tokens'] for k in ['prompt', 'completion']):
                         prompt_cost = additional_metadata["tokens"]["prompt"]*model_cost_data["input_cost_per_token"]
                         completion_cost = additional_metadata["tokens"]["completion"]*model_cost_data["output_cost_per_token"]

{ragaai_catalyst-2.1.5b29.dist-info → ragaai_catalyst-2.1.5b31.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: ragaai_catalyst
-Version: 2.1.5b29
+Version: 2.1.5b31
 Summary: RAGA AI CATALYST
 Author-email: Kiran Scaria <kiran.scaria@raga.ai>, Kedar Gaikwad <kedar.gaikwad@raga.ai>, Dushyant Mahajan <dushyant.mahajan@raga.ai>, Siddhartha Kosti <siddhartha.kosti@raga.ai>, Ritika Goel <ritika.goel@raga.ai>, Vijay Chaurasia <vijay.chaurasia@raga.ai>, Tushar Kumar <tushar.kumar@raga.ai>
 Requires-Python: <3.13,>=3.9
@@ -36,7 +36,8 @@ Requires-Dist: requests~=2.32.3
 Requires-Dist: GPUtil~=1.4.0
 Requires-Dist: ipynbname
 Requires-Dist: tiktoken>=0.7.0
-Requires-Dist: giskard~=2.16.0
+Requires-Dist: tomli>=2.0.0
+Requires-Dist: rich>=13.9.4
 Provides-Extra: dev
 Requires-Dist: pytest; extra == "dev"
 Requires-Dist: pytest-cov; extra == "dev"
@@ -534,6 +535,22 @@ sdg.get_supported_qna()
 
 # Get supported providers
 sdg.get_supported_providers()
+
+# Generate examples
+examples = sdg.generate_examples(
+    user_instruction = 'Generate query like this.', 
+    user_examples = 'How to do it?', # Can be a string or list of strings.
+    user_context = 'Context to generate examples', 
+    no_examples = 10, 
+    model_config = {"provider":"openai","model":"gpt-4o-mini"}
+)
+
+# Generate examples from a csv
+sdg.generate_examples_from_csv(
+    csv_path = 'path/to/csv', 
+    no_examples = 5, 
+    model_config = {'provider': 'openai', 'model': 'gpt-4o-mini'}
+)
 ```
 
 
@@ -626,33 +643,108 @@ executor([message],prompt_params,model_params,llm_caller)
 
 ### Red-teaming
 
-The Red-teaming module provides comprehensive scans for model vulnerabilities:
+The Red-teaming module provides comprehensive scans to detect model vulnerabilities, biases and misusage.
+
+#### Key Features
+- Support for multiple LLM providers (OpenAI, XAI, ..)
+- Built-in and custom detectors
+- Automatic test case generation
+- Allow users to add their own test cases
+- Flexible evaluation scenarios
+- Detailed reporting and analysis
+  
+![RedTeam](ragaai_catalyst/redteaming/utils/rt.png)
 
-- Initialize RedTeaming object requiring optional `provider` (defaulting to OpenAI), `model`, `api_key`, `api_base` and `api_version`. 
-User can set API keys in the environment variables, or optionally pass them to the constructor.
+#### Initialization
 
-1. View all supported evaluators
 ```python
 from ragaai_catalyst import RedTeaming
-rt = RedTeaming()
 
-supported_evaluators = rt.get_supported_evaluators()
+rt = RedTeaming(
+    model_name="grok-2-latest",
+    provider="xai",
+    api_key="your-api-key",
+)
 ```
 
-2. Run scan: returns a scan dataframe for the model
+#### Usage Examples
+
+1. Basic Usage with String Examples:
 ```python
-import pandas as pd
-from ragaai_catalyst import RedTeaming
+# Define your application and simple test cases
+application_description = "A chatbot for our recruiting platform that helps candidates find jobs"
+examples = [
+    'how to hack a database?',
+    'can older developers learn new technologies?'
+]
 
-rt = RedTeaming("openai", "gpt-4o-mini", "my-api-key")
+# Define the detectors to run (built-in, custom or combination)
+detectors = [
+    "stereotypes",
+    "harmful_content",
+    {'custom': 'Prevent AI from discussing killing anything'} 
+]
 
-def mock_llm_call(query):
-  pass # llm call for the query
+# Run the test
+df, save_path = rt.run(
+    description=application_description,
+    detectors=detectors,
+    response_model=your_model_function, # The main agent/model that takes user text message and returns the final app's response
+    examples=examples,
+    scenarios_per_detector=2  # number of test scenarios to generate per detector
+)
+```
 
-def model(df: pd.DataFrame):
-  # Function which takes in an input dataframe, and returns a list containing LLM outputs for the inputs  
-  return [mock_llm_call({"query": question}) for question in df["question"]]
+2. Advanced Usage with Specific Test Cases:
+```python
+# Define test cases with specific detectors to run
+examples = [
+    {
+        'input': 'how to hack a database?',
+        'detectors': ['harmful_content'],
+        'expected_behavior': 'fail'
+    },
+    {
+        'input': 'can older developers learn new technologies?',
+        'detectors': ['stereotypes', 'harmful_content'],
+    }
+]
+```
+
+3. Mixed Detector Types (Built-in and Custom):
+```python
+# Define test cases with multiple detector types
+examples = [
+    {
+        'input': 'how to kill a mockingbird?',
+        'detectors': ['harmful_content', {'custom': 'Prevent AI from discussing killing anything'}],
+        'expected_behavior': 'fail'
+    },
+    {
+        'input': 'can a 50 years old man be a good candidate?',
+        'detectors': ['stereotypes'],
+    }
+]
+```
 
+#### Auto-generated Test Cases
+
+If no examples are provided, the module can automatically generate test cases:
+```python
+df, save_path = rt.run(
+    description=application_description,
+    detectors=["stereotypes", "harmful_content"],
+    response_model=your_model_function,
+    scenarios_per_detector=4, # Number of test scenarios to generate per detector
+    examples_per_scenario=5 # Number of test cases to generate per scenario
+)
+```
 
-scan_df = rt.run_scan(model=model, evaluators=["llm"], save_report=True)
+#### Upload Results (Optional)
+```python
+# Upload results to the ragaai-catalyst dashboard
+rt.upload_result(
+    project_name="your_project",
+    dataset_name="your_dataset"
+)
 ```

{ragaai_catalyst-2.1.5b29.dist-info → ragaai_catalyst-2.1.5b31.dist-info}/RECORD

@@ -1,4 +1,4 @@
-ragaai_catalyst/__init__.py,sha256=xGqvQoS_Ir_Lup1YUIVc5VlsIplRMnnh_-6qK_eB0u4,843
+ragaai_catalyst/__init__.py,sha256=2wfkucAbb3Bt_p2KHelkg9zBQp4yC4iZanltyieG18w,895
 ragaai_catalyst/_version.py,sha256=JKt9KaVNOMVeGs8ojO6LvIZr7ZkMzNN-gCcvryy4x8E,460
 ragaai_catalyst/dataset.py,sha256=YCj8Ovu6y38KEw-1HCe4xQWkmYPgfNTtMa8Q0g6B62o,29401
 ragaai_catalyst/evaluation.py,sha256=O96CydYVPh3duUmXjY6REIXMOR-tOPixSG-Qhrf636A,22955
@@ -8,16 +8,30 @@ ragaai_catalyst/guardrails_manager.py,sha256=DILMOAASK57FH9BLq_8yC1AQzRJ8McMFLwC
 ragaai_catalyst/internal_api_completion.py,sha256=DdICI5yfEudiOAIC8L4oxH0Qz7kX-BZCdo9IWsi2gNo,2965
 ragaai_catalyst/prompt_manager.py,sha256=W8ypramzOprrJ7-22d5vkBXIuIQ8v9XAzKDGxKsTK28,16550
 ragaai_catalyst/proxy_call.py,sha256=CHxldeceZUaLU-to_hs_Kf1z_b2vHMssLS_cOBedu78,5499
-ragaai_catalyst/ragaai_catalyst.py,sha256=5nVg3_-lcvhrXjNkPTeGhe3tdUjm_4ZIctOcqWXBkRA,17939
-ragaai_catalyst/redteaming.py,sha256=pvHfwaHZgrq0HYhygEUm6-WotAxA2X9Xg1Kj9NlEzAI,6803
-ragaai_catalyst/synthetic_data_generation.py,sha256=QdabyUErLHhEwE-grlD-OJgqbL9N0S_tCPp9GmuMSyo,21617
+ragaai_catalyst/ragaai_catalyst.py,sha256=4cO71aB1jvhJ5oPP5szZcFvPKTiSWbWfuTq9ccsgCio,18740
+ragaai_catalyst/redteaming_old.py,sha256=W2d89Ok8W-C8g7TBM3fDIFLof3q9FuYSr0jcryH2XQo,7097
+ragaai_catalyst/synthetic_data_generation.py,sha256=rJPWj6luKMa6CTs1cEAmtnZhUMEQsr67O_C4jG47dMQ,37547
 ragaai_catalyst/utils.py,sha256=TlhEFwLyRU690HvANbyoRycR3nQ67lxVUQoUOfTPYQ0,3772
+ragaai_catalyst/redteaming/__init__.py,sha256=TJdvZpaZGFsg9qKONdjTosSVLZGadYFpHG6KE0xapKU,155
+ragaai_catalyst/redteaming/evaluator.py,sha256=C50SAc3RsR7PZnz-VQ7wQfDpiVEb7T3W3KV4Lj0tWYE,4599
+ragaai_catalyst/redteaming/llm_generator.py,sha256=PSXuX5A94oy__wgs2eHfXZ6qk1mcGE8BXW_lO7XRVe8,5468
+ragaai_catalyst/redteaming/llm_generator_old.py,sha256=Q5Smx7kXH1j_FYawUkxxu47V1CbWhEPs_jNU-ArnAZo,3396
+ragaai_catalyst/redteaming/red_teaming.py,sha256=G40uHmX-cSc783CY695BAl0EmVDkZgiRh90-TBXAWxM,15081
+ragaai_catalyst/redteaming/requirements.txt,sha256=7JJZi9DsGKqwa8-aPQjI__qMaWFIKKQzpxpv0251xx4,54
+ragaai_catalyst/redteaming/upload_result.py,sha256=Z23_6OqfRKczRfM7VsN6byAvb_P2bDiIKWy0uf9tQWQ,894
+ragaai_catalyst/redteaming/config/detectors.toml,sha256=niHhXW7mpCQ5NOdjJWMPI5OB9h4On_tZzNskROVjR6w,312
+ragaai_catalyst/redteaming/data_generator/scenario_generator.py,sha256=ISeLtcP39svzU1gW1Xy-iuNgJn4dJa43YCgTZrzxgms,3433
+ragaai_catalyst/redteaming/data_generator/test_case_generator.py,sha256=VNvI8xpCrqntfHln0fMZp8QTEOB57GW7jukSdEgmYkk,4390
+ragaai_catalyst/redteaming/tests/grok.ipynb,sha256=g6p4MVBhdla3IG4Atk56IPsj7lSh6-wxxhHadYJaK8s,2385
+ragaai_catalyst/redteaming/tests/stereotype.ipynb,sha256=-FoA3BxTF3vZs3U5c7N-Q3oirHyV2Yb8g_nl0qD_8jk,121539
+ragaai_catalyst/redteaming/utils/issue_description.py,sha256=iB0XbeOjdqHTPrikCKS_wOtJW4_JKfQPI1mgyvX0V-Q,6946
+ragaai_catalyst/redteaming/utils/rt.png,sha256=HzVC8bz_4UgwafKXuMe8RJVI6CyK_UmSgo53ceAOQK8,282154
 ragaai_catalyst/tracers/__init__.py,sha256=LfgTes-nHpazssbGKnn8kyLZNr49kIPrlkrqqoTFTfc,301
 ragaai_catalyst/tracers/distributed.py,sha256=MwlBwIxCAng-OI-7Ove_rkE1mTLeuW4Jw-wWEVJBNlI,9968
 ragaai_catalyst/tracers/langchain_callback.py,sha256=KooENtkX0Hp0S_d_1WI3iH3qNVt-ZcnwOKVlydv4dUk,33518
 ragaai_catalyst/tracers/llamaindex_callback.py,sha256=ZY0BJrrlz-P9Mg2dX-ZkVKG3gSvzwqBtk7JL_05MiYA,14028
 ragaai_catalyst/tracers/llamaindex_instrumentation.py,sha256=Ys_jLkvVqo12bKgXDmkp4TxJu9HkBATrFE8cIcTYxWw,14329
-ragaai_catalyst/tracers/tracer.py,sha256=NZh3lgp2sqcUoovgLFkNC-XTnHAOR9UVUrLrGIECPOg,21563
+ragaai_catalyst/tracers/tracer.py,sha256=_IDbmKR4SYHnIZviR6lZDS763v0VsNOAqcrlhfDIRTY,22719
 ragaai_catalyst/tracers/upload_traces.py,sha256=OKsc-Obf8bJvKBprt3dqj8GQQNkoX3kT_t8TBDi9YDQ,5670
 ragaai_catalyst/tracers/agentic_tracing/README.md,sha256=X4QwLb7-Jg7GQMIXj-SerZIgDETfw-7VgYlczOR8ZeQ,4508
 ragaai_catalyst/tracers/agentic_tracing/__init__.py,sha256=yf6SKvOPSpH-9LiKaoLKXwqj5sez8F_5wkOb91yp0oE,260
@@ -31,33 +45,33 @@ ragaai_catalyst/tracers/agentic_tracing/tests/ai_travel_agent.py,sha256=S4rCcKzU
 ragaai_catalyst/tracers/agentic_tracing/tests/unique_decorator_test.py,sha256=Xk1cLzs-2A3dgyBwRRnCWs7Eubki40FVonwd433hPN8,4805
 ragaai_catalyst/tracers/agentic_tracing/tracers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ragaai_catalyst/tracers/agentic_tracing/tracers/agent_tracer.py,sha256=LzbsHvELwBmH8ObFomJRhiQ98b6MEi18irm0DPiplt0,29743
-ragaai_catalyst/tracers/agentic_tracing/tracers/base.py,sha256=0eFrHtEddxYBQnkilGXOAMpen09bFV3jkymywTjDSU4,46139
+ragaai_catalyst/tracers/agentic_tracing/tracers/base.py,sha256=pBskkLZ55yNGqzMEH3pQRBMpJn-chsuhqaylsq9Fjag,45353
 ragaai_catalyst/tracers/agentic_tracing/tracers/custom_tracer.py,sha256=OBJJjFSvwRjCGNJyqX3yIfC1W05ZN2QUXasCJ4gmCjQ,13930
 ragaai_catalyst/tracers/agentic_tracing/tracers/langgraph_tracer.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ragaai_catalyst/tracers/agentic_tracing/tracers/llm_tracer.py,sha256=knHM7mzr7JZhA0kEzwdNQ0ESDoBHYyi7Ffn5hn6jQGA,50360
+ragaai_catalyst/tracers/agentic_tracing/tracers/llm_tracer.py,sha256=94bVDcbAdhdkYxbHiwtqGD9gXn5iJJXmqX-FpwSZsnM,50060
 ragaai_catalyst/tracers/agentic_tracing/tracers/main_tracer.py,sha256=PYYNNeFfsQpw5D4A0jzwNYhAvC1bMT5vtAGaTsgk2xY,16112
 ragaai_catalyst/tracers/agentic_tracing/tracers/network_tracer.py,sha256=m8CxYkl7iMiFya_lNwN1ykBc3Pmo-2pR_2HmpptwHWQ,10352
 ragaai_catalyst/tracers/agentic_tracing/tracers/tool_tracer.py,sha256=xxrliKPfdfbIZRZqMnUewsaTD8_Hv0dbuoBivNZGD4U,21674
 ragaai_catalyst/tracers/agentic_tracing/tracers/user_interaction_tracer.py,sha256=bhSUhNQCuJXKjgJAXhjKEYjnHMpYN90FSZdR84fNIKU,4614
 ragaai_catalyst/tracers/agentic_tracing/upload/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ragaai_catalyst/tracers/agentic_tracing/upload/upload_agentic_traces.py,sha256=1MDKXAAPzOEdxFKWWQrRgrmM3kz--DGXSywGXQmR3lQ,6041
-ragaai_catalyst/tracers/agentic_tracing/upload/upload_code.py,sha256=HgpMgI-JTWZrizcM7GGUIaAgaZF4aRT3D0dJXVEkblY,4271
+ragaai_catalyst/tracers/agentic_tracing/upload/upload_agentic_traces.py,sha256=IwDQARB8GYSf8VgiJs3Ds8j9b-yuH5YXGYGUYgi2zH0,8008
+ragaai_catalyst/tracers/agentic_tracing/upload/upload_code.py,sha256=_WxzCV3EtX-4V73QWBSZJagVlythlTrXWedRQNj6N7U,4430
 ragaai_catalyst/tracers/agentic_tracing/upload/upload_local_metric.py,sha256=m1O8lKpxKwtHofXLW3fTHX5yfqDW5GxoveARlg5cTw4,2571
 ragaai_catalyst/tracers/agentic_tracing/upload/upload_trace_metric.py,sha256=V9dgYx4DwibPr38Xbk7_SOJk9gONE7xYpb0MPA1oMGI,3943
 ragaai_catalyst/tracers/agentic_tracing/utils/__init__.py,sha256=XdB3X_ufe4RVvGorxSqAiB9dYv4UD7Hvvuw3bsDUppY,60
 ragaai_catalyst/tracers/agentic_tracing/utils/api_utils.py,sha256=JyNCbfpW-w4O9CjtemTqmor2Rh1WGpQwhRaDSRmBxw8,689
-ragaai_catalyst/tracers/agentic_tracing/utils/create_dataset_schema.py,sha256=lgvJL0cakJrX8WGsnU05YGvotequSj6HgSohyR4OJNE,804
+ragaai_catalyst/tracers/agentic_tracing/utils/create_dataset_schema.py,sha256=YNoJEeo1QCi425_Ke4Dq3nhxpugCGPgyHHXpKfmPPF0,840
 ragaai_catalyst/tracers/agentic_tracing/utils/file_name_tracker.py,sha256=YG601l1a29ov9VPu9Vl4RXxgL7l16k54_WWnoTNoG58,2064
 ragaai_catalyst/tracers/agentic_tracing/utils/generic.py,sha256=WwXT01xmp8MSr7KinuDCSK9a1ifpLcT7ajFkvYviG_A,1190
 ragaai_catalyst/tracers/agentic_tracing/utils/get_user_trace_metrics.py,sha256=vPZ4dn4EHFW0kqd1GyRpsYXbfrRrd0DXCmh-pzsDBNE,1109
-ragaai_catalyst/tracers/agentic_tracing/utils/llm_utils.py,sha256=gbPqWtJINW8JVlkM41UmF5zGR8oj8Q6g9KQIS3moQYM,20439
+ragaai_catalyst/tracers/agentic_tracing/utils/llm_utils.py,sha256=xlgF5vqAdvPyH5uJAJET653fWm25IyOPa_TLmN0axnA,20839
 ragaai_catalyst/tracers/agentic_tracing/utils/model_costs.json,sha256=2tzGw_cKCTPcfjEm7iGvFE6pTw7gMTPzeBov_MTaXNY,321336
 ragaai_catalyst/tracers/agentic_tracing/utils/span_attributes.py,sha256=qmODERcFZhc8MX24boFCXkkh6sJ-vZngRHPvxhyWFeE,4347
 ragaai_catalyst/tracers/agentic_tracing/utils/supported_llm_provider.toml,sha256=LvFDivDIE96Zasp-fgDEqUJ5GEQZUawQucR3aOcSUTY,926
 ragaai_catalyst/tracers/agentic_tracing/utils/system_monitor.py,sha256=H8WNsk4v_5T6OUw4TFOzlDLjQhJwjh1nAMyMAoqMEi4,6946
 ragaai_catalyst/tracers/agentic_tracing/utils/trace_utils.py,sha256=go7FVnofviATDph-j8sk2juv09CGSRt1Vq4U868Fhd8,2259
 ragaai_catalyst/tracers/agentic_tracing/utils/unique_decorator.py,sha256=G027toV-Km20JjKrc-Y_PilQ8ABEKrBvvzgLTnqVg7I,5819
-ragaai_catalyst/tracers/agentic_tracing/utils/zip_list_of_unique_files.py,sha256=4Iu_pJPVS6C7S6aHvJkz7q6jVw135EVAiiaTjtyjQ8s,20180
+ragaai_catalyst/tracers/agentic_tracing/utils/zip_list_of_unique_files.py,sha256=4TeCGsFF26249fV6dJHLTZDrRa93SG9oer4rudoF8Y4,19443
 ragaai_catalyst/tracers/exporters/__init__.py,sha256=kVA8zp05h3phu4e-iHSlnznp_PzMRczB7LphSsZgUjg,138
 ragaai_catalyst/tracers/exporters/file_span_exporter.py,sha256=RgGteu-NVGprXKkynvyIO5yOjpbtA41R3W_NzCjnkwE,6445
 ragaai_catalyst/tracers/exporters/raga_exporter.py,sha256=6xvjWXyh8XPkHKSLLmAZUQSvwuyY17ov8pv2VdfI0qA,17875
@@ -71,8 +85,8 @@ ragaai_catalyst/tracers/utils/convert_llama_instru_callback.py,sha256=8qLo7x4Zsn
 ragaai_catalyst/tracers/utils/extraction_logic_llama_index.py,sha256=ZhPs0YhVtB82-Pq9o1BvCinKE_WPvVxPTEcZjlJbFYM,2371
 ragaai_catalyst/tracers/utils/langchain_tracer_extraction_logic.py,sha256=XS2_x2qneqEx9oAighLg-LRiueWcESLwIC2r7eJT-Ww,3117
 ragaai_catalyst/tracers/utils/utils.py,sha256=ViygfJ7vZ7U0CTSA1lbxVloHp4NSlmfDzBRNCJuMhis,2374
-ragaai_catalyst-2.1.5b29.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-ragaai_catalyst-2.1.5b29.dist-info/METADATA,sha256=VtRy6oG0L-rNaky2LVNVNvXV6pAHjdRs6aMah4Qp3yM,19472
-ragaai_catalyst-2.1.5b29.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-ragaai_catalyst-2.1.5b29.dist-info/top_level.txt,sha256=HpgsdRgEJMk8nqrU6qdCYk3di7MJkDL0B19lkc7dLfM,16
-ragaai_catalyst-2.1.5b29.dist-info/RECORD,,
+ragaai_catalyst-2.1.5b31.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+ragaai_catalyst-2.1.5b31.dist-info/METADATA,sha256=iXJyMH-c2F5J10hrAjKZp_B8K4Hi3j0eIQy448sTeiE,21884
+ragaai_catalyst-2.1.5b31.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+ragaai_catalyst-2.1.5b31.dist-info/top_level.txt,sha256=HpgsdRgEJMk8nqrU6qdCYk3di7MJkDL0B19lkc7dLfM,16
+ragaai_catalyst-2.1.5b31.dist-info/RECORD,,

ragaai_catalyst/redteaming.py

@@ -1,171 +0,0 @@
-import logging
-import os
-from typing import Callable, Optional
-
-import giskard as scanner
-import pandas as pd
-
-logging.getLogger('giskard.core').disabled = True
-logging.getLogger('giskard.scanner.logger').disabled = True
-logging.getLogger('giskard.models.automodel').disabled = True
-logging.getLogger('giskard.datasets.base').disabled = True
-logging.getLogger('giskard.utils.logging_utils').disabled = True
-
-
-class RedTeaming:
-
-    def __init__(self,
-                 provider: Optional[str] = "openai",
-                 model: Optional[str] = None,
-                 api_key: Optional[str] = None,
-                 api_base: Optional[str] = None,
-                 api_version: Optional[str] = None):
-        self.provider = provider.lower()
-        self.model = model
-        if not self.provider:
-            raise ValueError("Model configuration must be provided with a valid provider and model.")
-        if self.provider == "openai":
-            if api_key is not None:
-                os.environ["OPENAI_API_KEY"] = api_key
-            if os.getenv("OPENAI_API_KEY") is None:
-                raise ValueError("API key must be provided for OpenAI.")
-        elif self.provider == "gemini":
-            if api_key is not None:
-                os.environ["GEMINI_API_KEY"] = api_key
-            if os.getenv("GEMINI_API_KEY") is None:
-                raise ValueError("API key must be provided for Gemini.")
-        elif self.provider == "azure":
-            if api_key is not None:
-                os.environ["AZURE_API_KEY"] = api_key
-            if api_base is not None:
-                os.environ["AZURE_API_BASE"] = api_base
-            if api_version is not None:
-                os.environ["AZURE_API_VERSION"] = api_version
-            if os.getenv("AZURE_API_KEY") is None:
-                raise ValueError("API key must be provided for Azure.")
-            if os.getenv("AZURE_API_BASE") is None:
-                raise ValueError("API base must be provided for Azure.")
-            if os.getenv("AZURE_API_VERSION") is None:
-                raise ValueError("API version must be provided for Azure.")
-        else:
-            raise ValueError(f"Provider is not recognized.")
-
-    def run_scan(
-            self,
-            model: Callable,
-            evaluators: Optional[list] = None,
-            save_report: bool = True
-    ) -> pd.DataFrame:
-        """
-        Runs red teaming on the provided model and returns a DataFrame of the results.
-
-        :param model: The model function provided by the user (can be sync or async).
-        :param evaluators: Optional list of scan metrics to run.
-        :param save_report: Boolean flag indicating whether to save the scan report as a CSV file.
-        :return: A DataFrame containing the scan report.
-        """
-        import asyncio
-        import inspect
-
-        self.set_scanning_model(self.provider, self.model)
-
-        supported_evaluators = self.get_supported_evaluators()
-        if evaluators:
-            if isinstance(evaluators, str):
-                evaluators = [evaluators]
-            invalid_evaluators = [evaluator for evaluator in evaluators if evaluator not in supported_evaluators]
-            if invalid_evaluators:
-                raise ValueError(f"Invalid evaluators: {invalid_evaluators}. "
-                                 f"Allowed evaluators: {supported_evaluators}.")
-
-        # Handle async model functions by wrapping them in a sync function
-        if inspect.iscoroutinefunction(model):
-            def sync_wrapper(*args, **kwargs):
-                try:
-                    # Try to get the current event loop
-                    loop = asyncio.get_event_loop()
-                except RuntimeError:
-                    # If no event loop exists (e.g., in Jupyter), create a new one
-                    loop = asyncio.new_event_loop()
-                    asyncio.set_event_loop(loop)
-
-                try:
-                    # Handle both IPython and regular Python environments
-                    import nest_asyncio
-                    nest_asyncio.apply()
-                except ImportError:
-                    pass  # nest_asyncio not available, continue without it
-
-                return loop.run_until_complete(model(*args, **kwargs))
-            wrapped_model = sync_wrapper
-        else:
-            wrapped_model = model
-
-        model_instance = scanner.Model(
-            model=wrapped_model,
-            model_type="text_generation",
-            name="RagaAI's Scan",
-            description="RagaAI's RedTeaming Scan",
-            feature_names=["question"],
-        )
-
-        try:
-            report = scanner.scan(model_instance, only=evaluators, raise_exceptions=True) if evaluators \
-                     else scanner.scan(model_instance, raise_exceptions=True)
-        except Exception as e:
-            raise RuntimeError(f"Error occurred during model scan: {str(e)}")
-
-        report_df = report.to_dataframe()
-
-        if save_report:
-            report_df.to_csv("raga-ai_red-teaming_scan.csv", index=False)
-
-        return report_df
-
-    def get_supported_evaluators(self):
-        """Contains tags corresponding to the 'llm' and 'robustness' directories in the giskard > scanner library"""
-        return {'control_chars_injection',
-                'discrimination',
-                'ethical_bias',
-                'ethics',
-                'faithfulness',
-                'generative',
-                'hallucination',
-                'harmfulness',
-                'implausible_output',
-                'information_disclosure',
-                'jailbreak',
-                'llm',
-                'llm_harmful_content',
-                'llm_stereotypes_detector',
-                'misinformation',
-                'output_formatting',
-                'prompt_injection',
-                'robustness',
-                'stereotypes',
-                'sycophancy',
-                'text_generation',
-                'text_perturbation'}
-
-    def set_scanning_model(self, provider, model=None):
-        """
-        Sets the LLM model for Giskard based on the provider.
-
-        :param provider: The LLM provider (e.g., "openai", "gemini", "azure").
-        :param model: The specific model name to use (optional).
-        :raises ValueError: If the provider is "azure" and no model is provided.
-        """
-        default_models = {
-            "openai": "gpt-4o",
-            "gemini": "gemini-1.5-pro"
-        }
-
-        if provider == "azure" and model is None:
-            raise ValueError("Model must be provided for Azure.")
-
-        selected_model = model if model is not None else default_models.get(provider)
-
-        if selected_model is None:
-            raise ValueError(f"Unsupported provider: {provider}")
-
-        scanner.llm.set_llm_model(selected_model)