ragaai-catalyst 2.1.5b21__py3-none-any.whl → 2.1.5b23__py3-none-any.whl

ragaai_catalyst/__init__.py

@@ -5,6 +5,7 @@ from .dataset import Dataset
 from .prompt_manager import PromptManager
 from .evaluation import Evaluation
 from .synthetic_data_generation import SyntheticDataGeneration
+from .redteaming import RedTeaming
 from .guardrails_manager import GuardrailsManager
 from .guard_executor import GuardExecutor
 from .tracers import Tracer, init_tracing, trace_agent, trace_llm, trace_tool, current_span, trace_custom
@@ -18,7 +19,8 @@ __all__ = [
     "Tracer", 
     "PromptManager", 
     "Evaluation",
-    "SyntheticDataGeneration", 
+    "SyntheticDataGeneration",
+    "RedTeaming",
     "GuardrailsManager", 
     "GuardExecutor",
     "init_tracing", 

ragaai_catalyst/dataset.py

@@ -1,5 +1,7 @@
 import os
+import csv
 import json
+import tempfile
 import requests
 from .utils import response_checker
 from typing import Union
@@ -653,4 +655,50 @@ class Dataset:
             return JOB_STATUS_FAILED
         except Exception as e:
             logger.error(f"An unexpected error occurred: {e}")
-            return JOB_STATUS_FAILED
+            return JOB_STATUS_FAILED
+
+    def _jsonl_to_csv(self, jsonl_file, csv_file):
+        """Convert a JSONL file to a CSV file."""
+        with open(jsonl_file, 'r', encoding='utf-8') as infile:
+            data = [json.loads(line) for line in infile]
+        
+        if not data:
+            print("Empty JSONL file.")
+            return
+        
+        with open(csv_file, 'w', newline='', encoding='utf-8') as outfile:
+            writer = csv.DictWriter(outfile, fieldnames=data[0].keys())
+            writer.writeheader()
+            writer.writerows(data)
+        
+        print(f"Converted {jsonl_file} to {csv_file}")
+
+    def create_from_jsonl(self, jsonl_path, dataset_name, schema_mapping):
+        tmp_csv_path = os.path.join(tempfile.gettempdir(), f"{dataset_name}.csv")
+        try:
+            self._jsonl_to_csv(jsonl_path, tmp_csv_path)
+            self.create_from_csv(tmp_csv_path, dataset_name, schema_mapping)
+        except (IOError, UnicodeError) as e:
+            logger.error(f"Error converting JSONL to CSV: {e}")
+            raise
+        finally:
+            if os.path.exists(tmp_csv_path):
+                try:
+                    os.remove(tmp_csv_path)
+                except Exception as e:
+                    logger.error(f"Error removing temporary CSV file: {e}")
+
+    def add_rows_from_jsonl(self, jsonl_path, dataset_name):
+        tmp_csv_path = os.path.join(tempfile.gettempdir(), f"{dataset_name}.csv")
+        try:
+            self._jsonl_to_csv(jsonl_path, tmp_csv_path)
+            self.add_rows(tmp_csv_path, dataset_name)
+        except (IOError, UnicodeError) as e:
+            logger.error(f"Error converting JSONL to CSV: {e}")
+            raise
+        finally:
+            if os.path.exists(tmp_csv_path):
+                try:
+                    os.remove(tmp_csv_path)
+                except Exception as e:
+                    logger.error(f"Error removing temporary CSV file: {e}")

ragaai_catalyst/redteaming.py

@@ -0,0 +1,171 @@
+import logging
+import os
+from typing import Callable, Optional
+
+import giskard as scanner
+import pandas as pd
+
+logging.getLogger('giskard.core').disabled = True
+logging.getLogger('giskard.scanner.logger').disabled = True
+logging.getLogger('giskard.models.automodel').disabled = True
+logging.getLogger('giskard.datasets.base').disabled = True
+logging.getLogger('giskard.utils.logging_utils').disabled = True
+
+
+class RedTeaming:
+
+    def __init__(self,
+                 provider: Optional[str] = "openai",
+                 model: Optional[str] = None,
+                 api_key: Optional[str] = None,
+                 api_base: Optional[str] = None,
+                 api_version: Optional[str] = None):
+        self.provider = provider.lower()
+        self.model = model
+        if not self.provider:
+            raise ValueError("Model configuration must be provided with a valid provider and model.")
+        if self.provider == "openai":
+            if api_key is not None:
+                os.environ["OPENAI_API_KEY"] = api_key
+            if os.getenv("OPENAI_API_KEY") is None:
+                raise ValueError("API key must be provided for OpenAI.")
+        elif self.provider == "gemini":
+            if api_key is not None:
+                os.environ["GEMINI_API_KEY"] = api_key
+            if os.getenv("GEMINI_API_KEY") is None:
+                raise ValueError("API key must be provided for Gemini.")
+        elif self.provider == "azure":
+            if api_key is not None:
+                os.environ["AZURE_API_KEY"] = api_key
+            if api_base is not None:
+                os.environ["AZURE_API_BASE"] = api_base
+            if api_version is not None:
+                os.environ["AZURE_API_VERSION"] = api_version
+            if os.getenv("AZURE_API_KEY") is None:
+                raise ValueError("API key must be provided for Azure.")
+            if os.getenv("AZURE_API_BASE") is None:
+                raise ValueError("API base must be provided for Azure.")
+            if os.getenv("AZURE_API_VERSION") is None:
+                raise ValueError("API version must be provided for Azure.")
+        else:
+            raise ValueError(f"Provider is not recognized.")
+
+    def run_scan(
+            self,
+            model: Callable,
+            evaluators: Optional[list] = None,
+            save_report: bool = True
+    ) -> pd.DataFrame:
+        """
+        Runs red teaming on the provided model and returns a DataFrame of the results.
+
+        :param model: The model function provided by the user (can be sync or async).
+        :param evaluators: Optional list of scan metrics to run.
+        :param save_report: Boolean flag indicating whether to save the scan report as a CSV file.
+        :return: A DataFrame containing the scan report.
+        """
+        import asyncio
+        import inspect
+
+        self.set_scanning_model(self.provider, self.model)
+
+        supported_evaluators = self.get_supported_evaluators()
+        if evaluators:
+            if isinstance(evaluators, str):
+                evaluators = [evaluators]
+            invalid_evaluators = [evaluator for evaluator in evaluators if evaluator not in supported_evaluators]
+            if invalid_evaluators:
+                raise ValueError(f"Invalid evaluators: {invalid_evaluators}. "
+                                 f"Allowed evaluators: {supported_evaluators}.")
+
+        # Handle async model functions by wrapping them in a sync function
+        if inspect.iscoroutinefunction(model):
+            def sync_wrapper(*args, **kwargs):
+                try:
+                    # Try to get the current event loop
+                    loop = asyncio.get_event_loop()
+                except RuntimeError:
+                    # If no event loop exists (e.g., in Jupyter), create a new one
+                    loop = asyncio.new_event_loop()
+                    asyncio.set_event_loop(loop)
+
+                try:
+                    # Handle both IPython and regular Python environments
+                    import nest_asyncio
+                    nest_asyncio.apply()
+                except ImportError:
+                    pass  # nest_asyncio not available, continue without it
+
+                return loop.run_until_complete(model(*args, **kwargs))
+            wrapped_model = sync_wrapper
+        else:
+            wrapped_model = model
+
+        model_instance = scanner.Model(
+            model=wrapped_model,
+            model_type="text_generation",
+            name="RagaAI's Scan",
+            description="RagaAI's RedTeaming Scan",
+            feature_names=["question"],
+        )
+
+        try:
+            report = scanner.scan(model_instance, only=evaluators, raise_exceptions=True) if evaluators \
+                     else scanner.scan(model_instance, raise_exceptions=True)
+        except Exception as e:
+            raise RuntimeError(f"Error occurred during model scan: {str(e)}")
+
+        report_df = report.to_dataframe()
+
+        if save_report:
+            report_df.to_csv("raga-ai_red-teaming_scan.csv", index=False)
+
+        return report_df
+
+    def get_supported_evaluators(self):
+        """Contains tags corresponding to the 'llm' and 'robustness' directories in the giskard > scanner library"""
+        return {'control_chars_injection',
+                'discrimination',
+                'ethical_bias',
+                'ethics',
+                'faithfulness',
+                'generative',
+                'hallucination',
+                'harmfulness',
+                'implausible_output',
+                'information_disclosure',
+                'jailbreak',
+                'llm',
+                'llm_harmful_content',
+                'llm_stereotypes_detector',
+                'misinformation',
+                'output_formatting',
+                'prompt_injection',
+                'robustness',
+                'stereotypes',
+                'sycophancy',
+                'text_generation',
+                'text_perturbation'}
+
+    def set_scanning_model(self, provider, model=None):
+        """
+        Sets the LLM model for Giskard based on the provider.
+
+        :param provider: The LLM provider (e.g., "openai", "gemini", "azure").
+        :param model: The specific model name to use (optional).
+        :raises ValueError: If the provider is "azure" and no model is provided.
+        """
+        default_models = {
+            "openai": "gpt-4o",
+            "gemini": "gemini-1.5-pro"
+        }
+
+        if provider == "azure" and model is None:
+            raise ValueError("Model must be provided for Azure.")
+
+        selected_model = model if model is not None else default_models.get(provider)
+
+        if selected_model is None:
+            raise ValueError(f"Unsupported provider: {provider}")
+
+        scanner.llm.set_llm_model(selected_model)

ragaai_catalyst/synthetic_data_generation.py

@@ -8,7 +8,9 @@ import markdown
 import pandas as pd
 import json
 from litellm import completion
+import litellm
 from tqdm import tqdm
+import tiktoken
 # import internal_api_completion
 # import proxy_call
 from .internal_api_completion import api_completion as internal_api_completion
@@ -48,13 +50,18 @@ class SyntheticDataGeneration:
         Raises:
             ValueError: If an invalid provider is specified or API key is missing.
         """
+        text_validity = self.validate_input(text)
+        if text_validity:
+            raise ValueError(text_validity)
+
         BATCH_SIZE = 5  # Optimal batch size for maintaining response quality
         provider = model_config.get("provider")
         model = model_config.get("model")
         api_base = model_config.get("api_base")
+        api_version = model_config.get("api_version")
 
         # Initialize the appropriate client based on provider
-        self._initialize_client(provider, api_key, api_base, internal_llm_proxy=kwargs.get("internal_llm_proxy", None))
+        self._initialize_client(provider, api_key, api_base, api_version, internal_llm_proxy=kwargs.get("internal_llm_proxy", None))
 
         # Initialize progress bar
         pbar = tqdm(total=n, desc="Generating QA pairs")
@@ -88,7 +95,7 @@ class SyntheticDataGeneration:
                     pbar.update(len(batch_df))
                     
             except Exception as e:
-                print(f"Batch generation failed.")
+                print(f"Batch generation failed:{str(e)}")
 
                 if any(error in str(e) for error in FAILURE_CASES):
                     raise Exception(f"{e}")
@@ -139,7 +146,7 @@ class SyntheticDataGeneration:
         
         return final_df
 
-    def _initialize_client(self, provider, api_key, api_base=None, internal_llm_proxy=None):
+    def _initialize_client(self, provider, api_key, api_base=None, api_version=None, internal_llm_proxy=None):
         """Initialize the appropriate client based on provider."""
         if not provider:
             raise ValueError("Model configuration must be provided with a valid provider and model.")
@@ -158,7 +165,17 @@ class SyntheticDataGeneration:
             if api_key is None and os.getenv("OPENAI_API_KEY") is None and internal_llm_proxy is None:
                 raise ValueError("API key must be provided for OpenAI.")
             openai.api_key = api_key or os.getenv("OPENAI_API_KEY")
-        
+
+        elif provider == "azure":
+            if api_key is None and os.getenv("AZURE_API_KEY") is None and internal_llm_proxy is None:
+                raise ValueError("API key must be provided for Azure.")
+            litellm.api_key = api_key or os.getenv("AZURE_API_KEY")
+            if api_base is None and os.getenv("AZURE_API_BASE") is None and internal_llm_proxy is None:
+                raise ValueError("API Base must be provided for Azure.")
+            litellm.api_base = api_base or os.getenv("AZURE_API_BASE")
+            if api_version is None and os.getenv("AZURE_API_VERSION") is None and internal_llm_proxy is None:
+                raise ValueError("API version must be provided for Azure.")
+            litellm.api_version = api_version or os.getenv("AZURE_API_VERSION")
         else:
             raise ValueError(f"Provider is not recognized.")
 
@@ -189,7 +206,15 @@ class SyntheticDataGeneration:
             kwargs=kwargs
         )
 
+    def validate_input(self,text):
 
+        if not text.strip():
+            return 'Empty Text provided for qna generation. Please provide valid text'
+        encoding = tiktoken.encoding_for_model("gpt-4")
+        tokens = encoding.encode(text)
+        if len(tokens)<5:
+            return 'Very Small Text provided for qna generation. Please provide longer text'
+        return False
         
           
     def _get_system_message(self, question_type, n):
@@ -274,10 +299,14 @@ class SyntheticDataGeneration:
         # Add optional parameters if they exist in model_config
         if "api_base" in model_config:
             completion_params["api_base"] = model_config["api_base"]
+        if "api_version" in model_config:
+            completion_params["api_version"] = model_config["api_version"]
         if "max_tokens" in model_config:
             completion_params["max_tokens"] = model_config["max_tokens"]
         if "temperature" in model_config:
             completion_params["temperature"] = model_config["temperature"]
+        if 'provider' in model_config:
+            completion_params['model'] = f'{model_config["provider"]}/{model_config["model"]}'
 
         # Make the API call using LiteLLM
         try:
@@ -318,9 +347,13 @@ class SyntheticDataGeneration:
             list_start_index = data.find('[')  # Find the index of the first '['
             substring_data = data[list_start_index:] if list_start_index != -1 else data  # Slice from the list start
             data = substring_data
-
+        elif provider == "azure":
+            data = response.choices[0].message.content.replace('\n', '')
+            list_start_index = data.find('[')  # Find the index of the first '['
+            substring_data = data[list_start_index:] if list_start_index != -1 else data  # Slice from the list start
+            data = substring_data
         else:
-            raise ValueError("Invalid provider. Choose 'groq', 'gemini', or 'openai'.")
+            raise ValueError("Invalid provider. Choose 'groq', 'gemini', 'azure' or 'openai'.")
         try:
             json_data = json.loads(data)
             return pd.DataFrame(json_data)
@@ -442,7 +475,7 @@ class SyntheticDataGeneration:
         Returns:
             list: A list of supported AI providers.
         """
-        return ['gemini', 'openai']
+        return ['gemini', 'openai','azure']
 
 # Usage:
 # from synthetic_data_generation import SyntheticDataGeneration

ragaai_catalyst/tracers/agentic_tracing/tracers/agent_tracer.py

@@ -48,15 +48,15 @@ class AgentTracerMixin:
         self.auto_instrument_network = False
 
     def trace_agent(
-        self,
-        name: str,
-        agent_type: str = None,
-        version: str = None,
-        capabilities: List[str] = None,
-        tags: List[str] = [],
-        metadata: Dict[str, Any] = {},
-        metrics: List[Dict[str, Any]] = [],
-        feedback: Optional[Any] = None,
+            self,
+            name: str,
+            agent_type: str = None,
+            version: str = None,
+            capabilities: List[str] = None,
+            tags: List[str] = [],
+            metadata: Dict[str, Any] = {},
+            metrics: List[Dict[str, Any]] = [],
+            feedback: Optional[Any] = None,
     ):
         if name not in self.span_attributes_dict:
             self.span_attributes_dict[name] = SpanAttributes(name)
@@ -101,7 +101,10 @@ class AgentTracerMixin:
                 original_init = target.__init__
 
                 def wrapped_init(self, *args, **kwargs):
-                    self.gt = kwargs.get("gt", None) if kwargs else None
+                    gt = kwargs.get("gt") if kwargs else None
+                    if gt is not None:
+                        span = self.span(name)
+                        span.add_gt(gt)
                     # Set agent context before initializing
                     component_id = str(uuid.uuid4())
                     hash_id = top_level_hash_id
@@ -159,7 +162,10 @@ class AgentTracerMixin:
                                 @self.file_tracker.trace_decorator
                                 @functools.wraps(method)
                                 def wrapped_method(self, *args, **kwargs):
-                                    self.gt = kwargs.get("gt", None) if kwargs else None
+                                    gt = kwargs.get("gt") if kwargs else None
+                                    if gt is not None:
+                                        span = tracer.span(name)
+                                        span.add_gt(gt)
                                     # Set this agent as current during method execution
                                     token = tracer.current_agent_id.set(
                                         self._agent_component_id
@@ -193,8 +199,8 @@ class AgentTracerMixin:
                                                 children = tracer.agent_children.get()
                                                 if children:
                                                     if (
-                                                        "children"
-                                                        not in component["data"]
+                                                            "children"
+                                                            not in component["data"]
                                                     ):
                                                         component["data"][
                                                             "children"
@@ -247,6 +253,7 @@ class AgentTracerMixin:
                             agent_type,
                             version,
                             capabilities,
+                            top_level_hash_id,
                             *args,
                             **kwargs,
                         )
@@ -256,10 +263,9 @@ class AgentTracerMixin:
         return decorator
 
     def _trace_sync_agent_execution(
-        self, func, name, agent_type, version, capabilities, *args, **kwargs
+            self, func, name, agent_type, version, capabilities, top_level_hash_id, *args, **kwargs
     ):
-        # Generate a unique hash_id for this execution context
-        hash_id = str(uuid.uuid4())
+        hash_id = top_level_hash_id
 
         """Synchronous version of agent tracing"""
         if not self.is_active:
@@ -276,6 +282,9 @@ class AgentTracerMixin:
 
         # Extract ground truth if present
         ground_truth = kwargs.pop("gt", None) if kwargs else None
+        if ground_truth is not None:
+            span = self.span(name)
+            span.add_gt(ground_truth)
 
         # Get parent agent ID if exists
         parent_agent_id = self.current_agent_id.get()
@@ -293,7 +302,7 @@ class AgentTracerMixin:
 
         try:
             # Execute the agent
-            result = func(*args, **kwargs)
+            result = self.file_tracker.trace_wrapper(func)(*args, **kwargs)
 
             # Calculate resource usage
             end_memory = psutil.Process().memory_info().rss
@@ -320,9 +329,6 @@ class AgentTracerMixin:
                 children=children,
                 parent_id=parent_agent_id,
             )
-            # Add ground truth to component data if present
-            if ground_truth is not None:
-                agent_component["data"]["gt"] = ground_truth
 
             # Add this component as a child to parent's children list
             parent_children.append(agent_component)
@@ -384,7 +390,7 @@ class AgentTracerMixin:
             self.agent_children.reset(children_token)
 
     async def _trace_agent_execution(
-        self, func, name, agent_type, version, capabilities, hash_id, *args, **kwargs
+            self, func, name, agent_type, version, capabilities, hash_id, *args, **kwargs
     ):
         """Asynchronous version of agent tracing"""
         if not self.is_active:
@@ -399,6 +405,9 @@ class AgentTracerMixin:
 
         # Extract ground truth if present
         ground_truth = kwargs.pop("gt", None) if kwargs else None
+        if ground_truth is not None:
+            span = self.span(name)
+            span.add_gt(ground_truth)
 
         # Get parent agent ID if exists
         parent_agent_id = self.current_agent_id.get()
@@ -414,7 +423,7 @@ class AgentTracerMixin:
 
         try:
             # Execute the agent
-            result = await func(*args, **kwargs)
+            result = await self.file_tracker.trace_wrapper(func)(*args, **kwargs)
 
             # Calculate resource usage
             end_memory = psutil.Process().memory_info().rss
@@ -441,10 +450,6 @@ class AgentTracerMixin:
                 parent_id=parent_agent_id,
             )
 
-            # Add ground truth to component data if present
-            if ground_truth is not None:
-                agent_component["data"]["gt"] = ground_truth
-
             # Add this component as a child to parent's children list
             parent_children.append(agent_component)
             self.agent_children.set(parent_children)
@@ -517,7 +522,7 @@ class AgentTracerMixin:
             for interaction in self.component_user_interaction.get(kwargs["component_id"], []):
                 if interaction["interaction_type"] in ["input", "output"]:
                     input_output_interactions.append(interaction)
-            interactions.extend(input_output_interactions) 
+            interactions.extend(input_output_interactions)
         if self.auto_instrument_file_io:
             file_io_interactions = []
             for interaction in self.component_user_interaction.get(kwargs["component_id"], []):
@@ -546,9 +551,10 @@ class AgentTracerMixin:
                 counter = sum(1 for x in self.visited_metrics if x.startswith(base_metric_name))
                 metric_name = f'{base_metric_name}_{counter}' if counter > 0 else base_metric_name
                 self.visited_metrics.append(metric_name)
-                metric["name"] = metric_name  
+                metric["name"] = metric_name
                 metrics.append(metric)
 
+        # TODO agent_trace execute metric
         component = {
             "id": kwargs["component_id"],
             "hash_id": kwargs["hash_id"],
@@ -576,8 +582,13 @@ class AgentTracerMixin:
             "interactions": interactions,
         }
 
-        if self.gt:
-            component["data"]["gt"] = self.gt
+        if name in self.span_attributes_dict:
+            span_gt = self.span_attributes_dict[name].gt
+            if span_gt is not None:
+                component["data"]["gt"] = span_gt
+            span_context = self.span_attributes_dict[name].context
+            if span_context:
+                component["data"]["context"] = span_context
 
         # Reset the SpanAttributes context variable
         self.span_attributes_dict[kwargs["name"]] = SpanAttributes(kwargs["name"])
@@ -599,22 +610,22 @@ class AgentTracerMixin:
         self.component_network_calls.set(component_network_calls)
 
     def _sanitize_input(self, args: tuple, kwargs: dict) -> dict:
-            """Sanitize and format input data, including handling of nested lists and dictionaries."""
-
-            def sanitize_value(value):
-                if isinstance(value, (int, float, bool, str)):
-                    return value
-                elif isinstance(value, list):
-                    return [sanitize_value(item) for item in value]
-                elif isinstance(value, dict):
-                    return {key: sanitize_value(val) for key, val in value.items()}
-                else:
-                    return str(value)  # Convert non-standard types to string
+        """Sanitize and format input data, including handling of nested lists and dictionaries."""
+
+        def sanitize_value(value):
+            if isinstance(value, (int, float, bool, str)):
+                return value
+            elif isinstance(value, list):
+                return [sanitize_value(item) for item in value]
+            elif isinstance(value, dict):
+                return {key: sanitize_value(val) for key, val in value.items()}
+            else:
+                return str(value)  # Convert non-standard types to string
 
-            return {
-                "args": [sanitize_value(arg) for arg in args],
-                "kwargs": {key: sanitize_value(val) for key, val in kwargs.items()},
-            }
+        return {
+            "args": [sanitize_value(arg) for arg in args],
+            "kwargs": {key: sanitize_value(val) for key, val in kwargs.items()},
+        }
 
     def _sanitize_output(self, output: Any) -> Any:
         """Sanitize and format output data"""
@@ -630,6 +641,6 @@ class AgentTracerMixin:
 
     def instrument_network_calls(self):
         self.auto_instrument_network = True
-        
+
     def instrument_file_io_calls(self):
         self.auto_instrument_file_io = True