ragaai-catalyst 2.1.5b22__py3-none-any.whl → 2.1.5b24__py3-none-any.whl

ragaai_catalyst/__init__.py

@@ -5,6 +5,7 @@ from .dataset import Dataset
 from .prompt_manager import PromptManager
 from .evaluation import Evaluation
 from .synthetic_data_generation import SyntheticDataGeneration
+from .redteaming import RedTeaming
 from .guardrails_manager import GuardrailsManager
 from .guard_executor import GuardExecutor
 from .tracers import Tracer, init_tracing, trace_agent, trace_llm, trace_tool, current_span, trace_custom
@@ -18,7 +19,8 @@ __all__ = [
     "Tracer", 
     "PromptManager", 
     "Evaluation",
-    "SyntheticDataGeneration", 
+    "SyntheticDataGeneration",
+    "RedTeaming",
     "GuardrailsManager", 
     "GuardExecutor",
     "init_tracing", 

ragaai_catalyst/dataset.py

@@ -1,5 +1,7 @@
 import os
+import csv
 import json
+import tempfile
 import requests
 from .utils import response_checker
 from typing import Union
@@ -653,4 +655,50 @@ class Dataset:
             return JOB_STATUS_FAILED
         except Exception as e:
             logger.error(f"An unexpected error occurred: {e}")
-            return JOB_STATUS_FAILED
+            return JOB_STATUS_FAILED
+
+    def _jsonl_to_csv(self, jsonl_file, csv_file):
+        """Convert a JSONL file to a CSV file."""
+        with open(jsonl_file, 'r', encoding='utf-8') as infile:
+            data = [json.loads(line) for line in infile]
+        
+        if not data:
+            print("Empty JSONL file.")
+            return
+        
+        with open(csv_file, 'w', newline='', encoding='utf-8') as outfile:
+            writer = csv.DictWriter(outfile, fieldnames=data[0].keys())
+            writer.writeheader()
+            writer.writerows(data)
+        
+        print(f"Converted {jsonl_file} to {csv_file}")
+
+    def create_from_jsonl(self, jsonl_path, dataset_name, schema_mapping):
+        tmp_csv_path = os.path.join(tempfile.gettempdir(), f"{dataset_name}.csv")
+        try:
+            self._jsonl_to_csv(jsonl_path, tmp_csv_path)
+            self.create_from_csv(tmp_csv_path, dataset_name, schema_mapping)
+        except (IOError, UnicodeError) as e:
+            logger.error(f"Error converting JSONL to CSV: {e}")
+            raise
+        finally:
+            if os.path.exists(tmp_csv_path):
+                try:
+                    os.remove(tmp_csv_path)
+                except Exception as e:
+                    logger.error(f"Error removing temporary CSV file: {e}")
+
+    def add_rows_from_jsonl(self, jsonl_path, dataset_name):
+        tmp_csv_path = os.path.join(tempfile.gettempdir(), f"{dataset_name}.csv")
+        try:
+            self._jsonl_to_csv(jsonl_path, tmp_csv_path)
+            self.add_rows(tmp_csv_path, dataset_name)
+        except (IOError, UnicodeError) as e:
+            logger.error(f"Error converting JSONL to CSV: {e}")
+            raise
+        finally:
+            if os.path.exists(tmp_csv_path):
+                try:
+                    os.remove(tmp_csv_path)
+                except Exception as e:
+                    logger.error(f"Error removing temporary CSV file: {e}")

ragaai_catalyst/redteaming.py

@@ -0,0 +1,171 @@
+import logging
+import os
+from typing import Callable, Optional
+
+import giskard as scanner
+import pandas as pd
+
+logging.getLogger('giskard.core').disabled = True
+logging.getLogger('giskard.scanner.logger').disabled = True
+logging.getLogger('giskard.models.automodel').disabled = True
+logging.getLogger('giskard.datasets.base').disabled = True
+logging.getLogger('giskard.utils.logging_utils').disabled = True
+
+
+class RedTeaming:
+
+    def __init__(self,
+                 provider: Optional[str] = "openai",
+                 model: Optional[str] = None,
+                 api_key: Optional[str] = None,
+                 api_base: Optional[str] = None,
+                 api_version: Optional[str] = None):
+        self.provider = provider.lower()
+        self.model = model
+        if not self.provider:
+            raise ValueError("Model configuration must be provided with a valid provider and model.")
+        if self.provider == "openai":
+            if api_key is not None:
+                os.environ["OPENAI_API_KEY"] = api_key
+            if os.getenv("OPENAI_API_KEY") is None:
+                raise ValueError("API key must be provided for OpenAI.")
+        elif self.provider == "gemini":
+            if api_key is not None:
+                os.environ["GEMINI_API_KEY"] = api_key
+            if os.getenv("GEMINI_API_KEY") is None:
+                raise ValueError("API key must be provided for Gemini.")
+        elif self.provider == "azure":
+            if api_key is not None:
+                os.environ["AZURE_API_KEY"] = api_key
+            if api_base is not None:
+                os.environ["AZURE_API_BASE"] = api_base
+            if api_version is not None:
+                os.environ["AZURE_API_VERSION"] = api_version
+            if os.getenv("AZURE_API_KEY") is None:
+                raise ValueError("API key must be provided for Azure.")
+            if os.getenv("AZURE_API_BASE") is None:
+                raise ValueError("API base must be provided for Azure.")
+            if os.getenv("AZURE_API_VERSION") is None:
+                raise ValueError("API version must be provided for Azure.")
+        else:
+            raise ValueError(f"Provider is not recognized.")
+
+    def run_scan(
+            self,
+            model: Callable,
+            evaluators: Optional[list] = None,
+            save_report: bool = True
+    ) -> pd.DataFrame:
+        """
+        Runs red teaming on the provided model and returns a DataFrame of the results.
+
+        :param model: The model function provided by the user (can be sync or async).
+        :param evaluators: Optional list of scan metrics to run.
+        :param save_report: Boolean flag indicating whether to save the scan report as a CSV file.
+        :return: A DataFrame containing the scan report.
+        """
+        import asyncio
+        import inspect
+
+        self.set_scanning_model(self.provider, self.model)
+
+        supported_evaluators = self.get_supported_evaluators()
+        if evaluators:
+            if isinstance(evaluators, str):
+                evaluators = [evaluators]
+            invalid_evaluators = [evaluator for evaluator in evaluators if evaluator not in supported_evaluators]
+            if invalid_evaluators:
+                raise ValueError(f"Invalid evaluators: {invalid_evaluators}. "
+                                 f"Allowed evaluators: {supported_evaluators}.")
+
+        # Handle async model functions by wrapping them in a sync function
+        if inspect.iscoroutinefunction(model):
+            def sync_wrapper(*args, **kwargs):
+                try:
+                    # Try to get the current event loop
+                    loop = asyncio.get_event_loop()
+                except RuntimeError:
+                    # If no event loop exists (e.g., in Jupyter), create a new one
+                    loop = asyncio.new_event_loop()
+                    asyncio.set_event_loop(loop)
+
+                try:
+                    # Handle both IPython and regular Python environments
+                    import nest_asyncio
+                    nest_asyncio.apply()
+                except ImportError:
+                    pass  # nest_asyncio not available, continue without it
+
+                return loop.run_until_complete(model(*args, **kwargs))
+            wrapped_model = sync_wrapper
+        else:
+            wrapped_model = model
+
+        model_instance = scanner.Model(
+            model=wrapped_model,
+            model_type="text_generation",
+            name="RagaAI's Scan",
+            description="RagaAI's RedTeaming Scan",
+            feature_names=["question"],
+        )
+
+        try:
+            report = scanner.scan(model_instance, only=evaluators, raise_exceptions=True) if evaluators \
+                     else scanner.scan(model_instance, raise_exceptions=True)
+        except Exception as e:
+            raise RuntimeError(f"Error occurred during model scan: {str(e)}")
+
+        report_df = report.to_dataframe()
+
+        if save_report:
+            report_df.to_csv("raga-ai_red-teaming_scan.csv", index=False)
+
+        return report_df
+
+    def get_supported_evaluators(self):
+        """Contains tags corresponding to the 'llm' and 'robustness' directories in the giskard > scanner library"""
+        return {'control_chars_injection',
+                'discrimination',
+                'ethical_bias',
+                'ethics',
+                'faithfulness',
+                'generative',
+                'hallucination',
+                'harmfulness',
+                'implausible_output',
+                'information_disclosure',
+                'jailbreak',
+                'llm',
+                'llm_harmful_content',
+                'llm_stereotypes_detector',
+                'misinformation',
+                'output_formatting',
+                'prompt_injection',
+                'robustness',
+                'stereotypes',
+                'sycophancy',
+                'text_generation',
+                'text_perturbation'}
+
+    def set_scanning_model(self, provider, model=None):
+        """
+        Sets the LLM model for Giskard based on the provider.
+
+        :param provider: The LLM provider (e.g., "openai", "gemini", "azure").
+        :param model: The specific model name to use (optional).
+        :raises ValueError: If the provider is "azure" and no model is provided.
+        """
+        default_models = {
+            "openai": "gpt-4o",
+            "gemini": "gemini-1.5-pro"
+        }
+
+        if provider == "azure" and model is None:
+            raise ValueError("Model must be provided for Azure.")
+
+        selected_model = model if model is not None else default_models.get(provider)
+
+        if selected_model is None:
+            raise ValueError(f"Unsupported provider: {provider}")
+
+        scanner.llm.set_llm_model(selected_model)

ragaai_catalyst/synthetic_data_generation.py

@@ -475,7 +475,7 @@ class SyntheticDataGeneration:
         Returns:
             list: A list of supported AI providers.
         """
-        return ['gemini', 'openai']
+        return ['gemini', 'openai','azure']
 
 # Usage:
 # from synthetic_data_generation import SyntheticDataGeneration

ragaai_catalyst/tracers/agentic_tracing/tracers/agent_tracer.py

@@ -9,6 +9,7 @@ import contextvars
 import asyncio
 from ..utils.file_name_tracker import TrackName
 from ..utils.span_attributes import SpanAttributes
+from .base import BaseTracer
 import logging
 
 logger = logging.getLogger(__name__)
@@ -48,15 +49,15 @@ class AgentTracerMixin:
         self.auto_instrument_network = False
 
     def trace_agent(
-        self,
-        name: str,
-        agent_type: str = None,
-        version: str = None,
-        capabilities: List[str] = None,
-        tags: List[str] = [],
-        metadata: Dict[str, Any] = {},
-        metrics: List[Dict[str, Any]] = [],
-        feedback: Optional[Any] = None,
+            self,
+            name: str,
+            agent_type: str = None,
+            version: str = None,
+            capabilities: List[str] = None,
+            tags: List[str] = [],
+            metadata: Dict[str, Any] = {},
+            metrics: List[Dict[str, Any]] = [],
+            feedback: Optional[Any] = None,
     ):
         if name not in self.span_attributes_dict:
             self.span_attributes_dict[name] = SpanAttributes(name)
@@ -199,8 +200,8 @@ class AgentTracerMixin:
                                                 children = tracer.agent_children.get()
                                                 if children:
                                                     if (
-                                                        "children"
-                                                        not in component["data"]
+                                                            "children"
+                                                            not in component["data"]
                                                     ):
                                                         component["data"][
                                                             "children"
@@ -263,7 +264,7 @@ class AgentTracerMixin:
         return decorator
 
     def _trace_sync_agent_execution(
-        self, func, name, agent_type, version, capabilities, top_level_hash_id, *args, **kwargs
+            self, func, name, agent_type, version, capabilities, top_level_hash_id, *args, **kwargs
     ):
         hash_id = top_level_hash_id
 
@@ -281,7 +282,7 @@ class AgentTracerMixin:
         component_id = str(uuid.uuid4())
 
         # Extract ground truth if present
-        ground_truth = kwargs.pop("gt") if kwargs else None
+        ground_truth = kwargs.pop("gt", None) if kwargs else None
         if ground_truth is not None:
             span = self.span(name)
             span.add_gt(ground_truth)
@@ -390,7 +391,7 @@ class AgentTracerMixin:
             self.agent_children.reset(children_token)
 
     async def _trace_agent_execution(
-        self, func, name, agent_type, version, capabilities, hash_id, *args, **kwargs
+            self, func, name, agent_type, version, capabilities, hash_id, *args, **kwargs
     ):
         """Asynchronous version of agent tracing"""
         if not self.is_active:
@@ -404,7 +405,7 @@ class AgentTracerMixin:
         component_id = str(uuid.uuid4())
 
         # Extract ground truth if present
-        ground_truth = kwargs.pop("gt") if kwargs else None
+        ground_truth = kwargs.pop("gt", None) if kwargs else None
         if ground_truth is not None:
             span = self.span(name)
             span.add_gt(ground_truth)
@@ -522,7 +523,7 @@ class AgentTracerMixin:
             for interaction in self.component_user_interaction.get(kwargs["component_id"], []):
                 if interaction["interaction_type"] in ["input", "output"]:
                     input_output_interactions.append(interaction)
-            interactions.extend(input_output_interactions) 
+            interactions.extend(input_output_interactions)
         if self.auto_instrument_file_io:
             file_io_interactions = []
             for interaction in self.component_user_interaction.get(kwargs["component_id"], []):
@@ -551,9 +552,14 @@ class AgentTracerMixin:
                 counter = sum(1 for x in self.visited_metrics if x.startswith(base_metric_name))
                 metric_name = f'{base_metric_name}_{counter}' if counter > 0 else base_metric_name
                 self.visited_metrics.append(metric_name)
-                metric["name"] = metric_name  
+                metric["name"] = metric_name
                 metrics.append(metric)
 
+        # TODO agent_trace execute metric
+        formatted_metrics = BaseTracer.get_formatted_metric(self.span_attributes_dict, self.project_id, name)
+        if formatted_metrics:
+            metrics.extend(formatted_metrics)
+
         component = {
             "id": kwargs["component_id"],
             "hash_id": kwargs["hash_id"],
@@ -609,22 +615,22 @@ class AgentTracerMixin:
         self.component_network_calls.set(component_network_calls)
 
     def _sanitize_input(self, args: tuple, kwargs: dict) -> dict:
-            """Sanitize and format input data, including handling of nested lists and dictionaries."""
-
-            def sanitize_value(value):
-                if isinstance(value, (int, float, bool, str)):
-                    return value
-                elif isinstance(value, list):
-                    return [sanitize_value(item) for item in value]
-                elif isinstance(value, dict):
-                    return {key: sanitize_value(val) for key, val in value.items()}
-                else:
-                    return str(value)  # Convert non-standard types to string
+        """Sanitize and format input data, including handling of nested lists and dictionaries."""
+
+        def sanitize_value(value):
+            if isinstance(value, (int, float, bool, str)):
+                return value
+            elif isinstance(value, list):
+                return [sanitize_value(item) for item in value]
+            elif isinstance(value, dict):
+                return {key: sanitize_value(val) for key, val in value.items()}
+            else:
+                return str(value)  # Convert non-standard types to string
 
-            return {
-                "args": [sanitize_value(arg) for arg in args],
-                "kwargs": {key: sanitize_value(val) for key, val in kwargs.items()},
-            }
+        return {
+            "args": [sanitize_value(arg) for arg in args],
+            "kwargs": {key: sanitize_value(val) for key, val in kwargs.items()},
+        }
 
     def _sanitize_output(self, output: Any) -> Any:
         """Sanitize and format output data"""
@@ -640,6 +646,6 @@ class AgentTracerMixin:
 
     def instrument_network_calls(self):
         self.auto_instrument_network = True
-        
+
     def instrument_file_io_calls(self):
         self.auto_instrument_file_io = True