ragaai-catalyst 2.1.5b29__py3-none-any.whl → 2.1.5b30__py3-none-any.whl

{ragaai_catalyst-2.1.5b29.dist-info → ragaai_catalyst-2.1.5b30.dist-info}/RECORD

@@ -1,4 +1,4 @@
-ragaai_catalyst/__init__.py,sha256=xGqvQoS_Ir_Lup1YUIVc5VlsIplRMnnh_-6qK_eB0u4,843
+ragaai_catalyst/__init__.py,sha256=2wfkucAbb3Bt_p2KHelkg9zBQp4yC4iZanltyieG18w,895
 ragaai_catalyst/_version.py,sha256=JKt9KaVNOMVeGs8ojO6LvIZr7ZkMzNN-gCcvryy4x8E,460
 ragaai_catalyst/dataset.py,sha256=YCj8Ovu6y38KEw-1HCe4xQWkmYPgfNTtMa8Q0g6B62o,29401
 ragaai_catalyst/evaluation.py,sha256=O96CydYVPh3duUmXjY6REIXMOR-tOPixSG-Qhrf636A,22955
@@ -9,15 +9,28 @@ ragaai_catalyst/internal_api_completion.py,sha256=DdICI5yfEudiOAIC8L4oxH0Qz7kX-B
 ragaai_catalyst/prompt_manager.py,sha256=W8ypramzOprrJ7-22d5vkBXIuIQ8v9XAzKDGxKsTK28,16550
 ragaai_catalyst/proxy_call.py,sha256=CHxldeceZUaLU-to_hs_Kf1z_b2vHMssLS_cOBedu78,5499
 ragaai_catalyst/ragaai_catalyst.py,sha256=5nVg3_-lcvhrXjNkPTeGhe3tdUjm_4ZIctOcqWXBkRA,17939
-ragaai_catalyst/redteaming.py,sha256=pvHfwaHZgrq0HYhygEUm6-WotAxA2X9Xg1Kj9NlEzAI,6803
-ragaai_catalyst/synthetic_data_generation.py,sha256=QdabyUErLHhEwE-grlD-OJgqbL9N0S_tCPp9GmuMSyo,21617
+ragaai_catalyst/redteaming_old.py,sha256=W2d89Ok8W-C8g7TBM3fDIFLof3q9FuYSr0jcryH2XQo,7097
+ragaai_catalyst/synthetic_data_generation.py,sha256=6PUHNlFFZ0WXoR2mnjT50uven7FeCvbNRmFtmK6ta1E,37057
 ragaai_catalyst/utils.py,sha256=TlhEFwLyRU690HvANbyoRycR3nQ67lxVUQoUOfTPYQ0,3772
+ragaai_catalyst/redteaming/__init__.py,sha256=TJdvZpaZGFsg9qKONdjTosSVLZGadYFpHG6KE0xapKU,155
+ragaai_catalyst/redteaming/evaluator.py,sha256=FZzSlKhNkGAnKKNAhyAb-hIWkKVckZefQETaaHDR-z8,4512
+ragaai_catalyst/redteaming/llm_generator.py,sha256=if8fqsPARF4Jjj_RS9GePLDwq5vh8VeVYjs3gnHuZLI,3380
+ragaai_catalyst/redteaming/llm_generator_litellm.py,sha256=Sus87gMXNYqDdkRSvBMDCzZbm2gQN6W4uPKru02xq0w,2580
+ragaai_catalyst/redteaming/red_teaming.py,sha256=a7JUDAYwMhjB-L0u0rexdYjX5IuhS1jOQWuZUoS1HVk,14871
+ragaai_catalyst/redteaming/requirements.txt,sha256=7JJZi9DsGKqwa8-aPQjI__qMaWFIKKQzpxpv0251xx4,54
+ragaai_catalyst/redteaming/upload_result.py,sha256=Z23_6OqfRKczRfM7VsN6byAvb_P2bDiIKWy0uf9tQWQ,894
+ragaai_catalyst/redteaming/config/detectors.toml,sha256=niHhXW7mpCQ5NOdjJWMPI5OB9h4On_tZzNskROVjR6w,312
+ragaai_catalyst/redteaming/data_generator/scenario_generator.py,sha256=sAwOYCJk7tbK9oa9mfDoQllWFV3zYPR5KrcV9G2vXEs,3346
+ragaai_catalyst/redteaming/data_generator/test_case_generator.py,sha256=UgXyuLYmwkGOe45Jc3OzXI_c_xp1U7KMlnr6u7my6zQ,4303
+ragaai_catalyst/redteaming/tests/grok.ipynb,sha256=g6p4MVBhdla3IG4Atk56IPsj7lSh6-wxxhHadYJaK8s,2385
+ragaai_catalyst/redteaming/tests/stereotype.ipynb,sha256=-FoA3BxTF3vZs3U5c7N-Q3oirHyV2Yb8g_nl0qD_8jk,121539
+ragaai_catalyst/redteaming/utils/issue_description.py,sha256=iB0XbeOjdqHTPrikCKS_wOtJW4_JKfQPI1mgyvX0V-Q,6946
 ragaai_catalyst/tracers/__init__.py,sha256=LfgTes-nHpazssbGKnn8kyLZNr49kIPrlkrqqoTFTfc,301
 ragaai_catalyst/tracers/distributed.py,sha256=MwlBwIxCAng-OI-7Ove_rkE1mTLeuW4Jw-wWEVJBNlI,9968
 ragaai_catalyst/tracers/langchain_callback.py,sha256=KooENtkX0Hp0S_d_1WI3iH3qNVt-ZcnwOKVlydv4dUk,33518
 ragaai_catalyst/tracers/llamaindex_callback.py,sha256=ZY0BJrrlz-P9Mg2dX-ZkVKG3gSvzwqBtk7JL_05MiYA,14028
 ragaai_catalyst/tracers/llamaindex_instrumentation.py,sha256=Ys_jLkvVqo12bKgXDmkp4TxJu9HkBATrFE8cIcTYxWw,14329
-ragaai_catalyst/tracers/tracer.py,sha256=NZh3lgp2sqcUoovgLFkNC-XTnHAOR9UVUrLrGIECPOg,21563
+ragaai_catalyst/tracers/tracer.py,sha256=iO5m4ev0ruXjp-YjRMuN_mhr0Cd499Phz6ZYlfJ1yak,22889
 ragaai_catalyst/tracers/upload_traces.py,sha256=OKsc-Obf8bJvKBprt3dqj8GQQNkoX3kT_t8TBDi9YDQ,5670
 ragaai_catalyst/tracers/agentic_tracing/README.md,sha256=X4QwLb7-Jg7GQMIXj-SerZIgDETfw-7VgYlczOR8ZeQ,4508
 ragaai_catalyst/tracers/agentic_tracing/__init__.py,sha256=yf6SKvOPSpH-9LiKaoLKXwqj5sez8F_5wkOb91yp0oE,260
@@ -34,7 +47,7 @@ ragaai_catalyst/tracers/agentic_tracing/tracers/agent_tracer.py,sha256=LzbsHvELw
 ragaai_catalyst/tracers/agentic_tracing/tracers/base.py,sha256=0eFrHtEddxYBQnkilGXOAMpen09bFV3jkymywTjDSU4,46139
 ragaai_catalyst/tracers/agentic_tracing/tracers/custom_tracer.py,sha256=OBJJjFSvwRjCGNJyqX3yIfC1W05ZN2QUXasCJ4gmCjQ,13930
 ragaai_catalyst/tracers/agentic_tracing/tracers/langgraph_tracer.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ragaai_catalyst/tracers/agentic_tracing/tracers/llm_tracer.py,sha256=knHM7mzr7JZhA0kEzwdNQ0ESDoBHYyi7Ffn5hn6jQGA,50360
+ragaai_catalyst/tracers/agentic_tracing/tracers/llm_tracer.py,sha256=BvC_jCN2EObfFZKu4-POnE_YmlH4Q_pjF9gZLV1AwZ0,50226
 ragaai_catalyst/tracers/agentic_tracing/tracers/main_tracer.py,sha256=PYYNNeFfsQpw5D4A0jzwNYhAvC1bMT5vtAGaTsgk2xY,16112
 ragaai_catalyst/tracers/agentic_tracing/tracers/network_tracer.py,sha256=m8CxYkl7iMiFya_lNwN1ykBc3Pmo-2pR_2HmpptwHWQ,10352
 ragaai_catalyst/tracers/agentic_tracing/tracers/tool_tracer.py,sha256=xxrliKPfdfbIZRZqMnUewsaTD8_Hv0dbuoBivNZGD4U,21674
@@ -50,14 +63,14 @@ ragaai_catalyst/tracers/agentic_tracing/utils/create_dataset_schema.py,sha256=lg
 ragaai_catalyst/tracers/agentic_tracing/utils/file_name_tracker.py,sha256=YG601l1a29ov9VPu9Vl4RXxgL7l16k54_WWnoTNoG58,2064
 ragaai_catalyst/tracers/agentic_tracing/utils/generic.py,sha256=WwXT01xmp8MSr7KinuDCSK9a1ifpLcT7ajFkvYviG_A,1190
 ragaai_catalyst/tracers/agentic_tracing/utils/get_user_trace_metrics.py,sha256=vPZ4dn4EHFW0kqd1GyRpsYXbfrRrd0DXCmh-pzsDBNE,1109
-ragaai_catalyst/tracers/agentic_tracing/utils/llm_utils.py,sha256=gbPqWtJINW8JVlkM41UmF5zGR8oj8Q6g9KQIS3moQYM,20439
+ragaai_catalyst/tracers/agentic_tracing/utils/llm_utils.py,sha256=h29o5xkEjE9v71pRTtJJTM_fPTj4GijCPYE0xypNF7E,21093
 ragaai_catalyst/tracers/agentic_tracing/utils/model_costs.json,sha256=2tzGw_cKCTPcfjEm7iGvFE6pTw7gMTPzeBov_MTaXNY,321336
 ragaai_catalyst/tracers/agentic_tracing/utils/span_attributes.py,sha256=qmODERcFZhc8MX24boFCXkkh6sJ-vZngRHPvxhyWFeE,4347
 ragaai_catalyst/tracers/agentic_tracing/utils/supported_llm_provider.toml,sha256=LvFDivDIE96Zasp-fgDEqUJ5GEQZUawQucR3aOcSUTY,926
 ragaai_catalyst/tracers/agentic_tracing/utils/system_monitor.py,sha256=H8WNsk4v_5T6OUw4TFOzlDLjQhJwjh1nAMyMAoqMEi4,6946
 ragaai_catalyst/tracers/agentic_tracing/utils/trace_utils.py,sha256=go7FVnofviATDph-j8sk2juv09CGSRt1Vq4U868Fhd8,2259
 ragaai_catalyst/tracers/agentic_tracing/utils/unique_decorator.py,sha256=G027toV-Km20JjKrc-Y_PilQ8ABEKrBvvzgLTnqVg7I,5819
-ragaai_catalyst/tracers/agentic_tracing/utils/zip_list_of_unique_files.py,sha256=4Iu_pJPVS6C7S6aHvJkz7q6jVw135EVAiiaTjtyjQ8s,20180
+ragaai_catalyst/tracers/agentic_tracing/utils/zip_list_of_unique_files.py,sha256=4TeCGsFF26249fV6dJHLTZDrRa93SG9oer4rudoF8Y4,19443
 ragaai_catalyst/tracers/exporters/__init__.py,sha256=kVA8zp05h3phu4e-iHSlnznp_PzMRczB7LphSsZgUjg,138
 ragaai_catalyst/tracers/exporters/file_span_exporter.py,sha256=RgGteu-NVGprXKkynvyIO5yOjpbtA41R3W_NzCjnkwE,6445
 ragaai_catalyst/tracers/exporters/raga_exporter.py,sha256=6xvjWXyh8XPkHKSLLmAZUQSvwuyY17ov8pv2VdfI0qA,17875
@@ -71,8 +84,8 @@ ragaai_catalyst/tracers/utils/convert_llama_instru_callback.py,sha256=8qLo7x4Zsn
 ragaai_catalyst/tracers/utils/extraction_logic_llama_index.py,sha256=ZhPs0YhVtB82-Pq9o1BvCinKE_WPvVxPTEcZjlJbFYM,2371
 ragaai_catalyst/tracers/utils/langchain_tracer_extraction_logic.py,sha256=XS2_x2qneqEx9oAighLg-LRiueWcESLwIC2r7eJT-Ww,3117
 ragaai_catalyst/tracers/utils/utils.py,sha256=ViygfJ7vZ7U0CTSA1lbxVloHp4NSlmfDzBRNCJuMhis,2374
-ragaai_catalyst-2.1.5b29.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-ragaai_catalyst-2.1.5b29.dist-info/METADATA,sha256=VtRy6oG0L-rNaky2LVNVNvXV6pAHjdRs6aMah4Qp3yM,19472
-ragaai_catalyst-2.1.5b29.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-ragaai_catalyst-2.1.5b29.dist-info/top_level.txt,sha256=HpgsdRgEJMk8nqrU6qdCYk3di7MJkDL0B19lkc7dLfM,16
-ragaai_catalyst-2.1.5b29.dist-info/RECORD,,
+ragaai_catalyst-2.1.5b30.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+ragaai_catalyst-2.1.5b30.dist-info/METADATA,sha256=qnPc-QVuCF7AesHJx66KVLwkIirI-2eIJfI0napvEOc,20005
+ragaai_catalyst-2.1.5b30.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+ragaai_catalyst-2.1.5b30.dist-info/top_level.txt,sha256=HpgsdRgEJMk8nqrU6qdCYk3di7MJkDL0B19lkc7dLfM,16
+ragaai_catalyst-2.1.5b30.dist-info/RECORD,,

ragaai_catalyst/redteaming.py

@@ -1,171 +0,0 @@
-import logging
-import os
-from typing import Callable, Optional
-
-import giskard as scanner
-import pandas as pd
-
-logging.getLogger('giskard.core').disabled = True
-logging.getLogger('giskard.scanner.logger').disabled = True
-logging.getLogger('giskard.models.automodel').disabled = True
-logging.getLogger('giskard.datasets.base').disabled = True
-logging.getLogger('giskard.utils.logging_utils').disabled = True
-
-
-class RedTeaming:
-
-    def __init__(self,
-                 provider: Optional[str] = "openai",
-                 model: Optional[str] = None,
-                 api_key: Optional[str] = None,
-                 api_base: Optional[str] = None,
-                 api_version: Optional[str] = None):
-        self.provider = provider.lower()
-        self.model = model
-        if not self.provider:
-            raise ValueError("Model configuration must be provided with a valid provider and model.")
-        if self.provider == "openai":
-            if api_key is not None:
-                os.environ["OPENAI_API_KEY"] = api_key
-            if os.getenv("OPENAI_API_KEY") is None:
-                raise ValueError("API key must be provided for OpenAI.")
-        elif self.provider == "gemini":
-            if api_key is not None:
-                os.environ["GEMINI_API_KEY"] = api_key
-            if os.getenv("GEMINI_API_KEY") is None:
-                raise ValueError("API key must be provided for Gemini.")
-        elif self.provider == "azure":
-            if api_key is not None:
-                os.environ["AZURE_API_KEY"] = api_key
-            if api_base is not None:
-                os.environ["AZURE_API_BASE"] = api_base
-            if api_version is not None:
-                os.environ["AZURE_API_VERSION"] = api_version
-            if os.getenv("AZURE_API_KEY") is None:
-                raise ValueError("API key must be provided for Azure.")
-            if os.getenv("AZURE_API_BASE") is None:
-                raise ValueError("API base must be provided for Azure.")
-            if os.getenv("AZURE_API_VERSION") is None:
-                raise ValueError("API version must be provided for Azure.")
-        else:
-            raise ValueError(f"Provider is not recognized.")
-
-    def run_scan(
-            self,
-            model: Callable,
-            evaluators: Optional[list] = None,
-            save_report: bool = True
-    ) -> pd.DataFrame:
-        """
-        Runs red teaming on the provided model and returns a DataFrame of the results.
-
-        :param model: The model function provided by the user (can be sync or async).
-        :param evaluators: Optional list of scan metrics to run.
-        :param save_report: Boolean flag indicating whether to save the scan report as a CSV file.
-        :return: A DataFrame containing the scan report.
-        """
-        import asyncio
-        import inspect
-
-        self.set_scanning_model(self.provider, self.model)
-
-        supported_evaluators = self.get_supported_evaluators()
-        if evaluators:
-            if isinstance(evaluators, str):
-                evaluators = [evaluators]
-            invalid_evaluators = [evaluator for evaluator in evaluators if evaluator not in supported_evaluators]
-            if invalid_evaluators:
-                raise ValueError(f"Invalid evaluators: {invalid_evaluators}. "
-                                 f"Allowed evaluators: {supported_evaluators}.")
-
-        # Handle async model functions by wrapping them in a sync function
-        if inspect.iscoroutinefunction(model):
-            def sync_wrapper(*args, **kwargs):
-                try:
-                    # Try to get the current event loop
-                    loop = asyncio.get_event_loop()
-                except RuntimeError:
-                    # If no event loop exists (e.g., in Jupyter), create a new one
-                    loop = asyncio.new_event_loop()
-                    asyncio.set_event_loop(loop)
-
-                try:
-                    # Handle both IPython and regular Python environments
-                    import nest_asyncio
-                    nest_asyncio.apply()
-                except ImportError:
-                    pass  # nest_asyncio not available, continue without it
-
-                return loop.run_until_complete(model(*args, **kwargs))
-            wrapped_model = sync_wrapper
-        else:
-            wrapped_model = model
-
-        model_instance = scanner.Model(
-            model=wrapped_model,
-            model_type="text_generation",
-            name="RagaAI's Scan",
-            description="RagaAI's RedTeaming Scan",
-            feature_names=["question"],
-        )
-
-        try:
-            report = scanner.scan(model_instance, only=evaluators, raise_exceptions=True) if evaluators \
-                     else scanner.scan(model_instance, raise_exceptions=True)
-        except Exception as e:
-            raise RuntimeError(f"Error occurred during model scan: {str(e)}")
-
-        report_df = report.to_dataframe()
-
-        if save_report:
-            report_df.to_csv("raga-ai_red-teaming_scan.csv", index=False)
-
-        return report_df
-
-    def get_supported_evaluators(self):
-        """Contains tags corresponding to the 'llm' and 'robustness' directories in the giskard > scanner library"""
-        return {'control_chars_injection',
-                'discrimination',
-                'ethical_bias',
-                'ethics',
-                'faithfulness',
-                'generative',
-                'hallucination',
-                'harmfulness',
-                'implausible_output',
-                'information_disclosure',
-                'jailbreak',
-                'llm',
-                'llm_harmful_content',
-                'llm_stereotypes_detector',
-                'misinformation',
-                'output_formatting',
-                'prompt_injection',
-                'robustness',
-                'stereotypes',
-                'sycophancy',
-                'text_generation',
-                'text_perturbation'}
-
-    def set_scanning_model(self, provider, model=None):
-        """
-        Sets the LLM model for Giskard based on the provider.
-
-        :param provider: The LLM provider (e.g., "openai", "gemini", "azure").
-        :param model: The specific model name to use (optional).
-        :raises ValueError: If the provider is "azure" and no model is provided.
-        """
-        default_models = {
-            "openai": "gpt-4o",
-            "gemini": "gemini-1.5-pro"
-        }
-
-        if provider == "azure" and model is None:
-            raise ValueError("Model must be provided for Azure.")
-
-        selected_model = model if model is not None else default_models.get(provider)
-
-        if selected_model is None:
-            raise ValueError(f"Unsupported provider: {provider}")
-
-        scanner.llm.set_llm_model(selected_model)