ragaai-catalyst 2.1.5b29__py3-none-any.whl → 2.1.5b30__py3-none-any.whl

ragaai_catalyst/synthetic_data_generation.py

@@ -1,25 +1,28 @@
 import os
-from groq import Groq
-import google.generativeai as genai
-import openai
-import PyPDF2
+import ast
 import csv
+import json
+import random
+import PyPDF2
 import markdown
 import pandas as pd
-import json
-from litellm import completion
-import litellm
 from tqdm import tqdm
+
+import openai
 import tiktoken
-# import internal_api_completion
-# import proxy_call
+import litellm
+import google.generativeai as genai
+from groq import Groq
+from litellm import completion
+
 from .internal_api_completion import api_completion as internal_api_completion
 from .proxy_call import api_completion as proxy_api_completion
-# from ragaai_catalyst import internal_api_completion
-# from ragaai_catalyst import proxy_call
-import ast
 
-# dotenv.load_dotenv()
+from typing import Optional, List, Dict, Any
+
+import logging
+
+logger = logging.getLogger(__name__)
 
 class SyntheticDataGeneration:
     """
@@ -329,6 +332,59 @@ class SyntheticDataGeneration:
 
         json_data = json.loads(content)
         return pd.DataFrame(json_data)
+    
+    def _generate_raw_llm_response(self, text, system_message: Optional[str] = None, model_config: Dict[str, Any] = dict(), api_key=None):
+        """
+        Generate questions using LiteLLM which supports multiple providers (OpenAI, Groq, Gemini, etc.).
+
+        Args:
+            text (str): The input text to generate questions from.
+            system_message (str): The system message for the AI model.
+            model_config (dict): Configuration dictionary containing model details.
+                Required keys:
+                - model: The model identifier (e.g., "gpt-4", "gemini-pro", "mixtral-8x7b-32768")
+                Optional keys:
+                - api_base: Custom API base URL if needed
+                - max_tokens: Maximum tokens in response
+                - temperature: Temperature for response generation
+            api_key (str, optional): The API key for the model provider.
+
+        Returns:
+            pandas.DataFrame: A DataFrame containing the generated questions and answers.
+
+        Raises:
+            Exception: If there's an error in generating the response.
+        """
+        messages = [
+            {"role": "system", "content": system_message},
+            {"role": "user", "content": text}
+        ]
+
+        completion_params = {
+            "model": model_config.get("model", 'gpt-4o'),
+            "messages": messages,
+            "api_key": api_key
+        }
+
+        if "api_base" in model_config:
+            completion_params["api_base"] = model_config["api_base"]
+        if "api_version" in model_config:
+            completion_params["api_version"] = model_config["api_version"]
+        if "max_tokens" in model_config:
+            completion_params["max_tokens"] = model_config["max_tokens"]
+        if "temperature" in model_config:
+            completion_params["temperature"] = model_config["temperature"]
+        if 'provider' in model_config:
+            completion_params['model'] = f'{model_config["provider"]}/{model_config["model"]}'
+
+        try:
+            response = completion(**completion_params)
+        except Exception as e:
+            if any(error in str(e).lower() for error in ["invalid api key", "incorrect api key", "unauthorized", "authentication"]):
+                raise ValueError(f"Invalid API key provided for {model_config.get('provider', 'the specified')} provider")
+            raise Exception(f"Error calling LLM API: {str(e)}")
+
+        return response.choices[0].message.content
 
     def _parse_response(self, response, provider):
         """
@@ -478,6 +534,281 @@ class SyntheticDataGeneration:
             list: A list of supported AI providers.
         """
         return ['gemini', 'openai','azure']
+    
+    def _get_init_ex_gen_prompt(self):
+        prompt = '''
+You are an expert example generator. Your task is to produce creative, relevant and varied examples according to the user instructions. 
+
+**Inputs**
+User Instruction: The user will provide guidance on how to generate examples, possibly accompanied by their own examples.
+User Examples[Optional]: The user may supply examples.
+User Context[Optional]: The user may supply context to generate the examples from.
+No of Examples: The total number of examples to produce.
+
+**Steps to follow**
+1. Carefully analyze the user's instruction
+2. If user examples are provided, check whether the user’s instructions refer to them specifically.
+3. If user context is provided, understand it thoroughly and identify relevant parts to generate examples.
+4. Comply with the system’s guidelines to generate examples, incorporating any user examples or user context as needed.
+
+**Output Format**:  
+- Present examples in a multiline string with each line a separate example.  
+- Avoid markdown or special formatting.
+- Omit any boilerplate texts.
+
+**Instructions for Diversity**:  
+- Vary the examples by context, tone, and (if applicable) technical complexity.
+- Include edge cases or unconventional scenarios.  
+- Ensure no two examples are conceptually identical.
+
+**Final Notes**:  
+- Focus on both originality and practical relevance.
+- Avoid repetitiveness in the examples.
+'''
+        return prompt
+    
+    def _get_iter_ex_gen_prompt(self):
+        prompt = '''
+You are an expert example generator. Your task is to produce creative, relevant and varied examples according to the user instructions. 
+
+**Inputs**
+User Instruction: The user will provide guidance on how to generate examples, possibly accompanied by their own examples.
+User Examples[Optional]: The user may supply examples.
+User Context[Optional]: The user may supply context to generate the examples from.
+No of Examples: The total number of examples to produce.
+Relevant Examples: Any examples that are relevant to the user's instruction.
+Irrelevant Examples: Any examples that are not relevant to the user's instruction.
+
+**Steps to follow**
+1. Carefully analyze the user's instruction
+2. If user examples are provided, check whether the user’s instructions refer to them specifically.
+3. If user context is provided, understand it thoroughly and identify relevant parts to generate examples.
+4. Review the relevant and irrelevant examples present, understanding the differences in them.
+5. Comply with the user's instruction to generate examples, similar to relevant examples and dissimilar to irrelevant ones.
+
+**Output Format**:  
+- Present examples in a multiline sting with each line a separate example.  
+- Avoid markdown or special formatting.
+- Omit any boilerplate texts.
+
+**Instructions for Diversity**:  
+- Vary the examples by context, tone, and (if applicable) technical complexity.
+- Include edge cases or unconventional scenarios.  
+- Ensure no two examples are conceptually identical.
+
+**Final Notes**:  
+- Focus on both originality and practical relevance.
+- Avoid repetitiveness in the examples.
+'''
+        return prompt
+    
+    def _generate_examples_iter(
+            self, 
+            user_instruction: str, 
+            user_examples: Optional[List[str] | str] = None, 
+            user_context: Optional[str] = None, 
+            relevant_examples: List[str]=[], irrelevant_examples: List[str]=[], 
+            no_examples: Optional[int] = None, 
+            model_config: Dict[str, Any] = dict(), 
+            api_key: Optional[str] = None
+            ):
+        if not no_examples:
+            no_examples = 5
+        relevant_examples_str = '\n'.join(relevant_examples)
+        irrelevant_examples_str = '\n'.join(irrelevant_examples)
+        user_message = f'**User Instruction:** {user_instruction}'
+        user_message += f'\n\n**No of Examples:** {no_examples}'
+        if user_examples:
+            if isinstance(user_examples, str):
+                user_examples_str = user_examples
+            elif isinstance(user_examples, list):
+                user_examples_str = "\n".join(user_examples)
+            else:
+                raise ValueError(f'Expected string or list of strings as user_examples got {type(user_examples)}')
+            user_message += f"\n\n**User Examples:** \n{user_examples_str}"
+        if relevant_examples:
+            user_message += f'\n\n**Relevant Examples:** \n{relevant_examples_str}'
+        if irrelevant_examples:
+            user_message += f'\n\n**Irrelevant Examples:** \n{irrelevant_examples_str}'
+        if user_context:
+            user_message += f'\n\n**User Context:** \n{user_context}'
+        system_prompt = self._get_iter_ex_gen_prompt()
+        return self._generate_raw_llm_response(user_message, system_prompt, model_config=model_config, api_key=api_key)
+    
+    def _generate_examples(
+            self, 
+            user_instruction:str, 
+            user_examples:Optional[List[str]|str]=None, 
+            user_context: Optional[str] = None, 
+            no_examples:Optional[int]=None, 
+            model_config: Dict[str, Any] = dict(), 
+            api_key: Optional[str] = None
+            ):
+        if not no_examples:
+            no_examples = 5
+        user_message = f"**User Instruction:** {user_instruction}"
+        if user_examples:
+            if isinstance(user_examples, str):
+                user_examples_str = user_examples
+            elif isinstance(user_examples, list):
+                user_examples_str = "\n".join(user_examples)
+            else:
+                raise ValueError(f'Expected string or list of strings as user_examples got {type(user_examples)}')
+            user_message += f"\n\n**User Examples:** \n{user_examples_str}"
+        if user_context:
+            user_message += f'\n\n**User Context:** \n{user_context}'
+        user_message += f'\n\n**No of Examples:** {no_examples}'
+        init_system_prompt = self._get_init_ex_gen_prompt()
+        return self._generate_raw_llm_response(user_message, init_system_prompt, model_config=model_config, api_key=api_key)
+    
+    def _get_valid_examples(self, user_indices_str: str, examples: List[str]):
+        valid_examples = []
+        try:
+            user_indices = user_indices_str.strip().split(',')
+            for index_str in user_indices:
+                try:
+                    index = int(index_str)
+                    if index <= 0 or index > len(examples):
+                        continue
+                except ValueError as e:
+                    continue
+                valid_examples.append(examples[index-1])
+        except Exception as e:
+            print(f'Error: {e}')
+        return valid_examples
+    
+    def generate_examples(
+        self, 
+        user_instruction: str, 
+        user_examples:Optional[List[str] | str] = None, 
+        no_examples: Optional[int] = None, 
+        model_config: Optional[Dict[str, Any]] = None, 
+        api_key: Optional[str] = None, 
+        max_iter: int = 0,
+        **kwargs
+        ):
+        if not model_config:
+            model_config = {}
+        provider = model_config.get("provider")
+        api_base = model_config.get("api_base")
+        api_version = model_config.get("api_version")
+        self._initialize_client(provider, api_key, api_base, api_version, internal_llm_proxy=kwargs.get("internal_llm_proxy", None))
+
+        if not no_examples:
+            no_examples = 5
+        relevant_examples = []
+        irrelevant_examples = []
+        max_relevant_examples = 5
+        max_irrelevant_examples = 10
+        while len(relevant_examples) <= max_relevant_examples or len(irrelevant_examples) <= max_irrelevant_examples:
+            if max_iter <= 0:
+                break
+            if len(relevant_examples) > max_relevant_examples:
+                relevant_examples = random.sample(relevant_examples, max_relevant_examples)
+            if len(irrelevant_examples) > max_irrelevant_examples:
+                irrelevant_examples = random.sample(irrelevant_examples, max_irrelevant_examples)
+            if relevant_examples or irrelevant_examples:
+                examples_str = self._generate_examples_iter(
+                    user_instruction = user_instruction, 
+                    user_examples = user_examples, 
+                    relevant_examples = relevant_examples, 
+                    irrelevant_examples = irrelevant_examples, 
+                    model_config = model_config, 
+                    api_key = api_key
+                    )
+            else:
+                examples_str = self._generate_examples(
+                    user_instruction = user_instruction, 
+                    user_examples = user_examples, 
+                    model_config = model_config, 
+                    api_key = api_key
+                )
+            examples = [example for example in examples_str.split('\n') if example.strip()]
+            print('Generated Examples:')
+            for i, example in enumerate(examples):
+                print(f'{i+1}. {example}')
+            relevant_indices = input('Enter the indices of relevant examples (comma-separated): ').strip()
+            if relevant_indices:
+                relevant_examples.extend(self._get_valid_examples(relevant_indices, examples))
+            irrelevant_indices = input('Enter the indices of irrelevant examples (comma-separated): ').strip()
+            if irrelevant_indices:
+                irrelevant_examples.extend(self._get_valid_examples(irrelevant_indices, examples))
+            max_iter -= 1
+        if len(relevant_examples) > max_relevant_examples:
+            fin_relevant_examples = random.sample(relevant_examples, max_relevant_examples)
+        else:
+            fin_relevant_examples = relevant_examples
+        if len(irrelevant_examples) > max_irrelevant_examples:
+            fin_irrelevant_examples = random.sample(irrelevant_examples, max_irrelevant_examples)
+        else:
+            fin_irrelevant_examples = irrelevant_examples
+        if relevant_examples or irrelevant_examples:
+            if len(relevant_examples) < no_examples:
+                more_no_examples = no_examples - len(relevant_examples)
+                final_examples_str = self._generate_examples_iter(
+                    user_instruction = user_instruction, 
+                    user_examples = user_examples, 
+                    relevant_examples = fin_relevant_examples, 
+                    irrelevant_examples = fin_irrelevant_examples, 
+                    no_examples = more_no_examples, 
+                    model_config = model_config, 
+                    api_key = api_key
+                    )
+                final_examples = [example for example in final_examples_str.split('\n') if example.strip()]
+                final_examples.extend(relevant_examples)
+            else:
+                final_examples = random.sample(relevant_examples, no_examples)
+        else:
+            final_examples_str = self._generate_examples(
+                user_instruction = user_instruction, 
+                user_examples = user_examples, 
+                no_examples = no_examples, 
+                model_config = model_config, 
+                api_key = api_key
+            )
+            final_examples = [example for example in final_examples_str.split('\n') if example.strip()]
+        return final_examples
+
+    
+    def generate_examples_from_csv(
+            self, 
+            csv_path: str, 
+            dst_csv_path: Optional[str] = None, 
+            no_examples: Optional[int] = None, 
+            model_config: Optional[Dict[str, Any]] = None, 
+            api_key: Optional[str] = None, 
+            **kwargs
+            ):
+        if not no_examples:
+            no_examples = 5
+        df = pd.read_csv(csv_path)
+        assert 'user_instruction' in df.columns, 'The csv must have a column named user_instruction'
+        fin_df_list = []
+        for i, row in df.iterrows():
+            user_instruction = row['user_instruction']
+            user_examples = row.get('user_examples')
+            user_context = row.get('user_context')
+            row_dict = row.to_dict()
+            examples = self.generate_examples(
+                user_instruction = user_instruction, 
+                user_examples = user_examples, 
+                user_context = user_context, 
+                no_examples = no_examples, 
+                model_config = model_config, 
+                api_key = api_key
+            )
+            row_dict['generated_examples'] = examples
+            fin_df_list.append(row_dict)
+        fin_df = pd.DataFrame(fin_df_list)
+        csv_file, csv_ext = os.path.splitext(csv_path)
+        if not dst_csv_path:
+            dst_csv_path = csv_file + '_with_examples' + csv_ext
+        dst_dir = os.path.dirname(dst_csv_path)
+        if dst_dir:
+            os.makedirs(dst_dir, exist_ok=True)
+        fin_df.to_csv(dst_csv_path)
+        logger.info(f'CSV with generated examples saved at {dst_csv_path}')
+
 
 # Usage:
 # from synthetic_data_generation import SyntheticDataGeneration

ragaai_catalyst/tracers/agentic_tracing/tracers/llm_tracer.py

@@ -626,12 +626,8 @@ class LLMTracerMixin:
             # TODO TO check i/p and o/p is according or not
             input = input_data["args"] if hasattr(input_data, "args") else input_data
             output = output_data.output_response if output_data else None
-            #print("Prompt input:",input)
             prompt = self.convert_to_content(input)
-            #print("Prompt Output: ",prompt)
-            #print("Response input: ",output)
             response = self.convert_to_content(output)
-            #print("Response output: ",response)
 
             # TODO: Execute & Add the User requested metrics here
             formatted_metrics = BaseTracer.get_formatted_metric(self.span_attributes_dict, self.project_id, name)
@@ -778,7 +774,7 @@ class LLMTracerMixin:
                     token_usage = extract_token_usage(result)
             else:
                 token_usage = extract_token_usage(result)
-            cost = calculate_llm_cost(token_usage, model_name, self.model_costs)
+            cost = calculate_llm_cost(token_usage, model_name, self.model_costs, self.model_custom_cost)
             parameters = extract_parameters(kwargs)
             input_data = extract_input_data(args, kwargs, result)
 
@@ -887,7 +883,7 @@ class LLMTracerMixin:
                     token_usage = extract_token_usage(result)
             else:
                 token_usage = extract_token_usage(result)
-            cost = calculate_llm_cost(token_usage, model_name, self.model_costs)
+            cost = calculate_llm_cost(token_usage, model_name, self.model_costs, self.model_custom_cost)
             parameters = extract_parameters(kwargs)
             input_data = extract_input_data(args, kwargs, result)
 

ragaai_catalyst/tracers/agentic_tracing/utils/llm_utils.py

@@ -45,6 +45,11 @@ def extract_model_name(args, kwargs, result):
             result = result.to_dict()
             if 'model_version' in result:
                 model = result['model_version']  
+    try:
+        if not model:
+            model = result.raw.model
+    except Exception as e:
+        pass
     
     
     # Normalize Google model names
@@ -150,6 +155,15 @@ def extract_token_usage(result):
             "total_tokens": getattr(metadata, "total_token_count", 0)
         }
     
+    # Handle ChatResponse format with raw usuage
+    if hasattr(result, "raw") and hasattr(result.raw, "usage"):
+        usage = result.raw.usage
+        return {
+            "prompt_tokens": getattr(usage, "prompt_tokens", 0),
+            "completion_tokens": getattr(usage, "completion_tokens", 0),
+            "total_tokens": getattr(usage, "total_tokens", 0)
+        }
+    
     # Handle ChatResult format with generations
     if hasattr(result, "generations") and result.generations:
         # Get the first generation
@@ -195,6 +209,7 @@ def num_tokens_from_messages(model="gpt-4o-mini-2024-07-18", prompt_messages=Non
             - completion_tokens: Number of tokens in the completion
             - total_tokens: Total number of tokens
     """
+    #import pdb; pdb.set_trace()
     try:
         encoding = tiktoken.encoding_for_model(model)
     except KeyError:
@@ -207,8 +222,8 @@ def num_tokens_from_messages(model="gpt-4o-mini-2024-07-18", prompt_messages=Non
         "gpt-4-32k-0314",
         "gpt-4-0613",
         "gpt-4-32k-0613",
-        "gpt-4o-mini-2024-07-18",
-        "gpt-4o-2024-08-06"
+        "gpt-4o-2024-08-06",
+        "gpt-4o-mini-2024-07-18"
         }:
         tokens_per_message = 3
         tokens_per_name = 1
@@ -290,15 +305,18 @@ def extract_input_data(args, kwargs, result):
     }
 
 
-def calculate_llm_cost(token_usage, model_name, model_costs):
+def calculate_llm_cost(token_usage, model_name, model_costs, model_custom_cost=None):
     """Calculate cost based on token usage and model"""
+    if model_custom_cost is None:
+        model_custom_cost = {}
+    model_costs.update(model_custom_cost)
     if not isinstance(token_usage, dict):
         token_usage = {
             "prompt_tokens": 0,
             "completion_tokens": 0,
             "total_tokens": token_usage if isinstance(token_usage, (int, float)) else 0
         }
-
+    
     # Get model costs, defaulting to default costs if unknown
     model_cost = model_cost = model_costs.get(model_name, {
         "input_cost_per_token": 0.0,   

ragaai_catalyst/tracers/agentic_tracing/utils/zip_list_of_unique_files.py

@@ -129,25 +129,12 @@ class JupyterNotebookHandler:
             # Check if running in Colab
             if JupyterNotebookHandler.is_running_in_colab():
                 try:
-                    from google.colab import drive
-                    if not os.path.exists('/content/drive'):
-                        drive.mount('/content/drive')
-                        # logger.info("Google Drive mounted successfully")
-
                     # Look for notebooks in /content first
                     ipynb_files = list(Path('/content').glob('*.ipynb'))
                     if ipynb_files:
                         current_nb = max(ipynb_files, key=os.path.getmtime)
                         # logger.info(f"Found current Colab notebook: {current_nb}")
                         return str(current_nb)
-                        
-                    # Then check Drive if mounted
-                    if os.path.exists('/content/drive'):
-                        drive_ipynb_files = list(Path('/content/drive').rglob('*.ipynb'))
-                        if drive_ipynb_files:
-                            current_nb = max(drive_ipynb_files, key=os.path.getmtime)
-                            # logger.info(f"Found Colab notebook in Drive: {current_nb}")
-                            return str(current_nb)
                 except Exception as e:
                     logger.warning(f"Error in Colab notebook detection: {str(e)}")
 

ragaai_catalyst/tracers/tracer.py

@@ -113,7 +113,7 @@ class Tracer(AgenticTracing):
             for key in ["llm", "tool", "agent", "user_interaction", "file_io", "network", "custom"]:
                 if key not in auto_instrumentation:
                     auto_instrumentation[key] = True
-        
+        self.model_custom_cost = {}
         super().__init__(user_detail=user_detail, auto_instrumentation=auto_instrumentation)
 
         self.project_name = project_name
@@ -176,7 +176,38 @@ class Tracer(AgenticTracing):
             self._upload_task = None
             # raise ValueError (f"Currently supported tracer types are 'langchain' and 'llamaindex'.")
 
-        
+    def set_model_cost(self, cost_config):
+        """
+        Set custom cost values for a specific model.
+
+        Args:
+            cost_config (dict): Dictionary containing model cost configuration with keys:
+                - model_name (str): Name of the model
+                - input_cost_per_token (float): Cost per input token
+                - output_cost_per_token (float): Cost per output token
+
+        Example:
+            tracer.set_model_cost({
+                "model_name": "gpt-4",
+                "input_cost_per_million_token": 6,
+                "output_cost_per_million_token": 2.40
+            })
+        """
+        if not isinstance(cost_config, dict):
+            raise TypeError("cost_config must be a dictionary")
+
+        required_keys = {"model_name", "input_cost_per_million_token", "output_cost_per_million_token"}
+        if not all(key in cost_config for key in required_keys):
+            raise ValueError(f"cost_config must contain all required keys: {required_keys}")
+
+        model_name = cost_config["model_name"]
+        self.model_custom_cost[model_name] = {
+            "input_cost_per_token": float(cost_config["input_cost_per_million_token"])/ 1000000,
+            "output_cost_per_token": float(cost_config["output_cost_per_million_token"]) /1000000
+        }
+
+
+
     def set_dataset_name(self, dataset_name):
         """
         Reinitialize the Tracer with a new dataset name while keeping all other parameters the same.

{ragaai_catalyst-2.1.5b29.dist-info → ragaai_catalyst-2.1.5b30.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: ragaai_catalyst
-Version: 2.1.5b29
+Version: 2.1.5b30
 Summary: RAGA AI CATALYST
 Author-email: Kiran Scaria <kiran.scaria@raga.ai>, Kedar Gaikwad <kedar.gaikwad@raga.ai>, Dushyant Mahajan <dushyant.mahajan@raga.ai>, Siddhartha Kosti <siddhartha.kosti@raga.ai>, Ritika Goel <ritika.goel@raga.ai>, Vijay Chaurasia <vijay.chaurasia@raga.ai>, Tushar Kumar <tushar.kumar@raga.ai>
 Requires-Python: <3.13,>=3.9
@@ -36,7 +36,8 @@ Requires-Dist: requests~=2.32.3
 Requires-Dist: GPUtil~=1.4.0
 Requires-Dist: ipynbname
 Requires-Dist: tiktoken>=0.7.0
-Requires-Dist: giskard~=2.16.0
+Requires-Dist: tomli>=2.0.0
+Requires-Dist: rich>=13.9.4
 Provides-Extra: dev
 Requires-Dist: pytest; extra == "dev"
 Requires-Dist: pytest-cov; extra == "dev"
@@ -534,6 +535,22 @@ sdg.get_supported_qna()
 
 # Get supported providers
 sdg.get_supported_providers()
+
+# Generate examples
+examples = sdg.generate_examples(
+    user_instruction = 'Generate query like this.', 
+    user_examples = 'How to do it?', # Can be a string or list of strings.
+    user_context = 'Context to generate examples', 
+    no_examples = 10, 
+    model_config = {"provider":"openai","model":"gpt-4o-mini"}
+)
+
+# Generate examples from a csv
+sdg.generate_examples_from_csv(
+    csv_path = 'path/to/csv', 
+    no_examples = 5, 
+    model_config = {'provider': 'openai', 'model': 'gpt-4o-mini'}
+)
 ```