SimplerLLM 0.1.0__py3-none-any.whl

SimplerLLM/prompts/prompt_builder.py

@@ -0,0 +1,86 @@
+class SimplePrompt:
+    """
+    A class for creating and manipulating simple prompt templates.
+    """
+
+    def __init__(self, template: str):
+        if not isinstance(template, str):
+            raise ValueError("Template must be a string")
+        self.template = template
+        self.content = ''  # Holds the latest filled template
+
+    def assign_parms(self, **kwargs) -> str:
+        """
+        Assigns parameters to the template and returns the filled template.
+        """
+        try:
+            self.content = self.template.format(**kwargs)
+        except KeyError as e:
+            raise KeyError(f"Missing a required key in the template: {e}")
+        except Exception as e:
+            # Catch-all for other exceptions related to string formatting
+            raise ValueError(f"Error processing the template: {e}")
+        return self.content
+
+    def update_template(self, new_template: str):
+        """
+        Updates the template and clears the latest content.
+        """
+        if not isinstance(new_template, str):
+            raise ValueError("New template must be a string")
+        self.template = new_template
+        self.content = ''
+
+    def __str__(self) -> str:
+        return self.content
+
+def create_prompt_template(template_string: str) -> SimplePrompt:
+    """
+    Factory function to create a SimpleTemplate instance.
+    """
+    if not isinstance(template_string, str):
+        raise ValueError("Template string must be a string")
+    return SimplePrompt(template_string)
+
+
+
+class MultiValuePrompt:
+    """
+    A class for creating and manipulating prompt templates with multiple sets of parameters.
+    """
+
+    def __init__(self, template: str):
+        if not isinstance(template, str):
+            raise ValueError("Template must be a string")
+        self.template = template
+        self.generated_prompts = []  # Holds the generated prompts
+
+    def generate_prompts(self, params_list: list) -> list:
+        """
+        Generates prompts for each set of parameters in the params_list.
+        """
+        if not all(isinstance(params, dict) for params in params_list):
+            raise ValueError("Each item in params_list must be a dictionary")
+
+        self.generated_prompts = []
+        for params in params_list:
+            try:
+                filled_prompt = self.template.format(**params)
+                self.generated_prompts.append(filled_prompt)
+            except KeyError as e:
+                raise KeyError(f"Missing a required key in the template: {e}")
+            except Exception as e:
+                raise ValueError(f"Error processing the template: {e}")
+
+        return self.generated_prompts
+
+    def __str__(self) -> str:
+        return "\n".join(self.generated_prompts)
+
+def create_multi_value_prompts(template_string: str) -> MultiValuePrompt:
+    """
+    Factory function to create a FewShotPrompt instance.
+    """
+    if not isinstance(template_string, str):
+        raise ValueError("Template string must be a string")
+    return MultiValuePrompt(template_string)

SimplerLLM/tools/generic_text_loader.py

@@ -0,0 +1,160 @@
+import newspaper
+import os
+import PyPDF2
+import docx
+from youtube_transcript_api import YouTubeTranscriptApi
+import re
+from urllib.parse import urlparse
+from pydantic import BaseModel
+from typing import Optional
+
+
+
+class TextDocument(BaseModel):
+    file_size: Optional[int] = None
+    word_count: int
+    character_count: int
+    content: str
+    title: Optional[str] = None
+    url_or_path: Optional[str] = None
+
+def load_text(input_path_or_url):
+    # Check if the input is a URL
+    input_path_or_url = str.lower(input_path_or_url)
+    if re.match(r'http[s]?://', input_path_or_url):
+        # Process based on URL content
+        if "youtube.com" in input_path_or_url or "youtu.be" in input_path_or_url:
+            content = __read_youtube_video_transcript(input_path_or_url)
+            file_size = len(content.encode('utf-8'))  # Size in bytes
+            return TextDocument(
+                word_count=len(content.split()),
+                character_count=len(content),
+                content=content,
+                file_size = file_size,
+                url_or_path=input_path_or_url
+            )
+        else:
+            article = __read_blog_from_url(input_path_or_url)
+            if article is not None:
+                file_size = len(article.text.encode('utf-8'))  # Size in bytes
+                return TextDocument(
+                    word_count=len(article.text.split()),
+                    character_count=len(article.text),
+                    content=article.text,
+                    title=article.title,
+                    file_size=file_size,
+                    url_or_path=input_path_or_url
+                )
+    else:
+        try:
+            # Process based on file extension
+            file_ext = os.path.splitext(input_path_or_url)[1].lower()
+            if file_ext in ['.txt']:
+                file_size, num_words, num_chars, content = __read_text_file(input_path_or_url)
+            elif file_ext in ['.docx']:
+                file_size, num_words, num_chars, content = __read_docx_file(input_path_or_url)
+            elif file_ext in ['.pdf']:
+                file_size, num_words, num_chars, content = __read_pdf_file(input_path_or_url)
+            else:
+                # Fallback: try reading as a text file
+                file_size, num_words, num_chars, content = __read_text_file(input_path_or_url)
+
+            return TextDocument(
+                    file_size=file_size,
+                    word_count=num_words,
+                    character_count=num_chars,
+                    content=content,
+                    url_or_path=input_path_or_url
+                )
+        except Exception as e:
+            raise ValueError(f"Error processing file: {e}")
+
+    raise ValueError("Unable to process the input")
+
+
+
+
+
+
+def __read_text_file(file_path):
+    with open(file_path, 'r',encoding='utf-8') as file:
+        content = file.read()
+
+    file_size = os.path.getsize(file_path)
+    words = content.split()
+    num_words = len(words)
+    num_chars = len(content)
+
+    return file_size, num_words, num_chars, content
+
+def __read_docx_file(file_path):
+    file_size = os.path.getsize(file_path)
+    doc = docx.Document(file_path)
+    content = "\n".join([para.text for para in doc.paragraphs])
+
+    words = content.split()
+    num_words = len(words)
+    num_chars = len(content)
+
+    return file_size, num_words, num_chars, content
+
+def __read_pdf_file(file_path):
+    file_size = os.path.getsize(file_path)
+
+    with open(file_path, 'rb') as file:
+        reader = PyPDF2.PdfReader(file)
+        content = "".join([reader.pages[i].extract_text() for i in range(len(reader.pages))])
+
+    words = content.split()
+    num_words = len(words)
+    num_chars = len(content)
+
+    return file_size, num_words, num_chars, content
+
+def __read_blog_from_url(url):
+    """
+    Extracts the text content from a given URL using the newspaper package.
+
+    Parameters:
+    url (str): The URL of the article to extract text from.
+
+    Returns:
+    str: The text content of the article if extraction is successful, None otherwise.
+    """
+    try:
+        article = newspaper.Article(url)
+        article.download()
+
+        if article.download_state == 2:
+            article.parse()
+            return article
+        else:
+            print(f"An error occurred while fetching the article")
+            return None
+    except newspaper.ArticleException as e:
+        print(f"An error occurred while fetching the article: {e}")
+        return None
+
+def __read_youtube_video_transcript(video_url):
+    """
+    Fetches the transcript of a YouTube video given its URL.
+
+    Parameters:
+    video_url (str): The URL of the YouTube video.
+
+    Returns:
+    str: The transcript of the video if available, raises an error otherwise.
+    """
+    # Enhanced regex to handle different YouTube URL formats
+    match = re.search(r"(?:youtube\.com/watch\?v=|youtu\.be/)([\w-]+)", video_url)
+    if match:
+        video_id = match.group(1)
+    else:
+        raise ValueError("Invalid YouTube URL")
+
+    try:
+        transcript = YouTubeTranscriptApi.get_transcript(video_id)
+        transcript_text = " ".join([line["text"] for line in transcript])
+        return transcript_text
+    except Exception as e:
+        raise f"An error occurred while fetching the transcript: {e}"

SimplerLLM/tools/json_helpers.py

@@ -0,0 +1,130 @@
+import re
+import json
+from pydantic import BaseModel, ValidationError
+from typing import get_type_hints
+from pydantic import BaseModel
+from typing import List, get_type_hints, Type
+
+
+
+def convert_pydantic_to_json(model_instance):
+    """
+    Converts a Pydantic model instance to a JSON string.
+
+    Args:
+        model_instance (YourModel): An instance of your Pydantic model.
+
+    Returns:
+        str: A JSON string representation of the model.
+    """
+    return model_instance.model_dump_json()
+
+def extract_json_from_text(text_response):
+    # This pattern matches a string that starts with '{' and ends with '}'
+    pattern = r'\{[^{}]*\}'
+
+    matches = re.finditer(pattern, text_response)
+    json_objects = []
+
+    for match in matches:
+        json_str = match.group(0)
+        try:
+            # Validate if the extracted string is valid JSON
+            json_obj = json.loads(json_str)
+            json_objects.append(json_obj)
+        except json.JSONDecodeError:
+            # Extend the search for nested structures
+            extended_json_str = extend_search(text_response, match.span())
+            try:
+                json_obj = json.loads(extended_json_str)
+                json_objects.append(json_obj)
+            except json.JSONDecodeError:
+                # Handle cases where the extraction is not valid JSON
+                continue
+
+    if json_objects:
+        return json_objects
+    else:
+        return None  # Or handle this case as you prefer
+
+def extend_search(text, span):
+    # Extend the search to try to capture nested structures
+    start, end = span
+    nest_count = 0
+    for i in range(start, len(text)):
+        if text[i] == '{':
+            nest_count += 1
+        elif text[i] == '}':
+            nest_count -= 1
+            if nest_count == 0:
+                return text[start:i+1]
+    return text[start:end]
+
+def validate_json_with_pydantic_model(model_class, json_data):
+    """
+    Validates JSON data against a specified Pydantic model.
+
+    Args:
+        model_class (BaseModel): The Pydantic model class to validate against.
+        json_data (dict or list): JSON data to validate. Can be a dict for a single JSON object, 
+                                  or a list for multiple JSON objects.
+
+    Returns:
+        list: A list of validated JSON objects that match the Pydantic model.
+        list: A list of errors for JSON objects that do not match the model.
+    """
+    validated_data = []
+    validation_errors = []
+
+    if isinstance(json_data, list):
+        for item in json_data:
+            try:
+                model_instance = model_class(**item)
+                validated_data.append(model_instance.dict())
+            except ValidationError as e:
+                validation_errors.append({"error": str(e), "data": item})
+    elif isinstance(json_data, dict):
+        try:
+            model_instance = model_class(**json_data)
+            validated_data.append(model_instance.dict())
+        except ValidationError as e:
+            validation_errors.append({"error": str(e), "data": json_data})
+    else:
+        raise ValueError("Invalid JSON data type. Expected dict or list.")
+
+    return validated_data, validation_errors
+
+def convert_json_to_pydantic_model(model_class, json_data):
+    try:
+        model_instance = model_class(**json_data)
+        return model_instance
+    except ValidationError as e:
+        print("Validation error:", e)
+        return None
+
+# Define a function to provide example values based on type
+def example_value_for_type(field_type: Type):
+    if field_type == str:
+        return "example_string"
+    elif field_type == int:
+        return 0
+    elif field_type == float:
+        return 0.0
+    elif field_type == bool:
+        return True
+    elif field_type == List[str]:
+        return ["generated text 1", "generated text 2"]
+    elif field_type == List[int]:
+        return [1, 2, 3]
+    else:
+        return "Unsupported type"
+
+# Function to generate a JSON example for any Pydantic model
+def generate_json_example_from_pydantic(model_class: Type[BaseModel]) -> str:
+    example_data = {}
+    for field_name, field_type in get_type_hints(model_class).items():
+        example_data[field_name] = example_value_for_type(field_type)
+    
+    model_instance = model_class(**example_data)
+    return model_instance.json()
+

SimplerLLM/tools/rapid_api.py

@@ -0,0 +1,109 @@
+from dotenv import load_dotenv
+import os
+import time
+import requests
+import aiohttp
+import asyncio
+from typing import Optional, Any, Dict
+
+load_dotenv()  # Load the environment variables
+
+class RapidAPIClient:
+    def __init__(self, api_key: Optional[str] = None, timeout: int = 30):
+        """
+        Initialize the RapidAPI client.
+
+        :param api_key: Optional API key. If not provided, it will be read from the environment variable 'RAPID_API_KEY'.
+        :param timeout: Request timeout in seconds.
+        """
+        self.api_key = api_key if api_key else os.getenv('RAPIDAPI_API_KEY')
+        self.timeout = timeout
+
+        if not self.api_key:
+            raise ValueError("API key must be provided or set as an environment variable 'RAPID_API_KEY'")
+
+    def _construct_headers(self, api_url: str, headers_extra: Optional[Dict[str, str]] = None) -> Dict[str, str]:
+        """
+        Construct headers for the API call.
+
+        :param api_url: URL of the RapidAPI endpoint
+        :param headers_extra: Additional headers if required by the API
+        :return: Dictionary of headers
+        """
+        headers = {
+            'x-rapidapi-key': self.api_key,
+            'x-rapidapi-host': api_url.split('/')[2]
+        }
+
+        if headers_extra:
+            headers.update(headers_extra)
+
+        return headers
+
+    def _check_response(self, response: requests.Response) -> Any:
+        """
+        Check the response status and return the JSON data if successful.
+
+        :param response: Response object from requests library.
+        :return: JSON response from the API
+        """
+        if response.status_code in [200, 201, 202, 204]:
+            return response.json() if response.text else None
+        response.raise_for_status()
+
+    def call_api(self, api_url: str, method: str = 'GET', headers_extra: Optional[Dict[str, str]] = None, params: Optional[Dict[str, str]] = None, data: Optional[Dict[str, str]] = None, json: Optional[Dict[str, Any]] = None, max_retries: int = 3, backoff_factor: int = 2) -> Any:
+        """
+        Make a synchronous API call to a RapidAPI endpoint.
+
+        :param api_url: URL of the RapidAPI endpoint
+        :param method: HTTP method ('GET' or 'POST')
+        :param headers_extra: Additional headers if required by the API
+        :param params: Query parameters for GET request
+        :param data: Form data for POST request
+        :param json: JSON data for POST request
+        :param max_retries: Maximum number of retries
+        :param backoff_factor: Factor by which the delay increases during each retry
+        :return: JSON response from the API
+        """
+        headers = self._construct_headers(api_url, headers_extra)
+        retries = 0
+
+        while retries < max_retries:
+            try:
+                with requests.request(method, api_url, headers=headers, params=params, data=data, json=json, timeout=self.timeout) as response:
+                    return self._check_response(response)
+            except requests.RequestException as e:
+                retries += 1
+                if retries >= max_retries:
+                    raise e
+                time.sleep(backoff_factor ** retries)
+
+    async def call_api_async(self, api_url: str, method: str = 'GET', headers_extra: Optional[Dict[str, str]] = None, params: Optional[Dict[str, str]] = None, data: Optional[Dict[str, str]] = None, json: Optional[Dict[str, Any]] = None, max_retries: int = 3, backoff_factor: int = 2) -> Any:
+        """
+        Make an asynchronous API call to a RapidAPI endpoint.
+
+        :param api_url: URL of the RapidAPI endpoint
+        :param method: HTTP method ('GET' or 'POST')
+        :param headers_extra: Additional headers if required by the API
+        :param params: Query parameters for GET request
+        :param data: Form data for POST request
+        :param json: JSON data for POST request
+        :param max_retries: Maximum number of retries
+        :param backoff_factor: Factor by which the delay increases during each retry
+        :return: JSON response from the API
+        """
+        headers = self._construct_headers(api_url, headers_extra)
+
+        async with aiohttp.ClientSession() as session:
+            retries = 0
+            while retries < max_retries:
+                try:
+                    async with session.request(method, api_url, headers=headers, params=params, data=data, json=json, timeout=self.timeout) as response:
+                        if response.status in [200, 201, 202, 204]:
+                            return await response.json() if response.text else None
+                        response.raise_for_status()
+                except aiohttp.ClientError as e:
+                    retries += 1
+                    if retries >= max_retries:
+                        raise e
+                    await asyncio.sleep(backoff_factor ** retries)

SimplerLLM/tools/serp.py

@@ -0,0 +1,79 @@
+from duckduckgo_search import DDGS, AsyncDDGS
+from dotenv import load_dotenv
+from urllib.parse import urlparse
+from pydantic import BaseModel, HttpUrl
+from typing import Optional, List
+
+class SearchResult(BaseModel):
+    URL: HttpUrl
+    Domain: Optional[str] = None
+    Title: Optional[str] = None
+    Description: Optional[str] = None
+
+
+def get_domain_from_url(url):
+    parsed_url = urlparse(url)
+    return parsed_url.netloc
+
+# Load environment variables
+load_dotenv()
+
+
+
+async def search_with_duck_duck_go_async(query, max_results=50):
+    """
+    Perform an asynchronous search using the DuckDuckGo search engine.
+
+    Args:
+    query (str): The search query string.
+    max_results (int, optional): The maximum number of results to return. Defaults to 50.
+
+    Returns:
+    str: A JSON string containing the search results, each result being a dictionary with URL, Title, and Description.
+    """
+    async with AsyncDDGS() as ddgs:
+        results = []
+        async for r in ddgs.text(query, max_results=max_results):
+            results.append(r)
+        result_data = []
+        for result in results:
+            # Ensure all keys exist to avoid key errors
+            url = result.get("href", None)
+            title = result.get("title", None)
+            description = result.get("body", None)
+            if url:
+                domain = get_domain_from_url(url)
+                result_data.append(SearchResult(URL=url, Domain=domain, Title=title, Description=description))
+            else:
+                result_data.append(SearchResult(URL=url, Title=title, Description=description))
+        
+        return result_data
+
+
+def search_with_duck_duck_go(query: str, max_results: int = 10) -> List[SearchResult]:
+    """
+    Perform a synchronous search using the DuckDuckGo search engine.
+
+    Args:
+        query (str): The search query string.
+        max_results (int, optional): The maximum number of results to return. Defaults to 50.
+
+    Returns:
+        List[SearchResult]: A list of SearchResult objects, each containing URL, Title, and Description from the search results.
+    """
+    with DDGS() as ddgs:
+        results = [r for r in ddgs.text(query, max_results=max_results)]
+        result_data = []
+        for result in results:
+            url = result.get("href", None)
+            title = result.get("title", None)
+            description = result.get("body", None)
+            if url:
+                domain = get_domain_from_url(url)
+                result_data.append(SearchResult(URL=url, Domain=domain, Title=title, Description=description))
+            else:
+                result_data.append(SearchResult(URL=url, Title=title, Description=description))
+        
+        return result_data
+
+