webscout 6.3__py3-none-any.whl → 6.5__py3-none-any.whl

webscout/Extra/autollama.py

@@ -1,196 +1,247 @@
-import warnings
-from datetime import time
-import os
-import sys
-import subprocess
-import logging
-import psutil
-from huggingface_hub import hf_hub_download  # Updated import
-import colorlog
-import ollama
-import argparse
-
-# Suppress specific warnings
-warnings.filterwarnings(
-    "ignore", category=FutureWarning, module="huggingface_hub.file_download"
-)
-
-# Configure logging with colors
-handler = colorlog.StreamHandler()
-handler.setFormatter(
-    colorlog.ColoredFormatter(
-        "%(log_color)s%(asctime)s - %(levelname)s - %(message)s",
-        datefmt="%Y-%m-%d %H:%M:%S",
-        log_colors={
-            "DEBUG": "cyan",
-            "INFO": "green",
-            "WARNING": "yellow",
-            "ERROR": "red",
-            "CRITICAL": "red,bg_white",
-        },
-    )
-)
-
-logger = colorlog.getLogger(__name__)
-if not logger.hasHandlers():
-    logger.addHandler(handler)
-    logger.setLevel(logging.INFO)
-
-logging.captureWarnings(True)
-py_warnings_logger = logging.getLogger("py.warnings")
-if not py_warnings_logger.hasHandlers():
-    py_warnings_logger.addHandler(handler)
-
-
-def show_art():
-    logger.info("Made with love in India")
-
-
-def usage():
-    logger.info("Usage: python script.py -m <model_path> -g <gguf_file>")
-    logger.info("Options:")
-    logger.info("  -m <model_path>    Set the path to the model")
-    logger.info("  -g <gguf_file>     Set the GGUF file name")
-    logger.info("  -h                 Display this help and exit")
-
-
-def is_model_downloaded(logging_name, download_log):
-    if not os.path.exists(download_log):
-        return False
-    with open(download_log, "r") as f:
-        for line in f:
-            if line.strip() == logging_name:
-                return True
-    return False
-
-
-def log_downloaded_model(logging_name, download_log):
-    with open(download_log, "a") as f:
-        f.write(logging_name + "\n")
-
-
-def is_model_created(model_name):
-    result = subprocess.run(["ollama", "list"], stdout=subprocess.PIPE)
-    return model_name in result.stdout.decode("utf-8")
-
-
-def download_model(repo_id, filename, token, cache_dir="downloads"):
-    """
-    Downloads a model file from the Hugging Face Hub using hf_hub_download.
-    """
-    try:
-        os.makedirs(cache_dir, exist_ok=True)
-        
-        # Download using hf_hub_download
-        filepath = hf_hub_download(
-            repo_id=repo_id,
-            filename=filename,
-            token=token,
-            cache_dir=cache_dir,
-            resume_download=True,
-            force_download=False,
-            local_files_only=False
-        )
-        
-        # Ensure file is in the expected location
-        expected_path = os.path.join(cache_dir, filename)
-        if filepath != expected_path:
-            os.makedirs(os.path.dirname(expected_path), exist_ok=True)
-            if not os.path.exists(expected_path):
-                import shutil
-                shutil.copy2(filepath, expected_path)
-            filepath = expected_path
-            
-        return filepath
-    
-    except Exception as e:
-        logger.error(f"Error downloading model: {str(e)}")
-        raise
-
-
-def is_ollama_running():
-    for proc in psutil.process_iter(["name"]):
-        if proc.info["name"] in ["ollama", "ollama.exe"]:
-            return True
-    return False
-
-
-def main(model_path=None, gguf_file=None):
-    show_art()
-
-    parser = argparse.ArgumentParser(description="Download and create an Ollama model")
-    parser.add_argument("-m", "--model_path", help="Path to the model on Hugging Face Hub")
-    parser.add_argument("-g", "--gguf_file", help="Name of the GGUF file")
-    args = parser.parse_args()
-
-    model_path = args.model_path if args.model_path else model_path
-    gguf_file = args.gguf_file if args.gguf_file else gguf_file
-
-    if not model_path or not gguf_file:
-        logger.error("Error: model_path and gguf_file are required.")
-        usage()
-        sys.exit(2)
-
-    model_name = gguf_file.split(".Q4")[0]
-    download_log = "downloaded_models.log"
-    logging_name = f"{model_path}_{model_name}"
-
-    if not os.path.exists(download_log):
-        with open(download_log, 'w') as f:
-            pass
-
-    try:
-        subprocess.check_output(['pip', 'show', 'huggingface-hub'])
-    except subprocess.CalledProcessError:
-        logger.info("Installing huggingface-hub...")
-        subprocess.check_call(['pip', 'install', '-U', 'huggingface_hub[cli]'])
-    else:
-        logger.info("huggingface-hub is already installed.")
-
-    if is_model_downloaded(logging_name, download_log):
-        logger.info(f"Model {logging_name} has already been downloaded. Skipping download.")
-    else:
-        logger.info(f"Downloading model {logging_name}...")
-        token = os.getenv('HUGGINGFACE_TOKEN', None)
-        if not token:
-            logger.warning("Warning: HUGGINGFACE_TOKEN environment variable is not set. Using None.")
-        
-        filepath = download_model(model_path, gguf_file, token)
-        log_downloaded_model(logging_name, download_log)
-        logger.info(f"Model {logging_name} downloaded and logged.")
-
-    try:
-        subprocess.check_output(['ollama', '--version'])
-    except subprocess.CalledProcessError:
-        logger.info("Installing Ollama...")
-        subprocess.check_call(['curl', '-fsSL', 'https://ollama.com/install.sh', '|', 'sh'])
-    else:
-        logger.info("Ollama is already installed.")
-
-    if is_ollama_running():
-        logger.info("Ollama is already running. Skipping the start.")
-    else:
-        logger.info("Starting Ollama...")
-        subprocess.Popen(['ollama', 'serve'])
-
-        while not is_ollama_running():
-            logger.info("Waiting for Ollama to start...")
-            time.sleep(1)
-
-        logger.info("Ollama has started.")
-
-    if is_model_created(model_name):
-        logger.info(f"Model {model_name} is already created. Skipping creation.")
-    else:
-        logger.info(f"Creating model {model_name}...")
-        with open('Modelfile', 'w') as f:
-            f.write(f"FROM ./downloads/{gguf_file}")
-        subprocess.check_call(['ollama', 'create', model_name, '-f', 'Modelfile'])
-        logger.info(f"Model {model_name} created.")
-
-    logger.info(f"model name is > {model_name}")
-    logger.info(f"Use Ollama run {model_name}")
-
-
-if __name__ == "__main__":
+"""
+Yo fam! 🔥 Welcome to AutoLlama - your go-to tool for downloading and setting up HelpingAI models! 💪
+
+Created by the legendary Abhay Koul, this script's got your back when it comes to:
+- Downloading models straight from HuggingFace Hub 🚀
+- Setting up Ollama with zero hassle 💯
+- Getting your AI assistant ready to vibe with you! ⚡
+
+Usage:
+>>> python -m webscout.Extra.autollama download -m "OEvortex/HelpingAI-Lite-1.5T" -g "HelpingAI-Lite-1.5T.q4_k_m.gguf"
+
+Features:
+- Smart model management 🧠
+- Automatic dependency installation 📦
+- Progress tracking that keeps it real 📈
+- Error handling that's got your back 💪
+
+Join the squad on Discord and level up your AI game! 🎮
+"""
+
+import warnings
+from datetime import time
+import os
+import sys
+import subprocess
+import psutil
+from huggingface_hub import hf_hub_download
+from ..Litlogger import LitLogger, LogFormat, ColorScheme
+from ..swiftcli import CLI, option
+# import ollama
+
+# Suppress specific warnings
+warnings.filterwarnings(
+    "ignore", category=FutureWarning, module="huggingface_hub.file_download"
+)
+
+# Initialize LitLogger with custom format and colors
+logger = LitLogger(
+    name="AutoLlama",
+    format=LogFormat.MODERN_EMOJI,
+    color_scheme=ColorScheme.OCEAN
+)
+
+def show_art():
+    """Dropping that signature HAI love! 💝 Made with passion in India! 🇮🇳"""
+    logger.info("Made with love in India")
+
+def usage():
+    logger.info("Usage: python script.py -m <model_path> -g <gguf_file>")
+    logger.info("Options:")
+    logger.info("  -m <model_path>    Set the path to the model")
+    logger.info("  -g <gguf_file>     Set the GGUF file name")
+    logger.info("  -h                 Display this help and exit")
+
+def is_model_downloaded(logging_name, download_log):
+    """
+    Checking if we already got that model downloaded! 🔍
+    
+    Args:
+        logging_name (str): The model's unique name in our records 📝
+        download_log (str): Where we keep track of our downloads 📋
+        
+    Returns:
+        bool: True if we got it, False if we need to grab it! 💯
+    """
+    if not os.path.exists(download_log):
+        return False
+    with open(download_log, "r") as f:
+        for line in f:
+            if line.strip() == logging_name:
+                return True
+    return False
+
+def log_downloaded_model(logging_name, download_log):
+    """
+    Keeping track of our downloaded models like a boss! 📝
+    
+    Args:
+        logging_name (str): Model's name to remember 🏷️
+        download_log (str): Our download history file 📋
+    """
+    with open(download_log, "a") as f:
+        f.write(logging_name + "\n")
+
+def is_model_created(model_name):
+    """
+    Checking if the model's already set up in Ollama! 🔍
+    
+    Args:
+        model_name (str): Name of the model we're looking for 🎯
+        
+    Returns:
+        bool: True if it's ready to roll, False if we need to set it up! 💪
+    """
+    result = subprocess.run(["ollama", "list"], stdout=subprocess.PIPE)
+    return model_name in result.stdout.decode("utf-8")
+
+def download_model(repo_id, filename, token, cache_dir="downloads"):
+    """
+    Pulling models straight from HuggingFace Hub! 🚀
+    
+    Args:
+        repo_id (str): Where to find the model on HF 🎯
+        filename (str): Name of the file we want 📄
+        token (str): Your HF access token (optional but recommended) 🔑
+        cache_dir (str): Where to save the downloads (default: 'downloads') 📂
+        
+    Returns:
+        str: Path to your downloaded model file 📍
+        
+    Raises:
+        Exception: If something goes wrong, we'll let you know what's up! ⚠️
+    """
+    try:
+        os.makedirs(cache_dir, exist_ok=True)
+        
+        # Download using hf_hub_download
+        filepath = hf_hub_download(
+            repo_id=repo_id,
+            filename=filename,
+            token=token,
+            cache_dir=cache_dir,
+            resume_download=True,
+            force_download=False,
+            local_files_only=False
+        )
+        
+        # Ensure file is in the expected location
+        expected_path = os.path.join(cache_dir, filename)
+        if filepath != expected_path:
+            os.makedirs(os.path.dirname(expected_path), exist_ok=True)
+            if not os.path.exists(expected_path):
+                import shutil
+                shutil.copy2(filepath, expected_path)
+            filepath = expected_path
+            
+        return filepath
+    
+    except Exception as e:
+        logger.error(f"Error downloading model: {str(e)}")
+        raise
+
+def is_ollama_running():
+    """
+    Checking if Ollama's up and running! 🏃‍♂️
+    
+    Returns:
+        bool: True if Ollama's vibing, False if it needs a kickstart! ⚡
+    """
+    for proc in psutil.process_iter(["name"]):
+        if proc.info["name"] in ["ollama", "ollama.exe"]:
+            return True
+    return False
+
+# Initialize CLI
+app = CLI(
+    name="autollama",
+    help="Download and create Ollama models",
+    version="1.0.0"
+)
+
+@app.command(name="download")
+@option("-m", "--model-path", help="Path to the model on Hugging Face Hub", required=True)
+@option("-g", "--gguf-file", help="Name of the GGUF file", required=True)
+def download_command(model_path: str, gguf_file: str):
+    """
+    Your one-stop command to download and set up HelpingAI models! 🚀
+    
+    Args:
+        model_path (str): Where to find your model on HuggingFace Hub 🎯
+        gguf_file (str): The GGUF file you want to download 📄
+        
+    Example:
+        >>> python -m webscout.Extra.autollama download \\
+        ...     -m "OEvortex/HelpingAI-Lite-1.5T" \\
+        ...     -g "HelpingAI-Lite-1.5T.q4_k_m.gguf"
+    """
+    show_art()
+
+    model_name = gguf_file.split(".Q4")[0]
+    download_log = "downloaded_models.log"
+    logging_name = f"{model_path}_{model_name}"
+
+    if not os.path.exists(download_log):
+        with open(download_log, 'w') as f:
+            pass
+
+    try:
+        subprocess.check_output(['pip', 'show', 'huggingface-hub'])
+    except subprocess.CalledProcessError:
+        logger.info("Installing huggingface-hub...")
+        subprocess.check_call(['pip', 'install', '-U', 'huggingface_hub[cli]'])
+    else:
+        logger.info("huggingface-hub is already installed.")
+
+    if is_model_downloaded(logging_name, download_log):
+        logger.info(f"Model {logging_name} has already been downloaded. Skipping download.")
+    else:
+        logger.info(f"Downloading model {logging_name}...")
+        token = os.getenv('HUGGINGFACE_TOKEN', None)
+        if not token:
+            logger.warning("Warning: HUGGINGFACE_TOKEN environment variable is not set. Using None.")
+        
+        filepath = download_model(model_path, gguf_file, token)
+        log_downloaded_model(logging_name, download_log)
+        logger.info(f"Model {logging_name} downloaded and logged.")
+
+    try:
+        subprocess.check_output(['ollama', '--version'])
+    except subprocess.CalledProcessError:
+        logger.info("Installing Ollama...")
+        subprocess.check_call(['curl', '-fsSL', 'https://ollama.com/install.sh', '|', 'sh'])
+    else:
+        logger.info("Ollama is already installed.")
+
+    if is_ollama_running():
+        logger.info("Ollama is already running. Skipping the start.")
+    else:
+        logger.info("Starting Ollama...")
+        subprocess.Popen(['ollama', 'serve'])
+
+        while not is_ollama_running():
+            logger.info("Waiting for Ollama to start...")
+            time.sleep(1)
+
+        logger.info("Ollama has started.")
+
+    if is_model_created(model_name):
+        logger.info(f"Model {model_name} is already created. Skipping creation.")
+    else:
+        logger.info(f"Creating model {model_name}...")
+        with open('Modelfile', 'w') as f:
+            f.write(f"FROM ./downloads/{gguf_file}")
+        subprocess.check_call(['ollama', 'create', model_name, '-f', 'Modelfile'])
+        logger.info(f"Model {model_name} created.")
+
+    logger.success(f"model name is > {model_name}")
+    logger.info(f"Use Ollama run {model_name}")
+
+def main():
+    """
+    Main function to run the AutoLlama CLI.
+    """
+    app.run()
+
+if __name__ == "__main__":
     main()

webscout/Extra/gguf.py

@@ -1,31 +1,49 @@
-# webscout/Extra/gguf.py
+"""
+Yo fam! 🔥 Welcome to GGUF Converter - your ultimate tool for converting models to GGUF format! 💪
+
+- Converting HuggingFace models to GGUF format 🚀
+- Multiple quantization methods for different needs 🎯
+- Easy upload back to HuggingFace Hub 📤
+
+Usage:
+>>> python -m webscout.Extra.gguf convert -m "OEvortex/HelpingAI-Lite-1.5T" -q "q4_k_m,q5_k_m"
+>>> # With upload options:
+>>> python -m webscout.Extra.gguf convert -m "your-model" -u "username" -t "token" -q "q4_k_m"
+
+Features:
+- Smart dependency checking 🔍
+- CUDA support detection ⚡
+- Progress tracking that keeps it real 📈
+- Multiple quantization options 🎮
+
+Join the squad on Discord and level up your AI game! 🎮
+"""
+
 import subprocess
 import os 
 import sys
-import logging
 import shutil
 from pathlib import Path
 from typing import List, Optional, Dict, Any
-from pyfiglet import figlet_format
+from webscout.zeroart import figlet_format
 from rich.console import Console
 from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn, TaskProgressColumn
-from rich.logging import RichHandler
 from rich.panel import Panel
 from rich.table import Table
-
-# Set up logging with Rich
-logging.basicConfig(
-    level=logging.INFO,
-    format="%(message)s",
-    datefmt="[%X]",
-    handlers=[RichHandler(rich_tracebacks=True)]
+from ..Litlogger import LitLogger, LogFormat, ColorScheme
+from ..swiftcli import CLI, option
+
+# Initialize LitLogger with ocean vibes
+logger = LitLogger(
+    name="GGUFConverter",
+    format=LogFormat.MODERN_EMOJI,
+    color_scheme=ColorScheme.OCEAN
 )
 
-log = logging.getLogger("rich")
 console = Console()
 
 class ConversionError(Exception):
-    """Custom exception for conversion errors"""
+    """Custom exception for when things don't go as planned! ⚠️"""
     pass
 
 class ModelConverter:
@@ -94,16 +112,16 @@ class ModelConverter:
         
         with console.status("[bold green]Setting up llama.cpp...") as status:
             if not llama_path.exists():
-                log.info("Cloning llama.cpp repository...")
+                logger.info("Cloning llama.cpp repository...")
                 subprocess.run(['git', 'clone', 'https://github.com/ggerganov/llama.cpp'], check=True)
             
             os.chdir(llama_path)
-            log.info("Installing requirements...")
+            logger.info("Installing requirements...")
             subprocess.run(['pip3', 'install', '-r', 'requirements.txt'], check=True)
             
             has_cuda = subprocess.run(['nvcc', '--version'], capture_output=True).returncode == 0
             
-            log.info("Building llama.cpp...")
+            logger.info("Building llama.cpp...")
             subprocess.run(['make', 'clean'], check=True)
             if has_cuda:
                 status.update("[bold green]Building with CUDA support...")
@@ -135,7 +153,7 @@ class ModelConverter:
         """Performs the model conversion process."""
         try:
             # Display banner and configuration
-            console.print(f"[bold green]{figlet_format('GGUF Converter')}[/]\n", justify="center")
+            console.print(f"[bold green]{figlet_format('GGUF Converter')}")
             self.display_config()
             
             # Validate inputs
@@ -186,11 +204,11 @@ class ModelConverter:
                         break
                     if output:
                         progress.update(task, description=output.strip())
-                        log.info(output.strip())
+                        logger.info(output.strip())
                 
                 stderr = process.stderr.read()
                 if stderr:
-                    log.warning(stderr)
+                    logger.warning(stderr)
                 
                 if process.returncode != 0:
                     raise ConversionError(f"Conversion failed with return code {process.returncode}")
@@ -372,45 +390,52 @@ echo "Script completed."
         script_path.write_text(script_content)
         script_path.chmod(0o755)
 
-def convert(
-    model_id: str,
-    username: Optional[str] = None,
-    token: Optional[str] = None,
-    quantization_methods: str = "q4_k_m,q5_k_m"
-) -> None:
-    """Converts and quantizes a Hugging Face model to GGUF format.
+# Initialize CLI with HAI vibes
+app = CLI(
+    name="gguf",
+    help="Convert HuggingFace models to GGUF format with style! 🔥",
+    version="1.0.0"
+)
 
+@app.command(name="convert")
+@option("-m", "--model-id", help="The HuggingFace model ID (e.g., 'OEvortex/HelpingAI-Lite-1.5T')", required=True)
+@option("-u", "--username", help="Your HuggingFace username for uploads", default=None)
+@option("-t", "--token", help="Your HuggingFace API token for uploads", default=None)
+@option("-q", "--quantization", help="Comma-separated quantization methods", default="q4_k_m,q5_k_m")
+def convert_command(model_id: str, username: Optional[str] = None, 
+                   token: Optional[str] = None, quantization: str = "q4_k_m,q5_k_m"):
+    """
+    Convert and quantize HuggingFace models to GGUF format! 🚀
+    
     Args:
-        model_id (str): The Hugging Face model ID (e.g., 'google/flan-t5-xl').
-        username (str, optional): Your Hugging Face username. Required for uploads.
-        token (str, optional): Your Hugging Face API token. Required for uploads. 
-        quantization_methods (str, optional): Comma-separated quantization methods. 
-                                            Defaults to "q4_k_m,q5_k_m".
-
-    Raises:
-        ConversionError: If any step in the conversion process fails.
-        ValueError: If invalid parameters are provided.
+        model_id (str): Your model's HF ID (like 'OEvortex/HelpingAI-Lite-1.5T') 🎯
+        username (str, optional): Your HF username for uploads 👤
+        token (str, optional): Your HF API token 🔑
+        quantization (str): Quantization methods (default: q4_k_m,q5_k_m) 🎮
+        
+    Example:
+        >>> python -m webscout.Extra.gguf convert \\
+        ...     -m "OEvortex/HelpingAI-Lite-1.5T" \\
+        ...     -q "q4_k_m,q5_k_m"
     """
-    converter = ModelConverter(model_id, username, token, quantization_methods)
-    converter.convert()
+    try:
+        converter = ModelConverter(
+            model_id=model_id,
+            username=username,
+            token=token,
+            quantization_methods=quantization
+        )
+        converter.convert()
+    except (ConversionError, ValueError) as e:
+        logger.error(f"Conversion failed: {str(e)}")
+        sys.exit(1)
+    except Exception as e:
+        logger.error(f"Unexpected error: {str(e)}")
+        sys.exit(1)
+
+def main():
+    """Fire up the GGUF converter! 🚀"""
+    app.run()
 
 if __name__ == "__main__":
-    import argparse
-    
-    parser = argparse.ArgumentParser(description="Convert Hugging Face models to GGUF format")
-    parser.add_argument("model_id", help="The Hugging Face model ID (e.g., 'google/flan-t5-xl')")
-    parser.add_argument("-u", "--username", help="Your Hugging Face username")
-    parser.add_argument("-t", "--token", help="Your Hugging Face API token")
-    parser.add_argument(
-        "-q", "--quantization-methods",
-        default="q4_k_m,q5_k_m",
-        help="Comma-separated quantization methods"
-    )
-    
-    args = parser.parse_args()
-    convert(
-        model_id=args.model_id,
-        username=args.username,
-        token=args.token,
-        quantization_methods=args.quantization_methods
-    )
+    main()