webtools-cli 1.3.3__tar.gz → 1.3.4__tar.gz

webtools_cli-1.3.4/ComfyUI/comfyUI.py

@@ -0,0 +1,126 @@
+#@title 1. System Initialization
+#@markdown **Run this cell first.** <br>
+#@markdown Mounts Drive, sets up ComfyUI, and extracts your high-speed Tarball cache to the local SSD for instant booting.
+
+import os
+import shutil
+import subprocess
+import sys
+
+# --- Input Resources (Environment Variable Overrides) ---
+try:
+    from google.colab import drive
+    COLAB_AVAILABLE = True
+except ImportError:
+    COLAB_AVAILABLE = False
+
+# Configuration from environment or defaults
+WORKSPACE_ROOT = os.environ.get("COMFYUI_WORKSPACE")
+if not WORKSPACE_ROOT:
+    WORKSPACE_ROOT = "/content/ComfyUI" if COLAB_AVAILABLE else os.path.join(os.getcwd(), "ComfyUI_repo")
+
+MOUNT_DRIVE = os.environ.get("MOUNT_DRIVE", "true").lower() == "true" if COLAB_AVAILABLE else False
+UPDATE_COMFY_UI = os.environ.get("UPDATE_COMFY_UI", "false").lower() == "true"
+INSTALL_COMFYUI_MANAGER = os.environ.get("INSTALL_COMFYUI_MANAGER", "true").lower() == "true"
+
+LOCAL_WORKSPACE = WORKSPACE_ROOT
+DRIVE_WORKSPACE = "/content/drive/MyDrive/ComfyUI" if COLAB_AVAILABLE else os.path.join(os.getcwd(), "ComfyUI_drive_sim")
+CACHE_TAR = os.path.join(DRIVE_WORKSPACE, "comfy_ui_cache.tar")
+
+def stream_cmd(cmd, cwd=None):
+    """Streams shell commands directly to the output"""
+    try:
+        process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, cwd=cwd, bufsize=1)
+        for line in iter(process.stdout.readline, ''):
+            sys.stdout.write(line)
+            sys.stdout.flush()
+        process.wait()
+        return process.returncode
+    except Exception as e:
+        print(f"Error executing command {cmd}: {e}")
+        return 1
+
+def setup_comfyui():
+    print("[SYSTEM] Initialization Protocol Started...\n")
+
+    # 1. Mount Google Drive
+    if MOUNT_DRIVE and COLAB_AVAILABLE:
+        print("💾 Requesting Google Drive Access...")
+        try:
+            drive.mount('/content/drive')
+        except Exception as e:
+            print(f"⚠️ Drive mount failed: {e}")
+
+    # 2. Setup ComfyUI Core
+    if not os.path.exists(LOCAL_WORKSPACE):
+        print(f"\n📦 Cloning ComfyUI repository to {LOCAL_WORKSPACE}...")
+        subprocess.run(["git", "clone", "https://github.com/comfyanonymous/ComfyUI", LOCAL_WORKSPACE])
+    else:
+        if UPDATE_COMFY_UI:
+            print("\n🔄 Checking for ComfyUI updates...")
+            subprocess.run(["git", "pull"], cwd=LOCAL_WORKSPACE)
+
+    # 3. Configure Hybrid Storage & Tarball Cache
+    heavy_dirs = ["models", "output", "input"]
+    print("\n🔗 Routing heavy model directories...")
+    for d in heavy_dirs:
+        local_path = os.path.join(LOCAL_WORKSPACE, d)
+        drive_path = os.path.join(DRIVE_WORKSPACE, d)
+        if not os.path.exists(drive_path): os.makedirs(drive_path, exist_ok=True)
+        
+        if os.path.exists(local_path) and not os.path.islink(local_path): 
+            try:
+                if os.path.isdir(local_path): shutil.rmtree(local_path)
+                else: os.remove(local_path)
+            except: pass
+            
+        if not os.path.exists(local_path):
+            try:
+                # Use absolute paths for symlinks to be safe
+                os.symlink(os.path.abspath(drive_path), os.path.abspath(local_path))
+            except Exception as e:
+                print(f"   ⚠️ Could not create symlink for {d}: {e}")
+
+    print("\n⚡ Deploying High-Speed Tarball Cache for UI Elements...")
+    if os.path.exists(CACHE_TAR):
+        print("   📦 Extracting cached UI nodes to local SSD...")
+        os.system(f"tar -xf '{CACHE_TAR}' -C '{LOCAL_WORKSPACE}' > /dev/null 2>&1")
+    else:
+        print("   ⚠️ No cache found. Creating fresh local directories...")
+        os.makedirs(os.path.join(LOCAL_WORKSPACE, "custom_nodes"), exist_ok=True)
+        os.makedirs(os.path.join(LOCAL_WORKSPACE, "user"), exist_ok=True)
+
+    # 4. Install ComfyUI Manager
+    if INSTALL_COMFYUI_MANAGER:
+        manager_path = os.path.join(LOCAL_WORKSPACE, "custom_nodes", "ComfyUI-Manager")
+        if not os.path.exists(manager_path):
+            print("\n📦 Installing ComfyUI Manager...")
+            subprocess.run(["git", "clone", "https://github.com/ltdrdata/ComfyUI-Manager.git", manager_path])
+        else:
+            print("\n✅ ComfyUI Manager verified.")
+            subprocess.run(["git", "pull"], cwd=manager_path)
+
+    # 5. Core Python Dependencies
+    print("\n🛠️ Installing base Python requirements...")
+    # Check if requirements.txt exists in LOCAL_WORKSPACE
+    req_file = os.path.join(LOCAL_WORKSPACE, "requirements.txt")
+    if os.path.exists(req_file):
+        stream_cmd([sys.executable, "-m", "pip", "install", "xformers!=0.0.18", "-r", "requirements.txt", "--extra-index-url", "https://download.pytorch.org/whl/cu121"], cwd=LOCAL_WORKSPACE)
+    
+    stream_cmd([sys.executable, "-m", "pip", "install", "insightface", "onnxruntime-gpu"])
+
+    # 6. AUTO-HEAL: Custom Node Dependencies
+    print("\n🔍 [AUTO-HEAL] Scanning local custom nodes for missing dependencies...")
+    custom_nodes_dir = os.path.join(LOCAL_WORKSPACE, "custom_nodes")
+    if os.path.exists(custom_nodes_dir):
+        for item in os.listdir(custom_nodes_dir):
+            node_path = os.path.join(custom_nodes_dir, item)
+            node_req = os.path.join(node_path, "requirements.txt")
+            if os.path.isdir(node_path) and os.path.exists(node_req):
+                print(f"\n   ⚙️ Restoring dependencies for: {item}")
+                stream_cmd([sys.executable, "-m", "pip", "install", "-r", "requirements.txt"], cwd=node_path)
+
+    print("\n✅ [SYSTEM READY] Initialization complete.")
+
+if __name__ == "__main__":
+    setup_comfyui()

webtools_cli-1.3.4/ComfyUI/comfyu.py

@@ -0,0 +1,120 @@
+import os
+import requests
+import subprocess
+import sys
+from urllib.parse import urlparse, unquote
+from tqdm.auto import tqdm
+
+# --- Input Resources (Environment Variable Overrides) ---
+WORKSPACE = os.environ.get("COMFYUI_WORKSPACE", "/content/ComfyUI")
+CHECKPOINT_URLS = os.environ.get("CHECKPOINT_URLS", "")
+UNET_DIFFUSION_URLS = os.environ.get("UNET_DIFFUSION_URLS", "")
+TEXT_ENCODER_URLS = os.environ.get("TEXT_ENCODER_URLS", "")
+CLIP_VISION_URLS = os.environ.get("CLIP_VISION_URLS", "")
+VAE_URLS = os.environ.get("VAE_URLS", "")
+LORA_URLS = os.environ.get("LORA_URLS", "")
+CONTROLNET_URLS = os.environ.get("CONTROLNET_URLS", "")
+UPSCALE_MODELS_URLS = os.environ.get("UPSCALE_MODELS_URLS", "")
+EMBEDDING_URLS = os.environ.get("EMBEDDING_URLS", "")
+CUSTOM_NODE_URLS = os.environ.get("CUSTOM_NODE_URLS", "")
+
+# --- Downloader Logic ---
+DIRS = {
+    "checkpoints":    os.path.join(WORKSPACE, "models/checkpoints"),
+    "unet":           os.path.join(WORKSPACE, "models/unet"),
+    "clip":           os.path.join(WORKSPACE, "models/clip"),
+    "clip_vision":    os.path.join(WORKSPACE, "models/clip_vision"),
+    "vae":            os.path.join(WORKSPACE, "models/vae"),
+    "loras":          os.path.join(WORKSPACE, "models/loras"),
+    "controlnet":     os.path.join(WORKSPACE, "models/controlnet"),
+    "upscale_models": os.path.join(WORKSPACE, "models/upscale_models"),
+    "embeddings":     os.path.join(WORKSPACE, "models/embeddings"),
+    "custom_nodes":   os.path.join(WORKSPACE, "custom_nodes")
+}
+
+def get_filename(url, response):
+    """Smartly determines filename from Content-Disposition or URL."""
+    if "Content-Disposition" in response.headers:
+        import re
+        fname = re.findall('filename="?([^"]+)"?', response.headers["Content-Disposition"])
+        if fname: return fname[0]
+    return unquote(os.path.basename(urlparse(url).path))
+
+def download_file(url, target_dir):
+    try:
+        # Stream the download to get headers first
+        response = requests.get(url, stream=True, allow_redirects=True)
+        response.raise_for_status()
+
+        filename = get_filename(url, response)
+        file_path = os.path.join(target_dir, filename)
+        total_size = int(response.headers.get('content-length', 0))
+
+        if os.path.exists(file_path):
+            print(f"   ⏩ Skipping (Exists): {filename}")
+            return
+
+        # Modern Progress Bar Log
+        print(f"   📥 Downloading: {filename}")
+
+        # The Progress Bar (Auto-Stretching)
+        with tqdm(
+            total=total_size,
+            unit='B',
+            unit_scale=True,
+            unit_divisor=1024,
+            desc="      🚀 Progress",
+            dynamic_ncols=True
+        ) as bar:
+            with open(file_path, 'wb') as f:
+                for chunk in response.iter_content(chunk_size=1024*1024): # 1MB chunks
+                    if chunk:
+                        f.write(chunk)
+                        bar.update(len(chunk))
+
+        print("      ✅ Download Complete\n")
+
+    except Exception as e:
+        print(f"   ❌ Failed to download: {url}")
+        print(f"      Error: {e}\n")
+
+def process_downloads(urls_str, target_dir, is_node=False):
+    if not urls_str.strip(): return
+
+    url_list = [u.strip() for u in urls_str.replace(',', '\n').split('\n') if u.strip()]
+    if not os.path.exists(target_dir): os.makedirs(target_dir, exist_ok=True)
+
+    print(f"📂 Category: {os.path.basename(target_dir)}")
+
+    for url in url_list:
+        if is_node:
+            node_name = url.split('/')[-1].replace('.git', '')
+            node_path = os.path.join(target_dir, node_name)
+            if not os.path.exists(node_path):
+                print(f"   ⬇️ Cloning Node: {node_name}...")
+                subprocess.run(["git", "clone", url, node_path])
+                # Auto-install requirements
+                req = os.path.join(node_path, "requirements.txt")
+                if os.path.exists(req):
+                    print(f"      📦 Installing requirements...")
+                    subprocess.run([sys.executable, "-m", "pip", "install", "-r", req])
+                print("      ✅ Installed\n")
+            else:
+                print(f"   ⏩ Node exists: {node_name}\n")
+        else:
+            download_file(url, target_dir)
+
+if __name__ == "__main__":
+    # --- Execution ---
+    process_downloads(CHECKPOINT_URLS,     DIRS["checkpoints"])
+    process_downloads(UNET_DIFFUSION_URLS, DIRS["unet"])
+    process_downloads(TEXT_ENCODER_URLS,   DIRS["clip"])
+    process_downloads(CLIP_VISION_URLS,    DIRS["clip_vision"])
+    process_downloads(VAE_URLS,            DIRS["vae"])
+    process_downloads(LORA_URLS,           DIRS["loras"])
+    process_downloads(CONTROLNET_URLS,     DIRS["controlnet"])
+    process_downloads(UPSCALE_MODELS_URLS, DIRS["upscale_models"])
+    process_downloads(EMBEDDING_URLS,      DIRS["embeddings"])
+    process_downloads(CUSTOM_NODE_URLS,    DIRS["custom_nodes"], is_node=True)
+
+    print("🎉 All tasks finished.")

webtools_cli-1.3.4/ComfyUI/ui.py

@@ -0,0 +1,98 @@
+#@title 4. Start ComfyUI Session
+#@markdown Installs missing runtime dependencies, establishes an optimized IPv4 Cloudflare tunnel, and launches the ComfyUI web interface with live standard logs.
+
+import subprocess
+import threading
+import time
+import socket
+import os
+import sys
+
+# --- Session Configuration ---
+#@markdown **Performance Profile:**
+MEMORY_PROFILE = "Standard (Auto-Detect)" #@param ["Standard (Auto-Detect)", "Low VRAM (T4 GPU / Heavy Models)", "High VRAM (A100 GPU Only)"]
+#@markdown **Visual Settings:**
+LIVE_GENERATION_PREVIEWS = True #@param {type:"boolean"}
+
+# Dynamic Workspace detection
+WORKSPACE = os.environ.get("COMFYUI_WORKSPACE")
+if not WORKSPACE:
+    if os.path.exists("/content/ComfyUI"):
+        WORKSPACE = "/content/ComfyUI"
+    else:
+        # Try to find it relative to this script
+        # Check if current directory has main.py (indicating we are inside ComfyUI)
+        if os.path.exists("main.py"):
+            WORKSPACE = os.getcwd()
+        else:
+            WORKSPACE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "ComfyUI_repo")
+
+ARGS = []
+
+if "Low VRAM" in MEMORY_PROFILE:
+    ARGS.append("--lowvram")
+elif "High VRAM" in MEMORY_PROFILE:
+    ARGS.append("--highvram")
+
+if LIVE_GENERATION_PREVIEWS:
+    ARGS.extend(["--preview-method", "auto"])
+
+# 0. Auto-Heal Missing Custom Node Dependencies
+print("[SYSTEM] Auto-healing runtime dependencies...")
+os.system("pip install -q gguf piexif > /dev/null 2>&1")
+
+# 1. Initialize Network Tunnel 
+print("[SYSTEM] Verifying Cloudflare Daemon...")
+if not os.path.exists("/usr/bin/cloudflared"):
+    print("[SYSTEM] Fetching Cloudflared via aria2c...")
+    os.system("apt-get install -y aria2 > /dev/null 2>&1")
+    os.system("aria2c -x 16 -s 16 -k 1M https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-amd64.deb -d /content -o cloudflared.deb > /dev/null 2>&1")
+    os.system("dpkg -i /content/cloudflared.deb > /dev/null 2>&1")
+
+def start_tunnel(port):
+    while True:
+        time.sleep(1)
+        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        result = sock.connect_ex(('127.0.0.1', port))
+        sock.close()
+        if result == 0:
+            break
+
+    print("\n[SYSTEM] ComfyUI Local Server Detected. Establishing Secure Tunnel...")
+    
+    p = subprocess.Popen(
+        ["cloudflared", "tunnel", "--url", f"http://127.0.0.1:{port}", "--edge-ip-version", "4", "--protocol", "http2"],
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        text=True
+    )
+
+    tunnel_established = False
+    for line in p.stderr:
+        if "trycloudflare.com" in line and not tunnel_established:
+            parts = line.split()
+            for part in parts:
+                if "trycloudflare.com" in part and part.startswith("http"):
+                    url = part.strip()
+                    print("\n" + "="*60)
+                    print(f"🚀 ACTIVE TUNNEL: {url}")
+                    print("="*60 + "\n")
+                    tunnel_established = True
+                    break
+
+threading.Thread(target=start_tunnel, daemon=True, args=(8188,)).start()
+
+# 2. Launch Application
+if os.path.exists(os.path.join(WORKSPACE, "main.py")):
+    os.chdir(WORKSPACE)
+    print(f"[SYSTEM] Booting ComfyUI [Profile: {MEMORY_PROFILE}]")
+    print("[SYSTEM] Streaming Logs...\n" + "-"*60)
+
+    cmd = ["python", "main.py", "--listen", "127.0.0.1", "--port", "8188"] + ARGS
+    process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1)
+
+    for line in iter(process.stdout.readline, ''):
+        sys.stdout.write(line)
+        sys.stdout.flush()
+else:
+    print("[FATAL ERROR] ComfyUI directory missing. Please re-run the initialization cell.")

{webtools_cli-1.3.3 → webtools_cli-1.3.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: webtools-cli
-Version: 1.3.3
+Version: 1.3.4
 Summary: Advanced Web Intelligence & Scraping Toolkit with CLI and Web UI
 Author: Abhinav Adarsh
 License-Expression: MIT

{webtools_cli-1.3.3 → webtools_cli-1.3.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "webtools-cli"
-version = "1.3.3"
+version = "1.3.4"
 description = "Advanced Web Intelligence & Scraping Toolkit with CLI and Web UI"
 readme = "README.md"
 license = "MIT"
@@ -52,7 +52,8 @@ webtools-install = "webtools.install:install_playwright_browsers"
 Homepage = "https://webtoolscli.pages.dev"
 
 [tool.setuptools.packages.find]
-include = ["webtools*"]
+include = ["webtools*", "ComfyUI*"]
 
 [tool.setuptools.package-data]
 webtools = ["web/*"]
+ComfyUI = ["*.py"]

{webtools_cli-1.3.3 → webtools_cli-1.3.4}/webtools/core.py

@@ -3497,6 +3497,27 @@ def print_image_forensics_report(data):
 
     print(f"{c_b}╚{'═'*(total_w-2)}╝{R}\n")
 
+def find_comfy_dir():
+    """Robustly find the ComfyUI directory"""
+    on_colab = is_colab()
+    possible_paths = [
+        os.path.join(os.getcwd(), "ComfyUI"),                 # Current directory (clone)
+        os.path.join(os.path.dirname(PACKAGE_DIR), "ComfyUI"), # Sibling of webtools
+        os.path.join(PACKAGE_DIR, "ComfyUI"),                 # Inside webtools
+        "/content/webtools-cli/ComfyUI" if on_colab else None, # Colab default
+        "/content/ComfyUI" if on_colab else None              # Direct colab
+    ]
+    for p in possible_paths:
+        if p and os.path.exists(os.path.join(p, "ui.py")):
+            return p
+    # Recursive search as last resort
+    for root, dirs, files in os.walk(os.getcwd()):
+        if "ComfyUI" in dirs:
+            p = os.path.join(root, "ComfyUI")
+            if os.path.exists(os.path.join(p, "ui.py")):
+                return p
+    return None
+
 def run_comfyui_mode():
     """Start ComfyUI session with GPU and Colab detection"""
     on_colab = is_colab()
@@ -3507,6 +3528,12 @@ def run_comfyui_mode():
         input("\nPress Enter to return to main menu...")
         return
 
+    comfy_dir = find_comfy_dir()
+    if not comfy_dir:
+        print(f"\n{Fore.RED}❌ ERROR: ComfyUI folder not found!{Style.RESET_ALL}")
+        input("\nPress Enter to return to main menu...")
+        return
+
     while True:
         os.system('cls' if os.name == 'nt' else 'clear')
         print(f"\n{Fore.CYAN}--- ComfyUI Launcher ---{Style.RESET_ALL}")
@@ -3529,8 +3556,17 @@ def run_comfyui_mode():
 
         print(f"\n{Fore.GREEN}🚀 Starting ComfyUI Setup & Session...{Style.RESET_ALL}")
         
-        comfy_dir = os.path.join(os.path.dirname(PACKAGE_DIR), "ComfyUI")
-        
+        # 0. Run System Initialization (comfyUI.py)
+        # This ensures the ComfyUI repo is cloned and dependencies are installed
+        init_script = os.path.join(comfy_dir, "comfyUI.py")
+        if os.path.exists(init_script):
+            print(f"\n{Fore.CYAN}--- Running System Initialization ---{Style.RESET_ALL}")
+            init_env = os.environ.copy()
+            init_env["COMFYUI_WORKSPACE"] = comfy_dir
+            subprocess.run([sys.executable, init_script], env=init_env)
+        else:
+            print(f"{Fore.YELLOW}Warning: {init_script} not found. Skipping initialization.{Style.RESET_ALL}")
+
         # 1. Handle Model Downloads if Anime chosen
         if cui_choice == '2':
             print(f"\n{Fore.MAGENTA}✨ Configuring Anime Model (Anima)...{Style.RESET_ALL}")
@@ -3538,24 +3574,27 @@ def run_comfyui_mode():
             download_env["UNET_DIFFUSION_URLS"] = "https://huggingface.co/circlestone-labs/Anima/resolve/main/split_files/diffusion_models/anima-preview3-base.safetensors"
             download_env["TEXT_ENCODER_URLS"] = "https://huggingface.co/circlestone-labs/Anima/resolve/main/split_files/text_encoders/qwen_3_06b_base.safetensors"
             download_env["VAE_URLS"] = "https://huggingface.co/circlestone-labs/Anima/resolve/main/split_files/vae/qwen_image_vae.safetensors"
+            download_env["COMFYUI_WORKSPACE"] = comfy_dir
             
             # Run comfyu.py with these env vars
             downloader_script = os.path.join(comfy_dir, "comfyu.py")
             if os.path.exists(downloader_script):
                 subprocess.run([sys.executable, downloader_script], env=download_env)
             else:
-                print(f"{Fore.RED}Error: ComfyUI/comfyu.py not found.{Style.RESET_ALL}")
+                print(f"{Fore.RED}Error: {downloader_script} not found.{Style.RESET_ALL}")
 
         # 2. Launch UI
         ui_script = os.path.join(comfy_dir, "ui.py")
         if os.path.exists(ui_script):
             print(f"\n{Fore.CYAN}--- Initializing ComfyUI Interface ---{Style.RESET_ALL}")
             try:
-                subprocess.run([sys.executable, ui_script])
+                ui_env = os.environ.copy()
+                ui_env["COMFYUI_WORKSPACE"] = comfy_dir
+                subprocess.run([sys.executable, ui_script], env=ui_env)
             except Exception as e:
                 print(f"{Fore.RED}Error running ComfyUI: {e}{Style.RESET_ALL}")
         else:
-            print(f"{Fore.RED}Error: ComfyUI/ui.py not found at {ui_script}{Style.RESET_ALL}")
+            print(f"{Fore.RED}Error: {ui_script} not found.{Style.RESET_ALL}")
         
         input("\nPress Enter to return to ComfyUI menu...")
 

{webtools_cli-1.3.3 → webtools_cli-1.3.4}/webtools_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: webtools-cli
-Version: 1.3.3
+Version: 1.3.4
 Summary: Advanced Web Intelligence & Scraping Toolkit with CLI and Web UI
 Author: Abhinav Adarsh
 License-Expression: MIT

{webtools_cli-1.3.3 → webtools_cli-1.3.4}/webtools_cli.egg-info/SOURCES.txt

@@ -1,6 +1,9 @@
 LICENSE
 README.md
 pyproject.toml
+ComfyUI/comfyUI.py
+ComfyUI/comfyu.py
+ComfyUI/ui.py
 webtools/__init__.py
 webtools/__main__.py
 webtools/cli.py

{webtools_cli-1.3.3 → webtools_cli-1.3.4}/webtools_cli.egg-info/top_level.txt

@@ -1 +1,2 @@
+ComfyUI
 webtools