machineconfig 6.82__py3-none-any.whl → 8.51__py3-none-any.whl

machineconfig/utils/installer_utils/github_release_bulk.py

@@ -5,12 +5,41 @@ Extracts GitHub repository URLs and fetches latest release data with rate limiti
 """
 
 import json
-import time
-import subprocess
+import requests
 from pathlib import Path
-from typing import Any, Dict, Optional, Set
+from typing import Any, Dict, Optional, Set, TypedDict
 from urllib.parse import urlparse
 
+from machineconfig.utils.installer_utils.github_release_scraper import scrape_github_release_page
+
+
+class AssetInfo(TypedDict):
+    """Type definition for GitHub release asset information."""
+    name: str
+    size: int
+    download_count: int
+    content_type: str
+    created_at: str
+    updated_at: str
+    browser_download_url: str
+
+
+class ReleaseInfo(TypedDict):
+    """Type definition for GitHub release information."""
+    tag_name: str
+    name: str
+    published_at: str
+    assets: list[AssetInfo]
+    assets_count: int
+
+
+class OutputData(TypedDict):
+    """Type definition for the output JSON data structure."""
+    generated_at: str
+    total_repositories: int
+    successful_fetches: int
+    releases: Dict[str, Optional[ReleaseInfo]]
+
 
 def is_github_repo(url: str) -> bool:
     """Check if URL is a GitHub repository URL."""
@@ -19,170 +48,98 @@ def is_github_repo(url: str) -> bool:
         return parsed.netloc == "github.com" and len(parsed.path.split("/")) >= 3
     except Exception:
         return False
-
-
 def extract_github_repos_from_json(json_file_path: Path) -> Set[str]:
     """Extract GitHub repository URLs from installer JSON file."""
     github_repos: Set[str] = set()
-    
     try:
         with open(json_file_path, 'r', encoding='utf-8') as file:
             data = json.load(file)
-            
         for installer in data.get("installers", []):
             repo_url = installer.get("repoURL", "")
             if is_github_repo(repo_url):
                 github_repos.add(repo_url)
-                
     except (json.JSONDecodeError, FileNotFoundError) as e:
         print(f"Error reading {json_file_path}: {e}")
-        
     return github_repos
-
-
-def get_repo_name_from_url(repo_url: str) -> str:
-    """Extract owner/repo from GitHub URL."""
+def get_repo_name_from_url(repo_url: str) -> Optional[tuple[str, str]]:
+    """Extract owner/repo from GitHub URL as a tuple (username, repo_name)."""
     try:
         parsed = urlparse(repo_url)
         path_parts = parsed.path.strip("/").split("/")
-        return f"{path_parts[0]}/{path_parts[1]}"
+        return (path_parts[0], path_parts[1])
     except (IndexError, AttributeError):
-        return ""
+        return None
+
 
+def fetch_github_release_data(
+    username: str,
+    repo_name: str,
+    version: Optional[str] = None,
+) -> Optional[Dict[str, Any]]:
+    """Fetch GitHub release data for the latest or a specific tag. Falls back to HTML scraping if API fails."""
 
-def fetch_github_release_data(repo_name: str) -> Optional[Dict[str, Any]]:
-    """Fetch latest release data from GitHub API using curl."""
     try:
-        cmd = [
-            "curl", "-s",
-            f"https://api.github.com/repos/{repo_name}/releases/latest"
-        ]
-        
-        result = subprocess.run(
-            cmd,
-            capture_output=True,
-            text=True,
-            timeout=30
-        )
-        
-        if result.returncode != 0:
-            print(f"❌ Failed to fetch data for {repo_name}: {result.stderr}")
-            return None
-            
-        response_data = json.loads(result.stdout)
-        
-        # Check if API returned an error
-        if "message" in response_data:
-            if "API rate limit exceeded" in response_data.get("message", ""):
-                print(f"🚫 Rate limit exceeded for {repo_name}")
-                return None
-            elif "Not Found" in response_data.get("message", ""):
-                print(f"🔍 No releases found for {repo_name}")
-                return None
-                
+        requested_version = (version or "").strip()
+        if requested_version and requested_version.lower() != "latest":
+            url = f"https://api.github.com/repos/{username}/{repo_name}/releases/tags/{requested_version}"
+        else:
+            url = f"https://api.github.com/repos/{username}/{repo_name}/releases/latest"
+
+        response = requests.get(url, timeout=30)
+        if response.status_code != 200:
+            print(f"⚠️ API failed for {username}/{repo_name}: HTTP {response.status_code}, trying HTML scraper...")
+            return scrape_github_release_page(username, repo_name, version)
+
+        response_data = response.json()
+        message = response_data.get("message")
+        if isinstance(message, str):
+            if "API rate limit exceeded" in message:
+                print(f"🚫 Rate limit exceeded for {username}/{repo_name}, trying HTML scraper...")
+                return scrape_github_release_page(username, repo_name, version)
+            if "Not Found" in message:
+                print(f"🔍 No releases found via API for {username}/{repo_name}, trying HTML scraper...")
+                return scrape_github_release_page(username, repo_name, version)
+
         return response_data
-        
-    except (subprocess.TimeoutExpired, json.JSONDecodeError, subprocess.SubprocessError) as e:
-        print(f"❌ Error fetching {repo_name}: {e}")
+
+    except (requests.RequestException, requests.Timeout, json.JSONDecodeError) as error:
+        print(f"⚠️ API error for {username}/{repo_name}: {error}, trying HTML scraper...")
+        return scrape_github_release_page(username, repo_name, version)
+
+
+def get_release_info(
+    username: str,
+    repo_name: str,
+    version: Optional[str] = None,
+) -> Optional[ReleaseInfo]:
+    """Return sanitized release information for the requested repository."""
+    release_data = fetch_github_release_data(username, repo_name, version)
+    if not release_data:
         return None
+    return extract_release_info(release_data)
 
 
-def extract_release_info(release_data: Dict[str, Any]) -> Dict[str, Any]:
+def extract_release_info(release_data: Dict[str, Any]) -> Optional[ReleaseInfo]:
     """Extract relevant information from GitHub release data."""
     if not release_data:
-        return {}
-        
-    asset_names = [asset["name"] for asset in release_data.get("assets", [])]
-    
+        return None
+    assets: list[AssetInfo] = []
+    for asset in release_data.get("assets", []):
+        asset_info: AssetInfo = {
+            "name": asset.get("name", ""),
+            "size": asset.get("size", 0),
+            "download_count": asset.get("download_count", 0),
+            "content_type": asset.get("content_type", ""),
+            "created_at": asset.get("created_at", ""),
+            "updated_at": asset.get("updated_at", ""),
+            "browser_download_url": asset.get("browser_download_url", "")
+        }
+        assets.append(asset_info)
     return {
         "tag_name": release_data.get("tag_name", ""),
         "name": release_data.get("name", ""),
         "published_at": release_data.get("published_at", ""),
-        "assets": asset_names,
-        "assets_count": len(asset_names)
+        "assets": assets,
+        "assets_count": len(assets)
     }
 
-
-def main() -> None:
-    """Main function to process installer JSON files and fetch GitHub release data."""
-    # Define paths
-    current_dir = Path(__file__).parent
-    installer_dir = current_dir.parent.parent / "jobs" / "installer"
-    
-    standard_json = installer_dir / "installer_data.json"
-    output_json = current_dir / "github_releases.json"
-    
-    print("🔍 Starting GitHub release data extraction...")
-    print(f"📁 Processing files from: {installer_dir}")
-    
-    # Extract GitHub repositories from both files
-    all_github_repos: Set[str] = set()
-    
-    if standard_json.exists():
-        print(f"📄 Reading {standard_json.name}...")
-        repos = extract_github_repos_from_json(standard_json)
-        all_github_repos.update(repos)
-        print(f"   Found {len(repos)} GitHub repos")
-    else:
-        print(f"⚠️  File not found: {standard_json}")    
-    print(f"🎯 Total unique GitHub repositories found: {len(all_github_repos)}")
-    
-    if not all_github_repos:
-        print("❌ No GitHub repositories found. Exiting.")
-        return
-    
-    # Fetch release data with rate limiting
-    release_mapping: Dict[str, Any] = {}
-    total_repos = len(all_github_repos)
-    
-    print(f"\n🚀 Fetching release data for {total_repos} repositories...")
-    print("⏰ Rate limiting: 5 seconds between requests")
-    print("-" * 60)
-    
-    for i, repo_url in enumerate(sorted(all_github_repos), 1):
-        repo_name = get_repo_name_from_url(repo_url)
-        
-        if not repo_name:
-            print(f"⚠️  [{i:3d}/{total_repos}] Invalid repo URL: {repo_url}")
-            continue
-            
-        print(f"📡 [{i:3d}/{total_repos}] Fetching: {repo_name}", end=" ... ")
-        
-        release_data = fetch_github_release_data(repo_name)
-        
-        if release_data:
-            release_info = extract_release_info(release_data)
-            release_mapping[repo_url] = release_info
-            assets_count = release_info.get("assets_count", 0)
-            tag = release_info.get("tag_name", "unknown")
-            print(f"✅ {tag} ({assets_count} assets)")
-        else:
-            release_mapping[repo_url] = {}
-            print("❌ No data")
-        
-        # Rate limiting - wait 5 seconds between requests (except for the last one)
-        if i < total_repos:
-            time.sleep(5)
-    
-    # Save results
-    output_data = {
-        "generated_at": time.strftime("%Y-%m-%d %H:%M:%S UTC", time.gmtime()),
-        "total_repositories": len(all_github_repos),
-        "successful_fetches": len([v for v in release_mapping.values() if v]),
-        "releases": release_mapping
-    }
-    
-    with open(output_json, 'w', encoding='utf-8') as f:
-        json.dump(output_data, f, indent=2, ensure_ascii=False)
-    
-    successful = len([v for v in release_mapping.values() if v])
-    print("\n📊 Summary:")
-    print(f"   Total repositories processed: {len(all_github_repos)}")
-    print(f"   Successful fetches: {successful}")
-    print(f"   Failed fetches: {len(all_github_repos) - successful}")
-    print(f"   Output saved to: {output_json}")
-    print("✅ Done!")
-
-
-if __name__ == "__main__":
-    main()

machineconfig/utils/installer_utils/github_release_scraper.py

@@ -0,0 +1,99 @@
+#!/usr/bin/env python3
+"""HTML scraper for GitHub release pages as fallback when API rate limit is exceeded."""
+
+import re
+from typing import Any, Optional
+import requests
+
+
+def extract_tag_from_html(html: str, owner: str, repo: str) -> str:
+    patterns = [
+        rf'/{re.escape(owner)}/{re.escape(repo)}/releases/tag/([^"\'<>\s]+)',
+        rf'/{re.escape(owner)}/{re.escape(repo)}/tree/([^"\'<>\s]+)',
+        r'<span[^>]*class="[^"]*ml-1[^"]*"[^>]*>([^<]+)</span>',
+    ]
+    for pattern in patterns:
+        match = re.search(pattern, html, re.IGNORECASE)
+        if match:
+            tag = match.group(1).strip()
+            if tag and not tag.startswith("http"):
+                return tag
+    return ""
+
+
+def extract_release_name(html: str) -> str:
+    patterns = [
+        r'<h1[^>]*class="[^"]*d-inline[^"]*"[^>]*>([^<]+)</h1>',
+        r'<bdi[^>]*class="[^"]*mr-2[^"]*"[^>]*>([^<]+)</bdi>',
+        r'<h1[^>]*>([^<]+)</h1>',
+    ]
+    for pattern in patterns:
+        match = re.search(pattern, html)
+        if match:
+            name = match.group(1).strip()
+            if name:
+                return name
+    return ""
+
+
+def extract_published_at(html: str) -> str:
+    pattern = r'<relative-time[^>]*datetime="([^"]+)"'
+    match = re.search(pattern, html)
+    if match:
+        return match.group(1)
+    return ""
+
+
+def fetch_expanded_assets(username: str, repo_name: str, tag_name: str, headers: dict[str, str]) -> list[dict[str, Any]]:
+    """Fetch assets from the expanded_assets endpoint which contains all downloadable files."""
+    assets: list[dict[str, Any]] = []
+    expanded_url = f"https://github.com/{username}/{repo_name}/releases/expanded_assets/{tag_name}"
+    try:
+        response = requests.get(expanded_url, timeout=30, headers=headers)
+        if response.status_code != 200:
+            print(f"⚠️ [Scraper] Could not fetch expanded assets for {username}/{repo_name}: HTTP {response.status_code}")
+            return assets
+        html = response.text
+        pattern = r'href="([^"]*?/releases/download/[^"]+)"[^>]*>.*?<span[^>]*class="[^"]*Truncate-text[^"]*text-bold[^"]*"[^>]*>([^<]+)</span>'
+        seen_urls: set[str] = set()
+        matches = re.findall(pattern, html, re.DOTALL)
+        for href, name in matches:
+            asset_name = name.strip()
+            if not asset_name or asset_name.isspace():
+                continue
+            download_url = f"https://github.com{href}" if href.startswith("/") else href
+            if download_url in seen_urls:
+                continue
+            seen_urls.add(download_url)
+            assets.append({"name": asset_name, "size": 0, "download_count": 0, "content_type": "", "created_at": "", "updated_at": "", "browser_download_url": download_url})
+    except requests.RequestException as error:
+        print(f"⚠️ [Scraper] Error fetching expanded assets for {username}/{repo_name}: {error}")
+    return assets
+
+
+def scrape_github_release_page(username: str, repo_name: str, version: Optional[str] = None) -> Optional[dict[str, Any]]:
+    """Scrape GitHub release page HTML to extract release information. Falls back to this when API rate limit is hit."""
+    try:
+        requested_version = (version or "").strip()
+        if requested_version and requested_version.lower() != "latest":
+            url = f"https://github.com/{username}/{repo_name}/releases/tag/{requested_version}"
+        else:
+            url = f"https://github.com/{username}/{repo_name}/releases/latest"
+        headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"}
+        response = requests.get(url, timeout=30, headers=headers, allow_redirects=True)
+        if response.status_code != 200:
+            print(f"❌ [Scraper] Failed to fetch page for {username}/{repo_name}: HTTP {response.status_code}")
+            return None
+        html = response.text
+        tag_name = extract_tag_from_html(html, username, repo_name)
+        if not tag_name:
+            print(f"🔍 [Scraper] No tag found for {username}/{repo_name}")
+            return None
+        release_name = extract_release_name(html) or tag_name
+        published_at = extract_published_at(html)
+        assets = fetch_expanded_assets(username, repo_name, tag_name, headers)
+        print(f"✅ [Scraper] Found {len(assets)} assets for {username}/{repo_name} @ {tag_name}")
+        return {"tag_name": tag_name, "name": release_name, "published_at": published_at, "assets": assets}
+    except requests.RequestException as error:
+        print(f"❌ [Scraper] Error fetching {username}/{repo_name}: {error}")
+        return None

machineconfig/utils/installer_utils/install_from_url.py

@@ -0,0 +1,183 @@
+
+
+import platform
+from typing import TYPE_CHECKING, Optional
+
+from machineconfig.utils.installer_utils.installer_helper import install_deb_package, download_and_prepare
+from machineconfig.utils.installer_utils.installer_locator_utils import find_move_delete_linux, find_move_delete_windows
+from machineconfig.utils.installer_utils.github_release_bulk import (
+    get_repo_name_from_url,
+    fetch_github_release_data,
+    extract_release_info,
+    AssetInfo,
+)
+from machineconfig.utils.path_extended import PathExtended
+from machineconfig.utils.source_of_truth import INSTALL_VERSION_ROOT
+
+if TYPE_CHECKING:
+    from rich.console import Console
+
+SUPPORTED_GITHUB_HOSTS = {"github.com", "www.github.com"}
+
+
+def _format_size(size_bytes: int) -> str:
+    if size_bytes <= 0:
+        return "0 B"
+    units = ("B", "KiB", "MiB", "GiB", "TiB")
+    value = float(size_bytes)
+    index = 0
+    while value >= 1024 and index < len(units) - 1:
+        value /= 1024
+        index += 1
+    return f"{value:.1f} {units[index]}"
+
+
+def _derive_tool_name(repo_name: str, asset_name: Optional[str]) -> Optional[str]:
+    repo_segment = repo_name.split("/", maxsplit=1)[-1]
+    repo_clean = repo_segment.replace(".git", "").lower()
+    repo_filtered = "".join(char for char in repo_clean if char.isalnum())
+    if repo_filtered:
+        return repo_filtered
+    if asset_name is None:
+        return None
+    asset_clean = asset_name.lower()
+    asset_filtered = "".join(char for char in asset_clean if char.isalnum())
+    if asset_filtered:
+        return asset_filtered
+    return None
+
+
+def _finalize_install(repo_name: str, asset_name: Optional[str], version: str, extracted_path: PathExtended, console: "Console") -> None:
+    from rich.panel import Panel
+    if extracted_path.suffix == ".deb":
+        install_deb_package(extracted_path)
+        tool_name_deb = _derive_tool_name(repo_name, asset_name)
+        if tool_name_deb is not None:
+            INSTALL_VERSION_ROOT.joinpath(tool_name_deb).parent.mkdir(parents=True, exist_ok=True)
+            INSTALL_VERSION_ROOT.joinpath(tool_name_deb).write_text(version, encoding="utf-8")
+        console.print(Panel(f"Installed Debian package for [green]{tool_name_deb}[/green]", title="✅ Complete", border_style="green"))
+        return
+    system_name = platform.system()
+    tool_name = _derive_tool_name(repo_name, asset_name)
+    rename_target = f"{tool_name}.exe" if system_name == "Windows" else tool_name
+    try:
+        if system_name == "Windows":
+            installed_path = find_move_delete_windows(downloaded_file_path=extracted_path, tool_name=tool_name, delete=True, rename_to=rename_target)
+        elif system_name in {"Linux", "Darwin"}:
+            installed_path = find_move_delete_linux(downloaded=extracted_path, tool_name=tool_name, delete=True, rename_to=rename_target)
+        else:
+            console.print(Panel(f"Unsupported operating system: {system_name}", title="❌ Error", border_style="red"))
+            return None
+    except IndexError:
+        if system_name == "Windows":
+            installed_path = find_move_delete_windows(downloaded_file_path=extracted_path, tool_name=None, delete=True, rename_to=rename_target)
+        elif system_name in {"Linux", "Darwin"}:
+            installed_path = find_move_delete_linux(downloaded=extracted_path, tool_name="", delete=True, rename_to=rename_target)
+        else:
+            raise
+    if tool_name is not None:
+        INSTALL_VERSION_ROOT.joinpath(tool_name).parent.mkdir(parents=True, exist_ok=True)
+        INSTALL_VERSION_ROOT.joinpath(tool_name).write_text(version, encoding="utf-8")
+    console.print(Panel(f"Installed [green]{tool_name}[/green] to {installed_path}\nVersion: {version}", title="✅ Complete", border_style="green"))
+
+
+def install_from_github_url(github_url: str) -> None:
+    from machineconfig.utils.options import choose_from_options
+    from rich.console import Console
+    from rich.panel import Panel
+
+    console = Console()
+    repo_info = get_repo_name_from_url(github_url)
+    if repo_info is None:
+        console.print(Panel(f"Invalid GitHub URL: {github_url}", title="❌ Error", border_style="red"))
+        return None
+    owner, repo = repo_info
+    repo_name = f"{owner}/{repo}"
+    console.print(Panel(f"Fetching latest release for [green]{repo_name}[/green]", title="🌐 GitHub", border_style="blue"))
+    release_raw = fetch_github_release_data(owner, repo)
+    if not release_raw:
+        console.print(Panel("No releases available for this repository.", title="❌ Error", border_style="red"))
+        return None
+    
+    release_info = extract_release_info(release_raw)
+    if not release_info:
+        console.print(Panel("Failed to parse release information.", title="❌ Error", border_style="red"))
+        return None
+    
+    assets = release_info["assets"]
+    if not assets:
+        console.print(Panel("No downloadable assets found in the latest release.", title="❌ Error", border_style="red"))
+        return None
+    binary_assets = assets
+    selection_pool = binary_assets if binary_assets else assets
+    if not selection_pool:
+        console.print(Panel("No assets available for installation.", title="❌ Error", border_style="red"))
+        return None
+    
+    # First pass: collect all formatted data and calculate column widths
+    asset_data = []
+    for asset in selection_pool:
+        name = asset["name"]
+        download_url = asset["browser_download_url"]
+        if name == "" or download_url == "":
+            continue
+        size = asset["size"]
+        download_count = asset.get("download_count", 0)
+        created_at = asset.get("created_at", "")
+        
+        # Format each field
+        size_str = f"[{_format_size(size)}]"
+        downloads_str = f"{download_count:,}"
+        date_str = created_at.split("T")[0] if created_at else "N/A"
+        
+        asset_data.append({
+            "name": name,
+            "size_str": size_str,
+            "downloads_str": downloads_str,
+            "date_str": date_str,
+            "asset": asset
+        })
+    
+    # Calculate maximum widths for alignment
+    max_name_len = max(len(item["name"]) for item in asset_data) if asset_data else 0
+    max_size_len = max(len(item["size_str"]) for item in asset_data) if asset_data else 0
+    max_downloads_len = max(len(item["downloads_str"]) for item in asset_data) if asset_data else 0
+    
+    # Second pass: build aligned labels
+    options_map: dict[str, AssetInfo] = {}
+    for item in asset_data:
+        name_padded = item["name"].ljust(max_name_len)
+        size_padded = item["size_str"].ljust(max_size_len)
+        downloads_padded = item["downloads_str"].rjust(max_downloads_len)
+        
+        label = f"{name_padded} {size_padded} | #🔽 {downloads_padded} | 📅 {item['date_str']}"
+        options_map[label] = item["asset"]
+    
+    if not options_map:
+        console.print(Panel("Release assets lack download URLs.", title="❌ Error", border_style="red"))
+        return None
+    selection_label = choose_from_options(options=list(options_map.keys()), msg="Select a release asset", multi=False, header="📦 GitHub Release Assets", tv=True)
+    selected_asset = options_map[selection_label]
+    download_url_value = selected_asset["browser_download_url"]
+    asset_name_value = selected_asset["name"]
+    if download_url_value == "":
+        console.print(Panel("Selected asset lacks a download URL.", title="❌ Error", border_style="red"))
+        return None
+    asset_name = asset_name_value if asset_name_value != "" else "github_binary"
+    version = release_info["tag_name"] if release_info["tag_name"] != "" else "latest"
+    console.print(Panel(f"Downloading [cyan]{asset_name}[/cyan]", title="⬇️ Download", border_style="magenta"))
+    extracted_path = download_and_prepare(download_url_value)
+    _finalize_install(repo_name=repo_name, asset_name=asset_name, version=version, extracted_path=extracted_path, console=console)
+
+
+def install_from_binary_url(binary_url: str) -> None:
+    from rich.console import Console
+    # from rich.panel import Panel
+    console = Console()
+    # parsed = urlparse(binary_url)
+    # asset_candidate = parsed.path.split("/")[-1] if parsed.path else ""
+    # asset_name = asset_candidate if asset_candidate != "" else "binary_asset"
+    # host = parsed.netloc if parsed.netloc != "" else "remote host"
+    # console.print(Panel(f"Downloading from [green]{binary_url}[/green]", title="⬇️ Download", border_style="magenta"))
+    extracted_path = download_and_prepare(binary_url)
+    _finalize_install(repo_name="", asset_name=None, version="latest", extracted_path=extracted_path, console=console)