sima-cli 0.0.20__py3-none-any.whl → 0.0.22__py3-none-any.whl

sima_cli/install/metadata_installer.py

@@ -0,0 +1,447 @@
+import os
+import re
+import tempfile
+import click
+import json
+import sys
+import shutil
+import tarfile
+import zipfile
+from urllib.parse import urlparse
+
+from typing import Dict
+from tqdm import tqdm
+from urllib.parse import urljoin
+from pathlib import Path
+import subprocess
+
+from rich.console import Console
+from rich.panel import Panel
+
+from huggingface_hub import snapshot_download
+
+from sima_cli.utils.disk import check_disk_space
+from sima_cli.utils.env import get_environment_type, get_exact_devkit_type
+from sima_cli.download.downloader import download_file_from_url
+from sima_cli.install.metadata_validator import validate_metadata, MetadataValidationError
+from sima_cli.install.metadata_info import print_metadata_summary, parse_size_string_to_bytes
+
+console = Console()
+
+def _download_assets(metadata: dict, base_url: str, dest_folder: str, internal: bool = False, skip_models: bool = False) -> list:
+    """
+    Downloads resources defined in metadata to a local destination folder.
+
+    Args:
+        metadata (dict): Parsed and validated metadata
+        base_url (str): Base URL of the metadata file (used to resolve relative resource paths)
+        dest_folder (str): Local path to download resources into
+        internal (bool): Whether to use internal download routing (if applicable)
+        skip_models (bool): If True, skips downloading any file path starting with 'models/'
+
+    Returns:
+        list: Paths to the downloaded local files
+    """
+    resources = metadata.get("resources", [])
+    if not resources:
+        raise click.ClickException("❌ No 'resources' defined in metadata.")
+
+    os.makedirs(dest_folder, exist_ok=True)
+    local_paths = []
+
+    filtered_resources = []
+    for r in resources:
+        if skip_models and r.strip().lower().startswith("models/"):
+            click.echo(f"⏭️  Skipping model file: {r}")
+            continue
+        filtered_resources.append(r)
+
+    if not filtered_resources:
+        click.echo("ℹ️ No non-model resources to download.")
+        return []
+
+    click.echo(f"📥 Downloading {len(filtered_resources)} resource(s) to: {dest_folder}\n")
+
+    for resource in filtered_resources:
+        try:
+            # Handle Hugging Face snapshot-style URL: "hf:<repo_id>@version"
+            if resource.startswith("hf:"):
+                # Strip prefix and split by @
+                resource_spec = resource[3:]
+                if "@" in resource_spec:
+                    repo_id, revision = resource_spec.split("@", 1)
+                else:
+                    repo_id, revision = resource_spec, None
+
+                if "/" not in repo_id:
+                    raise click.ClickException(f"❌ Invalid Hugging Face repo spec: {resource}")
+
+                org, name = repo_id.split("/", 1)
+                target_dir = os.path.join(dest_folder, name)
+
+                click.echo(f"🤗 Downloading Hugging Face repo: {repo_id}" + (f"@{revision}" if revision else ""))
+                model_path = snapshot_download(
+                    repo_id=repo_id,
+                    local_dir=target_dir,
+                    revision=revision  # None if not specified
+                )
+                local_paths.append(model_path)
+                continue
+
+            # Handle normal relative or absolute URLs
+            resource_url = urljoin(base_url, resource)
+            local_path = download_file_from_url(
+                url=resource_url,
+                dest_folder=dest_folder,
+                internal=internal
+            )
+            click.echo(f"✅ Downloaded: {resource}")
+            local_paths.append(local_path)
+
+        except Exception as e:
+            raise click.ClickException(f"❌ Failed to download resource '{resource}': {e}")
+
+    return local_paths
+
+def _download_and_validate_metadata(metadata_url, internal=False):
+    """
+    Downloads (if remote), validates, and parses metadata from a given URL or local file path.
+
+    Args:
+        metadata_url (str): URL or local path to a metadata.json file
+        internal (bool): Whether to use internal mirrors or logic in downloader
+
+    Returns:
+        tuple: (parsed metadata dict, folder containing the metadata file)
+    """
+    try:
+        parsed = urlparse(metadata_url)
+
+        # Case 1: Local file (e.g., /path/to/file or ./file)
+        if parsed.scheme == "" or parsed.scheme == "file":
+            metadata_path = parsed.path
+            if not os.path.isfile(metadata_path):
+                raise FileNotFoundError(f"File not found: {metadata_path}")
+            click.echo(f"📄 Using local metadata file: {metadata_path}")
+
+        # Case 2: Remote URL
+        else:
+            with tempfile.TemporaryDirectory() as tmpdir:
+                metadata_path = download_file_from_url(
+                    url=metadata_url,
+                    dest_folder=tmpdir,
+                    internal=internal
+                )
+                click.echo(f"⬇️  Downloaded metadata to: {metadata_path}")
+                
+                # Must copy to outside tmpdir since tmpdir will be deleted
+                # But since we're returning contents only, no need to keep file
+                with open(metadata_path, "r", encoding="utf-8") as f:
+                    metadata = json.load(f)
+                validate_metadata(metadata)
+                click.echo("✅ Metadata validated successfully.")
+                return metadata, os.path.dirname(metadata_path)
+
+        # Common validation logic for local file
+        with open(metadata_path, "r", encoding="utf-8") as f:
+            metadata = json.load(f)
+
+        validate_metadata(metadata)
+        click.echo("✅ Metadata validated successfully.")
+        return metadata, os.path.dirname(os.path.abspath(metadata_path))
+
+    except MetadataValidationError as e:
+        click.echo(f"❌ Metadata validation failed: {e}")
+        raise click.Abort()
+
+    except Exception as e:
+        click.echo(f"❌ Failed to retrieve or parse metadata from {metadata_url}: {e}")
+        raise click.Abort()
+    
+def _check_whether_disk_is_big_enough(metadata: dict):
+    # Step 3: Disk space check
+    try:
+        install_size_str = metadata.get("size", {}).get("install")
+        if install_size_str:
+            required_bytes = parse_size_string_to_bytes(install_size_str)
+            if not check_disk_space(required_bytes, folder="."):
+                required_gb = required_bytes / 1e9
+                raise click.ClickException(
+                    f"Not enough disk space. At least {required_gb:.2f} GB required the in current directory."
+                )
+
+            available_bytes = shutil.disk_usage(".").free
+            available_gb = available_bytes / 1e9
+            required_gb = required_bytes / 1e9
+            click.echo(f"🗄️  Available disk space: {available_gb:.2f} GB")
+            click.echo(f"✅ Enough disk space for installation: requires {required_gb:.2f} GB")
+            return True
+    except Exception as e:
+        click.echo(f"❌ Failed to validate disk space: {e}")
+        raise click.Abort()
+
+    return False
+
+def _extract_tar_streaming(tar_path: Path, extract_dir: Path):
+    """
+    Extract tar while preserving full folder structure.
+    """
+    extracted_files = 0
+    with tarfile.open(tar_path, "r:*") as tar:
+        with tqdm(desc=f"📦 Extracting {tar_path.name}", unit=" file") as pbar:
+            while True:
+                member = tar.next()
+                if member is None:
+                    break
+
+                # Don't strip anything — preserve full path
+                if not member.name.strip():
+                    print(f"⚠️ Skipping empty member in archive: {member}")
+                    continue
+
+                tar.extract(member, path=extract_dir)
+                extracted_files += 1
+                pbar.update(1)
+
+    print(f"✅ Extracted {extracted_files} files to {extract_dir}/")
+
+def _extract_zip_streaming(zip_path: Path, extract_dir: Path):
+    """
+    Extract a .zip file using streaming to avoid NFS slowness from metadata calls,
+    and flatten one top-level directory if present.
+    """
+    with zipfile.ZipFile(zip_path, "r") as zipf:
+        members = zipf.infolist()
+        with tqdm(total=len(members), desc=f"📦 Extracting {zip_path.name}", unit="file") as pbar:
+            for member in members:
+                # Strip one top-level directory if it exists
+                parts = Path(member.filename).parts
+                if len(parts) > 1:
+                    stripped_path = Path(*parts[1:])
+                else:
+                    stripped_path = Path(parts[-1])
+
+                target_path = extract_dir / stripped_path
+                target_path.parent.mkdir(parents=True, exist_ok=True)
+
+                with zipf.open(member) as src, open(target_path, "wb") as dst:
+                    shutil.copyfileobj(src, dst)
+
+                pbar.update(1)
+
+    print(f"✅ Extracted {len(members)} files to {extract_dir}/")
+
+def _combine_multipart_files(folder: str):
+    """
+    Scan a folder for multipart files like name-split-aa, -ab, etc.,
+    combine them into a single file, and remove the split parts.
+    Then auto-extract .tar files with progress.
+    """
+    folder = Path(folder)
+    parts_by_base = {}
+
+    # Step 1: Group parts by base name
+    for file in folder.iterdir():
+        if not file.is_file():
+            continue
+
+        match = re.match(r"(.+)-split-([a-z]{2})$", file.name)
+        if match:
+            base, part = match.groups()
+            parts_by_base.setdefault(base, []).append((part, file))
+
+    # Step 2: Process each group
+    for base, parts in parts_by_base.items():
+        parts.sort(key=lambda x: x[0])
+        output_file = folder / f"{base}.tar"
+        total_size = sum(part_file.stat().st_size for _, part_file in parts)
+
+        print(f"\n🧩 Reassembling: {output_file.name} from {len(parts)} parts")
+
+        if not output_file.exists():
+            with open(output_file, "wb") as outfile, tqdm(
+                total=total_size,
+                unit="B",
+                unit_scale=True,
+                unit_divisor=1024,
+                desc=f"Combining {output_file.name}",
+            ) as pbar:
+                for _, part_file in parts:
+                    with open(part_file, "rb") as infile:
+                        while True:
+                            chunk = infile.read(1024 * 1024)  # 1MB
+                            if not chunk:
+                                break
+                            outfile.write(chunk)
+                            pbar.update(len(chunk))
+
+        # Step 3: Remove original parts
+        # for _, part_file in parts:
+        #     part_file.unlink()
+
+        print(f"✅ Created: {output_file.name} ({output_file.stat().st_size / 1e6:.2f} MB)")
+
+        # Step 4: Auto-extract .tar
+        extract_dir = folder / base
+        print(f"📦 Extracting {output_file.name} to {extract_dir}/")
+        _extract_tar_streaming(output_file, extract_dir)
+
+        print(f"✅ Extracted to: {extract_dir}/")
+
+def _extract_archives_in_folder(folder: str):
+    """
+    Extract all .tar.gz and .zip files in the given folder into subdirectories.
+    Uses streaming to avoid NFS performance issues.
+    """
+    folder = Path(folder)
+    for file in folder.iterdir():
+        if not file.is_file():
+            continue
+
+        # TAR.GZ
+        if file.suffixes == [".tar", ".gz"] or file.name.endswith(".tar.gz"):
+            extract_dir = folder / file.stem.replace(".tar", "")
+            print(f"📦 Extracting TAR.GZ: {file.name} to {extract_dir}/")
+            _extract_tar_streaming(file, extract_dir)
+
+        # ZIP
+        elif file.suffix == ".zip":
+            extract_dir = folder / file.stem
+            print(f"📦 Extracting ZIP: {file.name} to {extract_dir}/")
+            _extract_zip_streaming(file, extract_dir)
+
+def _is_platform_compatible(metadata: dict) -> bool:
+    """
+    Determines if the current environment is compatible with the package metadata.
+
+    Args:
+        metadata (dict): Metadata that includes a 'platforms' section
+
+    Returns:
+        bool: True if compatible, False otherwise
+    """
+    env_type, env_subtype = get_environment_type()
+    exact_devkit_type = get_exact_devkit_type()
+    platforms = metadata.get("platforms", [])
+
+    for i, platform_entry in enumerate(platforms):
+        platform_type = platform_entry.get("type")
+        if platform_type != env_type:
+            continue
+
+        # For board/devkit: check compatible_with list
+        if env_type == "board":
+            compat = platform_entry.get("compatible_with", [])
+            if env_subtype not in compat and exact_devkit_type not in compat:
+                continue
+
+        # For host/sdk/generic: optionally check OS match
+        if "os" in platform_entry:
+            supported_oses = [os_name.lower() for os_name in platform_entry["os"]]
+            if env_subtype.lower() not in supported_oses:
+                continue
+
+        # Passed all checks
+        return True
+
+    click.echo(f"❌ Current environment [{env_type}:{env_subtype}] is not compatible with the package")
+    return False
+
+
+def _print_post_install_message(metadata: Dict):
+    """
+    Print post-installation instructions from the metadata in a compact box.
+
+    Args:
+        metadata (Dict): The package metadata dictionary.
+    """
+    msg = metadata.get("installation", {}).get("post-message", "").strip()
+
+    if msg:
+        panel = Panel.fit(
+            msg,
+            title="[bold green]Post-Installation Instructions[/bold green]",
+            title_align="left",
+            border_style="green",
+            padding=(1, 2)
+        )
+        console.print(panel)
+
+def _run_installation_script(metadata: Dict, extract_path: str = "."):
+    """
+    Run the installation script specified in the metadata.
+
+    Args:
+        metadata (dict): Metadata dictionary with an 'installation' key.
+        extract_path (str): Path where the files were extracted.
+    """
+    script = metadata.get("installation", {}).get("script", "").strip()
+    if not script:
+        print("⚠️ No installation script provided. Follow package documentation to install the package.")
+        return
+
+    print(f"🚀 Running installation script in: {os.path.abspath(extract_path)}")
+    print(f"📜 Script: {script}")
+
+    # Determine shell type based on platform
+    shell_executable = os.environ.get("COMSPEC") if os.name == "nt" else None
+
+    try:
+        subprocess.run(
+            script,
+            shell=True,
+            executable=shell_executable,
+            cwd=extract_path,
+            check=True
+        )
+        _print_post_install_message(metadata=metadata)        
+    except subprocess.CalledProcessError as e:
+        print("❌ Installation failed with return code:", e.returncode)
+        sys.exit(e.returncode)
+
+    print("✅ Installation completed successfully.")
+
+def install_from_metadata(metadata_url: str, internal: bool, install_dir: str = '.'):
+    try:
+        metadata, _ = _download_and_validate_metadata(metadata_url, internal)
+        print_metadata_summary(metadata=metadata)
+
+        if _check_whether_disk_is_big_enough(metadata):
+            if _is_platform_compatible(metadata):
+                local_paths = _download_assets(metadata, metadata_url, install_dir, internal)
+                if len(local_paths) > 0:
+                    _combine_multipart_files(install_dir)
+                    _extract_archives_in_folder(install_dir)
+                    _run_installation_script(metadata=metadata, extract_path=install_dir)
+
+    except Exception as e:
+        click.echo(f"❌ Failed to install from metadata URL {metadata_url}: {e}")
+        exit(1)
+
+    return False
+
+def metadata_resolver(component: str, version: str = None, tag: str = None) -> str:
+    """
+    Resolve the metadata.json URL for a given component and version/tag.
+
+    Args:
+        component (str): Component name (e.g., "examples.llima")
+        version (str): Optional SDK version string (e.g., "1.7.0")
+        tag (str): Optional tag to use (e.g., "dev")
+
+    Returns:
+        str: Fully qualified metadata URL
+    """
+    if not version:
+        raise ValueError("Version (-v) is required for non-hardcoded components.")
+
+    # Normalize version for URL path
+    sdk_path = f"SDK{version}"
+    base = f"https://docs.sima.ai/pkg_downloads/{sdk_path}/{component}"
+
+    if tag:
+        metadata_name = f"metadata-{tag}.json"
+    else:
+        metadata_name = "metadata.json"
+
+    return f"{base}/{metadata_name}"

sima_cli/install/metadata_validator.py

@@ -0,0 +1,138 @@
+import re
+import sys
+import json
+from pathlib import Path
+
+class MetadataValidationError(Exception):
+    pass
+
+VALID_TYPES = {"board", "palette", "host"}
+VALID_OS = {"linux", "windows", "mac"}
+
+def validate_metadata(data: dict):
+    # Top-level required fields
+    required_fields = ["name", "version", "release", "platforms", "resources"]
+    for field in required_fields:
+        if field not in data:
+            raise MetadataValidationError(f"Missing required field: '{field}'")
+
+    # Validate platforms
+    if not isinstance(data["platforms"], list):
+        raise MetadataValidationError("'platforms' must be a list")
+
+    for i, platform in enumerate(data["platforms"]):
+        if "type" not in platform:
+            raise MetadataValidationError(f"Missing 'type' in platform entry {i}")
+        if platform["type"] not in VALID_TYPES:
+            raise MetadataValidationError(
+                f"Invalid platform type '{platform['type']}' in entry {i}. Must be one of {VALID_TYPES}"
+            )
+
+        if platform["type"] == "board":
+            if "compatible_with" not in platform:
+                raise MetadataValidationError(f"'compatible_with' is required for board in entry {i}")
+            if not isinstance(platform["compatible_with"], list):
+                raise MetadataValidationError(f"'compatible_with' must be a list in entry {i}")
+
+        if "os" in platform:
+            if not isinstance(platform["os"], list):
+                raise MetadataValidationError(f"'os' must be a list in entry {i}")
+            for os_value in platform["os"]:
+                if os_value.lower() not in VALID_OS:
+                    raise MetadataValidationError(
+                        f"Invalid OS '{os_value}' in platform entry {i}. Supported: {VALID_OS}"
+                    )
+
+    # Validate resources
+    if not isinstance(data["resources"], list) or not data["resources"]:
+        raise MetadataValidationError("'resources' must be a non-empty list")
+
+    # Validate prerequisite (optional)
+    if "prerequisite" in data:
+        prereq = data["prerequisite"]
+        if "wheel_url" not in prereq or "entry_point" not in prereq:
+            raise MetadataValidationError("Both 'wheel_url' and 'entry_point' are required in 'prerequisite'")
+        _validate_entry_point_format(prereq["entry_point"], field="prerequisite.entry_point")
+
+    # Validate installation (optional)
+    if "installation" in data:
+        install = data["installation"]
+        if "script" not in install:
+            raise MetadataValidationError("Missing 'script' in 'installation'")
+        if not isinstance(install["script"], str):
+            raise MetadataValidationError("'installation.script' must be a string")
+
+    # Validate size (optional)
+    if "size" in data:
+        size = data["size"]
+        if not isinstance(size, dict):
+            raise MetadataValidationError("'size' must be a dictionary with 'download' and 'install' fields")
+
+        for key in ["download", "install"]:
+            if key not in size:
+                raise MetadataValidationError(f"Missing '{key}' in 'size'")
+            if not isinstance(size[key], str):
+                raise MetadataValidationError(f"'size.{key}' must be a string")
+
+            size_str = size[key].strip().upper()
+            if not any(size_str.endswith(unit) for unit in ["KB", "MB", "GB"]):
+                raise MetadataValidationError(
+                    f"'size.{key}' must end with one of: KB, MB, GB (e.g., '30GB')"
+                )
+
+            try:
+                # Extract number (e.g., "30GB" → 30.0)
+                float(size_str[:-2].strip())
+            except ValueError:
+                raise MetadataValidationError(
+                    f"'size.{key}' must start with a numeric value (e.g., '30GB')"
+                )
+
+    return True
+
+
+def _validate_entry_point_format(entry_point: str, field: str):
+    if not re.match(r"^[a-zA-Z0-9_.\-]+:[a-zA-Z0-9_]+$", entry_point):
+        raise MetadataValidationError(
+            f"Invalid format for {field}. Must be in the form 'module:function'"
+        )
+
+
+def validate_file(filepath):
+    try:
+        with open(filepath, "r") as f:
+            metadata = json.load(f)
+        validate_metadata(metadata)
+        print(f"✅ {filepath} is valid.")
+    except FileNotFoundError:
+        print(f"❌ File not found: {filepath}")
+    except json.JSONDecodeError as e:
+        print(f"❌ JSON parse error in {filepath}: {e}")
+    except MetadataValidationError as e:
+        print(f"❌ Validation failed in {filepath}: {e}")
+
+def main():
+    if len(sys.argv) != 2:
+        print("Usage: python validate_metadata.py <file-or-folder>")
+        sys.exit(1)
+
+    path = Path(sys.argv[1])
+
+    if not path.exists():
+        print(f"❌ Path does not exist: {path}")
+        sys.exit(1)
+
+    if path.is_file():
+        validate_file(path)
+    elif path.is_dir():
+        json_files = list(path.rglob("*.json"))
+        if not json_files:
+            print(f"⚠️ No JSON files found in directory: {path}")
+        for file in json_files:
+            validate_file(file)
+    else:
+        print(f"❌ Unsupported path type: {path}")
+        sys.exit(1)
+
+if __name__ == "__main__":
+    main()