qm-template 0.1.0__py3-none-any.whl

qm_template/__init__.py

@@ -0,0 +1,10 @@
+"""Download cloud images and create Proxmox VE VM templates."""
+
+from importlib.metadata import PackageNotFoundError, version
+
+PROGRAM = "qm-template"
+
+try:
+    __version__ = version(PROGRAM)
+except PackageNotFoundError:
+    __version__ = "0.0.0"

qm_template/__main__.py

@@ -0,0 +1,6 @@
+import sys
+
+from qm_template.cli import main
+
+if __name__ == "__main__":
+    sys.exit(main())

qm_template/checksum.py

@@ -0,0 +1,65 @@
+import hashlib
+import re
+from pathlib import Path
+
+from qm_template.errors import QmTemplateError
+from qm_template.http import http_get_text
+from qm_template.log import log
+
+_CHECKSUM_SUFFIXES = {"sha256": ".sha256", "sha512": ".sha512"}
+
+_CHECKSUM_PATTERNS = (
+    re.compile(r"^(?P<digest>[0-9a-fA-F]{32,128})[ \t]+\*?(?P<name>\S.*?)\s*$"),
+    re.compile(
+        r"^(?:SHA\d+|MD5)\s*\((?P<name>.+?)\)\s*=\s*(?P<digest>[0-9a-fA-F]+)\s*$"
+    ),
+    re.compile(r"^(?P<digest>[0-9a-fA-F]{32,128})$"),
+)
+
+
+def parse_checksum(text: str, filename: str) -> str:
+    entries: list[tuple[str, str]] = []
+    for raw_line in text.splitlines():
+        line = raw_line.strip()
+        if not line or line.startswith(("#", "-", "Hash:")):
+            continue
+        for pattern in _CHECKSUM_PATTERNS:
+            match = pattern.match(line)
+            if match:
+                groups = match.groupdict()
+                entries.append((groups.get("name") or "", groups["digest"].lower()))
+                break
+    for name, digest in entries:
+        if Path(name).name == filename:
+            return digest
+    if len(entries) == 1:
+        return entries[0][1]
+    raise QmTemplateError(f"no checksum for {filename!r} in checksum file")
+
+
+def fetch_checksum(url: str, filename: str) -> str:
+    log.debug("Fetching checksum from: %s", url)
+    return parse_checksum(http_get_text(url), filename)
+
+
+def verify_checksum(path: Path, expected: str, algorithm: str) -> bool:
+    with path.open("rb") as handle:
+        actual = hashlib.file_digest(handle, algorithm).hexdigest()
+    if actual != expected.lower():
+        log.debug(
+            "Checksum mismatch for %s: expected %s, got %s", path, expected, actual
+        )
+        return False
+    return True
+
+
+def save_checksum(path: Path, digest: str, algorithm: str) -> Path:
+    """Write a sha256sum-compatible checksum file next to the image."""
+    try:
+        suffix = _CHECKSUM_SUFFIXES[algorithm]
+    except KeyError:
+        raise QmTemplateError(f"unsupported checksum algorithm: {algorithm}") from None
+    target = path.with_name(path.name + suffix)
+    target.write_text(f"{digest.lower()}  {path.name}\n", encoding="ascii")
+    log.debug("Wrote checksum file: %s", target)
+    return target

qm_template/cli.py

@@ -0,0 +1,64 @@
+import argparse
+import sys
+from collections.abc import Sequence
+
+from qm_template import PROGRAM, __version__
+from qm_template.commands import COMMANDS
+from qm_template.config import load_settings, resolve_config_path, write_default_config
+from qm_template.errors import QmTemplateError, UserCancelled
+from qm_template.log import log, setup_logging
+
+
+def build_parser() -> argparse.ArgumentParser:
+    common = argparse.ArgumentParser(add_help=False)
+    common.add_argument(
+        "--config",
+        metavar="PATH",
+        default=argparse.SUPPRESS,
+        help="configuration file (default: /etc/qm-template/config.toml)",
+    )
+    parser = argparse.ArgumentParser(
+        prog=PROGRAM,
+        description="Download cloud images and create Proxmox VE VM templates.",
+        parents=[common],
+    )
+    parser.add_argument(
+        "--version", action="version", version=f"%(prog)s {__version__}"
+    )
+    subparsers = parser.add_subparsers(dest="command", metavar="COMMAND", required=True)
+    for command in COMMANDS:
+        subparser = subparsers.add_parser(
+            command.name,
+            aliases=list(command.aliases),
+            parents=[common],
+            help=command.help,
+            description=command.description,
+        )
+        command.add_arguments(subparser)
+        subparser.set_defaults(handler=command.run)
+    return parser
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    args = build_parser().parse_args(argv)
+    setup_logging()
+    try:
+        config_path, explicit = resolve_config_path(getattr(args, "config", None))
+        if not explicit:
+            write_default_config(config_path)
+        settings = load_settings(config_path, explicit=explicit)
+        args.handler(args, settings)
+    except UserCancelled:
+        log.info("Operation cancelled by user")
+        return 0
+    except QmTemplateError as exc:
+        log.error("%s", exc)
+        return 1
+    except KeyboardInterrupt:
+        log.warning("Interrupted")
+        return 130
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

qm_template/commands.py

@@ -0,0 +1,204 @@
+import argparse
+from collections.abc import Callable
+from dataclasses import dataclass, replace
+
+from qm_template import PROGRAM
+from qm_template.checksum import fetch_checksum, save_checksum, verify_checksum
+from qm_template.config import Settings
+from qm_template.distros import DISTROS
+from qm_template.download import download_image, part_path, select_downloaders
+from qm_template.errors import QmTemplateError
+from qm_template.log import log
+from qm_template.pve import (
+    build_qm_create,
+    check_storage,
+    choose_image,
+    choose_vm_id,
+    default_vm_name,
+    find_images,
+    run_qm,
+    sshkeys_file,
+    vm_config_path,
+)
+from qm_template.shell import pretty
+
+
+def add_download_arguments(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument(
+        "distro",
+        nargs="?",
+        help="distro name (default from configuration)",
+    )
+    parser.add_argument("--release", help="distro release or codename")
+    parser.add_argument("--variant", help="image variant")
+    parser.add_argument("--arch", help="image architecture")
+    parser.add_argument("--tag", help="specific image tag/version")
+    parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="print the download command and exit",
+    )
+
+
+def run_download(args: argparse.Namespace, settings: Settings) -> None:
+    name = args.distro or settings.download.default_distro
+    distro = DISTROS.get(name)
+    if distro is None:
+        raise QmTemplateError(
+            f"unknown distro {name!r} (run `{PROGRAM} distros` for a list)"
+        )
+    overrides = {
+        "release": args.release,
+        "variant": args.variant,
+        "arch": args.arch,
+        "tag": args.tag,
+    }
+    if args.tag and not distro.supports_tag:
+        log.warning("%s does not support --tag, ignoring it", distro.name)
+        overrides["tag"] = None
+    params = distro.merge(settings.download.defaults.get(distro.name, {}), overrides)
+    image = distro.resolve(params)
+    log.info("Image: %s", image.local_path)
+    log.debug("URL: %s", image.url)
+    log.debug("Checksum: %s (%s)", image.checksum_url, image.algorithm)
+    destination = settings.images_dir / image.local_path
+    if args.dry_run:
+        downloader = select_downloaders(
+            settings.download.preferred, settings.download.connections
+        )[0]
+        print(pretty(downloader.build_command(image.url, part_path(destination))))
+        return
+    destination.parent.mkdir(parents=True, exist_ok=True)
+    expected = fetch_checksum(image.checksum_url, image.filename)
+    if destination.is_file():
+        if verify_checksum(destination, expected, image.algorithm):
+            save_checksum(destination, expected, image.algorithm)
+            log.info("Already up to date: %s", destination)
+            return
+        log.warning("Checksum mismatch for %s, removing it", destination)
+        destination.unlink()
+    part = download_image(
+        image,
+        destination,
+        select_downloaders(settings.download.preferred, settings.download.connections),
+    )
+    log.info("Verifying checksum")
+    if not verify_checksum(part, expected, image.algorithm):
+        part.unlink(missing_ok=True)
+        raise QmTemplateError("downloaded image failed checksum verification")
+    part.replace(destination)
+    save_checksum(destination, expected, image.algorithm)
+    log.info("Saved %s", destination)
+
+
+def add_create_arguments(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument("pattern", nargs="?", help="regex to filter local images")
+    parser.add_argument("--vm-id", type=int, help="VM ID (prompted when omitted)")
+    parser.add_argument(
+        "--vm-name", help="VM name (derived from the image when omitted)"
+    )
+    parser.add_argument("--storage", help="Proxmox storage for VM disks")
+    parser.add_argument("--cores", type=int, help="number of CPU cores")
+    parser.add_argument("--memory", type=int, help="memory in MiB")
+    parser.add_argument("--bridge", help="network bridge")
+    parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="print the assembled qm command and exit",
+    )
+
+
+def run_create(args: argparse.Namespace, settings: Settings) -> None:
+    create = replace(
+        settings.create,
+        storage=(args.storage if args.storage is not None else settings.create.storage),
+        cores=args.cores if args.cores is not None else settings.create.cores,
+        memory=args.memory if args.memory is not None else settings.create.memory,
+        bridge=args.bridge if args.bridge is not None else settings.create.bridge,
+    )
+    if create.cores < 1:
+        raise QmTemplateError("cores must be a positive integer")
+    if create.memory < 1:
+        raise QmTemplateError("memory must be a positive integer")
+
+    images = find_images(settings.images_dir, args.pattern)
+    if len(images) == 1:
+        image = images[0]
+        log.info("Selected image: %s", image.name)
+    else:
+        image = choose_image(images, settings.images_dir)
+
+    vm_name = args.vm_name or default_vm_name(image)
+    if args.vm_id is not None:
+        if args.vm_id <= 0:
+            raise QmTemplateError("VM ID must be a positive integer")
+        if vm_config_path(args.vm_id).exists():
+            raise QmTemplateError(f"VM ID {args.vm_id} is already in use")
+        vm_id = args.vm_id
+    else:
+        vm_id = choose_vm_id()
+    log.info("Creating VM %d (%s)", vm_id, vm_name)
+    log.debug(
+        "storage=%s cores=%d memory=%d bridge=%s",
+        create.storage,
+        create.cores,
+        create.memory,
+        create.bridge,
+    )
+
+    check_storage(create.storage)
+    with sshkeys_file(create) as sshkeys:
+        command = build_qm_create(vm_id, vm_name, image, sshkeys, create)
+        if args.dry_run:
+            print(pretty(command))
+            return
+        run_qm(command)
+    log.info("Template %s (ID %d) created", vm_name, vm_id)
+
+
+def add_distros_arguments(parser: argparse.ArgumentParser) -> None:
+    pass
+
+
+def run_distros(args: argparse.Namespace, settings: Settings) -> None:
+    for distro in DISTROS.values():
+        params = distro.merge(settings.download.defaults.get(distro.name, {}), {})
+        rendered = " ".join(f"{key}={value}" for key, value in sorted(params.items()))
+        print(f"{distro.name:<12} {distro.description:<26} {rendered}")
+
+
+@dataclass(frozen=True)
+class Command:
+    name: str
+    help: str
+    add_arguments: Callable[[argparse.ArgumentParser], None]
+    run: Callable[[argparse.Namespace, Settings], None]
+    description: str = ""
+    aliases: tuple[str, ...] = ()
+
+
+COMMANDS: tuple[Command, ...] = (
+    Command(
+        name="download",
+        help="download a cloud image",
+        add_arguments=add_download_arguments,
+        run=run_download,
+        description="Resolve, download and verify a distro cloud image.",
+        aliases=("get",),
+    ),
+    Command(
+        name="create",
+        help="create a Proxmox VE VM template",
+        add_arguments=add_create_arguments,
+        run=run_create,
+        description="Create a Proxmox VE VM template from a downloaded cloud image.",
+        aliases=("template",),
+    ),
+    Command(
+        name="distros",
+        help="list supported distros",
+        add_arguments=add_distros_arguments,
+        run=run_distros,
+        description="List supported distros with their configured defaults.",
+    ),
+)

qm_template/config.default.toml

@@ -0,0 +1,90 @@
+# qm-template configuration
+#
+# Default location:
+#   /etc/qm-template/config.toml
+#
+# Every setting is optional; the values shown here are the built-in defaults.
+
+[paths]
+# Directory where downloaded cloud images are stored.
+images_dir = "/var/lib/qm-template"
+
+[download]
+# Downloader preference order; the first available command is used.
+preferred = ["axel", "aria2c", "wget", "curl"]
+
+# Parallel connections per file for axel and aria2c; ignored by wget and curl.
+connections = 8
+
+# Distro used when `qm-template download` is called without a name.
+default_distro = "debian"
+
+# Per-distro overrides for `qm-template download`. Any table can be removed to
+# fall back to the built-in defaults shown by `qm-template distros`.
+
+[download.debian]
+release = "trixie"       # codename, optionally with -backports
+variant = "genericcloud" # genericcloud, generic, nocloud
+arch = "amd64"           # amd64, arm64
+
+[download.ubuntu]
+release = "resolute"     # codename, e.g. noble, jammy
+variant = "server"       # server, minimal
+arch = "amd64"
+
+[download.rocky]
+release = "10"
+variant = "GenericCloud" # GenericCloud, GenericCloud-LVM
+arch = "x86_64"
+
+[download.almalinux]
+release = "10"
+variant = "GenericCloud" # GenericCloud, GenericCloud-ext4
+arch = "x86_64"
+
+[download.fedora]
+release = "44"
+variant = "Generic"      # Generic, UEFI-UKI
+arch = "x86_64"
+
+[download.centos]
+release = "10"
+variant = "GenericCloud"
+arch = "x86_64"
+
+[download.alpine]
+release = "3.24"
+variant = "generic"      # generic, nocloud
+arch = "x86_64"
+
+[download.opensuse]
+release = "tumbleweed"   # tumbleweed or a Leap version, e.g. "15.6"
+variant = "Minimal"
+arch = "x86_64"
+
+[download.archlinux]
+release = "latest"
+variant = "cloudimg"     # cloudimg, basic
+arch = "x86_64"
+
+[create]
+# Proxmox storage for the imported disk and the Cloud-Init drive.
+storage = "local-lvm"
+cores = 1
+memory = 1024
+bridge = "vmbr0"
+ciuser = "debian"
+# Kept as a fixed default so the console rescue password stays predictable;
+# change it here to use something else.
+cipassword = "debian"
+
+# SSH public keys injected via Cloud-Init. Inline keys in sshkeys and the
+# contents of every sshkeys_files entry are merged and deduplicated by SSH key
+# fingerprint; at least one key is required. Each inline entry must look like
+# an SSH public key. No key file is read unless configured here.
+sshkeys = [
+    # "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAA... user@example.com",
+]
+sshkeys_files = [
+    # "~/.ssh/id_ed25519.pub",
+]

qm_template/config.py

@@ -0,0 +1,239 @@
+import base64
+import binascii
+import hashlib
+import os
+import tomllib
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass, field
+from importlib import resources
+from pathlib import Path
+from typing import Any
+
+from qm_template import PROGRAM
+from qm_template.distros import DISTROS
+from qm_template.errors import QmTemplateError
+from qm_template.log import log
+
+
+def default_config_path() -> Path:
+    return Path("/etc") / PROGRAM / "config.toml"
+
+
+def default_images_dir() -> Path:
+    return Path("/var/lib") / PROGRAM
+
+
+def resolve_config_path(cli_value: str | None) -> tuple[Path, bool]:
+    if cli_value:
+        return Path(cli_value).expanduser(), True
+    env_value = os.environ.get("QM_TEMPLATE_CONFIG")
+    if env_value:
+        return Path(env_value).expanduser(), True
+    return default_config_path(), False
+
+
+def _packaged_config_text() -> str:
+    config = resources.files("qm_template") / "config.default.toml"
+    return config.read_text(encoding="utf-8")
+
+
+def write_default_config(path: Path) -> bool:
+    """Write the packaged default configuration unless a file already exists."""
+    if path.exists():
+        return False
+    try:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(_packaged_config_text(), encoding="utf-8")
+    except OSError as exc:
+        log.debug("Could not write default configuration to %s: %s", path, exc)
+        return False
+    log.info("Wrote default configuration to %s", path)
+    return True
+
+
+@dataclass(frozen=True)
+class DownloadSettings:
+    preferred: tuple[str, ...] = ("axel", "aria2c", "wget", "curl")
+    connections: int = 8
+    default_distro: str = "debian"
+    defaults: Mapping[str, Mapping[str, str]] = field(default_factory=dict)
+
+
+@dataclass(frozen=True)
+class CreateSettings:
+    storage: str = "local-lvm"
+    cores: int = 1
+    memory: int = 1024
+    bridge: str = "vmbr0"
+    ciuser: str = "debian"
+    cipassword: str = "debian"
+    sshkeys: tuple[str, ...] = ()
+    sshkeys_files: tuple[str, ...] = ()
+
+
+@dataclass(frozen=True)
+class Settings:
+    images_dir: Path = field(default_factory=default_images_dir)
+    download: DownloadSettings = field(default_factory=DownloadSettings)
+    create: CreateSettings = field(default_factory=CreateSettings)
+
+
+_DOWNLOAD_DEFAULTS = DownloadSettings()
+_CREATE_DEFAULTS = CreateSettings()
+_DOWNLOAD_KEYS = frozenset(DownloadSettings.__dataclass_fields__) - {"defaults"}
+_CREATE_KEYS = frozenset(CreateSettings.__dataclass_fields__)
+
+
+def _table(data: Mapping[str, Any], key: str, source: Path) -> Mapping[str, Any]:
+    value = data.get(key, {})
+    if not isinstance(value, dict):
+        raise QmTemplateError(f"[{key}] must be a table in {source}")
+    return value
+
+
+def _reject_unknown(
+    table: Mapping[str, Any],
+    allowed: set[str] | frozenset[str],
+    section: str | None,
+    source: Path,
+) -> None:
+    unknown = sorted(set(table) - allowed)
+    if unknown:
+        names = ", ".join(repr(key) for key in unknown)
+        where = f"in [{section}]" if section else "at the top level"
+        raise QmTemplateError(f"unknown keys {names} {where} of {source}")
+
+
+def _str(table: Mapping[str, Any], key: str, default: str, source: Path) -> str:
+    value = table.get(key, default)
+    if not isinstance(value, str):
+        raise QmTemplateError(f"{key!r} must be a string in {source}")
+    return value
+
+
+def _int(table: Mapping[str, Any], key: str, default: int, source: Path) -> int:
+    value = table.get(key, default)
+    if not isinstance(value, int) or isinstance(value, bool):
+        raise QmTemplateError(f"{key!r} must be an integer in {source}")
+    return value
+
+
+def _string_list(
+    table: Mapping[str, Any], key: str, default: Sequence[str], source: Path
+) -> tuple[str, ...]:
+    value = table.get(key, default)
+    if not isinstance(value, (list, tuple)) or not all(
+        isinstance(item, str) for item in value
+    ):
+        raise QmTemplateError(f"{key!r} must be a list of strings in {source}")
+    return tuple(value)
+
+
+SSH_KEY_TYPE_PREFIXES = ("ssh-", "ecdsa-sha2-", "sk-")
+
+
+def ssh_key_fingerprint(line: str) -> str | None:
+    fields = line.split()
+    if len(fields) < 2:
+        return None
+    try:
+        blob = base64.b64decode(fields[1], validate=True)
+    except binascii.Error:
+        return None
+    digest = hashlib.sha256(blob).digest()
+    return base64.b64encode(digest).rstrip(b"=").decode("ascii")
+
+
+def _ssh_keys(table: Mapping[str, Any], key: str, source: Path) -> tuple[str, ...]:
+    keys: list[str] = []
+    for value in _string_list(table, key, _CREATE_DEFAULTS.sshkeys, source):
+        stripped = value.strip()
+        if not stripped.startswith(SSH_KEY_TYPE_PREFIXES) or not ssh_key_fingerprint(
+            stripped
+        ):
+            raise QmTemplateError(
+                f"{key!r} entry {value!r} does not look like an SSH public key "
+                f"in {source}"
+            )
+        keys.append(stripped)
+    return tuple(keys)
+
+
+def _parse_download(table: Mapping[str, Any], source: Path) -> DownloadSettings:
+    _reject_unknown(table, _DOWNLOAD_KEYS | set(DISTROS), "download", source)
+    defaults: dict[str, dict[str, str]] = {}
+    for key, value in table.items():
+        if key in _DOWNLOAD_KEYS:
+            continue
+        if not isinstance(value, dict):
+            raise QmTemplateError(f"[download.{key}] must be a table in {source}")
+        allowed = set(DISTROS[key].defaults) | {"tag"}
+        params: dict[str, str] = {}
+        for param, raw in value.items():
+            if param not in allowed:
+                raise QmTemplateError(
+                    f"unknown parameter {param!r} in [download.{key}] of {source}"
+                )
+            if isinstance(raw, bool) or not isinstance(raw, (str, int)):
+                raise QmTemplateError(
+                    f"[download.{key}].{param} must be a string or integer in {source}"
+                )
+            params[param] = str(raw)
+        defaults[key] = params
+    connections = _int(table, "connections", _DOWNLOAD_DEFAULTS.connections, source)
+    if connections < 1:
+        raise QmTemplateError(f"'connections' must be a positive integer in {source}")
+    return DownloadSettings(
+        preferred=_string_list(
+            table, "preferred", _DOWNLOAD_DEFAULTS.preferred, source
+        ),
+        connections=connections,
+        default_distro=_str(
+            table, "default_distro", _DOWNLOAD_DEFAULTS.default_distro, source
+        ),
+        defaults=defaults,
+    )
+
+
+def _parse_create(table: Mapping[str, Any], source: Path) -> CreateSettings:
+    _reject_unknown(table, _CREATE_KEYS, "create", source)
+    return CreateSettings(
+        storage=_str(table, "storage", _CREATE_DEFAULTS.storage, source),
+        cores=_int(table, "cores", _CREATE_DEFAULTS.cores, source),
+        memory=_int(table, "memory", _CREATE_DEFAULTS.memory, source),
+        bridge=_str(table, "bridge", _CREATE_DEFAULTS.bridge, source),
+        ciuser=_str(table, "ciuser", _CREATE_DEFAULTS.ciuser, source),
+        cipassword=_str(table, "cipassword", _CREATE_DEFAULTS.cipassword, source),
+        sshkeys=_ssh_keys(table, "sshkeys", source),
+        sshkeys_files=_string_list(
+            table, "sshkeys_files", _CREATE_DEFAULTS.sshkeys_files, source
+        ),
+    )
+
+
+def parse_settings(data: Mapping[str, Any], source: Path) -> Settings:
+    _reject_unknown(data, {"paths", "download", "create"}, None, source)
+    paths = _table(data, "paths", source)
+    _reject_unknown(paths, {"images_dir"}, "paths", source)
+    images_dir_value = _str(paths, "images_dir", str(default_images_dir()), source)
+    images_dir = Path(os.path.expandvars(images_dir_value)).expanduser()
+    return Settings(
+        images_dir=images_dir,
+        download=_parse_download(_table(data, "download", source), source),
+        create=_parse_create(_table(data, "create", source), source),
+    )
+
+
+def load_settings(path: Path, *, explicit: bool) -> Settings:
+    if not path.is_file():
+        if explicit:
+            raise QmTemplateError(f"configuration file not found: {path}")
+        log.debug("No configuration file at %s, using built-in defaults", path)
+        return Settings()
+    log.debug("Loading configuration from: %s", path)
+    try:
+        with path.open("rb") as handle:
+            data = tomllib.load(handle)
+    except tomllib.TOMLDecodeError as exc:
+        raise QmTemplateError(f"invalid TOML in {path}: {exc}") from exc
+    return parse_settings(data, source=path)