wimcast 0.1.0__py3-none-any.whl

wimcast/__init__.py

@@ -0,0 +1,3 @@
+"""wimcast: Linux-native Windows image builder for Foreman."""
+
+__version__ = "0.1.0"

wimcast/__main__.py

@@ -0,0 +1,6 @@
+"""wimcast CLI entry point."""
+
+from wimcast.cli import main
+
+if __name__ == "__main__":
+    raise SystemExit(main())

wimcast/bootfiles.py

@@ -0,0 +1,151 @@
+"""Boot payload: pinned wimboot and the ISO's boot files (plan 5.2 stage 8).
+
+wimboot is downloaded from the GitHub release matching the pinned version
+(config [wimboot] version, default 2.9.0, see README "Why wimboot is
+pinned"). bootmgr, bootmgr.efi, boot\\boot.sdi and boot\\bcd come from the
+source ISO, preserving the exact bits the image was built from.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import os
+import tempfile
+import urllib.request
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from wimcast.cache import Cache
+
+from wimcast.iso import IsoReader, open_iso
+
+WIMBOOT_BASE_URL = "https://github.com/ipxe/wimboot/releases/download/v{version}/wimboot"
+WIMBOOT_SHA256 = {
+    "2.9.0": "5f067ccdc4d084d5bf77b6c853bd0f8402dfc2b4cd1b103d358993ae97fae8e3",
+}
+_WIMBOOT_CHUNK = 1024 * 1024
+
+# ISO members copied to the final layout, (iso member, relative dest)
+BOOT_MEMBERS: tuple[tuple[str, str], ...] = (
+    ("bootmgr", "bootmgr"),
+    ("bootmgr.efi", "bootmgr.efi"),
+    ("boot/boot.sdi", "boot/boot.sdi"),
+    ("boot/bcd", "boot/bcd"),
+)
+
+
+class BootfilesError(RuntimeError):
+    """A boot-file fetch or extraction failure."""
+
+    def __init__(self, message: str) -> None:
+        super().__init__(message)
+        self.message = message
+
+
+def _sha256_file(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as fd:
+        for chunk in iter(lambda: fd.read(_WIMBOOT_CHUNK), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+class Bootfiles:
+    def __init__(
+        self,
+        cache: Cache | None = None,
+        sevenzip: str | None = None,
+        sha256: str | None = None,
+    ) -> None:
+        self._cache = cache
+        self._sevenzip = sevenzip
+        self._sha256 = sha256.lower() if sha256 else None
+
+    def _expected_sha256(self, version: str) -> str:
+        if self._sha256 is not None:
+            return self._sha256
+        digest = WIMBOOT_SHA256.get(version)
+        if digest is None:
+            raise BootfilesError(
+                f"no pinned SHA-256 for wimboot v{version}; "
+                "set [wimboot] sha256 to the digest of the release asset"
+            )
+        return digest
+
+    async def wimboot(self, version: str, dest: Path) -> Path:
+        """Pinned wimboot binary, cached; downloads on miss."""
+        version = version.lstrip("v")
+        expected = self._expected_sha256(version)
+        key = f"wimboot:v{version}:{expected}"
+        if self._cache is not None and self._cache.materialize(key, dest):
+            return dest
+        url = WIMBOOT_BASE_URL.format(version=version)
+        data = await asyncio.to_thread(self._download, url)
+        if not data:
+            raise BootfilesError(f"wimboot v{version} downloaded empty payload")
+        sha = hashlib.sha256(data).hexdigest()
+        if sha != expected:
+            raise BootfilesError(
+                f"wimboot v{version} checksum mismatch: expected {expected}, got {sha}"
+            )
+        dest.parent.mkdir(parents=True, exist_ok=True)
+        fd, tmp_name = tempfile.mkstemp(prefix=".wimboot-", suffix=".tmp", dir=dest.parent)
+        tmp = Path(tmp_name)
+        try:
+            with os.fdopen(fd, "wb") as out:
+                out.write(data)
+                out.flush()
+                os.fsync(out.fileno())
+            if self._cache is not None:
+                blob = self._cache.put(key, tmp, sha, len(data))
+                if self._cache.enabled:
+                    if not self._cache.materialize(key, dest):
+                        raise BootfilesError(f"cache lost wimboot entry {key}")
+                else:
+                    blob.replace(dest)
+            else:
+                tmp.replace(dest)
+        finally:
+            tmp.unlink(missing_ok=True)
+        return dest
+
+    @staticmethod
+    def _download(url: str) -> bytes:
+        try:
+            with urllib.request.urlopen(url, timeout=60) as response:
+                return bytes(response.read())
+        except OSError as exc:
+            raise BootfilesError(f"cannot download {url}: {exc}") from exc
+
+    async def iso_boot_files(self, iso_path: Path, dest_dir: Path) -> None:
+        """Copy the boot payload members from the ISO into dest_dir."""
+        await asyncio.to_thread(self._iso_boot_files_sync, iso_path, dest_dir)
+
+    def _iso_boot_files_sync(self, iso_path: Path, dest_dir: Path) -> None:
+        reader = open_iso(iso_path, self._sevenzip)
+        try:
+            self._copy_members(reader, dest_dir)
+        finally:
+            reader.close()
+
+    def _copy_members(self, reader: IsoReader, dest_dir: Path) -> None:
+        for member, rel in BOOT_MEMBERS:
+            if not reader.exists(member):
+                raise BootfilesError(f"boot member not found in ISO: {member}")
+            self._copy_member(reader, member, dest_dir / rel)
+
+    def _copy_member(self, reader: IsoReader, member: str, dest: Path) -> None:
+        dest.parent.mkdir(parents=True, exist_ok=True)
+        fd, tmp_name = tempfile.mkstemp(prefix=".boot-", suffix=".tmp", dir=dest.parent)
+        tmp = Path(tmp_name)
+        try:
+            with os.fdopen(fd, "wb") as out:
+                for chunk in reader.stream_file(member):
+                    out.write(chunk)
+                out.flush()
+                os.fsync(out.fileno())
+            tmp.replace(dest)
+        finally:
+            tmp.unlink(missing_ok=True)

wimcast/build.py

@@ -0,0 +1,453 @@
+"""Build orchestrator (plan 5.2).
+
+One source leaf = one ISO = one finalized/<arch>/<name>/ tree. Stage
+ordering, caching and verification:
+
+  probe -> hash -> extract -> info -> filter -> export -> winpe -> bootfiles
+  -> publish -> manifest -> verify
+
+Every expensive stage is keyed in the content-addressed store; a hit skips
+the work entirely and only republishes the files.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import logging
+import shutil
+import tempfile
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from wimcast.cache import Cache
+    from wimcast.config import Config
+
+from wimcast import __version__
+from wimcast.bootfiles import Bootfiles
+from wimcast.cache import stage_key
+from wimcast.drivers import Drivers
+from wimcast.iso import open_iso
+from wimcast.manifest import (
+    BuildMeta,
+    edition_filenames,
+    write_build_json,
+    write_images_ini,
+)
+from wimcast.model import Artifact, Source, WimImage
+from wimcast.probe import BOOT_MEMBER, ISO_MEMBERS, SourceError, discover
+from wimcast.publish import prune_tree, publish_file, publish_tree
+from wimcast.run import Runner, ToolError
+from wimcast.wim import WimError, Wimlib, WimOperations, parse_wimlib_version
+from wimcast.winpe import build_winpe
+
+log = logging.getLogger("wimcast.build")
+
+_LAYOUT_VERSION = "v1"
+_MIN_WIMLIB = (1, 13, 0)
+_CHUNK = 1024 * 1024
+
+
+class BuildError(RuntimeError):
+    pass
+
+
+@dataclass(frozen=True)
+class Stage:
+    name: str
+    duration_s: float
+
+
+@dataclass(frozen=True)
+class SourceResult:
+    arch: str
+    name: str
+    images: tuple[WimImage, ...]
+    exported: tuple[tuple[int, str, str], ...]
+    artifacts: tuple[Artifact, ...]
+    stages: tuple[Stage, ...]
+
+
+@dataclass
+class BuildSummary:
+    sources: list[SourceResult] = field(default_factory=list)
+    durations_s: float = 0.0
+
+
+def config_hash(config: Config) -> str:
+    return hashlib.sha256(repr(config).encode("utf-8")).hexdigest()
+
+
+def tree_hash(directory: Path) -> str:
+    """Deterministic hash of a directory tree (relative paths + contents)."""
+    digest = hashlib.sha256()
+    for path in sorted(directory.rglob("*")):
+        if path.is_file():
+            rel = path.relative_to(directory).as_posix().encode("utf-8")
+            digest.update(rel)
+            with path.open("rb") as fd:
+                for chunk in iter(lambda: fd.read(_CHUNK), b""):
+                    digest.update(chunk)
+    return digest.hexdigest()
+
+
+def file_sha256(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as fd:
+        for chunk in iter(lambda: fd.read(_CHUNK), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+class IsoHashMemo:
+    """In-process memo of ISO hashes, keyed on (path, size, mtime)."""
+
+    def __init__(self) -> None:
+        self._entries: dict[tuple[str, int, int], str] = {}
+
+    def get(self, path: Path) -> str:
+        stat = path.stat()
+        key = (str(path), stat.st_size, stat.st_mtime_ns)
+        cached = self._entries.get(key)
+        if cached is not None:
+            return cached
+        sha = file_sha256(path)
+        self._entries[key] = sha
+        return sha
+
+
+class Builder:
+    def __init__(
+        self,
+        config: Config,
+        runner: Runner,
+        cache: Cache | None = None,
+        wimlib: WimOperations | None = None,
+        workdir_base: Path | None = None,
+        finalized_root: Path = Path("finalized"),
+    ) -> None:
+        from wimcast.cache import Cache
+
+        config.validate()
+        self._config = config
+        self._runner = runner
+        self._cache = cache if cache is not None else Cache(config.cache.directory)
+        self._wimlib = wimlib or Wimlib(runner, config.tools.wimlib_imagex)
+        self._drivers = Drivers(config)
+        self._memo = IsoHashMemo()
+        self._workdir_base = workdir_base
+        self._finalized_root = finalized_root
+        self._wimlib_version: str | None = None
+
+    async def _tool_version(self) -> str:
+        if self._wimlib_version is None:
+            exe = self._config.tools.wimlib_imagex or "wimlib-imagex"
+            try:
+                result = await self._runner.run(exe, "--version")
+            except ToolError as exc:
+                raise BuildError(f"wimlib-imagex not usable: {exc}") from exc
+            version = parse_wimlib_version(result.stdout)
+            if version is None:
+                raise BuildError(f"cannot parse wimlib-imagex version from {result.stdout!r}")
+            if version < _MIN_WIMLIB:
+                raise BuildError(
+                    "wimlib-imagex >= 1.13 required, found "
+                    f"{'.'.join(str(part) for part in version)}"
+                )
+            self._wimlib_version = f"wimlib-imagex {'.'.join(str(part) for part in version)}"
+        return self._wimlib_version
+
+    async def build(self, path: Path | None = None) -> BuildSummary:
+        path = path or Path("sources")
+        try:
+            sources = discover(path)
+        except SourceError as exc:
+            raise BuildError(str(exc)) from exc
+        if not sources:
+            raise BuildError(f"no sources found under {path}")
+        summary = BuildSummary()
+        for source in sources:
+            started = time.monotonic()
+            result = await self.build_source(source)
+            summary.sources.append(result)
+            summary.durations_s += time.monotonic() - started
+        return summary
+
+    async def build_source(self, source: Source) -> SourceResult:
+        stages: list[Stage] = []
+        workdir = self._new_workdir(source)
+        try:
+            cfg_hash = config_hash(self._config)
+            tool_version = await self._tool_version()
+
+            iso_sha = await asyncio.to_thread(self._memo.get, source.iso_path)
+
+            started = time.monotonic()
+            install_wim, boot_wim, backend = await asyncio.to_thread(
+                self._extract, source.iso_path, workdir
+            )
+            stages.append(Stage("extract", time.monotonic() - started))
+            install_sha = await asyncio.to_thread(file_sha256, install_wim)
+
+            started = time.monotonic()
+            images = await self._wimlib.info(install_wim)
+            stages.append(Stage("info", time.monotonic() - started))
+
+            selected = self._select_editions(images)
+            if not selected:
+                raise BuildError(
+                    f"no edition matched [editions] allowed={self._config.editions.allowed!r}"
+                )
+
+            started = time.monotonic()
+            exported = await self._export(
+                selected, install_wim, install_sha, cfg_hash, tool_version, workdir
+            )
+            stages.append(Stage("export", time.monotonic() - started))
+
+            started = time.monotonic()
+            boot_wim_final = await self._build_winpe(
+                source, boot_wim, cfg_hash, tool_version, workdir
+            )
+            stages.append(Stage("winpe", time.monotonic() - started))
+
+            # Emit driver reports (boot.wim injected, install.wim deferred).
+            self._drivers.report_for(source)
+
+            started = time.monotonic()
+            bootfiles = Bootfiles(
+                self._cache,
+                sevenzip=self._config.tools.sevenzip,
+                sha256=self._config.wimboot.sha256,
+            )
+            boot_dir = workdir / "boot"
+            await bootfiles.wimboot(self._config.wimboot.version, boot_dir / "wimboot")
+            await bootfiles.iso_boot_files(source.iso_path, boot_dir)
+            stages.append(Stage("bootfiles", time.monotonic() - started))
+
+            started = time.monotonic()
+            published = await asyncio.to_thread(
+                self._publish,
+                source,
+                workdir,
+                exported,
+                iso_sha,
+                tool_version,
+                backend,
+                boot_wim_final,
+            )
+            stages.append(Stage("publish", time.monotonic() - started))
+
+            started = time.monotonic()
+            await self._verify(source, exported)
+            stages.append(Stage("verify", time.monotonic() - started))
+
+            started = time.monotonic()
+            artifacts = await asyncio.to_thread(self._published_artifacts, source)
+            dest = self._finalized_root / source.arch / source.name
+            meta = BuildMeta(
+                layout_version=_LAYOUT_VERSION,
+                wimcast_version=__version__,
+                tool_wimlib_imagex=tool_version,
+                iso_backend=backend,
+                iso=source.iso_path.name,
+                iso_sha256=iso_sha,
+                arch=source.arch,
+                name=source.name,
+                edition_images=tuple(exported),
+                artifacts=artifacts,
+                start_offset_s=0.0,
+                end_offset_s=sum(stage.duration_s for stage in stages),
+            )
+            await asyncio.to_thread(write_build_json, dest / "build.json", meta)
+            expected = published | {dest / "build.json"}
+            await asyncio.to_thread(prune_tree, dest, expected)
+            stages.append(Stage("manifest", time.monotonic() - started))
+
+            return SourceResult(
+                arch=source.arch,
+                name=source.name,
+                images=images,
+                exported=tuple(exported),
+                artifacts=artifacts,
+                stages=tuple(stages),
+            )
+        finally:
+            if workdir != self._workdir_base:
+                try:
+                    shutil.rmtree(workdir)
+                except OSError as exc:
+                    log.warning("cannot remove workdir %s: %s", workdir, exc)
+
+    def _new_workdir(self, source: Source) -> Path:
+        if self._workdir_base is not None:
+            workdir = self._workdir_base / source.arch / source.name
+        else:
+            workdir = Path(tempfile.mkdtemp(prefix="wimcast-"))
+        workdir.mkdir(parents=True, exist_ok=True)
+        return workdir
+
+    def _extract(self, iso_path: Path, workdir: Path) -> tuple[Path, Path | None, str]:
+        """Extract install.wim (or install.esd) and boot.wim from the ISO."""
+        reader = open_iso(iso_path, self._config.tools.sevenzip)
+        dest_dir = workdir / "iso"
+        dest_dir.mkdir(parents=True, exist_ok=True)
+        try:
+            member = next((m for m in ISO_MEMBERS if reader.exists(m)), None)
+            if member is None:
+                raise BuildError(f"neither {' nor '.join(ISO_MEMBERS)} found in {iso_path}")
+            install_dest = dest_dir / "install.wim"
+            reader.extract_file(member, install_dest)
+            boot_dest: Path | None = None
+            if reader.exists(BOOT_MEMBER):
+                boot_dest = dest_dir / "boot.wim"
+                reader.extract_file(BOOT_MEMBER, boot_dest)
+            return install_dest, boot_dest, reader.backend
+        finally:
+            reader.close()
+
+    def _select_editions(self, images: tuple[WimImage, ...]) -> tuple[WimImage, ...]:
+        allowed = self._config.editions.allowed
+        if not allowed:
+            return images
+        return tuple(
+            image for image in images if any(part.lower() in image.name.lower() for part in allowed)
+        )
+
+    async def _export(
+        self,
+        images: tuple[WimImage, ...],
+        install_wim: Path,
+        install_sha: str,
+        cfg_hash: str,
+        tool_version: str,
+        workdir: Path,
+    ) -> list[tuple[int, str, str]]:
+        exported: list[tuple[int, str, str]] = []
+        sources_dir = workdir / "sources"
+        plan = edition_filenames(images, self._config.output.filename_scheme)
+        for image, filename in plan:
+            key = stage_key("export", install_sha, image.index, filename, cfg_hash, tool_version)
+            dest = sources_dir / filename
+            if dest.parent != sources_dir:
+                raise BuildError(f"unsafe export filename: {filename!r}")
+            if not self._cache.materialize(key, dest):
+                dest.parent.mkdir(parents=True, exist_ok=True)
+                await self._wimlib.export(install_wim, image.index, dest, compress="LZX")
+                sha = await asyncio.to_thread(file_sha256, dest)
+                blob = self._cache.put(key, dest, sha, dest.stat().st_size)
+                if self._cache.enabled:
+                    self._cache.materialize(key, dest)
+                else:
+                    dest = blob
+            exported.append((image.index, image.name, filename))
+        return exported
+
+    async def _build_winpe(
+        self,
+        source: Source,
+        boot_wim: Path | None,
+        cfg_hash: str,
+        tool_version: str,
+        workdir: Path,
+    ) -> Path | None:
+        config = self._config
+        if config.winpe.source == "file":
+            if config.winpe.path is None or config.winpe.sha256 is None:
+                raise BuildError('[winpe] source = "file" requires path and sha256')
+            base = Path(config.winpe.path)
+            if not base.is_file():
+                raise BuildError(f"[winpe] path does not exist: {base}")
+            if await asyncio.to_thread(file_sha256, base) != config.winpe.sha256.lower():
+                raise BuildError(f"[winpe] sha256 mismatch for {base}")
+            base_index = 1
+        else:
+            if boot_wim is None:
+                raise BuildError(f"{source.iso_path}: sources/boot.wim missing from ISO")
+            base = boot_wim
+            base_index = config.winpe.iso_index
+        base_sha = await asyncio.to_thread(file_sha256, base)
+        overlay = Path(config.winpe.overlay_dir) if config.winpe.overlay_dir else None
+        overlay_hash = (
+            await asyncio.to_thread(tree_hash, overlay)
+            if overlay is not None and overlay.is_dir()
+            else ""
+        )
+        drivers_dir = self._bootpe_drivers_dir(source)
+        drivers_hash = (
+            await asyncio.to_thread(tree_hash, drivers_dir) if drivers_dir is not None else ""
+        )
+        key = stage_key(
+            "winpe", base_sha, base_index, overlay_hash, drivers_hash, cfg_hash, tool_version
+        )
+        out = workdir / "boot.wim"
+        if not self._cache.materialize(key, out):
+            await build_winpe(
+                self._wimlib,
+                base,
+                base_index,
+                workdir,
+                overlay_dir=overlay,
+                drivers_dir=drivers_dir,
+            )
+            sha = await asyncio.to_thread(file_sha256, out)
+            self._cache.put(key, out, sha, out.stat().st_size)
+            if self._cache.enabled:
+                self._cache.materialize(key, out)
+        return out
+
+    def _bootpe_drivers_dir(self, source: Source) -> Path | None:
+        drivers_root = self._config.winpe.drivers_dir
+        if drivers_root is None:
+            return None
+        arch_root = Path(drivers_root) / source.arch
+        return arch_root if arch_root.is_dir() else None
+
+    def _publish(
+        self,
+        source: Source,
+        workdir: Path,
+        exported: list[tuple[int, str, str]],
+        iso_sha: str,
+        tool_version: str,
+        backend: str,
+        boot_wim: Path | None,
+    ) -> set[Path]:
+        dest = self._finalized_root / source.arch / source.name
+        published = publish_tree(workdir / "boot", dest)
+        sources_dest = dest / "sources"
+        for _, _, filename in exported:
+            published.add(publish_file(workdir / "sources" / filename, sources_dest / filename))
+        if boot_wim is not None and boot_wim.is_file():
+            published.add(publish_file(boot_wim, sources_dest / "boot.wim"))
+        if source.drivers_dir is not None:
+            published |= publish_tree(source.drivers_dir, sources_dest / "drivers")
+        write_images_ini(
+            sources_dest / "images.ini",
+            tuple((name, filename) for _, name, filename in exported),
+        )
+        published.add(sources_dest / "images.ini")
+        return published
+
+    def _published_artifacts(self, source: Source) -> tuple[Artifact, ...]:
+        base = self._finalized_root / source.arch / source.name
+        artifacts: list[Artifact] = []
+        for path in sorted(base.rglob("*.wim")):
+            artifacts.append(
+                Artifact(path=str(path), sha256=file_sha256(path), size=path.stat().st_size)
+            )
+        return tuple(artifacts)
+
+    async def _verify(self, source: Source, exported: list[tuple[int, str, str]]) -> None:
+        base = self._finalized_root / source.arch / source.name
+        for wim in sorted(base.rglob("*.wim")):
+            try:
+                await self._wimlib.verify(wim)
+            except WimError as exc:
+                raise BuildError(f"verify failed for {wim}: {exc}") from exc
+        for artifact in self._published_artifacts(source):
+            if artifact.sha256 != await asyncio.to_thread(file_sha256, Path(artifact.path)):
+                raise BuildError(f"hash mismatch after publish: {artifact.path}")