boatyard 0.6.0.dev170__py3-none-any.whl

boatyard/config/lock.py

@@ -0,0 +1,152 @@
+from __future__ import annotations
+
+import re
+from pathlib import Path
+from typing import Self
+
+from pydantic import BaseModel, ConfigDict, Field, ValidationInfo, model_validator
+
+from boatyard.config.config import ConfigFile, bundle_lock_key, chart_lock_key
+from boatyard.config.values import PLATFORM_KEY
+from boatyard.paths import LOCK_PATH, VALUES_FILE
+from boatyard.temp import PLATFORM_PATTERN, SHA256_DIGEST_REGEX
+from boatyard.yaml import load_yaml_model
+
+
+class LockFile(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    platforms: list[str] = Field(default_factory=list)
+    images: dict[str, ConfigLockEntry] = Field(default_factory=dict)
+    charts: dict[str, str] = Field(default_factory=dict)
+    bundles: dict[str, ConfigLockBundle] = Field(default_factory=dict)
+
+    @model_validator(mode="after")
+    def _validate(self, info: ValidationInfo) -> Self:
+        context: dict[str, object] = info.context or {}
+        source = str(context.get("path", "the image lock"))
+        for name, reference in self.charts.items():
+            if not _digest_pinned(reference):
+                raise ValueError(f"{source}: chart {name} is not digest-pinned ({reference!r})")
+        shared = sorted(
+            (self.images.keys() & self.charts.keys())
+            | (self.images.keys() & self.bundles.keys())
+            | (self.charts.keys() & self.bundles.keys())
+        )
+        if shared:
+            raise ValueError(f"{source}: names {shared} must appear under at most one of images, charts, bundles")
+        config = context.get("config")
+        if isinstance(config, ConfigFile):
+            self._require_paired(config, source)
+        return self
+
+    def _require_paired(self, config: ConfigFile, source: str) -> None:
+        recorded = sorted(self.platforms)
+        declared = sorted(config.platforms)
+        if recorded != declared:
+            raise ValueError(
+                f"{source}: records platforms {recorded} but boatyard.yml declares {declared} (run boatyard lock)"
+            )
+        for name, entry in self.images.items():
+            if sorted(entry.platforms) != recorded:
+                raise ValueError(
+                    f"{source}: image entry {name} records platforms {sorted(entry.platforms)} "
+                    f"but the lock records {recorded} (run boatyard lock)"
+                )
+        missing_images = sorted(set(config.base_images) - self.images.keys())
+        if missing_images:
+            raise ValueError(f"{source}: lacks the {missing_images} entries base_images declares (run boatyard lock)")
+        missing_charts = sorted({chart_lock_key(chart) for chart in config.declared_charts} - self.charts.keys())
+        if missing_charts:
+            raise ValueError(f"{source}: lacks the {missing_charts} entries k8s.charts declares (run boatyard lock)")
+        declared_names = (
+            set(config.base_images)
+            | {chart_lock_key(chart) for chart in config.declared_charts}
+            | {bundle_lock_key(name) for name, _reference, _entry in config.remote_declared_entries()}
+        )
+        orphans = sorted((self.images.keys() | self.charts.keys() | self.bundles.keys()) - declared_names)
+        if orphans:
+            raise ValueError(
+                f"{source}: records entries boatyard.yml no longer declares ({orphans}); run boatyard lock"
+            )
+
+    @classmethod
+    def load(cls, path: Path, config: ConfigFile) -> LockFile:
+        context: dict[str, object] = {"path": path.as_posix(), "config": config}
+        if not path.is_file():
+            # Every consumer reads a missing lock as the empty lock paired against the config.
+            return cls.model_validate({}, context=context)
+        return load_yaml_model(cls, path, context)
+
+    def require(self, name: str, subject: str) -> str:
+        if name in self.images:
+            return self.images[name].index
+        if name in self.charts:
+            return self.charts[name]
+        if name in self.bundles:
+            return self.bundles[name].reference
+        raise ValueError(f"{subject}: {LOCK_PATH.as_posix()} lacks the {name} entry (run boatyard lock)")
+
+    def index_references(self) -> dict[str, str]:
+        return {**{name: entry.index for name, entry in self.images.items()}, **self.charts}
+
+    def manifest_references(self, platform: str | None) -> dict[str, str]:
+        resolved: str | None
+        if platform is None:
+            if not self.platforms:
+                resolved = None
+            elif len(self.platforms) == 1:
+                resolved = self.platforms[0]
+            else:
+                raise ValueError(
+                    f"platforms {self.platforms} are recorded; {PLATFORM_KEY}: must be named in {VALUES_FILE.name}"
+                )
+        elif self.platforms and platform not in self.platforms:
+            raise ValueError(f"{PLATFORM_KEY}: {platform!r} must be one of the recorded platforms {self.platforms}")
+        else:
+            resolved = platform
+        if resolved is None:
+            return self.index_references()
+        return {**{name: entry.platforms[resolved] for name, entry in self.images.items()}, **self.charts}
+
+
+class ConfigLockEntry(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    index: str
+    platforms: dict[str, str] = Field(default_factory=dict)
+
+    @model_validator(mode="after")
+    def _require_digest_pins(self, info: ValidationInfo) -> Self:
+        context: dict[str, object] = info.context or {}
+        source = str(context.get("path", "the image lock"))
+        for site, reference in [("index", self.index), *self.platforms.items()]:
+            if not _digest_pinned(reference):
+                raise ValueError(f"{source}: entry field {site} is not digest-pinned ({reference!r})")
+        for platform in self.platforms:
+            if PLATFORM_PATTERN.fullmatch(platform) is None:
+                raise ValueError(f"{source}: platform key {platform!r} must be an os/arch pair like linux/arm64")
+        return self
+
+    @property
+    def recorded_digests(self) -> list[str]:
+        return [self.index, *self.platforms.values()]
+
+
+class ConfigLockBundle(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    reference: str
+    requires: list[str] = Field(default_factory=list)
+
+    @model_validator(mode="after")
+    def _require_digest_pin(self, info: ValidationInfo) -> Self:
+        context: dict[str, object] = info.context or {}
+        source = str(context.get("path", "the image lock"))
+        if not _digest_pinned(self.reference):
+            raise ValueError(f"{source}: bundle entry is not digest-pinned ({self.reference!r})")
+        return self
+
+
+def _digest_pinned(reference: str) -> bool:
+    return re.fullmatch(SHA256_DIGEST_REGEX, reference.rpartition("@")[2]) is not None

boatyard/config/values.py

@@ -0,0 +1,188 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Self
+
+from pydantic import BaseModel, ConfigDict, ValidationInfo, field_validator, model_validator
+
+from boatyard.config.config import (
+    ENV_NAME_PATTERN,
+    LOCK_SLOT_MARKER,
+    ConfigFile,
+    Yaml,
+    is_mapping,
+    qualified_option_name,
+)
+from boatyard.yaml import load_yaml_model
+from boatyard.paths import VALUES_FILE
+
+PLATFORM_KEY = "platform"
+TYPED_KEYS = frozenset({"bundles", "path_overrides", "public_url", PLATFORM_KEY})
+
+OVERRIDE_LAYOUT = ("stack", "generated", "compose")
+
+
+class PublicUrl(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    forwarded_proto: str
+    forwarded_port: str
+
+    @model_validator(mode="before")
+    @classmethod
+    def _parse(cls, value: str) -> dict[str, str]:
+        scheme, separator, rest = value.partition("://")
+        if not separator:
+            raise ValueError(f"public_url: {value!r} must carry a scheme (https://example.dev)")
+        hostport = rest.partition("/")[0]
+        if ":" in hostport:
+            forwarded_port = hostport.rpartition(":")[2]
+        elif scheme == "https":
+            forwarded_port = "443"
+        else:
+            forwarded_port = "80"
+        return {"forwarded_proto": scheme, "forwarded_port": forwarded_port}
+
+    @property
+    def forwarded(self) -> dict[str, Yaml]:
+        return {"FORWARDED_PROTO": self.forwarded_proto, "FORWARDED_PORT": self.forwarded_port}
+
+
+class ValuesFile(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    choices: dict[str, str]
+    path_overrides: dict[str, Path]
+    public_url: PublicUrl | None
+    platform: str | None
+    variables: dict[str, str]
+
+    @model_validator(mode="before")
+    @classmethod
+    def _fold_selection(cls, data: object, info: ValidationInfo) -> object:
+        if not is_mapping(data):
+            return data
+        context: dict[str, object] = info.context or {}
+        config = context.get("config")
+        raw_bundles = data.get("bundles", [])
+        if not isinstance(raw_bundles, list):
+            raise TypeError("bundles: must be a list of selection names")
+        bundles: list[str] = []
+        for name in raw_bundles:
+            if not isinstance(name, str):
+                raise TypeError("bundles: must be a list of selection names")
+            bundles.append(name)
+        choices: dict[str, str] = {}
+        if not isinstance(config, ConfigFile) or not config.services:
+            if bundles:
+                raise ValueError("bundles: is set but this repo declares no services")
+        else:
+            selectable = config.selectable_options
+            bare = config.bare_entries
+            unknown = [name for name in bundles if name not in selectable]
+            if unknown:
+                bare_entries_note = f"; bare entries, always included: {bare}" if bare else ""
+                raise ValueError(
+                    f"bundles: names {unknown}; the selectable options are {sorted(selectable)}{bare_entries_note}"
+                )
+            # Fold the selection into one choice per group
+            for name in bundles:
+                group, option_name = selectable[name]
+                if group in choices:
+                    raise ValueError(
+                        f"bundles: selects {qualified_option_name(group, choices[group])} and {name}; "
+                        f"at most one option per group ({group})"
+                    )
+                choices[group] = option_name
+        variables: dict[str, str] = {
+            str(key): (str(value).lower() if isinstance(value, bool) else str(value))
+            for key, value in data.items()
+            if str(key) not in TYPED_KEYS
+        }
+        return {
+            "choices": choices,
+            "path_overrides": data.get("path_overrides", {}),
+            "public_url": data.get("public_url"),
+            "platform": data.get("platform"),
+            "variables": variables,
+        }
+
+    @model_validator(mode="after")
+    def _require_ref_entry_overrides(self, info: ValidationInfo) -> Self:
+        context: dict[str, object] = info.context or {}
+        config = context.get("config")
+        if not isinstance(config, ConfigFile) or not config.services:
+            return self
+        vocabulary = set(config.selectable_options) | set(config.bare_entries)
+        remote_names = {name for name, _declared, _entry in config.remote_declared_entries()}
+        unknown_overrides: list[str] = []
+        local_names: list[str] = []
+        for name in self.path_overrides:
+            if name not in vocabulary:
+                unknown_overrides.append(name)
+            elif name not in remote_names:
+                local_names.append(name)
+        if unknown_overrides:
+            raise ValueError(
+                f"path_overrides: names {sorted(unknown_overrides)}; the bundle vocabulary is {sorted(vocabulary)}"
+            )
+        if local_names:
+            raise ValueError(
+                f"path_overrides: names {sorted(local_names)}; an override targets a ref entry; "
+                f"the ref entries are {sorted(remote_names)}"
+            )
+        return self
+
+    @field_validator("path_overrides", mode="before")
+    @classmethod
+    def _require_path_values(cls, raw_overrides: object) -> object:
+        if not is_mapping(raw_overrides):
+            return raw_overrides
+        path_overrides: dict[str, Path] = {}
+        for name, value in raw_overrides.items():
+            stripped = str(value).strip()
+            if not stripped:
+                raise ValueError(f"path_overrides.{name}: empty; a value is a filesystem path")
+            if "://" in stripped:
+                raise ValueError(
+                    f"path_overrides.{name}: {stripped!r} carries a URL scheme; a value is a filesystem path"
+                )
+            path_overrides[name] = Path(stripped)
+        return path_overrides
+
+    @field_validator("path_overrides")
+    @classmethod
+    def _require_output_layout(cls, path_overrides: dict[str, Path]) -> dict[str, Path]:
+        for name, path in path_overrides.items():
+            if path.parts[-4:-1] != OVERRIDE_LAYOUT:
+                raise ValueError(
+                    f"path_overrides.{name}: {path.as_posix()!r} must point into a stack/generated/compose/ bundle output layout"
+                )
+        return path_overrides
+
+    @field_validator("variables")
+    @classmethod
+    def _require_env_names(cls, variables: dict[str, str], info: ValidationInfo) -> dict[str, str]:
+        context: dict[str, object] = info.context or {}
+        source = context.get("path", VALUES_FILE.name)
+        unknown = sorted(key for key in variables if ENV_NAME_PATTERN.fullmatch(key) is None)
+        if unknown:
+            raise ValueError(f"{source}: keys {unknown} are not ALL_CAPS environment names")
+        return variables
+
+    @field_validator("variables")
+    @classmethod
+    def _require_lock_marker_free_names(cls, variables: dict[str, str], info: ValidationInfo) -> dict[str, str]:
+        context: dict[str, object] = info.context or {}
+        source = context.get("path", VALUES_FILE.name)
+        reserved = sorted(name for name in variables if LOCK_SLOT_MARKER in name)
+        if reserved:
+            raise ValueError(
+                f"{source}: keys {reserved} carry the reserved {LOCK_SLOT_MARKER} marker; "
+                f"the lock fills those slots — never set them by hand"
+            )
+        return variables
+
+    @classmethod
+    def load(cls, path: Path, config: ConfigFile | None) -> ValuesFile:
+        return load_yaml_model(cls, path, {"path": path.as_posix(), "config": config})

boatyard/crane.py

@@ -0,0 +1,23 @@
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+from bashrun.bash import bash
+
+CRANE_VERSION = "v0.22.1"
+
+_installed = False
+
+
+def install_crane() -> None:
+    global _installed
+    if _installed:
+        return
+    print(f"Install crane {CRANE_VERSION}")
+    sudo = "sudo " if os.geteuid() != 0 else ""
+    archive = "go-containerregistry_Linux_x86_64.tar.gz"
+    bash(f"curl -fsSLO https://github.com/google/go-containerregistry/releases/download/{CRANE_VERSION}/{archive}")
+    bash(f"{sudo}tar -xzf {archive} -C /usr/local/bin crane")
+    Path(archive).unlink(missing_ok=True)
+    _installed = True

boatyard/dockerfiles.py

@@ -0,0 +1,201 @@
+from __future__ import annotations
+
+import re
+from pathlib import Path, PurePosixPath
+from tempfile import TemporaryDirectory
+from typing import Self
+
+from bashrun.bash import bash, bash_output
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+DOCKERFILE_NAME = "Dockerfile"
+FROM_LINE_PATTERN = re.compile(r"^FROM\s+(?:--platform=\S+\s+)?(\S+)(?:\s+AS\s+(\S+))?", re.IGNORECASE)
+ARG_LINE_PATTERN = re.compile(r"^ARG\s+([A-Za-z_][A-Za-z0-9_]*)")
+COPY_FROM_LINE_PATTERN = re.compile(r"^COPY\s+--from=(\S+)", re.IGNORECASE)
+BARE_IDENTIFIER_PATTERN = re.compile(r"^[a-z0-9][a-z0-9_-]*$")
+BUILD_ARG_REFERENCE_PATTERN = re.compile(r"^\$\{[A-Za-z0-9_]+\}$")
+
+
+class ServiceBuild(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    name: str
+    dockerfile: str
+    contexts: list[str] = Field(default_factory=list)
+    axis: str | None = None
+    variants: list[str] = Field(default_factory=list)
+    sha_args: list[str] = Field(default_factory=list)
+
+    @model_validator(mode="after")
+    def _validate(self) -> Self:
+        if (self.axis is None) != (not self.variants):
+            raise ValueError("axis and variants must be declared together")
+        return self
+
+    @property
+    def target_variants(self) -> list[tuple[str, str | None]]:
+        variants: list[str | None] = list(self.variants) or [None]
+        return [(f"{self.name}-{variant}" if variant is not None else self.name, variant) for variant in variants]
+
+
+def derive_services(root: Path, axes: dict[str, list[str]]) -> dict[str, ServiceBuild]:
+    service_dirs: dict[str, Path] = {}
+    tops: set[str] = set()
+    for top in sorted(root.iterdir()):
+        if not top.is_dir() or top.name.startswith("."):
+            continue
+        for child in sorted(top.iterdir()):
+            if child.is_dir() and (child / DOCKERFILE_NAME).is_file():
+                service_dirs[child.name] = child
+                tops.add(top.name)
+    _require_single_top(tops, "Dockerfile service directories")
+
+    builds: dict[str, ServiceBuild] = {}
+    for name, directory in service_dirs.items():
+        dockerfile = directory / DOCKERFILE_NAME
+        relative = dockerfile.relative_to(root).as_posix()
+        from_images: list[str] = []
+        copy_from_values: list[str] = []
+        stages: set[str] = set()
+        args: list[str] = []
+        for line in dockerfile.read_text(encoding="utf-8").splitlines():
+            from_match = FROM_LINE_PATTERN.match(line.strip())
+            if from_match is not None:
+                from_images.append(from_match.group(1))
+                if from_match.group(2) is not None:
+                    stages.add(from_match.group(2))
+                continue
+            copy_from_match = COPY_FROM_LINE_PATTERN.match(line.strip())
+            if copy_from_match is not None:
+                copy_from_values.append(copy_from_match.group(1))
+                continue
+            arg_match = ARG_LINE_PATTERN.match(line.strip())
+            if arg_match is not None:
+                args.append(arg_match.group(1))
+        contexts: list[str] = []
+        for image in from_images:
+            if image in stages:
+                continue
+            if BARE_IDENTIFIER_PATTERN.fullmatch(image) is None:
+                if BUILD_ARG_REFERENCE_PATTERN.fullmatch(image) is None:
+                    raise ValueError(f"{relative}: FROM {image} must reference a base_images build arg")
+                continue
+            if image not in service_dirs:
+                raise ValueError(f"{relative}: FROM {image} names no service directory or stage")
+            contexts.append(image)
+        for value in copy_from_values:
+            if "/" in value:
+                raise ValueError(f"{relative}: COPY --from={value} must reference a base_images build arg")
+        declared_axis_args = [argument for argument in args if argument in axes]
+        if len(declared_axis_args) > 1:
+            raise ValueError(
+                f"{relative}: declares two variant-axis ARGs ({', '.join(declared_axis_args)}); "
+                "a service carries one axis"
+            )
+        axis_arg = declared_axis_args[0] if declared_axis_args else None
+        builds[name] = ServiceBuild(
+            name=name,
+            dockerfile=relative,
+            contexts=sorted(set(contexts)),
+            axis=axis_arg,
+            variants=list(axes[axis_arg]) if axis_arg is not None else [],
+            sha_args=sorted({arg for arg in args if arg.endswith("_SHA")}),
+        )
+
+    effective_axes = {name: _effective_axis(name, builds, ()) for name in builds}
+    for name, build in builds.items():
+        axis = effective_axes[name]
+        if axis is not None:
+            build.axis = axis
+            build.variants = list(axes[axis])
+    return builds
+
+
+def _require_single_top(tops: set[str], subject: str) -> None:
+    if len(tops) > 1:
+        raise ValueError(f"{subject} span multiple top-level directories: {sorted(tops)}")
+
+
+def service_sha_env(name: str) -> str:
+    return name.upper().replace("-", "_") + "_SHA"
+
+
+def compute_service_shas(repo_root: Path, service_builds: dict[str, ServiceBuild]) -> dict[str, str]:
+    repo_root = repo_root.resolve()
+
+    service_dirs = {str(PurePosixPath(service.dockerfile).parent) for service in service_builds.values()}
+
+    if not service_dirs:
+        # Skip the disk walk when no services exist.
+        return {}
+
+    # Derive the workload root as the single top-level directory holding every service.
+    tops = {PurePosixPath(service_dir).parts[0] for service_dir in service_dirs}
+    _require_single_top(tops, "services")
+    workload_prefix = f"{next(iter(tops))}/"
+
+    # Require that .dockerignore be an allowlist
+    rules = [line.strip() for line in (repo_root / ".dockerignore").read_text(encoding="utf-8").splitlines()]
+    if sum(1 for rule in rules if rule == "*") != 1:
+        raise ValueError(".dockerignore: must contain exactly one '*' line")
+    includes: list[str] = []
+    for number, rule in enumerate(rules, start=1):
+        if not rule or rule.startswith("#") or rule == "*":
+            continue
+        if not rule.startswith("!"):
+            raise ValueError(
+                f".dockerignore:{number}: only allowlists are supported ('*' + '!path' lines); got {rule!r}"
+            )
+        path = rule[1:].strip("/")
+        if not path or any(character in path for character in "*?[]\\"):
+            raise ValueError(
+                f".dockerignore:{number}: re-include lines must be literal paths without wildcards; got {rule!r}"
+            )
+        includes.append(path)
+
+    # Collect allowlisted files
+    allowed = [
+        path
+        for path in (entry.relative_to(repo_root).as_posix() for entry in repo_root.rglob("*") if entry.is_file())
+        if any(path == include or path.startswith(include + "/") for include in includes)
+    ]
+
+    # Partition into shared and per-service paths, dropping workload-root files owned by no service.
+    shared_paths = [path for path in allowed if not path.startswith(workload_prefix)]
+    per_service = {d: [p for p in allowed if p.startswith(d + "/")] for d in service_dirs}
+
+    result: dict[str, str] = {}
+    for service_dir in sorted(service_dirs):
+        with TemporaryDirectory() as tmpdir:
+            index_env = {"GIT_INDEX_FILE": str(Path(tmpdir) / "index")}
+            # Conversion pinned off so blobs hash the raw disk bytes the build compiles.
+            bash(
+                "git -c core.autocrlf=false update-index --add --stdin",
+                cwd=repo_root,
+                env=index_env,
+                stdin_text="".join(f"{path}\n" for path in shared_paths + per_service[service_dir]),
+            )
+            tree_hash = bash_output("git write-tree", cwd=repo_root, env=index_env).strip()
+
+        result[service_sha_env(PurePosixPath(service_dir).name)] = f"tree-{tree_hash}"
+
+    return result
+
+
+def _effective_axis(name: str, builds: dict[str, ServiceBuild], path: tuple[str, ...]) -> str | None:
+    if name in path:
+        raise ValueError(f"FROM context cycle: {' -> '.join((*path, name))}")
+    build = builds[name]
+    parent_axes = [_effective_axis(parent, builds, (*path, name)) for parent in build.contexts]
+    if len(set(parent_axes)) > 1:
+        raise ValueError(
+            f"workloads/{name}/{DOCKERFILE_NAME}: parents ({', '.join(build.contexts)}) disagree on variants"
+        )
+    if parent_axes:
+        if build.axis is not None:
+            raise ValueError(
+                f"workloads/{name}/{DOCKERFILE_NAME}: declares ARG {build.axis} over a "
+                f"variant-bearing base (parents: {', '.join(build.contexts)})"
+            )
+        return parent_axes[0]
+    return build.axis