boatyard 0.6.0.dev170__py3-none-any.whl

boatyard/lock.py

@@ -0,0 +1,233 @@
+from __future__ import annotations
+
+import base64
+import json
+import re
+import shlex
+
+import yaml
+from bashrun.bash import bash_output
+from pydantic import BaseModel, ConfigDict, Field
+
+from boatyard.config.config import (
+    OCI_REFERENCE_PREFIX,
+    ConfigFile,
+    Yaml,
+    bundle_lock_key,
+    chart_lock_key,
+    is_list,
+    is_mapping,
+    mirrored_ref,
+    scan_bundle_texts,
+)
+from boatyard.crane import install_crane
+from boatyard.config.lock import (
+    LOCK_PATH,
+    LockFile,
+    ConfigLockBundle,
+    ConfigLockEntry,
+)
+from boatyard.temp import SHA256_DIGEST_REGEX
+from boatyard.yaml import load_yaml_model
+
+DIGEST_TAIL_PATTERN = re.compile(rf"@{SHA256_DIGEST_REGEX}$")
+COMPOSE_FILE_MEDIA_TYPE = "application/vnd.docker.compose.file+yaml"
+
+
+# Model the OCI shapes crane answers with: index children and artifact layers.
+class OciPlatform(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    os: str | None = None
+    architecture: str | None = None
+
+
+class OciDescriptor(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    digest: str
+    media_type: str | None = Field(default=None, alias="mediaType")
+    data: str | None = None
+    platform: OciPlatform | None = None
+    annotations: dict[str, str] = Field(default_factory=dict)
+
+
+class OciIndex(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    manifests: list[OciDescriptor] = Field(default_factory=list)
+
+
+class OciArtifactManifest(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    layers: list[OciDescriptor] = Field(default_factory=list)
+
+    @classmethod
+    def fetch_compose_texts(cls, pinned: str) -> list[str]:
+        # Read the bundle's compose files as inline base64 layer data, with crane blob as the fallback.
+        print(f"Reading bundle content for: {pinned}")
+        artifact = cls.model_validate(json.loads(bash_output(f"crane manifest {shlex.quote(pinned)}")))
+
+        # Strip the tag, treating a colon as a tag only when no slash follows it.
+        repository = pinned.rpartition("@")[0]
+        slash = repository.rfind("/")
+        colon = repository.rfind(":")
+        repository = repository[:colon] if colon > slash else repository
+        compose_texts = [
+            base64.b64decode(layer.data).decode("utf-8")
+            if layer.data is not None
+            else bash_output(f"crane blob {shlex.quote(repository + '@' + layer.digest)}")
+            for layer in artifact.layers
+            if layer.media_type == COMPOSE_FILE_MEDIA_TYPE
+        ]
+
+        # Refuse a bundle that carries no compose file layer.
+        if not compose_texts:
+            raise ValueError(f"{pinned}: the compose bundle carries no compose file layer")
+        return compose_texts
+
+
+def relock(config: ConfigFile, *, upgrade: bool = False, check: bool = False) -> LockFile:
+    # Install the pinned crane and derive the mirror prefix.
+    install_crane()
+    prefix = config.mirror_prefix
+    # Read the committed lock schema-only; drift is relock's input, never refused.
+    committed_path = config.root / LOCK_PATH
+    committed = load_yaml_model(LockFile, committed_path) if committed_path.is_file() else LockFile()
+
+    # Resolve first-party bundles direct, recomputing requires from the pinned content and validating declared references against it.
+    bundles: dict[str, ConfigLockBundle] = {}
+    for name, reference, entry in config.remote_declared_entries():
+        key = bundle_lock_key(name)
+        committed_entry = committed.bundles.get(key)
+
+        # Pin the frozen reference or resolve the declaration's fresh digest.
+        pinned = (
+            committed_entry.reference
+            if committed_entry is not None and not upgrade and committed_entry.reference.startswith(reference + "@")
+            else f"{reference}@{resolve_remote_digest(reference)}"
+        )
+
+        # Record the pinned reference with its required slots.
+        bundles[key] = ConfigLockBundle(
+            reference=pinned,
+            requires=scan_bundle_texts(
+                name, entry.routes, entry.ports, OciArtifactManifest.fetch_compose_texts(pinned)
+            ).requires,
+        )
+
+    images: dict[str, ConfigLockEntry] = {}
+    for name, reference in config.base_images.items():
+        base = mirrored_ref(reference, prefix)
+        committed_entry = committed.images.get(name)
+
+        # Record a digest-pinned declaration as-is under the mirror prefix.
+        if DIGEST_TAIL_PATTERN.search(reference):
+            images[name] = ConfigLockEntry(index=base, platforms=dict.fromkeys(config.platforms, base))
+            continue
+
+        # Freeze the committed entry forward while its pinned tag and platform set hold.
+        if (
+            committed_entry is not None
+            and not upgrade
+            and committed_entry.index.startswith(base + "@")
+            and set(committed_entry.platforms) == set(config.platforms)
+        ):
+            images[name] = committed_entry
+            continue
+
+        # Resolve the reference into an index digest and its child manifests.
+        print(f"Resolving digests for: {reference}")
+        index_digest = resolve_remote_digest(reference)
+        children = OciIndex.model_validate(
+            json.loads(bash_output(f"crane manifest {shlex.quote(reference)}"))
+        ).manifests
+
+        if not children:
+            # Treat a manifest answer as every platform's sole manifest.
+            matches = {platform: [OciDescriptor(digest=index_digest)] for platform in config.platforms}
+        else:
+            # Match each declared platform against its concrete, non-attestation children.
+            matches = {
+                platform: [
+                    child
+                    for child in children
+                    if child.annotations.get("vnd.docker.reference.type") != "attestation-manifest"
+                    if child.platform is not None and f"{child.platform.os}/{child.platform.architecture}" == platform
+                ]
+                for platform in config.platforms
+            }
+
+        # Refuse any platform without exactly one match and record the survivor's digest.
+        selected: dict[str, str] = {}
+        for platform in config.platforms:
+            if len(matches[platform]) != 1:
+                raise ValueError(
+                    f"base_images entry {name}: platform {platform} "
+                    f"matches {len(matches[platform])} upstream manifests, expected exactly one"
+                )
+            child = matches[platform][0]
+            if not child.digest.startswith("sha256:"):
+                raise ValueError(f"base_images entry {name}: upstream index child must carry a digest ({child!r})")
+            selected[platform] = child.digest
+
+        # Record the mirror-prefixed index and one reference per platform.
+        images[name] = ConfigLockEntry(
+            index=f"{base}@{index_digest}",
+            platforms={platform: f"{base}@{digest}" for platform, digest in selected.items()},
+        )
+
+    # Freeze unchanged charts and resolve fresh tags to mirror-pinned digests.
+    charts: dict[str, str] = {}
+    for name, reference in config.declared_charts.items():
+        key = chart_lock_key(name)
+        reference = reference.removeprefix(OCI_REFERENCE_PREFIX)
+        base = mirrored_ref(reference, prefix)
+        committed_entry = committed.charts.get(key)
+
+        # Record a digest-pinned declaration as-is under the mirror prefix.
+        if DIGEST_TAIL_PATTERN.search(reference):
+            charts[key] = base
+            continue
+
+        # Freeze the committed entry forward while its pinned tag holds.
+        if committed_entry is not None and not upgrade and committed_entry.startswith(base + "@"):
+            charts[key] = committed_entry
+            continue
+
+        charts[key] = f"{base}@{resolve_remote_digest(reference)}"
+
+    # Assemble the lock and render its canonical form: sections in field order, nested keys sorted.
+    lock = LockFile(platforms=list(config.platforms), images=images, charts=charts, bundles=bundles)
+    document = {key: _canonical(value) for key, value in lock.model_dump(exclude_defaults=True).items()}
+    rendered = (
+        yaml.safe_dump(document, sort_keys=False, default_flow_style=False, width=10_000_000) if document else "{}\n"
+    )
+    # Write the rendered lock over the committed file.
+    if not check:
+        committed_path.write_text(rendered, encoding="utf-8")
+        return lock
+    # Refuse staleness when the committed lock is not the computed canonical bytes.
+    committed_text = committed_path.read_text(encoding="utf-8") if committed_path.is_file() else ""
+    if committed_text != rendered:
+        raise SystemExit("image lock is stale. Run 'boatyard lock' and commit the result.")
+    return lock
+
+
+def resolve_remote_digest(reference: str) -> str:
+    # Refuse crane digest output that is not one sha256 digest.
+    print(f"Resolving digest for: {reference}")
+    output = bash_output(f"crane digest {shlex.quote(reference)}").strip()
+    if re.fullmatch(SHA256_DIGEST_REGEX, output) is None:
+        raise ValueError(f"crane digest output for {reference} must be one sha256 digest")
+    return output
+
+
+def _canonical(value: Yaml) -> Yaml:
+    # Return the value with every nested dict's keys sorted.
+    if is_mapping(value):
+        return {key: _canonical(item) for key, item in sorted(value.items())}
+    if is_list(value):
+        return [_canonical(item) for item in value]
+    return value

boatyard/markers.py

@@ -0,0 +1,201 @@
+from __future__ import annotations
+
+import re
+from pathlib import Path
+from typing import Literal, NamedTuple
+
+import yaml
+from pydantic import BaseModel, ConfigDict
+
+from boatyard.config.config import Yaml, is_mapping
+
+TEXT_MARKER = "include"
+YAML_MARKER = "include-yaml"
+LOCK_MARKER = "lock"
+LOCK_IMAGE_MARKER = "lock-image"
+LOCK_MARKERS = (LOCK_MARKER, LOCK_IMAGE_MARKER)
+IMAGE_FIELD = "image"
+IMAGE_SLOT_PATTERN = re.compile(r"\$\{|{{")
+
+
+class LockImageSpec(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    key: str
+    digest_field: Literal["digest", "sha"] = "digest"
+
+
+class MarkerContext(NamedTuple):
+    root: Path
+    images: dict[str, str]
+    source: str
+    ancestors: tuple[Path, ...] = ()
+
+
+class ImageReference(NamedTuple):
+    registry: str
+    repository: str
+    tag: str | None
+    digest: str
+
+
+def resolve_markers(node: Yaml, context: MarkerContext) -> Yaml:
+    # Expand includes before locks because spliced content may carry lock markers
+    return resolve_locks(expand_includes(node, context), context)
+
+
+def expand_includes(node: Yaml, context: MarkerContext) -> Yaml:
+    match node:
+        case list():
+            # Resolve every list element
+            expanded = [expand_includes(item, context) for item in node]
+        case dict():
+            match tuple(node.items()):
+                # Splice in the included bytes
+                case [(marker, value)] if marker == TEXT_MARKER:
+                    expanded = include_text(value, context)
+                # Splice in the included structure
+                case [(marker, value)] if marker == YAML_MARKER:
+                    expanded = include_yaml(value, context)
+                # Leave lock markers whole for the lock pass to consume or refuse
+                case [(marker, _)] if marker in LOCK_MARKERS:
+                    expanded = node
+                # Resolve plain mappings child by child
+                case entries:
+                    expanded = {key: expand_includes(child, context) for key, child in entries}
+        case _:
+            # Pass scalars through untouched
+            expanded = node
+
+    return expanded
+
+
+def include_text(value: Yaml, context: MarkerContext) -> str:
+    path = included_path(value, TEXT_MARKER, context)
+    return path.read_text(encoding="utf-8")
+
+
+def include_yaml(value: Yaml, context: MarkerContext) -> Yaml:
+    path = included_path(value, YAML_MARKER, context)
+
+    try:
+        parsed = yaml.safe_load(path.read_text(encoding="utf-8"))
+    except yaml.YAMLError as error:
+        raise ValueError(f"{context.source}: include {value!r} is not valid YAML: {error}") from error
+
+    if parsed is None:
+        raise ValueError(f"{context.source}: include {value!r} is empty")
+
+    return expand_includes(parsed, context._replace(ancestors=(*context.ancestors, path)))
+
+
+def included_path(value: Yaml, marker: str, context: MarkerContext) -> Path:
+    source = context.source
+
+    match value:
+        case str():
+            relative = Path(value)
+        case _:
+            raise ValueError(f"{source}: {marker} takes a repo-root-relative path string, not {type(value).__name__}")
+
+    # Anchor include paths at the repo root
+    if relative.is_absolute() or ".." in relative.parts:
+        raise ValueError(f"{source}: include path {value!r} must anchor at the repo root")
+
+    path = context.root / relative
+
+    # Refuse cycles and missing targets
+    if path in context.ancestors:
+        raise ValueError(f"{source}: include {value!r} participates in a cycle")
+    if not path.is_file():
+        raise ValueError(f"{source}: include {value!r} does not exist at the repo root")
+
+    return path
+
+
+def resolve_locks(node: Yaml, context: MarkerContext) -> Yaml:
+    match node:
+        case list():
+            # Resolve every list element
+            resolved = [resolve_locks(item, context) for item in node]
+        case dict():
+            match tuple(node.items()):
+                # Substitute the full locked reference as one string
+                case [(marker, value)] if marker == LOCK_MARKER:
+                    resolved = locked_reference(value, context)
+                # Split the locked reference into chart fields
+                case [(marker, value)] if marker == LOCK_IMAGE_MARKER:
+                    resolved = lock_image_fields(value, context.images, context.source)
+                # Refuse literal images, then resolve plain mappings child by child
+                case entries:
+                    _check_literal_image(node, context.source)
+                    resolved = {key: resolve_locks(child, context) for key, child in entries}
+        case _:
+            # Pass scalars through untouched
+            resolved = node
+
+    return resolved
+
+
+def locked_reference(value: Yaml, context: MarkerContext) -> str:
+    match value:
+        case str() if value in context.images:
+            reference = context.images[value]
+        case str():
+            raise ValueError(f"{context.source}: lock entry {value!r} does not exist in the image lock")
+        case _:
+            raise ValueError(f"{context.source}: lock takes a lock-entry name string, not {type(value).__name__}")
+
+    return reference
+
+
+def lock_image_fields(value: Yaml, images: dict[str, str], source: str) -> dict[str, Yaml]:
+    # Accept a bare lock-entry name or a key/digest_field mapping
+    if isinstance(value, str):
+        image_spec = LockImageSpec(key=value)
+    elif is_mapping(value):
+        image_spec = LockImageSpec.model_validate(value)
+    else:
+        raise ValueError(
+            f"{source}: lock-image takes a lock-entry name or a key/digest_field mapping, not {type(value).__name__}"
+        )
+
+    if image_spec.key not in images:
+        raise ValueError(f"{source}: lock entry {image_spec.key!r} does not exist in the image lock")
+
+    reference = split_image_reference(images[image_spec.key], source)
+
+    # Assemble the chart fields in chart-convention key order under the digest spelling
+    fields: dict[str, Yaml] = {"registry": reference.registry, "repository": reference.repository}
+    if reference.tag is not None:
+        fields["tag"] = reference.tag
+
+    digest = reference.digest
+    fields[image_spec.digest_field] = digest.removeprefix("sha256:") if image_spec.digest_field == "sha" else digest
+
+    return fields
+
+
+def _check_literal_image(node: dict[str, Yaml], source: str) -> None:
+    image = node.get(IMAGE_FIELD)
+    if isinstance(image, str) and IMAGE_SLOT_PATTERN.search(image) is None:
+        raise ValueError(f"{source}: image {image!r} must consume the lock (lock: NAME / lock-image: NAME)")
+
+
+def split_image_reference(reference: str, source: str) -> ImageReference:
+    # Peel the trailing digest
+    path_part, _, digest = reference.rpartition("@")
+
+    # Treat a colon as a tag only when no slash follows so registry ports stay whole
+    tagged = path_part.rpartition(":")
+    tag: str | None = None
+    registry_part = path_part
+    if tagged[1] and "/" not in tagged[2]:
+        registry_part, tag = tagged[0], tagged[2]
+
+    # Split the registry from the repository at the last slash
+    registry, separator, repository = registry_part.rpartition("/")
+    if not separator or not registry or not repository:
+        raise ValueError(f"{source}: lock reference {reference!r} has no registry/repository split")
+
+    return ImageReference(registry, repository, tag, digest)

boatyard/paths.py

@@ -0,0 +1,5 @@
+from pathlib import Path
+
+DEFAULT_CONFIG_PATH = Path("boatyard.yml")
+VALUES_FILE = Path("boatyard.values.yaml")
+LOCK_PATH = Path("boatyard.lock.yml")

boatyard/publish.py

@@ -0,0 +1,72 @@
+from __future__ import annotations
+
+import re
+from functools import partial
+from pathlib import Path
+from typing import NamedTuple
+
+import yaml
+
+from boatyard.config.config import Yaml, double_dollars
+from boatyard.score_compose import ComposeDocument
+from boatyard.lock import resolve_remote_digest
+
+VAR_PATTERN = re.compile(r"\$\{([A-Z][A-Z0-9_]*)(:[?\-+][^}]*)?\}")
+STUB_VALUE = "publish-stub"
+
+
+class PublishedEntry(NamedTuple):
+    artifact: str
+    sha_reference: str
+    branch_reference: str
+    digest: str
+
+
+def bake_sources(baked_directory: Path, source_name: str, substitutions: dict[str, str]) -> Path:
+    print("Bake compose YAML")
+    source_path = Path(source_name)
+    baked_text = VAR_PATTERN.sub(
+        lambda match: substitutions.get(match.group(1), match.group(0)),
+        source_path.read_text(encoding="utf-8"),
+    )
+    baked_path = baked_directory / source_path
+    baked_path.parent.mkdir(parents=True, exist_ok=True)
+    baked_path.write_text(baked_text, encoding="utf-8")
+    return baked_path
+
+
+def finalize_baked(baked_path: Path, repository: str) -> None:
+    print("Inline configs content")
+    # Inline file-sourced configs as content entries.
+    own_image_pattern = re.compile(rf"{re.escape(repository)}/[a-z0-9][a-z0-9-]*:[^\s\"'@$]+")
+    document = yaml.safe_load(baked_path.read_text(encoding="utf-8"))
+    parsed = ComposeDocument.model_validate(document)
+    if parsed.configs is not None:
+        inlined: dict[str, dict[str, Yaml]] = {}
+        for config_name, compose_config in parsed.configs.items():
+            if compose_config.file is None:
+                inlined[config_name] = document["configs"][config_name]
+                continue
+            entry = compose_config.model_dump(exclude_defaults=True)
+            entry.pop("file", None)
+            entry["content"] = double_dollars(Path(compose_config.file).read_text(encoding="utf-8"))
+            inlined[config_name] = entry
+        document["configs"] = inlined
+        baked_path.write_text(
+            yaml.dump(document, default_flow_style=False, sort_keys=False, width=1_000_000),
+            encoding="utf-8",
+        )
+
+    # Pin image refs under the repository to digests.
+    digests: dict[str, str] = {}
+    baked_path.write_text(
+        own_image_pattern.sub(partial(_pinned_reference, digests), baked_path.read_text(encoding="utf-8")),
+        encoding="utf-8",
+    )
+
+
+def _pinned_reference(digests: dict[str, str], match: re.Match[str]) -> str:
+    reference = match.group(0)
+    if reference not in digests:
+        digests[reference] = resolve_remote_digest(reference)
+    return f"{reference.rsplit(':', 1)[0]}@{digests[reference]}"