docker-devkit 0.2.1.dev37490360141__py3-none-any.whl → 0.3.0.dev113__py3-none-any.whl

docker_devkit/build_docker.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import hashlib
 import json
 import os
 import platform
@@ -7,12 +8,12 @@ import re
 from pathlib import Path
 from subprocess import CalledProcessError
 from tempfile import TemporaryDirectory
-from typing import Annotated, Any, Literal, TypedDict
+from typing import Annotated, Any, TypedDict
 
 import typer
 from bashrun.bash import bash, bash_output
-from ci_devkit.builds import push_build
-from ci_devkit.setup_oras import install_oras
+from build_artifact_registry.builds import push_build
+from build_artifact_registry.setup_oras import install_oras
 from .detect_gpu import GPU_TYPES, Gpu, detect_gpu
 from .documents import BakeDocument, digest_pinned, parse_bake
 from pydantic_settings import BaseSettings
@@ -32,8 +33,6 @@ settings = Settings.model_validate({})
 
 METADATA_PATH = Path("metadata.json")
 
-Mode = Literal["local", "ci"]
-
 DIGEST_PROJECT = "images-digests"
 DIGEST_PLATFORM = "all"
 DIGEST_FILE_NAME = "images-digests.json"
@@ -50,55 +49,43 @@ app = typer.Typer(add_completion=False, pretty_exceptions_show_locals=False)
 
 @app.command()
 def build(
+    push: Annotated[
+        bool, typer.Option("--push", help="Push images (+ registry caches) and shelve the digest manifest.")
+    ] = False,
+    targets_opt: Annotated[
+        list[str] | None,
+        typer.Option("--targets", "-t", help="Build only these services (from the image manifest)."),
+    ] = None,
+    gpu: Annotated[Gpu, typer.Option("--gpu", help="auto|cuda|rocm|none")] = "auto",
     upgrade: bool = typer.Option(False, "--upgrade", "-u", help="Re-resolve and rewrite base digests."),
     lock_only: bool = typer.Option(False, "--lock-only", help="Update lock file without building images."),
-    mode: Annotated[
-        Mode, typer.Option("--mode", help="local: --load images; ci: --push images + registry caches.")
-    ] = "local",
-    gpu: Annotated[Gpu, typer.Option("--gpu", help="auto|cuda|rocm|none")] = "auto",
     gpu_only: bool = typer.Option(
         False, "--gpu-only", help="Build only the services suffixed for this gpu (requires a concrete --gpu)."
     ),
     no_cache: bool = typer.Option(False, "--no-cache", help="Force rebuild by disabling cache usage."),
-    targets_opt: Annotated[
-        list[str] | None,
-        typer.Option("--targets", "-t", help="Build only these services (from the image manifest)."),
-    ] = None,
-    builds_registry: Annotated[
-        str | None,
-        typer.Option(
-            "--builds-registry", help="OCI registry path to push the digest manifest to (CI build artifacts)."
-        ),
-    ] = None,
-    run_number: Annotated[
-        int, typer.Option("--run-number", help="CI run number baked into the digest manifest's builds-shelf tag.")
-    ] = 0,
 ) -> None:
+    targets = [service for value in targets_opt or [] for service in value.split()] or None
     run_build(
+        push=push,
+        targets_opt=targets,
+        gpu=gpu,
         upgrade=upgrade,
         lock_only=lock_only,
-        mode=mode,
-        gpu=gpu,
         gpu_only=gpu_only,
         no_cache=no_cache,
-        targets_opt=targets_opt,
-        builds_registry=builds_registry,
-        run_number=run_number,
     )
 
 
 def run_build(
     *,
+    push: bool = False,
+    targets_opt: list[str] | None = None,
+    gpu: Gpu = "auto",
     upgrade: bool = False,
     lock_only: bool = False,
-    mode: Mode = "local",
-    gpu: Gpu = "auto",
     gpu_only: bool = False,
     no_cache: bool = False,
-    targets_opt: list[str] | None = None,
     env_overlay: dict[str, str] | None = None,
-    builds_registry: str | None = None,
-    run_number: int = 0,
 ) -> None:
     root = Path.cwd()
     manifest = require_manifest(root)
@@ -113,18 +100,17 @@ def run_build(
     # Load any existing lock
     lock_data = parse_env_file(lock_file) if lock_file.exists() else {}
 
-    # TOOD: Create separate commands for ci and local modes so typer can do this validation instead of us
-    if mode == "ci" and gpu == "auto":
-        raise typer.BadParameter("In CI mode, --gpu cannot be 'auto'; specify 'cuda' or 'rocm'.")
+    if push and gpu == "auto":
+        raise typer.BadParameter("With --push, --gpu cannot be 'auto'; specify 'cuda' or 'rocm'.")
 
-    if mode == "ci" and bake.registry_cache is None:
-        raise typer.BadParameter("In CI mode, the bake file must declare x-registry-cache.")
+    if push and bake.registry_cache is None:
+        raise typer.BadParameter("With --push, the bake file must declare x-registry-cache.")
 
     if gpu_only and gpu not in GPU_TYPES:
         raise typer.BadParameter("--gpu-only requires a concrete gpu (cuda or rocm), not 'auto' or 'none'.")
 
     # For local builds, ensure Docker GC limits are high enough that GPU builds don't cause cache evictions
-    if mode == "local" and not lock_only and not targets_opt:
+    if not push and not lock_only and not targets_opt:
         _check_gc_limits(min_gb=60)
 
     # Resolve gpu
@@ -176,8 +162,8 @@ def run_build(
     else:
         targets = compute_default_targets(bake, gpu, gpu_only)
 
-    # Configure registry caches in CI mode
-    if mode == "ci":
+    # Configure registry caches for push builds
+    if push:
         for target in targets:
             target_cache = f"{bake.registry_cache}:{target}"
             command_arguments.append(
@@ -187,9 +173,9 @@ def run_build(
 
     # `docker buildx --load` only writes a single arch to the host image store,
     # so bake targets declared multi-platform get overridden to host arch in
-    # local builds. CI uses `--push` and produces the full manifest list.
+    # local builds. --push produces the full manifest list.
     # Override key is `platform` (singular) even though the bake field is plural.
-    if mode == "local":
+    if not push:
         machine = platform.machine().lower()
         if machine in ("x86_64", "amd64"):
             host_platform = "linux/amd64"
@@ -201,8 +187,8 @@ def run_build(
             if len(bake.services[target].build.platforms) > 1:
                 command_arguments.append(f"--set {target}.platform={host_platform}")
 
-    # Load or push images based on mode
-    command_arguments.append("--load" if mode == "local" else "--push")
+    # Load or push images
+    command_arguments.append("--push" if push else "--load")
 
     # Handle no-cache option
     if no_cache:
@@ -230,29 +216,39 @@ def run_build(
     if not set(targets) <= baked_images.keys():
         raise RuntimeError("Bake output does not cover the requested targets")
 
-    push_image_digests(builds_registry, run_number, baked_images, targets)
+    # Push the digest manifest to the repo's builds shelf (registry supplied explicitly, SHA from the checkout)
+    if push:
+        registry = os.environ.get("REGISTRY", "")
+        if not registry:
+            raise SystemExit("REGISTRY not set — --push shelves the digest manifest under it")
+        shelf = f"{registry}/builds"
+        sha = bash_output("git rev-parse HEAD").strip()
+        push_image_digests(shelf, sha, baked_images, targets)
+
 
+def leg_key(targets: list[str]) -> str:
+    return hashlib.sha256(",".join(sorted(targets)).encode("utf-8")).hexdigest()[:8]
 
+
+# Each matrix leg pushes its own manifest shard under sha-{sha}-{key}: legs run in
+# parallel, so a shared tag would be last-writer-wins. The pull side enumerates a
+# SHA's shards and merges — consumption happens only after every leg finished.
 def push_image_digests(
-    builds_registry: str | None,
-    run_number: int,
+    shelf: str,
+    sha: str,
     baked_images: dict[str, Any],
     targets: list[str],
 ) -> None:
-    if builds_registry is None:
-        return
-    if run_number <= 0:
-        raise typer.BadParameter("--builds-registry requires a positive --run-number")
     install_oras()
     manifest = distill_digest_manifest(baked_images, targets)
     with TemporaryDirectory(prefix="digest-manifest-") as staging:
         manifest_path = Path(staging) / DIGEST_FILE_NAME
         manifest_path.write_text(json.dumps(manifest, indent=2), encoding="utf-8")
         push_build(
-            builds_registry,
+            shelf,
             DIGEST_PROJECT,
             DIGEST_PLATFORM,
-            f"run-{run_number}",
+            f"sha-{sha}-{leg_key(targets)}",
             Path(staging),
             [DIGEST_FILE_NAME],
         )
@@ -273,6 +269,8 @@ def distill_digest_manifest(baked_images: dict[str, Any], targets: list[str]) ->
             if separator:
                 image_ref = repository
                 tags.append(tag)
+        if not tags:
+            raise ValueError(f"target '{target}' produced no tagged image references from '{image_name}'")
         manifest[target] = {"ref": image_ref, "digest": digest, "tags": tags}
     return manifest
 

docker_devkit/matrix.py

@@ -18,32 +18,49 @@ PLAIN_VARIANT = "common"
 @app.command()
 def matrix_main() -> None:
     bake = parse_bake(require_manifest(Path.cwd()))
-    print(f"matrix={json.dumps({'include': compute_matrix_entries(bake)})}")
+    print(json.dumps(compute_matrix_entries(bake)))
 
 
-# One entry per variant: targets are the dependency-closure groups of that variant's
-# explicitly-buildable services, coalesced into a single leg — the shape a single
-# gpu-resolved build invocation would produce, with dependents grouped with the base
-# their additional_contexts `target:` references pull in.
+# One entry per variant and cohort cell: targets are the dependency-closure groups
+# of that cell's explicitly-buildable services, coalesced into a single leg — the
+# shape a single gpu-resolved build invocation would produce, with dependents
+# grouped with the base their additional_contexts `target:` references pull in.
+# The cross-compile cohort gets its own legs: per-arch builds are the QEMU-slow
+# ones, so they parallelize away from native builds instead of serializing them
+# into one invocation.
 def compute_matrix_entries(bake: BakeDocument) -> list[dict[str, str]]:
-    entries_by_variant: dict[str, list[str]] = {}
+    cohort = cross_compile_cohort(bake)
+    entries_by_cell: dict[tuple[str, bool], list[str]] = {}
     for group in dependency_groups(bake):
-        explicit = sorted(
-            service
-            for service in group
-            if bake.services[service].build.tags and service not in set(bake.cross_compile_targets)
-        )
+        explicit = sorted(service for service in group if bake.services[service].build.tags)
         if not explicit:
             continue
         variant = group_variant(bake, explicit)
-        entries_by_variant.setdefault(variant, []).extend(explicit)
+        memberships = {service in cohort for service in explicit}
+        if len(memberships) > 1:
+            raise RuntimeError(
+                f"dependency group spans cross-compile cohort membership ({', '.join(explicit)}) — "
+                "additional_contexts must stay within one cohort"
+            )
+        entries_by_cell.setdefault((variant, memberships.pop()), []).extend(explicit)
 
     return [
-        {"targets": " ".join(sorted(entries_by_variant[variant])), "variant": variant}
-        for variant in sorted(entries_by_variant)
+        {"targets": " ".join(sorted(targets)), "variant": variant}
+        for (variant, _), targets in sorted(entries_by_cell.items())
     ]
 
 
+# The cross-compile cohort: every service declaring `platforms:` (per-arch
+# treatment is a manifest fact, not an operator judgment) plus the hand-listed
+# x-cross-compile-targets members.
+def cross_compile_cohort(bake: BakeDocument) -> set[str]:
+    unknown = set(bake.cross_compile_targets) - set(bake.services)
+    if unknown:
+        raise RuntimeError(f"x-cross-compile-targets names unknown services: {', '.join(sorted(unknown))}")
+    pinned = {service for service, config in bake.services.items() if config.build.platforms}
+    return pinned | set(bake.cross_compile_targets)
+
+
 def dependency_groups(bake: BakeDocument) -> list[list[str]]:
     parent = {service: service for service in bake.services}
 

{docker_devkit-0.2.1.dev37490360141.dist-info → docker_devkit-0.3.0.dev113.dist-info}/METADATA

@@ -1,12 +1,12 @@
 Metadata-Version: 2.5
 Name: docker-devkit
-Version: 0.2.1.dev37490360141
+Version: 0.3.0.dev113
 Summary: Docker-stack lifecycle commands (up/down/build) driven by a pyproject-declared compose assembly
 License-File: LICENSE
 License-File: NOTICE
 Requires-Python: >=3.13
 Requires-Dist: bashrun>=0.1.0
-Requires-Dist: ci-devkit>=0.1.15
+Requires-Dist: build-artifact-registry>=0.2.0.dev0
 Requires-Dist: packaging>=24.1
 Requires-Dist: pathspec>=1.0.4
 Requires-Dist: pydantic-settings>=2.9.1

{docker_devkit-0.2.1.dev37490360141.dist-info → docker_devkit-0.3.0.dev113.dist-info}/RECORD

@@ -1,5 +1,5 @@
 docker_devkit/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-docker_devkit/build_docker.py,sha256=mpYSZgWHyLvtwBscqBAn_j8oL8l2Xuc6z5QWGIs02XY,12813
+docker_devkit/build_docker.py,sha256=DtvTUTGHdQAkpBVelGObWyYD5Ldc05iPWXkNpCMQt-E,12927
 docker_devkit/config.py,sha256=UMAS6yF1E9PIbsRs2TVig9XwSNCvQe5w9eTLUHOPKq0,3112
 docker_devkit/context_sha.py,sha256=lhCxUcYM98xCdDum3hsFRFw5WYrjvZUSMbOu_7ZmQjE,3041
 docker_devkit/detect_gpu.py,sha256=C4e3vixvSSJS6xQiMVWp2IdWSAgMr551KrBTuIrkTk4,456
@@ -7,15 +7,15 @@ docker_devkit/documents.py,sha256=c6jqbBNUKyJOltex-8u536_niWhEL7ScOab_MwZmM9k,21
 docker_devkit/down.py,sha256=9OMXPZQ3SIdaSELc46hzkqNpLSwOJ1athCD8Bmlz2BI,2189
 docker_devkit/image_refs.py,sha256=6pHYPWh4lL6qMqHtQBHBbeWijVsT0PhsMS6CMDho5nY,5294
 docker_devkit/lifecycle.py,sha256=FuwaCSRK7qVx_qj-HmIFqMm8Ak4cpAQEqqXRPDNCLmo,2259
-docker_devkit/matrix.py,sha256=__qZhGdCOdYqjeexDYHsm55ehMyyD1Nw67m2T-DTvFg,3489
+docker_devkit/matrix.py,sha256=6NyGL4Ol849dYlcMRzAL2yjFoN6sGkGwVaaxcDm3bG4,4530
 docker_devkit/mirror.py,sha256=tissqnoqMwRA6s0ykwXZOEgy6F1d3AAebPKGEkSpJRg,3224
 docker_devkit/modes.py,sha256=Lwhblzu5Kn7qhfy_QMqE28XuflSIJtCRBYNX7Iq8Qfo,1510
 docker_devkit/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 docker_devkit/score_generate.py,sha256=psbQm-MWf9A5YP0ov0rLPUiuVAQbUWn6yfUmB-CXurU,9434
 docker_devkit/up.py,sha256=32A2_6mxAF00XILecxYTzCQw0QvSjevBWCnayF5X1Xs,4284
-docker_devkit-0.2.1.dev37490360141.dist-info/METADATA,sha256=LrhBD-2sSaPdnDYpQeFABN4d7FvkcQwU8p4c6-2xbCc,530
-docker_devkit-0.2.1.dev37490360141.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
-docker_devkit-0.2.1.dev37490360141.dist-info/entry_points.txt,sha256=6cAHqlMvuXI00uTJLf73MTqWnLd1-m5sDgJrSyHWv4U,244
-docker_devkit-0.2.1.dev37490360141.dist-info/licenses/LICENSE,sha256=RFhQPdSOiMTguUX7JSoIuTxA7HVzCbj_p8WU36HjUQQ,10947
-docker_devkit-0.2.1.dev37490360141.dist-info/licenses/NOTICE,sha256=jvglZza3GCdN7v4_TLuXUqX9C_uABlwAKUkcIeKwk_8,26
-docker_devkit-0.2.1.dev37490360141.dist-info/RECORD,,
+docker_devkit-0.3.0.dev113.dist-info/METADATA,sha256=0NT3OLGPZKW3hL5cvP-avLSORfSbo57cbDvwrZV2aOs,540
+docker_devkit-0.3.0.dev113.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+docker_devkit-0.3.0.dev113.dist-info/entry_points.txt,sha256=6cAHqlMvuXI00uTJLf73MTqWnLd1-m5sDgJrSyHWv4U,244
+docker_devkit-0.3.0.dev113.dist-info/licenses/LICENSE,sha256=RFhQPdSOiMTguUX7JSoIuTxA7HVzCbj_p8WU36HjUQQ,10947
+docker_devkit-0.3.0.dev113.dist-info/licenses/NOTICE,sha256=jvglZza3GCdN7v4_TLuXUqX9C_uABlwAKUkcIeKwk_8,26
+docker_devkit-0.3.0.dev113.dist-info/RECORD,,