rosotacom-dev 2.5.dev21__tar.gz → 2.5.dev24__tar.gz

{rosotacom_dev-2.5.dev21 → rosotacom_dev-2.5.dev24}/.github/workflows/pr-merge-gate.yml

@@ -192,6 +192,96 @@ jobs:
           fi
           echo "All preflight checks passed"
 
+  image:
+    # Publish the images the e2e suite builds, once per set of build inputs,
+    # instead of rebuilding them in each of thirteen jobs. #236 measured that
+    # build at 154s per job: 57s pulling the multi-GB ROS base from Docker Hub,
+    # 40s exporting layers, 47s of apt and pip. A pull from GHCR — co-located
+    # with the runners, no Hub rate limit, one transfer instead of a transfer
+    # plus a build — replaces all of it, and takes twelve of the thirteen Hub
+    # pulls off the merge gate's critical path.
+    #
+    # The tag is a hash of everything that decides the image (base digest,
+    # package lists, Dockerfile, entrypoint, ros2docker version, build UID):
+    # `rosotacom image references` derives it, nothing here chooses it. A
+    # build-input change is therefore a different tag, so "the published image
+    # is stale" is not a state this can reach — which is the objection #226
+    # raised against caching the image inline.
+    #
+    # Runs beside preflight rather than after it: on a hit it inspects two
+    # manifests and stops, so it is never on the critical path.
+    name: image
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    if: github.event_name != 'pull_request' || github.event.pull_request.draft == false
+    permissions:
+      contents: read
+      packages: write
+    outputs:
+      repository: ${{ steps.publish.outputs.repository }}
+
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v6
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+          cache: pip
+
+      - name: Install just
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y just
+
+      - name: Set up local environment
+        run: just setup
+
+      - name: Log in to GHCR
+        run: echo '${{ secrets.GITHUB_TOKEN }}' | docker login ghcr.io -u '${{ github.actor }}' --password-stdin
+
+      - name: Publish every image the e2e suite builds
+        id: publish
+        run: |
+          set -euo pipefail
+          repository="ghcr.io/$(echo '${{ github.repository_owner }}' | tr '[:upper:]' '[:lower:]')/rosotacom-e2e"
+          # The packaged example project, because that is the one the e2e tests
+          # copy. The repository's own rosotacom.yaml points at profiles and
+          # artifacts that live beside the checkout in the private harness, so it
+          # does not resolve here.
+          project="$(.venv/bin/rosotacom resources path examples)/rosotacom.yaml"
+          .venv/bin/rosotacom image references --project "$project" --repository "$repository" > references.txt
+          cat references.txt
+
+          while read -r reference; do
+            if docker manifest inspect "$reference" >/dev/null 2>&1; then
+              echo "present: $reference"
+              continue
+            fi
+            echo "missing, building: $reference"
+            .venv/bin/rosotacom image build --project "$project" --reference "$reference"
+            docker push "$reference" || echo "::warning::could not push $reference"
+          done < references.txt
+
+          # Report the repository only when every reference resolves. A
+          # read-only GITHUB_TOKEN (a fork PR, Dependabot) can log in and pull
+          # but not push, and the slices must then build as they always did
+          # rather than fail on a pull that was never going to work. Probed
+          # rather than predicted from the event, so it stays right when
+          # GitHub's token rules change.
+          available=true
+          while read -r reference; do
+            docker manifest inspect "$reference" >/dev/null 2>&1 || available=false
+          done < references.txt
+
+          if [[ "$available" == "true" ]]; then
+            echo "repository=$repository" >> "$GITHUB_OUTPUT"
+          else
+            echo "repository=" >> "$GITHUB_OUTPUT"
+            echo "::warning::published images unavailable; each e2e slice will build its own"
+          fi
+
   e2e:
     # One matrix job, not six near-identical copies. The copies were how
     # `e2e-media` ended up listed in ci-success's `needs` with no check on its
@@ -210,8 +300,11 @@ jobs:
     name: e2e-${{ matrix.slice }}
     runs-on: ubuntu-latest
     timeout-minutes: 90
-    needs: preflight-success
-    if: always() && !cancelled() && needs.preflight-success.result == 'success'
+    needs: [preflight-success, image]
+    if: always() && !cancelled() && needs.preflight-success.result == 'success' && needs.image.result == 'success'
+    permissions:
+      contents: read
+      packages: read
     strategy:
       fail-fast: false
       matrix:
@@ -240,7 +333,18 @@ jobs:
       - name: Show Docker environment
         run: docker info
 
+      - name: Log in to GHCR
+        if: needs.image.outputs.repository != ''
+        run: echo '${{ secrets.GITHUB_TOKEN }}' | docker login ghcr.io -u '${{ github.actor }}' --password-stdin
+
       - name: Run E2E slice
+        env:
+          # Empty when the `image` job could not publish, and then rosotacom
+          # builds exactly as before. Set, and each image build becomes a pull
+          # of the tag this tree's own build inputs hash to — rosotacom derives
+          # that tag itself and never accepts one, so it cannot be pointed at an
+          # image built from something else.
+          ROSOTACOM_IMAGE_CACHE: ${{ needs.image.outputs.repository }}
         run: just test-e2e-slice ${{ matrix.slice }}
 
       - name: Upload session artifacts
@@ -261,6 +365,7 @@ jobs:
       - merge-lightweight
       - package
       - preflight-success
+      - image
       - e2e
     if: always() && (github.event_name != 'pull_request' || github.event.pull_request.draft == false)
 
@@ -291,6 +396,10 @@ jobs:
             echo "preflight-success failed or was skipped: ${{ needs.preflight-success.result }}"
             exit 1
           fi
+          if [[ "${{ needs.image.result }}" != "success" ]]; then
+            echo "image failed or was skipped: ${{ needs.image.result }}"
+            exit 1
+          fi
           if [[ "${{ needs.e2e.result }}" != "success" ]]; then
             echo "one or more e2e slices failed or were skipped: ${{ needs.e2e.result }}"
             exit 1

{rosotacom_dev-2.5.dev21 → rosotacom_dev-2.5.dev24}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rosotacom-dev
-Version: 2.5.dev21
+Version: 2.5.dev24
 Summary: Host CLI for ROS OTA communication sessions.
 Author: develNor
 License-Expression: BSD-3-Clause
@@ -27,7 +27,7 @@ Requires-Dist: argcomplete<4,>=3.6
 Requires-Dist: mcap<2,>=1.4
 Requires-Dist: Pillow<13,>=10
 Requires-Dist: PyYAML>=6
-Requires-Dist: ros2docker<0.2,>=0.1.4
+Requires-Dist: ros2docker<0.2,>=0.1.5.dev8
 Provides-Extra: dev
 Requires-Dist: build>=1.2; extra == "dev"
 Requires-Dist: check-wheel-contents>=0.6; extra == "dev"
@@ -244,6 +244,32 @@ example to bake `com_msgs` into its own container image. Ask for the resource by
 name rather than reconstructing a path: the names are part of the CLI contract,
 the layout behind them is not.
 
+### Reusing an image instead of rebuilding it
+
+Somewhere that starts many rosotacom sessions from the same tree — CI above all
+— can build each image once, publish it, and have every later run pull it:
+
+```bash
+# What this project's images are called: <repository>:<hash of the build inputs>
+rosotacom image references --repository ghcr.io/owner/rosotacom-e2e
+
+# Build the one a reference names, then push it
+rosotacom image build --reference ghcr.io/owner/rosotacom-e2e:<hash>
+docker push ghcr.io/owner/rosotacom-e2e:<hash>
+
+# Adopt published images anywhere rosotacom would otherwise build
+export ROSOTACOM_IMAGE_CACHE=ghcr.io/owner/rosotacom-e2e
+```
+
+The tag is a hash of everything that decides the image — base image and its
+pinned digest, apt and pip packages, the Dockerfile and entrypoint ros2docker
+stages, the build UID — so a build-input change is a different tag and a
+published image can never be stale for the tree that adopts it. Nothing accepts
+a reference: rosotacom derives the one its own inputs allow and pulls exactly
+that, and fails rather than quietly rebuilding if it is not there. With
+`ROSOTACOM_IMAGE_CACHE` unset, nothing changes. See
+[docs/ci.md](docs/ci.md) for how the merge gate uses this.
+
 See the [example project README](src/rosotacom/resources/examples/README.md)
 for the copyable example layout, the
 [session configuration reference](session-configuration.md) for the session

{rosotacom_dev-2.5.dev21 → rosotacom_dev-2.5.dev24}/README.md

@@ -199,6 +199,32 @@ example to bake `com_msgs` into its own container image. Ask for the resource by
 name rather than reconstructing a path: the names are part of the CLI contract,
 the layout behind them is not.
 
+### Reusing an image instead of rebuilding it
+
+Somewhere that starts many rosotacom sessions from the same tree — CI above all
+— can build each image once, publish it, and have every later run pull it:
+
+```bash
+# What this project's images are called: <repository>:<hash of the build inputs>
+rosotacom image references --repository ghcr.io/owner/rosotacom-e2e
+
+# Build the one a reference names, then push it
+rosotacom image build --reference ghcr.io/owner/rosotacom-e2e:<hash>
+docker push ghcr.io/owner/rosotacom-e2e:<hash>
+
+# Adopt published images anywhere rosotacom would otherwise build
+export ROSOTACOM_IMAGE_CACHE=ghcr.io/owner/rosotacom-e2e
+```
+
+The tag is a hash of everything that decides the image — base image and its
+pinned digest, apt and pip packages, the Dockerfile and entrypoint ros2docker
+stages, the build UID — so a build-input change is a different tag and a
+published image can never be stale for the tree that adopts it. Nothing accepts
+a reference: rosotacom derives the one its own inputs allow and pulls exactly
+that, and fails rather than quietly rebuilding if it is not there. With
+`ROSOTACOM_IMAGE_CACHE` unset, nothing changes. See
+[docs/ci.md](docs/ci.md) for how the merge gate uses this.
+
 See the [example project README](src/rosotacom/resources/examples/README.md)
 for the copyable example layout, the
 [session configuration reference](session-configuration.md) for the session

{rosotacom_dev-2.5.dev21 → rosotacom_dev-2.5.dev24}/docs/ci.md

@@ -83,6 +83,63 @@ config, catmux pane logs, Docker logs when available, and the smoke verification
 log under `session-instances/`; collect that directory as the first debugging
 artifact when an E2E job fails.
 
+### The e2e image is published once, not rebuilt per job
+
+Every e2e job used to build the project image from scratch. #236 measured that
+at 154s per job — 57s pulling the multi-GB ROS base from Docker Hub, 40s
+exporting layers, 47s of apt and pip — which at thirteen slices is 33 minutes of
+runner time and thirteen Hub pulls on the critical path of every merge gate.
+
+The `image` job now publishes those images to
+`ghcr.io/<owner>/rosotacom-e2e` and each slice pulls instead of building. GHCR
+is co-located with the runners and has no Hub rate limit, so this is one
+transfer where there used to be a transfer plus a build. Measured on run
+31678648935: **154s → 70s per job**, and the slowest slice's pytest run fell
+from 8m19s to 6m50s.
+
+The tag is the whole safety argument. It is a SHA-256 of everything that decides
+what the image contains — the `docker build` command ros2docker renders from a
+config (base image and its pinned digest, `APT_PACKAGES`, `PIP_PACKAGES`, the
+build UID) and every file in the context it stages (Dockerfile, entrypoint, any
+baked packages). Change any of them and the name changes, so "the published
+image is stale" is not a state this can reach, which is the objection #226
+raised against caching the image inline.
+
+Nothing ever *accepts* a reference. `rosotacom image references` derives them;
+the publisher builds and pushes only what is missing, and only under the name
+its own inputs hash to (`rosotacom image build --reference`). A consumer is told
+a repository, in `ROSOTACOM_IMAGE_CACHE`, and derives the tag itself:
+
+```bash
+# What this project's build inputs are called
+rosotacom image references --repository ghcr.io/owner/rosotacom-e2e
+
+# Adopt them instead of building, anywhere rosotacom would build
+ROSOTACOM_IMAGE_CACHE=ghcr.io/owner/rosotacom-e2e just test-e2e-slice heartbeat
+```
+
+With the variable unset, nothing changes: rosotacom builds as it always did. Set
+and the image absent, the job fails rather than falling back — a miss means the
+publisher did not run or did not agree with this tree, and a silent rebuild
+would hide both while costing exactly what this removes. The one place that
+decision is made is the `image` job, which reports its repository only after
+confirming every reference resolves; a read-only `GITHUB_TOKEN` (a fork PR,
+Dependabot) therefore leaves the slices building as before.
+
+Two properties are worth keeping in mind when changing this:
+
+- **Publisher and consumer read one list.** `2_native_chatter` builds a second
+  image from a different package list, so `rosotacom image references` walks the
+  project config *and* every scenario application. Publishing only the project
+  image would leave that slice adopting something nobody published.
+- **The release and nightly lanes still build from scratch**, as does the
+  nightly `image-scan`. That is deliberate: they are what would notice if a
+  published image and a fresh build ever stopped agreeing.
+
+Content-addressed tags accumulate in the GHCR package — one per distinct set of
+build inputs, so roughly one per change to the base digest or package lists.
+Nothing prunes them yet.
+
 ### Balancing the e2e slices
 
 A slice is not its own pytest invocation. `just test-e2e-slice <name>` runs the
@@ -102,18 +159,24 @@ with a usage error, and a test owned twice fails
 
 The costs are warm pytest `call` durations — medians over seven merge-gate runs
 of 2026-08-11/12. On top of them each job pays a fixed
-`RUNNER_SETUP_SECONDS + IMAGE_BUILD_SECONDS` (3m29s: 55s of runner setup, then the
-project image built inside whichever test runs first). Because that constant is
-per job and not per test, balancing warm costs balances wall clock.
+`RUNNER_SETUP_SECONDS + IMAGE_BUILD_SECONDS` (2m05s: 55s of runner setup, then
+the project image obtained inside whichever test runs first). Because that
+constant is per job and not per test, balancing warm costs balances wall clock.
+
+`IMAGE_BUILD_SECONDS` models the merge gate, where that image is pulled rather
+than built (see above); the release and nightly lanes still build and pay the
+older 154s. One slice does not currently reproduce its recorded split: `media`
+spends about 24s more than its two costs predict, and it is the critical path,
+so it is the first thing to remeasure.
 
 ### How many slices
 
 `floor_seconds()` is `fixed + slowest single test` — the fastest any partition
 of this suite could be, because one job has to run that test and pays the fixed
-cost like every other. It is currently **7m56s**, of which 2m34s is the image
-build (#236 is about removing it).
+cost like every other. It is currently **6m32s**, of which 1m10s is the image
+(7m56s before #236 published it instead of rebuilding it per job).
 
-Thirteen slices sit at 8m38s, 1.09x the floor. That is the point of choosing N
+Thirteen slices sit at 7m14s, 1.11x the floor. That is the point of choosing N
 from the numbers rather than from taste: six balanced slices were 13m37s,
 twelve reach the floor, and past twelve every extra job is pure cost. The
 contract test asserts distance to the floor rather than the spread between

{rosotacom_dev-2.5.dev21 → rosotacom_dev-2.5.dev24}/pyproject.toml

@@ -19,7 +19,7 @@ dependencies = [
   "mcap>=1.4,<2",
   "Pillow>=10,<13",
   "PyYAML>=6",
-  "ros2docker>=0.1.4,<0.2",
+  "ros2docker>=0.1.5.dev8,<0.2",
 ]
 classifiers = [
   "Development Status :: 3 - Alpha",

rosotacom_dev-2.5.dev24/requirements.txt

@@ -0,0 +1 @@
+ros2docker==0.1.5.dev8

{rosotacom_dev-2.5.dev21 → rosotacom_dev-2.5.dev24}/src/rosotacom/cli.py

@@ -55,6 +55,15 @@ from .deployment import (
     parse_assignments,
     resolve_peer_bindings,
 )
+from .image_cache import (
+    IMAGE_CACHE_ENV,
+    ImageCacheError,
+    adopt_cached_image,
+    cache_repository,
+    cached_reference,
+    image_fingerprint,
+    reference_tag,
+)
 
 anonymize_lib = importlib.import_module("rosotacom.anonymize")
 
@@ -597,6 +606,53 @@ def _scoped_image_name_from_base(base: str, suffix: str) -> str:
     return f"{base}-{suffix}"
 
 
+def _build_image(config_file: Path, override: Mapping[str, object]) -> None:
+    """Build the image these inputs describe, or adopt the published one.
+
+    With ``ROSOTACOM_IMAGE_CACHE`` unset this is exactly ``ros2docker build``.
+    With it set, the build is replaced by a pull of ``<repository>:<hash of
+    these build inputs>`` — see `rosotacom.image_cache` for why that name is
+    the whole safety argument.
+
+    The fingerprint deliberately ignores ``override``: it carries the container
+    name, the scoped image name, and sometimes isolated-network run args, none
+    of which reach ``docker build``. Feeding them in would give the same image
+    a different name per session.
+    """
+    repository = cache_repository()
+    if repository is None:
+        ros2docker_build(config_file=config_file, override=override)
+        return
+
+    image_name = str(override["image_name"])
+    reference = cached_reference(repository, image_fingerprint(config_file))
+    print(f"Adopting published image {reference} as {image_name} (build skipped)")
+    adopt_cached_image(reference, image_name)
+
+
+def _project_build_inputs(runtime: RuntimeConfig) -> list[tuple[str, Path]]:
+    """Every ``(label, ros2docker config)`` this project can build an image from.
+
+    The project image is one of several: each scenario application names its own
+    config, and `2_native_chatter` builds a second image from a different
+    package list. Publishing only the project image would leave the adopt path
+    strict about an image nobody published, so the publisher and the consumer
+    read the same list.
+    """
+    inputs: list[tuple[str, Path]] = [("project", runtime.ros2docker_config)]
+    seen = {runtime.ros2docker_config.resolve()}
+    for scenario_name in _scenario_names(runtime):
+        definition = _load_scenario_definition(_resolve_scenario(scenario_name, runtime))
+        for identity, applications in sorted(definition.applications.items()):
+            for application in applications:
+                config = application.ros2docker_config.resolve()
+                if config in seen:
+                    continue
+                seen.add(config)
+                inputs.append((f"{scenario_name}/{identity}/{application.name}", config))
+    return inputs
+
+
 def _instance_name_token(instance_id: str) -> str:
     # Underscore-free so the instance component of a container name can be split
     # off unambiguously (the surrounding name parts are underscore-separated).
@@ -4911,7 +4967,7 @@ def start_session(args: argparse.Namespace) -> str:
 
     common_override = {"container_name": container_name, "image_name": image_name}
     if mode == "detached":
-        ros2docker_build(config_file=runtime.ros2docker_config, override=common_override)
+        _build_image(runtime.ros2docker_config, common_override)
         ros2docker_run(
             config_file=runtime.ros2docker_config,
             override={**common_override, "run_type": "up", **run_override},
@@ -4926,7 +4982,7 @@ def start_session(args: argparse.Namespace) -> str:
         )
         _write_docker_log(container_name, instance, identity)
     else:
-        ros2docker_build(config_file=runtime.ros2docker_config, override=common_override)
+        _build_image(runtime.ros2docker_config, common_override)
         ros2docker_run(
             config_file=runtime.ros2docker_config,
             override={
@@ -5258,7 +5314,7 @@ def run_scenario_application(args: argparse.Namespace) -> int:
             network_name,
             getattr(args, "network_ip", None),
         )
-    ros2docker_build(config_file=application.ros2docker_config, override=override)
+    _build_image(application.ros2docker_config, override)
     ros2docker_run(config_file=application.ros2docker_config, override=override)
     return 0
 
@@ -5332,6 +5388,55 @@ def examples_create_command(args: argparse.Namespace) -> int:
     return 0
 
 
+def image_references_command(args: argparse.Namespace) -> int:
+    """Print the content-addressed reference of every image this project builds.
+
+    The publisher's whole job is this list: build and push what is missing. It
+    never chooses a tag, so it cannot publish an image under a name that
+    describes different inputs.
+    """
+    _require_ros2docker()
+    runtime = _load_runtime_config(args)
+    repository = args.repository or cache_repository()
+    if not repository:
+        raise RuntimeError(f"Pass --repository or set {IMAGE_CACHE_ENV} to the repository that holds published images.")
+
+    printed: set[str] = set()
+    for _label, config in _project_build_inputs(runtime):
+        reference = cached_reference(repository, image_fingerprint(config))
+        # Two scenario applications with the same build inputs are one image.
+        if reference in printed:
+            continue
+        printed.add(reference)
+        print(reference)
+    return 0
+
+
+def image_build_command(args: argparse.Namespace) -> int:
+    """Build the image whose build inputs hash to the tag in ``--reference``.
+
+    Taking a reference rather than a config keeps the publisher honest: the only
+    thing it can push is an image whose inputs actually hash to the name it is
+    pushing under.
+    """
+    _require_ros2docker()
+    runtime = _load_runtime_config(args)
+    reference = args.reference
+    wanted = reference_tag(reference)
+
+    for label, config in _project_build_inputs(runtime):
+        if image_fingerprint(config) != wanted:
+            continue
+        print(f"Building {label} ({config}) as {reference}")
+        ros2docker_build(config_file=config, override={"image_name": reference})
+        return 0
+
+    raise ImageCacheError(
+        f"No image this project builds has the fingerprint {wanted}. Take the reference from "
+        f"`rosotacom image references`; it changes whenever the build inputs do."
+    )
+
+
 def resources_path_command(args: argparse.Namespace) -> int:
     """Print the host path of a packaged resource, for external tooling."""
 
@@ -8648,6 +8753,7 @@ TOP_LEVEL_COMMANDS = {
     "list-sessions",
     "scenario",
     "examples",
+    "image",
     "resources",
     "config",
     "bundle",
@@ -9376,6 +9482,30 @@ def main(argv: list[str] | None = None) -> int:
     examples_create_parser.add_argument("--force", action="store_true", help="Replace the target if it exists.")
     examples_create_parser.set_defaults(func=examples_create_command)
 
+    image_parser = subparsers.add_parser("image", help="Publish and consume the project's container images.")
+    image_subparsers = image_parser.add_subparsers(dest="image_action", required=True)
+
+    image_references_parser = image_subparsers.add_parser(
+        "references", help="Print the content-addressed reference of every image this project builds."
+    )
+    _add_common_config_args(image_references_parser)
+    image_references_parser.add_argument(
+        "--repository",
+        help=f"Repository holding published images, e.g. ghcr.io/owner/name. Defaults to {IMAGE_CACHE_ENV}.",
+    )
+    image_references_parser.set_defaults(func=image_references_command)
+
+    image_build_parser = image_subparsers.add_parser(
+        "build", help="Build the image whose build inputs hash to the tag in --reference."
+    )
+    _add_common_config_args(image_build_parser)
+    image_build_parser.add_argument(
+        "--reference",
+        required=True,
+        help="Full reference from `rosotacom image references`. Its tag decides which image is built.",
+    )
+    image_build_parser.set_defaults(func=image_build_command)
+
     resources_parser = subparsers.add_parser("resources", help="Locate resources packaged with rosotacom.")
     resources_subparsers = resources_parser.add_subparsers(dest="resources_action", required=True)
     resources_path_parser = resources_subparsers.add_parser("path", help="Print the host path of a packaged resource.")