docker-squash 1.3.0__py3-none-any.whl

docker_squash/lib/common.py

@@ -0,0 +1,81 @@
+# -*- coding: utf-8 -*-
+
+import os
+from typing import Any, Dict
+
+import docker
+import docker.errors
+import docker.utils
+import requests
+
+from docker_squash.errors import Error
+
+# First try to import Docker client using the API
+# available in version 2 of the library and fall
+# back to version 1
+try:
+    from docker.api.client import APIClient as APIClientClass
+except ImportError:
+    from docker.client import Client as APIClientClass
+
+DEFAULT_TIMEOUT_SECONDS = 600
+
+
+def docker_client(log):
+    log.debug("Preparing Docker client...")
+
+    # Default timeout 10 minutes
+    try:
+        timeout = int(os.getenv("DOCKER_TIMEOUT", 600))
+    except ValueError:
+        raise Error(
+            "Provided timeout value: %s cannot be parsed as integer, exiting."
+            % os.getenv("DOCKER_TIMEOUT")
+        )
+
+    if not timeout > 0:
+        raise Error(
+            "Provided timeout value needs to be greater than zero, currently: %s, exiting."
+            % timeout
+        )
+
+    # backwards compat
+    try:
+        os.environ["DOCKER_HOST"] = os.environ["DOCKER_CONNECTION"]
+        log.warn("DOCKER_CONNECTION is deprecated, please use DOCKER_HOST instead")
+    except KeyError:
+        pass
+
+    params: Dict[str, Any] = {"version": "auto"}
+    params.update(docker.utils.kwargs_from_env())
+    params["timeout"] = timeout
+
+    try:
+        client = APIClientClass(**params)
+    except docker.errors.DockerException as e:
+        log.error(
+            "Could not create Docker client, please make sure that you specified valid parameters in the 'DOCKER_HOST' environment variable."
+        )
+        raise Error("Error while creating the Docker client: %s" % e)
+
+    if client and valid_docker_connection(client):
+        log.debug("Docker client ready")
+        return client
+    else:
+        log.error(
+            "Could not connect to the Docker daemon, please make sure the Docker daemon is running."
+        )
+
+        if os.environ.get("DOCKER_HOST"):
+            log.error(
+                "If Docker daemon is running, please make sure that you specified valid parameters in the 'DOCKER_HOST' environment variable."
+            )
+
+        raise Error("Cannot connect to Docker daemon")
+
+
+def valid_docker_connection(client):
+    try:
+        return client.ping()
+    except requests.exceptions.ConnectionError:
+        return False

docker_squash/squash.py

@@ -0,0 +1,170 @@
+# -*- coding: utf-8 -*-
+
+import os
+from logging import Logger
+from typing import Optional
+
+import docker.errors as docker_errors
+from docker.api.client import APIClient
+from packaging import version as packaging_version
+
+from docker_squash.errors import SquashError
+from docker_squash.image import Image
+from docker_squash.lib import common
+from docker_squash.v1_image import V1Image
+from docker_squash.v2_image import V2Image
+from docker_squash.version import __version__
+
+
+class Squash(object):
+    def __init__(
+        self,
+        log,
+        image,
+        docker: Optional[APIClient] = None,
+        from_layer: Optional[str] = None,
+        tag: Optional[str] = None,
+        comment: Optional[str] = "",
+        tmp_dir: Optional[str] = None,
+        output_path: Optional[str] = None,
+        load_image: Optional[bool] = True,
+        cleanup: Optional[bool] = False,
+    ):
+        self.log: Logger = log
+        self.docker: APIClient = docker if docker else common.docker_client(self.log)
+        self.image: str = image
+        self.from_layer: Optional[str] = from_layer
+        self.tag: Optional[str] = tag
+        self.comment: Optional[str] = comment
+        self.tmp_dir: Optional[str] = tmp_dir
+        self.output_path: Optional[str] = output_path
+        self.load_image: Optional[bool] = load_image
+        self.cleanup: Optional[bool] = cleanup
+        self.development = False
+
+        if tag == image and cleanup:
+            log.warning("Tag is the same as image; preventing cleanup")
+            self.cleanup = False
+        if tmp_dir:
+            self.development = True
+
+    def run(self):
+        docker_version = self.docker.version()
+        self.log.info(
+            "docker-squash version %s, Docker %s, API %s..."
+            % (__version__, docker_version["Version"], docker_version["ApiVersion"])
+        )
+
+        if self.image is None:
+            raise SquashError("Image is not provided")
+
+        if not (self.output_path or self.load_image):
+            self.log.warning(
+                "No output path specified and loading into Docker is not selected either; squashed image would not accessible, proceeding with squashing doesn't make sense"
+            )
+            return
+
+        if self.output_path and os.path.exists(self.output_path):
+            self.log.warning(
+                "Path '%s' specified as output path where the squashed image should be saved already exists, it'll be overriden"
+                % self.output_path
+            )
+
+        image: Image
+
+        if packaging_version.parse(
+            docker_version["ApiVersion"]
+        ) >= packaging_version.parse("1.22"):
+            image = V2Image(
+                self.log,
+                self.docker,
+                self.image,
+                self.from_layer,
+                self.tmp_dir,
+                self.tag,
+                self.comment,
+            )
+        else:
+            image = V1Image(
+                self.log,
+                self.docker,
+                self.image,
+                self.from_layer,
+                self.tmp_dir,
+                self.tag,
+            )
+
+        self.log.info("Using %s image format" % image.FORMAT)
+
+        try:
+            return self.squash(image)
+        except Exception:
+            # https://github.com/goldmann/docker-scripts/issues/44
+            # If development mode is not enabled, make sure we clean up the
+            # temporary directory
+            if not self.development:
+                image.cleanup()
+
+            raise
+
+    def _cleanup(self):
+        try:
+            image_id = self.docker.inspect_image(self.image)["Id"]
+        except docker_errors.APIError as ex:
+            self.log.warning(
+                "Could not get the image ID for {} image: {}, skipping cleanup after squashing".format(
+                    self.image, str(ex)
+                )
+            )
+            return
+
+        self.log.info("Removing old {} image...".format(self.image))
+
+        try:
+            self.docker.remove_image(image_id, force=False, noprune=False)
+            self.log.info("Image {} removed!".format(self.image))
+        except docker_errors.APIError as ex:
+            self.log.warning(
+                "Could not remove image {}: {}, skipping cleanup after squashing".format(
+                    self.image, str(ex)
+                )
+            )
+
+    def squash(self, image: Image):
+        # Do the actual squashing
+        new_image_id = image.squash()
+
+        self.log.info("New squashed image ID is %s" % new_image_id)
+
+        if self.output_path:
+            # Move the tar archive to the specified path
+            image.export_tar_archive(self.output_path)
+
+        if self.load_image:
+            # Load squashed image into Docker; use the ID assigned by the
+            # daemon, which differs from the computed one on some image stores
+            loaded_image_id = image.load_squashed_image()
+
+            if loaded_image_id:
+                if loaded_image_id != new_image_id:
+                    self.log.info(
+                        "Image ID assigned by the Docker daemon is %s" % loaded_image_id
+                    )
+                new_image_id = loaded_image_id
+
+        # If development mode is not enabled, make sure we clean up the
+        # temporary directory
+        if not self.development:
+            # Clean up all temporary files
+            image.cleanup()
+
+        # Remove the source image - this is the only possible way
+        # to remove orphaned layers from Docker daemon at the build time.
+        # We cannot use here a tag name because it could be used as the target,
+        # squashed image tag - we need to use the image ID.
+        if self.cleanup:
+            self._cleanup()
+
+        self.log.info("Done")
+
+        return new_image_id

docker_squash/v1_image.py

@@ -0,0 +1,87 @@
+import hashlib
+import os
+import random
+import shutil
+
+from docker_squash.image import Image
+
+
+class V1Image(Image):
+    FORMAT = "v1"
+
+    def _generate_image_id(self):
+        while True:
+            image_id = hashlib.sha256(
+                str(random.getrandbits(128)).encode("utf8")
+            ).hexdigest()
+
+            try:
+                int(image_id[0:10])
+            except ValueError:
+                # All good!
+                return image_id
+
+    def _before_squashing(self):
+        super(V1Image, self)._before_squashing()
+
+        if self.layers_to_move:
+            self.squash_id = self.layers_to_move[-1]
+
+    def _squash(self) -> str:
+        # Prepare the directory
+        os.makedirs(self.squashed_dir)
+        self._squash_layers(self.layers_to_squash, self.layers_to_move)
+        self._write_version_file(self.squashed_dir)
+        # Move all the layers that should be untouched
+        self._move_layers(self.layers_to_move, self.old_image_dir, self.new_image_dir)
+
+        config_file = os.path.join(self.old_image_dir, self.old_image_id, "json")
+
+        image_id = self._update_squashed_layer_metadata(config_file, self.squashed_dir)
+        shutil.move(self.squashed_dir, os.path.join(self.new_image_dir, image_id))
+        repositories_file = os.path.join(self.new_image_dir, "repositories")
+        self._generate_repositories_json(
+            repositories_file, image_id, self.image_name, self.image_tag
+        )
+
+        return image_id
+
+    def _update_squashed_layer_metadata(self, old_json_file, squashed_dir):
+        image_id = self._generate_image_id()
+
+        metadata = self._read_old_metadata(old_json_file)
+
+        # Modify common metadata fields
+        if self.squash_id:
+            metadata["config"]["Image"] = self.squash_id
+        else:
+            metadata["config"].pop("Image", None)
+
+        if "parent_id" in metadata and self.squash_id:
+            metadata["parent_id"] = "sha256:%s" % self.squash_id
+        else:
+            metadata.pop("parent_id", None)
+
+        metadata.pop("layer_id", None)
+
+        metadata["created"] = self.date
+
+        # Remove unnecessary fields
+        del metadata["container_config"]
+        del metadata["container"]
+        del metadata["config"]["Hostname"]
+
+        if self.squash_id:
+            metadata["parent"] = self.squash_id
+        else:
+            metadata.pop("parent", None)
+
+        metadata["id"] = image_id
+        metadata["Size"] = os.path.getsize(os.path.join(squashed_dir, "layer.tar"))
+        json_metadata = self._dump_json(metadata)[0]
+
+        self._write_json_metadata(
+            "%s" % json_metadata, os.path.join(squashed_dir, "json")
+        )
+
+        return image_id