docker-squash 1.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- docker_squash/__init__.py +1 -0
- docker_squash/cli.py +163 -0
- docker_squash/errors.py +10 -0
- docker_squash/image.py +1130 -0
- docker_squash/lib/__init__.py +0 -0
- docker_squash/lib/common.py +81 -0
- docker_squash/squash.py +170 -0
- docker_squash/v1_image.py +87 -0
- docker_squash/v2_image.py +425 -0
- docker_squash/version.py +1 -0
- docker_squash-1.3.0.dist-info/METADATA +242 -0
- docker_squash-1.3.0.dist-info/RECORD +16 -0
- docker_squash-1.3.0.dist-info/WHEEL +5 -0
- docker_squash-1.3.0.dist-info/entry_points.txt +2 -0
- docker_squash-1.3.0.dist-info/licenses/LICENSE +21 -0
- docker_squash-1.3.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,425 @@
|
|
|
1
|
+
import gzip
|
|
2
|
+
import hashlib
|
|
3
|
+
import json
|
|
4
|
+
import os
|
|
5
|
+
import shutil
|
|
6
|
+
from collections import OrderedDict
|
|
7
|
+
from typing import Any, Dict, List, Tuple
|
|
8
|
+
|
|
9
|
+
from docker_squash.errors import SquashError
|
|
10
|
+
from docker_squash.image import Image
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class V2Image(Image):
|
|
14
|
+
FORMAT = "v2"
|
|
15
|
+
|
|
16
|
+
def _before_squashing(self):
|
|
17
|
+
super(V2Image, self)._before_squashing()
|
|
18
|
+
|
|
19
|
+
# Read old image manifest file
|
|
20
|
+
self.old_image_manifest = self._get_manifest()
|
|
21
|
+
self.log.debug(
|
|
22
|
+
f"Retrieved manifest {json.dumps(self.old_image_manifest, indent=4)}"
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
# Read old image config file
|
|
26
|
+
self.old_image_config = self._read_json_file(
|
|
27
|
+
os.path.join(self.old_image_dir, self.old_image_manifest["Config"])
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
# Read layer paths inside of the tar archive
|
|
31
|
+
# We split it into layers that needs to be squashed
|
|
32
|
+
# and layers that needs to be moved as-is
|
|
33
|
+
self.layer_paths_to_squash: List[str] = []
|
|
34
|
+
self.layer_paths_to_move: List[str] = []
|
|
35
|
+
self.layer_paths_to_squash, self.layer_paths_to_move = self._read_layer_paths(
|
|
36
|
+
self.old_image_config, self.old_image_manifest, self.layers_to_move
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
if self.layer_paths_to_move:
|
|
40
|
+
self.squash_id = self.layer_paths_to_move[-1]
|
|
41
|
+
|
|
42
|
+
self.log.debug(f"Layers paths to squash: {self.layer_paths_to_squash}")
|
|
43
|
+
self.log.debug(f"Layers paths to move: {self.layer_paths_to_move}")
|
|
44
|
+
|
|
45
|
+
def _squash(self) -> str:
|
|
46
|
+
if self.layer_paths_to_squash:
|
|
47
|
+
# Prepare the directory
|
|
48
|
+
os.makedirs(self.squashed_dir)
|
|
49
|
+
# Merge data layers
|
|
50
|
+
self._squash_layers(self.layer_paths_to_squash, self.layer_paths_to_move)
|
|
51
|
+
|
|
52
|
+
self.diff_ids = self._generate_diff_ids()
|
|
53
|
+
self.chain_ids = self._generate_chain_ids(self.diff_ids)
|
|
54
|
+
|
|
55
|
+
if self.layer_paths_to_squash:
|
|
56
|
+
# Must run after the diff ID is computed: the diff ID is the digest
|
|
57
|
+
# of the uncompressed tar.
|
|
58
|
+
self._compress_squashed_layer()
|
|
59
|
+
|
|
60
|
+
metadata = self._generate_image_metadata()
|
|
61
|
+
image_id = self._write_image_metadata(metadata)
|
|
62
|
+
|
|
63
|
+
layer_path_id = None
|
|
64
|
+
|
|
65
|
+
if self.layer_paths_to_squash:
|
|
66
|
+
# Compute layer id to use to name the directory where
|
|
67
|
+
# we store the layer data inside of the tar archive
|
|
68
|
+
layer_path_id = self._generate_squashed_layer_path_id()
|
|
69
|
+
|
|
70
|
+
if self.oci_format:
|
|
71
|
+
old_layer_path = self.old_image_manifest["Config"]
|
|
72
|
+
else:
|
|
73
|
+
if self.layer_paths_to_squash[0]:
|
|
74
|
+
old_layer_path = self.layer_paths_to_squash[0]
|
|
75
|
+
else:
|
|
76
|
+
old_layer_path = layer_path_id
|
|
77
|
+
old_layer_path = os.path.join(old_layer_path, "json")
|
|
78
|
+
|
|
79
|
+
metadata = self._generate_last_layer_metadata(layer_path_id, old_layer_path)
|
|
80
|
+
self._write_squashed_layer_metadata(metadata)
|
|
81
|
+
|
|
82
|
+
# Write version file to the squashed layer
|
|
83
|
+
# Even Docker doesn't know why it's needed...
|
|
84
|
+
self._write_version_file(self.squashed_dir)
|
|
85
|
+
|
|
86
|
+
# Move the temporary squashed layer directory to the correct one
|
|
87
|
+
shutil.move(
|
|
88
|
+
self.squashed_dir, os.path.join(self.new_image_dir, layer_path_id)
|
|
89
|
+
)
|
|
90
|
+
|
|
91
|
+
manifest = self._generate_manifest_metadata(
|
|
92
|
+
image_id,
|
|
93
|
+
self.image_name,
|
|
94
|
+
self.image_tag,
|
|
95
|
+
self.old_image_manifest,
|
|
96
|
+
self.layer_paths_to_move,
|
|
97
|
+
layer_path_id,
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
self._write_manifest_metadata(manifest)
|
|
101
|
+
|
|
102
|
+
repository_image_id = manifest[0]["Layers"][-1].split("/")[0]
|
|
103
|
+
|
|
104
|
+
# Move all the layers that should be untouched
|
|
105
|
+
self._move_layers(
|
|
106
|
+
self.layer_paths_to_move, self.old_image_dir, self.new_image_dir
|
|
107
|
+
)
|
|
108
|
+
|
|
109
|
+
repositories_file = os.path.join(self.new_image_dir, "repositories")
|
|
110
|
+
self._generate_repositories_json(
|
|
111
|
+
repositories_file, repository_image_id, self.image_name, self.image_tag
|
|
112
|
+
)
|
|
113
|
+
|
|
114
|
+
return image_id
|
|
115
|
+
|
|
116
|
+
def _write_image_metadata(self, metadata):
|
|
117
|
+
# Create JSON from the metadata
|
|
118
|
+
# Docker adds new line at the end
|
|
119
|
+
json_metadata, image_id = self._dump_json(metadata, True)
|
|
120
|
+
image_metadata_file = os.path.join(self.new_image_dir, "%s.json" % image_id)
|
|
121
|
+
|
|
122
|
+
self._write_json_metadata(json_metadata, image_metadata_file)
|
|
123
|
+
|
|
124
|
+
return image_id
|
|
125
|
+
|
|
126
|
+
def _write_squashed_layer_metadata(self, metadata):
|
|
127
|
+
layer_metadata_file = os.path.join(self.squashed_dir, "json")
|
|
128
|
+
json_metadata = self._dump_json(metadata)[0]
|
|
129
|
+
|
|
130
|
+
self._write_json_metadata(json_metadata, layer_metadata_file)
|
|
131
|
+
|
|
132
|
+
def _write_manifest_metadata(self, manifest):
|
|
133
|
+
manifest_file = os.path.join(self.new_image_dir, "manifest.json")
|
|
134
|
+
json_manifest = self._dump_json(manifest, True)[0]
|
|
135
|
+
|
|
136
|
+
self._write_json_metadata(json_manifest, manifest_file)
|
|
137
|
+
|
|
138
|
+
def _generate_manifest_metadata(
|
|
139
|
+
self,
|
|
140
|
+
image_id,
|
|
141
|
+
image_name,
|
|
142
|
+
image_tag,
|
|
143
|
+
old_image_manifest,
|
|
144
|
+
layer_paths_to_move,
|
|
145
|
+
layer_path_id=None,
|
|
146
|
+
):
|
|
147
|
+
manifest: Dict[str, Any] = OrderedDict()
|
|
148
|
+
manifest["Config"] = "%s.json" % image_id
|
|
149
|
+
|
|
150
|
+
if image_name and image_tag:
|
|
151
|
+
manifest["RepoTags"] = ["%s:%s" % (image_name, image_tag)]
|
|
152
|
+
|
|
153
|
+
manifest["Layers"] = old_image_manifest["Layers"][: len(layer_paths_to_move)]
|
|
154
|
+
|
|
155
|
+
if layer_path_id:
|
|
156
|
+
manifest["Layers"].append("%s/layer.tar" % layer_path_id)
|
|
157
|
+
|
|
158
|
+
return [manifest]
|
|
159
|
+
|
|
160
|
+
def _read_json_file(self, json_file):
|
|
161
|
+
"""Helper function to read JSON file as OrderedDict"""
|
|
162
|
+
|
|
163
|
+
self.log.debug(f"Reading '{json_file}' JSON file...")
|
|
164
|
+
|
|
165
|
+
with open(json_file, "r") as f:
|
|
166
|
+
return json.load(f, object_pairs_hook=OrderedDict)
|
|
167
|
+
|
|
168
|
+
def _read_layer_paths(
|
|
169
|
+
self, old_image_config, old_image_manifest, layers_to_move: List[str]
|
|
170
|
+
) -> Tuple[List[str], List[str]]:
|
|
171
|
+
"""
|
|
172
|
+
In case of v2 format, layer id's are not the same as the id's
|
|
173
|
+
used in the exported tar archive to name directories for layers.
|
|
174
|
+
These id's can be found in the configuration files saved with
|
|
175
|
+
the image - we need to read them.
|
|
176
|
+
"""
|
|
177
|
+
|
|
178
|
+
# In manifest.json we do not have listed all layers
|
|
179
|
+
# but only layers that do contain some data.
|
|
180
|
+
current_manifest_layer = 0
|
|
181
|
+
|
|
182
|
+
layer_paths_to_move = []
|
|
183
|
+
layer_paths_to_squash = []
|
|
184
|
+
|
|
185
|
+
# Iterate over image history, from base image to top layer
|
|
186
|
+
for i, layer in enumerate(old_image_config["history"]):
|
|
187
|
+
# If it's not an empty layer get the id
|
|
188
|
+
# (directory name) where the layer's data is
|
|
189
|
+
# stored
|
|
190
|
+
if not layer.get("empty_layer", False):
|
|
191
|
+
# Under <25 layers look like
|
|
192
|
+
# 27f9b97654306a5389e8e48ba3486a11026d34055e1907672231cbd8e1380481/layer.tar
|
|
193
|
+
# while >=25 layers look like
|
|
194
|
+
# blobs/sha256/d6a7fc1fb44b63324d3fc67f016e1ef7ecc1a5ae6668ae3072d2e17230e3cfbc
|
|
195
|
+
if self.oci_format:
|
|
196
|
+
layer_id = old_image_manifest["Layers"][current_manifest_layer]
|
|
197
|
+
else:
|
|
198
|
+
layer_id = old_image_manifest["Layers"][
|
|
199
|
+
current_manifest_layer
|
|
200
|
+
].rsplit("/")[0]
|
|
201
|
+
|
|
202
|
+
# Check if this layer should be moved or squashed
|
|
203
|
+
if len(layers_to_move) > i:
|
|
204
|
+
layer_paths_to_move.append(layer_id)
|
|
205
|
+
else:
|
|
206
|
+
layer_paths_to_squash.append(layer_id)
|
|
207
|
+
|
|
208
|
+
current_manifest_layer += 1
|
|
209
|
+
|
|
210
|
+
return layer_paths_to_squash, layer_paths_to_move
|
|
211
|
+
|
|
212
|
+
def _generate_chain_id(self, chain_ids, diff_ids, parent_chain_id):
|
|
213
|
+
if parent_chain_id is None:
|
|
214
|
+
return self._generate_chain_id(chain_ids, diff_ids[1:], diff_ids[0])
|
|
215
|
+
|
|
216
|
+
chain_ids.append(parent_chain_id)
|
|
217
|
+
|
|
218
|
+
if len(diff_ids) == 0:
|
|
219
|
+
return parent_chain_id
|
|
220
|
+
|
|
221
|
+
# This probably should not be hardcoded
|
|
222
|
+
to_hash = "sha256:%s sha256:%s" % (parent_chain_id, diff_ids[0])
|
|
223
|
+
digest = hashlib.sha256(str(to_hash).encode("utf8")).hexdigest()
|
|
224
|
+
|
|
225
|
+
return self._generate_chain_id(chain_ids, diff_ids[1:], digest)
|
|
226
|
+
|
|
227
|
+
def _generate_chain_ids(self, diff_ids):
|
|
228
|
+
chain_ids: List[str] = []
|
|
229
|
+
|
|
230
|
+
self._generate_chain_id(chain_ids, diff_ids, None)
|
|
231
|
+
|
|
232
|
+
return chain_ids
|
|
233
|
+
|
|
234
|
+
def _generate_diff_ids(self):
|
|
235
|
+
diff_ids = []
|
|
236
|
+
|
|
237
|
+
for path in self.layer_paths_to_move:
|
|
238
|
+
sha256 = self._compute_sha256(self._extract_tar_name(path))
|
|
239
|
+
diff_ids.append(sha256)
|
|
240
|
+
|
|
241
|
+
if self.layer_paths_to_squash:
|
|
242
|
+
sha256 = self._compute_sha256(os.path.join(self.squashed_dir, "layer.tar"))
|
|
243
|
+
diff_ids.append(sha256)
|
|
244
|
+
|
|
245
|
+
return diff_ids
|
|
246
|
+
|
|
247
|
+
def _compress_squashed_layer(self):
|
|
248
|
+
"""
|
|
249
|
+
Replace the squashed layer.tar with its gzip.
|
|
250
|
+
|
|
251
|
+
The containerd image store keeps a loaded layer as given and pushes it
|
|
252
|
+
as is, so a raw tar is pushed uncompressed. The daemon derives the
|
|
253
|
+
compressed digest itself and the config's diff ID still refers to the
|
|
254
|
+
uncompressed tar, so no other metadata changes. The mtime is fixed to
|
|
255
|
+
keep the output reproducible.
|
|
256
|
+
"""
|
|
257
|
+
compressed = self.squashed_tar + ".gz"
|
|
258
|
+
|
|
259
|
+
with open(self.squashed_tar, "rb") as src, open(compressed, "wb") as raw:
|
|
260
|
+
with gzip.GzipFile(
|
|
261
|
+
filename="", mode="wb", fileobj=raw, compresslevel=9, mtime=0
|
|
262
|
+
) as dst:
|
|
263
|
+
shutil.copyfileobj(src, dst)
|
|
264
|
+
|
|
265
|
+
os.replace(compressed, self.squashed_tar)
|
|
266
|
+
|
|
267
|
+
def _compute_sha256(self, layer_tar):
|
|
268
|
+
sha256 = hashlib.sha256()
|
|
269
|
+
|
|
270
|
+
with open(layer_tar, "rb") as f:
|
|
271
|
+
while True:
|
|
272
|
+
# Read in 10MB chunks
|
|
273
|
+
data = f.read(10485760)
|
|
274
|
+
|
|
275
|
+
if not data:
|
|
276
|
+
break
|
|
277
|
+
|
|
278
|
+
sha256.update(data)
|
|
279
|
+
|
|
280
|
+
return sha256.hexdigest()
|
|
281
|
+
|
|
282
|
+
def _generate_squashed_layer_path_id(self):
|
|
283
|
+
"""
|
|
284
|
+
This function generates the id used to name the directory to
|
|
285
|
+
store the squashed layer content in the archive.
|
|
286
|
+
|
|
287
|
+
This mimics what Docker does here: https://github.com/docker/docker/blob/v1.10.0-rc1/image/v1/imagev1.go#L42
|
|
288
|
+
To make it simpler we do reuse old image metadata and
|
|
289
|
+
modify it to what it should look which means to be exact
|
|
290
|
+
as https://github.com/docker/docker/blob/v1.10.0-rc1/image/v1/imagev1.go#L64
|
|
291
|
+
"""
|
|
292
|
+
|
|
293
|
+
# Using OrderedDict, because order of JSON elements is important
|
|
294
|
+
v1_metadata = OrderedDict(self.old_image_config)
|
|
295
|
+
|
|
296
|
+
# Update image creation date
|
|
297
|
+
v1_metadata["created"] = self.date
|
|
298
|
+
|
|
299
|
+
# Remove unnecessary elements
|
|
300
|
+
# Do not fail if key is not found
|
|
301
|
+
for key in "history", "rootfs", "container":
|
|
302
|
+
v1_metadata.pop(key, None)
|
|
303
|
+
|
|
304
|
+
# Docker internally changes the order of keys between
|
|
305
|
+
# exported metadata (why oh why?!). We need to add 'os'
|
|
306
|
+
# element after 'layer_id'
|
|
307
|
+
operating_system = v1_metadata.pop("os", None)
|
|
308
|
+
|
|
309
|
+
# The 'layer_id' element is the chain_id of the
|
|
310
|
+
# squashed layer
|
|
311
|
+
v1_metadata["layer_id"] = "sha256:%s" % self.chain_ids[-1]
|
|
312
|
+
|
|
313
|
+
# Add back 'os' element
|
|
314
|
+
if operating_system:
|
|
315
|
+
v1_metadata["os"] = operating_system
|
|
316
|
+
|
|
317
|
+
# The 'parent' element is the name of the directory (inside the
|
|
318
|
+
# exported tar archive) of the last layer that we move
|
|
319
|
+
# (layer below squashed layer)
|
|
320
|
+
|
|
321
|
+
if self.layer_paths_to_move:
|
|
322
|
+
if self.layer_paths_to_squash:
|
|
323
|
+
parent = self.layer_paths_to_move[-1]
|
|
324
|
+
else:
|
|
325
|
+
parent = self.layer_paths_to_move[0]
|
|
326
|
+
|
|
327
|
+
v1_metadata["parent"] = "sha256:%s" % parent
|
|
328
|
+
|
|
329
|
+
# The 'Image' element is the id of the layer from which we squash
|
|
330
|
+
if self.squash_id:
|
|
331
|
+
# Update image id, should be one layer below squashed layer
|
|
332
|
+
v1_metadata["config"]["Image"] = self.squash_id
|
|
333
|
+
else:
|
|
334
|
+
v1_metadata["config"]["Image"] = ""
|
|
335
|
+
|
|
336
|
+
# Get the sha256sum of the JSON exported metadata,
|
|
337
|
+
# we do not care about the metadata anymore
|
|
338
|
+
sha = self._dump_json(v1_metadata)[1]
|
|
339
|
+
|
|
340
|
+
return sha
|
|
341
|
+
|
|
342
|
+
def _generate_last_layer_metadata(self, layer_path_id, old_layer_path: str):
|
|
343
|
+
config_file = os.path.join(self.old_image_dir, old_layer_path)
|
|
344
|
+
with open(config_file, "r") as f:
|
|
345
|
+
config = json.load(f, object_pairs_hook=OrderedDict)
|
|
346
|
+
|
|
347
|
+
config["created"] = self.date
|
|
348
|
+
|
|
349
|
+
if self.squash_id:
|
|
350
|
+
# Update image id, should be one layer below squashed layer
|
|
351
|
+
config["config"]["Image"] = self.squash_id
|
|
352
|
+
else:
|
|
353
|
+
config["config"]["Image"] = ""
|
|
354
|
+
|
|
355
|
+
# Update 'parent' - it should be path to the last layer to move
|
|
356
|
+
if self.layer_paths_to_move:
|
|
357
|
+
config["parent"] = self.layer_paths_to_move[-1]
|
|
358
|
+
else:
|
|
359
|
+
config.pop("parent", None)
|
|
360
|
+
# Update 'id' - it should be the path to the layer
|
|
361
|
+
config["id"] = layer_path_id
|
|
362
|
+
config.pop("container", None)
|
|
363
|
+
return config
|
|
364
|
+
|
|
365
|
+
def _generate_image_metadata(self):
|
|
366
|
+
# First - read old image config, we'll update it instead of
|
|
367
|
+
# generating one from scratch
|
|
368
|
+
metadata = OrderedDict(self.old_image_config)
|
|
369
|
+
# Update image creation date
|
|
370
|
+
metadata["created"] = self.date
|
|
371
|
+
|
|
372
|
+
# Remove unnecessary or old fields
|
|
373
|
+
metadata.pop("container", None)
|
|
374
|
+
|
|
375
|
+
# Remove squashed layers from history
|
|
376
|
+
metadata["history"] = metadata["history"][: len(self.layers_to_move)]
|
|
377
|
+
# Remove diff_ids for squashed layers
|
|
378
|
+
metadata["rootfs"]["diff_ids"] = metadata["rootfs"]["diff_ids"][
|
|
379
|
+
: len(self.layer_paths_to_move)
|
|
380
|
+
]
|
|
381
|
+
|
|
382
|
+
history: Dict[str, Any] = {"comment": self.comment, "created": self.date}
|
|
383
|
+
|
|
384
|
+
if self.layer_paths_to_squash:
|
|
385
|
+
# Add diff_ids for the squashed layer
|
|
386
|
+
metadata["rootfs"]["diff_ids"].append("sha256:%s" % self.diff_ids[-1])
|
|
387
|
+
else:
|
|
388
|
+
history["empty_layer"] = True
|
|
389
|
+
|
|
390
|
+
# Add new entry for squashed layer to history
|
|
391
|
+
metadata["history"].append(history)
|
|
392
|
+
|
|
393
|
+
if self.squash_id:
|
|
394
|
+
# Update image id, should be one layer below squashed layer
|
|
395
|
+
metadata["config"]["Image"] = self.squash_id
|
|
396
|
+
else:
|
|
397
|
+
metadata["config"]["Image"] = ""
|
|
398
|
+
|
|
399
|
+
return metadata
|
|
400
|
+
|
|
401
|
+
def _get_manifest(self):
|
|
402
|
+
if os.path.exists(os.path.join(self.old_image_dir, "index.json")):
|
|
403
|
+
# New OCI Archive format type
|
|
404
|
+
self.oci_format = True
|
|
405
|
+
# Not using index.json to extract manifest details as while the config
|
|
406
|
+
# sha could be extracted via some indirection i.e.
|
|
407
|
+
#
|
|
408
|
+
# index.json:manifest/digest::sha256:<intermediary>
|
|
409
|
+
# blobs/sha256/<intermediary>:config/digest::sha256:<config>
|
|
410
|
+
#
|
|
411
|
+
# Docker spec currently will always include a manifest.json so will standardise
|
|
412
|
+
# on using that. Further we rely upon the original manifest format in order to write
|
|
413
|
+
# it back.
|
|
414
|
+
if os.path.exists(os.path.join(self.old_image_dir, "manifest.json")):
|
|
415
|
+
return (
|
|
416
|
+
self._read_json_file(
|
|
417
|
+
os.path.join(self.old_image_dir, "manifest.json")
|
|
418
|
+
)
|
|
419
|
+
)[0]
|
|
420
|
+
else:
|
|
421
|
+
raise SquashError("Unable to locate manifest.json")
|
|
422
|
+
else:
|
|
423
|
+
return (
|
|
424
|
+
self._read_json_file(os.path.join(self.old_image_dir, "manifest.json"))
|
|
425
|
+
)[0]
|
docker_squash/version.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "1.3.0"
|
|
@@ -0,0 +1,242 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: docker-squash
|
|
3
|
+
Version: 1.3.0
|
|
4
|
+
Summary: Docker layer squashing tool
|
|
5
|
+
Home-page: https://github.com/goldmann/docker-squash
|
|
6
|
+
Download-URL: https://github.com/goldmann/docker-squash/archive/1.3.0.tar.gz
|
|
7
|
+
Author: Marek Goldmann
|
|
8
|
+
Author-email: marek.goldmann@gmail.com
|
|
9
|
+
License: MIT
|
|
10
|
+
Keywords: docker
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Requires-Dist: docker
|
|
13
|
+
Requires-Dist: packaging
|
|
14
|
+
Dynamic: author
|
|
15
|
+
Dynamic: author-email
|
|
16
|
+
Dynamic: description
|
|
17
|
+
Dynamic: download-url
|
|
18
|
+
Dynamic: home-page
|
|
19
|
+
Dynamic: keywords
|
|
20
|
+
Dynamic: license
|
|
21
|
+
Dynamic: license-file
|
|
22
|
+
Dynamic: requires-dist
|
|
23
|
+
Dynamic: summary
|
|
24
|
+
|
|
25
|
+
``docker-squash``
|
|
26
|
+
==================
|
|
27
|
+
|
|
28
|
+
.. image:: https://github.com/goldmann/docker-squash/actions/workflows/squash.yml/badge.svg
|
|
29
|
+
:target: https://github.com/goldmann/docker-squash/actions/workflows/squash.yml
|
|
30
|
+
|
|
31
|
+
The problem
|
|
32
|
+
-----------
|
|
33
|
+
|
|
34
|
+
Docker creates many layers while building the image. Sometimes it's not necessary or desireable
|
|
35
|
+
to have them in the image. For example a Dockerfile `ADD` instruction creates a single layer
|
|
36
|
+
with files you want to make available in the image. The problem arises when these files are
|
|
37
|
+
only temporary files (for example product distribution that you want to unpack). Docker will
|
|
38
|
+
carry this unnecessary layer always with the image, even if you delete these files in next
|
|
39
|
+
layer. This a waste of time (more data to push/load/save) and resources (bigger image).
|
|
40
|
+
|
|
41
|
+
Squashing helps with organizing images in logical layers. Instead of
|
|
42
|
+
having an image with multiple (in almost all cases) unnecessary layers -
|
|
43
|
+
we can control the structure of the image.
|
|
44
|
+
|
|
45
|
+
Features
|
|
46
|
+
--------
|
|
47
|
+
|
|
48
|
+
- Can squash last n layers from an image
|
|
49
|
+
- Can squash from a selected layer to the end (not always possible, depends on the image)
|
|
50
|
+
- Support for Docker 1.9 or newer (older releases may run perfectly fine too, try it!)
|
|
51
|
+
- Squashed image can be loaded back to the Docker daemon or stored as tar archive somewhere
|
|
52
|
+
|
|
53
|
+
Installation
|
|
54
|
+
------------
|
|
55
|
+
|
|
56
|
+
From source code
|
|
57
|
+
|
|
58
|
+
::
|
|
59
|
+
|
|
60
|
+
$ pip install --user https://github.com/goldmann/docker-squash/archive/master.zip
|
|
61
|
+
|
|
62
|
+
From PyPi
|
|
63
|
+
|
|
64
|
+
::
|
|
65
|
+
|
|
66
|
+
$ pip install docker-squash
|
|
67
|
+
|
|
68
|
+
It is supported on Python 3.7 and above.
|
|
69
|
+
|
|
70
|
+
Usage
|
|
71
|
+
-----
|
|
72
|
+
|
|
73
|
+
::
|
|
74
|
+
|
|
75
|
+
$ docker-squash -h
|
|
76
|
+
usage: cli.py [-h] [-v] [--version] [-d] [-f FROM_LAYER] [-t TAG]
|
|
77
|
+
[--tmp-dir TMP_DIR] [--output-path OUTPUT_PATH]
|
|
78
|
+
image
|
|
79
|
+
|
|
80
|
+
Docker layer squashing tool
|
|
81
|
+
|
|
82
|
+
positional arguments:
|
|
83
|
+
image Image to be squashed
|
|
84
|
+
|
|
85
|
+
optional arguments:
|
|
86
|
+
-h, --help show this help message and exit
|
|
87
|
+
-v, --verbose Verbose output
|
|
88
|
+
--version Show version and exit
|
|
89
|
+
-f FROM_LAYER, --from-layer FROM_LAYER
|
|
90
|
+
Number of layers to squash or ID of the layer (or image ID or image name) to squash from.
|
|
91
|
+
In case the provided value is an integer, specified number of layers will be squashed.
|
|
92
|
+
Every layer in the image will be squashed if the parameter is not provided.
|
|
93
|
+
-t TAG, --tag TAG Specify the tag to be used for the new image. If not specified no tag will be applied
|
|
94
|
+
-m MESSAGE, --message MESSAGE
|
|
95
|
+
Specify a commit message (comment) for the new image.
|
|
96
|
+
-c, --cleanup Remove source image from Docker after squashing
|
|
97
|
+
--tmp-dir TMP_DIR Temporary directory to be created and used. This will NOT be deleted afterwards for
|
|
98
|
+
easier debugging.
|
|
99
|
+
--output-path OUTPUT_PATH
|
|
100
|
+
Path where the image may be stored after squashing.
|
|
101
|
+
--load-image [LOAD_IMAGE]
|
|
102
|
+
Whether to load the image into Docker daemon after squashing
|
|
103
|
+
Default: true
|
|
104
|
+
|
|
105
|
+
Note that environment variables may be set as documented in `here <docs/environment_variables.adoc>`_.
|
|
106
|
+
|
|
107
|
+
License
|
|
108
|
+
-------
|
|
109
|
+
|
|
110
|
+
MIT
|
|
111
|
+
|
|
112
|
+
Examples
|
|
113
|
+
--------
|
|
114
|
+
|
|
115
|
+
We start with image like this:
|
|
116
|
+
|
|
117
|
+
::
|
|
118
|
+
|
|
119
|
+
$ docker history jboss/wildfly:latest
|
|
120
|
+
IMAGE CREATED CREATED BY SIZE COMMENT
|
|
121
|
+
25954e6d2300 3 weeks ago /bin/sh -c #(nop) CMD ["/opt/jboss/wildfly/bi 0 B
|
|
122
|
+
5ae69cb454a5 3 weeks ago /bin/sh -c #(nop) EXPOSE 8080/tcp 0 B
|
|
123
|
+
dc24712f35c4 3 weeks ago /bin/sh -c #(nop) ENV LAUNCH_JBOSS_IN_BACKGRO 0 B
|
|
124
|
+
d929129d4c8e 3 weeks ago /bin/sh -c cd $HOME && curl -O https://do 160.8 MB
|
|
125
|
+
b8fa3caf7d6d 3 weeks ago /bin/sh -c #(nop) ENV JBOSS_HOME=/opt/jboss/w 0 B
|
|
126
|
+
38b8f85e74bf 3 weeks ago /bin/sh -c #(nop) ENV WILDFLY_SHA1=c0dd7552c5 0 B
|
|
127
|
+
ae79b646b9a9 3 weeks ago /bin/sh -c #(nop) ENV WILDFLY_VERSION=10.0.0. 0 B
|
|
128
|
+
2b4606dc9dc7 3 weeks ago /bin/sh -c #(nop) ENV JAVA_HOME=/usr/lib/jvm/ 0 B
|
|
129
|
+
118fa9e33576 3 weeks ago /bin/sh -c #(nop) USER [jboss] 0 B
|
|
130
|
+
5f7e8f36c3bb 3 weeks ago /bin/sh -c yum -y install java-1.8.0-openjdk- 197.4 MB
|
|
131
|
+
3d4d0228f161 3 weeks ago /bin/sh -c #(nop) USER [root] 0 B
|
|
132
|
+
f7ab4ea19708 3 weeks ago /bin/sh -c #(nop) MAINTAINER Marek Goldmann < 0 B
|
|
133
|
+
4bb15f3b6977 3 weeks ago /bin/sh -c #(nop) USER [jboss] 0 B
|
|
134
|
+
5dc1e49f4361 3 weeks ago /bin/sh -c #(nop) WORKDIR /opt/jboss 0 B
|
|
135
|
+
7f0f9eb31174 3 weeks ago /bin/sh -c groupadd -r jboss -g 1000 && usera 4.349 kB
|
|
136
|
+
bd515f044af7 3 weeks ago /bin/sh -c yum update -y && yum -y install xm 25.18 MB
|
|
137
|
+
b78336099045 3 weeks ago /bin/sh -c #(nop) MAINTAINER Marek Goldmann < 0 B
|
|
138
|
+
4816a298548c 3 weeks ago /bin/sh -c #(nop) CMD ["/bin/bash"] 0 B
|
|
139
|
+
6ee235cf4473 3 weeks ago /bin/sh -c #(nop) LABEL name=CentOS Base Imag 0 B
|
|
140
|
+
474c2ee77fa3 3 weeks ago /bin/sh -c #(nop) ADD file:72852fc7626d233343 196.6 MB
|
|
141
|
+
1544084fad81 6 months ago /bin/sh -c #(nop) MAINTAINER The CentOS Proje 0 B
|
|
142
|
+
|
|
143
|
+
And we want to squash all the layers down to layer ``4bb15f3b6977``.
|
|
144
|
+
|
|
145
|
+
::
|
|
146
|
+
|
|
147
|
+
$ docker-squash -f 4bb15f3b6977 -t jboss/wildfly:squashed jboss/wildfly:latest
|
|
148
|
+
2016-04-01 13:11:02,358 root INFO docker-scripts version 1.0.0dev, Docker 7206621, API 1.21...
|
|
149
|
+
2016-04-01 13:11:02,358 root INFO Using v1 image format
|
|
150
|
+
2016-04-01 13:11:02,374 root INFO Old image has 21 layers
|
|
151
|
+
2016-04-01 13:11:02,378 root INFO Checking if squashing is necessary...
|
|
152
|
+
2016-04-01 13:11:02,378 root INFO Attempting to squash last 12 layers...
|
|
153
|
+
2016-04-01 13:11:02,378 root INFO Saving image 25954e6d230006235eecb7f0cc560264d73146985c2d2e663bac953d660b8730 to /tmp/docker-squash-fbxZz4/old/image.tar file...
|
|
154
|
+
2016-04-01 13:11:08,003 root INFO Image saved!
|
|
155
|
+
2016-04-01 13:11:08,031 root INFO Unpacking /tmp/docker-squash-fbxZz4/old/image.tar tar file to /tmp/docker-squash-fbxZz4/old directory
|
|
156
|
+
2016-04-01 13:11:08,588 root INFO Archive unpacked!
|
|
157
|
+
2016-04-01 13:11:08,636 root INFO Squashing image 'jboss/wildfly:latest'...
|
|
158
|
+
2016-04-01 13:11:08,637 root INFO Starting squashing...
|
|
159
|
+
2016-04-01 13:11:08,637 root INFO Squashing file '/tmp/docker-squash-fbxZz4/old/25954e6d230006235eecb7f0cc560264d73146985c2d2e663bac953d660b8730/layer.tar'...
|
|
160
|
+
2016-04-01 13:11:08,637 root INFO Squashing file '/tmp/docker-squash-fbxZz4/old/5ae69cb454a5a542f63e148ce40fb9e01de5bb01805b4ded238841bc2ce8e895/layer.tar'...
|
|
161
|
+
2016-04-01 13:11:08,637 root INFO Squashing file '/tmp/docker-squash-fbxZz4/old/dc24712f35c40e958be8aca2731e7bf8353b9b18baa6a94ad84c6952cbc77004/layer.tar'...
|
|
162
|
+
2016-04-01 13:11:08,638 root INFO Squashing file '/tmp/docker-squash-fbxZz4/old/d929129d4c8e61ea3661eb42c30d01f4c152418689178afc7dc8185a37814528/layer.tar'...
|
|
163
|
+
2016-04-01 13:11:09,113 root INFO Squashing file '/tmp/docker-squash-fbxZz4/old/b8fa3caf7d6dc228bf2499a3af86e5073ad0c17304c3900fa341e9d2fe4e5655/layer.tar'...
|
|
164
|
+
2016-04-01 13:11:09,115 root INFO Squashing file '/tmp/docker-squash-fbxZz4/old/38b8f85e74bfa773a0ad69da2205dc0148945e6f5a7ceb04fa4e8619e1de425b/layer.tar'...
|
|
165
|
+
2016-04-01 13:11:09,115 root INFO Squashing file '/tmp/docker-squash-fbxZz4/old/ae79b646b9a9a287c5f6a01871cc9d9ee596dafee2db942714ca3dea0c06eef3/layer.tar'...
|
|
166
|
+
2016-04-01 13:11:09,115 root INFO Squashing file '/tmp/docker-squash-fbxZz4/old/2b4606dc9dc773aa220a65351fe8d54f03534c58fea230960e95915222366074/layer.tar'...
|
|
167
|
+
2016-04-01 13:11:09,115 root INFO Squashing file '/tmp/docker-squash-fbxZz4/old/118fa9e33576ecc625ebbbfdf2809c1527e716cb4fd5cb40548eb6d3503a75a9/layer.tar'...
|
|
168
|
+
2016-04-01 13:11:09,115 root INFO Squashing file '/tmp/docker-squash-fbxZz4/old/5f7e8f36c3bb20c9db7470a22f828710b4d28aede64966c425add48a1b14fe23/layer.tar'...
|
|
169
|
+
2016-04-01 13:11:10,127 root INFO Squashing file '/tmp/docker-squash-fbxZz4/old/3d4d0228f161b67eb46fdb425ad148c31d9944dcb822f67eac3e2ac2effefc73/layer.tar'...
|
|
170
|
+
2016-04-01 13:11:10,129 root INFO Squashing file '/tmp/docker-squash-fbxZz4/old/f7ab4ea197084ab7483a2ca5409bdcf5473141bfb61b8687b1329943359cc3fe/layer.tar'...
|
|
171
|
+
2016-04-01 13:11:10,732 root INFO Squashing finished!
|
|
172
|
+
2016-04-01 13:11:10,737 root INFO New squashed image ID is 52255e75d3eb83123e074f897e8c971dec9d1168a5c82d7c1496a190da2e40ef
|
|
173
|
+
2016-04-01 13:11:14,563 root INFO Image registered in Docker daemon as jboss/wildfly:squashed
|
|
174
|
+
2016-04-01 13:11:14,652 root INFO Done
|
|
175
|
+
|
|
176
|
+
We can now confirm the layer structure:
|
|
177
|
+
|
|
178
|
+
::
|
|
179
|
+
|
|
180
|
+
$ docker history jboss/wildfly:squashed
|
|
181
|
+
IMAGE CREATED CREATED BY SIZE COMMENT
|
|
182
|
+
52255e75d3eb 40 seconds ago 358.2 MB
|
|
183
|
+
4bb15f3b6977 3 weeks ago /bin/sh -c #(nop) USER [jboss] 0 B
|
|
184
|
+
5dc1e49f4361 3 weeks ago /bin/sh -c #(nop) WORKDIR /opt/jboss 0 B
|
|
185
|
+
7f0f9eb31174 3 weeks ago /bin/sh -c groupadd -r jboss -g 1000 && usera 4.349 kB
|
|
186
|
+
bd515f044af7 3 weeks ago /bin/sh -c yum update -y && yum -y install xm 25.18 MB
|
|
187
|
+
b78336099045 3 weeks ago /bin/sh -c #(nop) MAINTAINER Marek Goldmann < 0 B
|
|
188
|
+
4816a298548c 3 weeks ago /bin/sh -c #(nop) CMD ["/bin/bash"] 0 B
|
|
189
|
+
6ee235cf4473 3 weeks ago /bin/sh -c #(nop) LABEL name=CentOS Base Imag 0 B
|
|
190
|
+
474c2ee77fa3 3 weeks ago /bin/sh -c #(nop) ADD file:72852fc7626d233343 196.6 MB
|
|
191
|
+
1544084fad81 6 months ago /bin/sh -c #(nop) MAINTAINER The CentOS Proje 0 B
|
|
192
|
+
|
|
193
|
+
Other option is to specify how many layers (counting from the newest layer) we want to squash.\
|
|
194
|
+
Let's squash last 10 layers from the ``jboss/wildfly:latest`` image:
|
|
195
|
+
|
|
196
|
+
::
|
|
197
|
+
|
|
198
|
+
$ docker-squash -f 10 -t jboss/wildfly:squashed jboss/wildfly:latest
|
|
199
|
+
2016-04-01 13:15:06,488 root INFO docker-scripts version 1.0.0dev, Docker 7206621, API 1.21...
|
|
200
|
+
2016-04-01 13:15:06,488 root INFO Using v1 image format
|
|
201
|
+
2016-04-01 13:15:06,504 root INFO Old image has 21 layers
|
|
202
|
+
2016-04-01 13:15:06,504 root INFO Checking if squashing is necessary...
|
|
203
|
+
2016-04-01 13:15:06,504 root INFO Attempting to squash last 10 layers...
|
|
204
|
+
2016-04-01 13:15:06,505 root INFO Saving image 25954e6d230006235eecb7f0cc560264d73146985c2d2e663bac953d660b8730 to /tmp/docker-squash-fu80CX/old/image.tar file...
|
|
205
|
+
2016-04-01 13:15:12,136 root INFO Image saved!
|
|
206
|
+
2016-04-01 13:15:12,167 root INFO Unpacking /tmp/docker-squash-fu80CX/old/image.tar tar file to /tmp/docker-squash-fu80CX/old directory
|
|
207
|
+
2016-04-01 13:15:12,706 root INFO Archive unpacked!
|
|
208
|
+
2016-04-01 13:15:12,756 root INFO Squashing image 'jboss/wildfly:latest'...
|
|
209
|
+
2016-04-01 13:15:12,756 root INFO Starting squashing...
|
|
210
|
+
2016-04-01 13:15:12,756 root INFO Squashing file '/tmp/docker-squash-fu80CX/old/25954e6d230006235eecb7f0cc560264d73146985c2d2e663bac953d660b8730/layer.tar'...
|
|
211
|
+
2016-04-01 13:15:12,757 root INFO Squashing file '/tmp/docker-squash-fu80CX/old/5ae69cb454a5a542f63e148ce40fb9e01de5bb01805b4ded238841bc2ce8e895/layer.tar'...
|
|
212
|
+
2016-04-01 13:15:12,757 root INFO Squashing file '/tmp/docker-squash-fu80CX/old/dc24712f35c40e958be8aca2731e7bf8353b9b18baa6a94ad84c6952cbc77004/layer.tar'...
|
|
213
|
+
2016-04-01 13:15:12,757 root INFO Squashing file '/tmp/docker-squash-fu80CX/old/d929129d4c8e61ea3661eb42c30d01f4c152418689178afc7dc8185a37814528/layer.tar'...
|
|
214
|
+
2016-04-01 13:15:13,234 root INFO Squashing file '/tmp/docker-squash-fu80CX/old/b8fa3caf7d6dc228bf2499a3af86e5073ad0c17304c3900fa341e9d2fe4e5655/layer.tar'...
|
|
215
|
+
2016-04-01 13:15:13,235 root INFO Squashing file '/tmp/docker-squash-fu80CX/old/38b8f85e74bfa773a0ad69da2205dc0148945e6f5a7ceb04fa4e8619e1de425b/layer.tar'...
|
|
216
|
+
2016-04-01 13:15:13,235 root INFO Squashing file '/tmp/docker-squash-fu80CX/old/ae79b646b9a9a287c5f6a01871cc9d9ee596dafee2db942714ca3dea0c06eef3/layer.tar'...
|
|
217
|
+
2016-04-01 13:15:13,235 root INFO Squashing file '/tmp/docker-squash-fu80CX/old/2b4606dc9dc773aa220a65351fe8d54f03534c58fea230960e95915222366074/layer.tar'...
|
|
218
|
+
2016-04-01 13:15:13,236 root INFO Squashing file '/tmp/docker-squash-fu80CX/old/118fa9e33576ecc625ebbbfdf2809c1527e716cb4fd5cb40548eb6d3503a75a9/layer.tar'...
|
|
219
|
+
2016-04-01 13:15:13,236 root INFO Squashing file '/tmp/docker-squash-fu80CX/old/5f7e8f36c3bb20c9db7470a22f828710b4d28aede64966c425add48a1b14fe23/layer.tar'...
|
|
220
|
+
2016-04-01 13:15:14,848 root INFO Squashing finished!
|
|
221
|
+
2016-04-01 13:15:14,853 root INFO New squashed image ID is fde7edd2e5683c97bedf9c0bf52ad5150db5650e421de3d9293ce5223b256455
|
|
222
|
+
2016-04-01 13:15:18,963 root INFO Image registered in Docker daemon as jboss/wildfly:squashed
|
|
223
|
+
2016-04-01 13:15:19,059 root INFO Done
|
|
224
|
+
|
|
225
|
+
Let's confirm the image structure now:
|
|
226
|
+
|
|
227
|
+
::
|
|
228
|
+
|
|
229
|
+
$ docker history jboss/wildfly:squashed
|
|
230
|
+
IMAGE CREATED CREATED BY SIZE COMMENT
|
|
231
|
+
fde7edd2e568 32 seconds ago 358.2 MB
|
|
232
|
+
3d4d0228f161 3 weeks ago /bin/sh -c #(nop) USER [root] 0 B
|
|
233
|
+
f7ab4ea19708 3 weeks ago /bin/sh -c #(nop) MAINTAINER Marek Goldmann < 0 B
|
|
234
|
+
4bb15f3b6977 3 weeks ago /bin/sh -c #(nop) USER [jboss] 0 B
|
|
235
|
+
5dc1e49f4361 3 weeks ago /bin/sh -c #(nop) WORKDIR /opt/jboss 0 B
|
|
236
|
+
7f0f9eb31174 3 weeks ago /bin/sh -c groupadd -r jboss -g 1000 && usera 4.349 kB
|
|
237
|
+
bd515f044af7 3 weeks ago /bin/sh -c yum update -y && yum -y install xm 25.18 MB
|
|
238
|
+
b78336099045 3 weeks ago /bin/sh -c #(nop) MAINTAINER Marek Goldmann < 0 B
|
|
239
|
+
4816a298548c 3 weeks ago /bin/sh -c #(nop) CMD ["/bin/bash"] 0 B
|
|
240
|
+
6ee235cf4473 3 weeks ago /bin/sh -c #(nop) LABEL name=CentOS Base Imag 0 B
|
|
241
|
+
474c2ee77fa3 3 weeks ago /bin/sh -c #(nop) ADD file:72852fc7626d233343 196.6 MB
|
|
242
|
+
1544084fad81 6 months ago /bin/sh -c #(nop) MAINTAINER The CentOS Proje 0 B
|