occystrap 0.3.0__py3-none-any.whl → 0.4.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- occystrap/_version.py +34 -0
- occystrap/filters/__init__.py +10 -0
- occystrap/filters/base.py +67 -0
- occystrap/filters/exclude.py +136 -0
- occystrap/filters/inspect.py +179 -0
- occystrap/filters/normalize_timestamps.py +123 -0
- occystrap/filters/search.py +177 -0
- occystrap/inputs/__init__.py +1 -0
- occystrap/inputs/base.py +40 -0
- occystrap/inputs/docker.py +171 -0
- occystrap/inputs/registry.py +260 -0
- occystrap/inputs/tarfile.py +88 -0
- occystrap/main.py +330 -31
- occystrap/outputs/__init__.py +1 -0
- occystrap/outputs/base.py +46 -0
- occystrap/{output_directory.py → outputs/directory.py} +10 -9
- occystrap/outputs/docker.py +137 -0
- occystrap/{output_mounts.py → outputs/mounts.py} +2 -1
- occystrap/{output_ocibundle.py → outputs/ocibundle.py} +1 -1
- occystrap/outputs/registry.py +240 -0
- occystrap/{output_tarfile.py → outputs/tarfile.py} +18 -2
- occystrap/pipeline.py +297 -0
- occystrap/tarformat.py +122 -0
- occystrap/tests/test_inspect.py +355 -0
- occystrap/tests/test_tarformat.py +199 -0
- occystrap/uri.py +231 -0
- occystrap/util.py +67 -38
- occystrap-0.4.1.dist-info/METADATA +444 -0
- occystrap-0.4.1.dist-info/RECORD +38 -0
- {occystrap-0.3.0.dist-info → occystrap-0.4.1.dist-info}/WHEEL +1 -1
- {occystrap-0.3.0.dist-info → occystrap-0.4.1.dist-info}/entry_points.txt +0 -1
- occystrap/docker_extract.py +0 -36
- occystrap/docker_registry.py +0 -192
- occystrap-0.3.0.dist-info/METADATA +0 -131
- occystrap-0.3.0.dist-info/RECORD +0 -20
- occystrap-0.3.0.dist-info/pbr.json +0 -1
- {occystrap-0.3.0.dist-info → occystrap-0.4.1.dist-info/licenses}/AUTHORS +0 -0
- {occystrap-0.3.0.dist-info → occystrap-0.4.1.dist-info/licenses}/LICENSE +0 -0
- {occystrap-0.3.0.dist-info → occystrap-0.4.1.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
import io
|
|
2
|
+
import json
|
|
3
|
+
import logging
|
|
4
|
+
import os
|
|
5
|
+
import tarfile
|
|
6
|
+
|
|
7
|
+
from occystrap import constants
|
|
8
|
+
from occystrap.inputs.base import ImageInput
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
LOG = logging.getLogger(__name__)
|
|
12
|
+
LOG.setLevel(logging.INFO)
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def always_fetch(digest):
|
|
16
|
+
return True
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class Image(ImageInput):
|
|
20
|
+
def __init__(self, tarfile_path):
|
|
21
|
+
self.tarfile_path = tarfile_path
|
|
22
|
+
self._manifest = None
|
|
23
|
+
self._image = None
|
|
24
|
+
self._tag = None
|
|
25
|
+
self._load_manifest()
|
|
26
|
+
|
|
27
|
+
def _load_manifest(self):
|
|
28
|
+
with tarfile.open(self.tarfile_path, 'r') as tf:
|
|
29
|
+
manifest_member = tf.getmember('manifest.json')
|
|
30
|
+
manifest_file = tf.extractfile(manifest_member)
|
|
31
|
+
self._manifest = json.loads(manifest_file.read().decode('utf-8'))
|
|
32
|
+
|
|
33
|
+
# Parse image and tag from RepoTags
|
|
34
|
+
# Format is typically ["image:tag"] or ["registry/image:tag"]
|
|
35
|
+
repo_tags = self._manifest[0].get('RepoTags', [])
|
|
36
|
+
if repo_tags:
|
|
37
|
+
repo_tag = repo_tags[0]
|
|
38
|
+
if ':' in repo_tag:
|
|
39
|
+
self._image, self._tag = repo_tag.rsplit(':', 1)
|
|
40
|
+
else:
|
|
41
|
+
self._image = repo_tag
|
|
42
|
+
self._tag = 'latest'
|
|
43
|
+
else:
|
|
44
|
+
# Fallback if no RepoTags
|
|
45
|
+
self._image = 'unknown'
|
|
46
|
+
self._tag = 'unknown'
|
|
47
|
+
|
|
48
|
+
@property
|
|
49
|
+
def image(self):
|
|
50
|
+
return self._image
|
|
51
|
+
|
|
52
|
+
@property
|
|
53
|
+
def tag(self):
|
|
54
|
+
return self._tag
|
|
55
|
+
|
|
56
|
+
def fetch(self, fetch_callback=always_fetch):
|
|
57
|
+
LOG.info('Reading image from tarball %s' % self.tarfile_path)
|
|
58
|
+
|
|
59
|
+
with tarfile.open(self.tarfile_path, 'r') as tf:
|
|
60
|
+
# Yield config file
|
|
61
|
+
config_filename = self._manifest[0]['Config']
|
|
62
|
+
LOG.info('Reading config file %s' % config_filename)
|
|
63
|
+
config_member = tf.getmember(config_filename)
|
|
64
|
+
config_file = tf.extractfile(config_member)
|
|
65
|
+
config_data = config_file.read()
|
|
66
|
+
yield (constants.CONFIG_FILE, config_filename,
|
|
67
|
+
io.BytesIO(config_data))
|
|
68
|
+
|
|
69
|
+
# Yield each layer
|
|
70
|
+
layers = self._manifest[0]['Layers']
|
|
71
|
+
LOG.info('There are %d image layers' % len(layers))
|
|
72
|
+
|
|
73
|
+
for layer_path in layers:
|
|
74
|
+
# Layer path is like "abc123/layer.tar"
|
|
75
|
+
layer_digest = os.path.dirname(layer_path)
|
|
76
|
+
if not fetch_callback(layer_digest):
|
|
77
|
+
LOG.info('Fetch callback says skip layer %s' % layer_digest)
|
|
78
|
+
yield (constants.IMAGE_LAYER, layer_digest, None)
|
|
79
|
+
continue
|
|
80
|
+
|
|
81
|
+
LOG.info('Reading layer %s' % layer_path)
|
|
82
|
+
layer_member = tf.getmember(layer_path)
|
|
83
|
+
layer_file = tf.extractfile(layer_member)
|
|
84
|
+
layer_data = layer_file.read()
|
|
85
|
+
yield (constants.IMAGE_LAYER, layer_digest,
|
|
86
|
+
io.BytesIO(layer_data))
|
|
87
|
+
|
|
88
|
+
LOG.info('Done')
|
occystrap/main.py
CHANGED
|
@@ -4,11 +4,16 @@ import os
|
|
|
4
4
|
from shakenfist_utilities import logs
|
|
5
5
|
import sys
|
|
6
6
|
|
|
7
|
-
from occystrap import
|
|
8
|
-
from occystrap import
|
|
9
|
-
from occystrap import
|
|
10
|
-
from occystrap import
|
|
11
|
-
from occystrap import
|
|
7
|
+
from occystrap.inputs import docker as input_docker
|
|
8
|
+
from occystrap.inputs import registry as input_registry
|
|
9
|
+
from occystrap.inputs import tarfile as input_tarfile
|
|
10
|
+
from occystrap.outputs import directory as output_directory
|
|
11
|
+
from occystrap.outputs import mounts as output_mounts
|
|
12
|
+
from occystrap.outputs import ocibundle as output_ocibundle
|
|
13
|
+
from occystrap.outputs import tarfile as output_tarfile
|
|
14
|
+
from occystrap.filters import SearchFilter, TimestampNormalizer
|
|
15
|
+
from occystrap.pipeline import PipelineBuilder, PipelineError
|
|
16
|
+
from occystrap import uri
|
|
12
17
|
|
|
13
18
|
|
|
14
19
|
LOG = logs.setup_console(__name__)
|
|
@@ -19,8 +24,15 @@ LOG = logs.setup_console(__name__)
|
|
|
19
24
|
@click.option('--os', default='linux')
|
|
20
25
|
@click.option('--architecture', default='amd64')
|
|
21
26
|
@click.option('--variant', default='')
|
|
27
|
+
@click.option('--username', default=None, envvar='OCCYSTRAP_USERNAME',
|
|
28
|
+
help='Username for registry authentication')
|
|
29
|
+
@click.option('--password', default=None, envvar='OCCYSTRAP_PASSWORD',
|
|
30
|
+
help='Password for registry authentication')
|
|
31
|
+
@click.option('--insecure', is_flag=True, default=False,
|
|
32
|
+
help='Use HTTP instead of HTTPS for registry connections')
|
|
22
33
|
@click.pass_context
|
|
23
|
-
def cli(ctx, verbose=None, os=None, architecture=None, variant=None
|
|
34
|
+
def cli(ctx, verbose=None, os=None, architecture=None, variant=None,
|
|
35
|
+
username=None, password=None, insecure=None):
|
|
24
36
|
if verbose:
|
|
25
37
|
logging.basicConfig(level=logging.DEBUG)
|
|
26
38
|
LOG.setLevel(logging.DEBUG)
|
|
@@ -30,31 +42,145 @@ def cli(ctx, verbose=None, os=None, architecture=None, variant=None):
|
|
|
30
42
|
ctx.obj['OS'] = os
|
|
31
43
|
ctx.obj['ARCHITECTURE'] = architecture
|
|
32
44
|
ctx.obj['VARIANT'] = variant
|
|
45
|
+
ctx.obj['USERNAME'] = username
|
|
46
|
+
ctx.obj['PASSWORD'] = password
|
|
47
|
+
ctx.obj['INSECURE'] = insecure
|
|
33
48
|
|
|
34
49
|
|
|
35
|
-
def _fetch(
|
|
36
|
-
img = docker_registry.Image(
|
|
37
|
-
registry, image, tag, os, architecture, variant, secure=secure)
|
|
50
|
+
def _fetch(img, output):
|
|
38
51
|
for image_element in img.fetch(fetch_callback=output.fetch_callback):
|
|
39
52
|
output.process_image_element(*image_element)
|
|
40
53
|
output.finalize()
|
|
41
54
|
|
|
42
55
|
|
|
43
|
-
|
|
56
|
+
# =============================================================================
|
|
57
|
+
# New URI-style commands
|
|
58
|
+
# =============================================================================
|
|
59
|
+
|
|
60
|
+
@click.command('process')
|
|
61
|
+
@click.argument('source')
|
|
62
|
+
@click.argument('destination')
|
|
63
|
+
@click.option('--filter', '-f', 'filters', multiple=True,
|
|
64
|
+
help='Apply filter (can be specified multiple times)')
|
|
65
|
+
@click.pass_context
|
|
66
|
+
def process_cmd(ctx, source, destination, filters):
|
|
67
|
+
"""Process container images through a pipeline.
|
|
68
|
+
|
|
69
|
+
SOURCE and DESTINATION are URIs specifying where to read from
|
|
70
|
+
and write to.
|
|
71
|
+
|
|
72
|
+
\b
|
|
73
|
+
Input URI schemes:
|
|
74
|
+
registry://HOST/IMAGE:TAG - Docker/OCI registry
|
|
75
|
+
docker://IMAGE:TAG - Local Docker daemon
|
|
76
|
+
tar://PATH - Docker-save tarball
|
|
77
|
+
|
|
78
|
+
\b
|
|
79
|
+
Output URI schemes:
|
|
80
|
+
tar://PATH - Create tarball
|
|
81
|
+
dir://PATH - Extract to directory
|
|
82
|
+
oci://PATH - Create OCI bundle
|
|
83
|
+
mounts://PATH - Create overlay mounts
|
|
84
|
+
|
|
85
|
+
\b
|
|
86
|
+
Filters (use -f, can chain multiple):
|
|
87
|
+
normalize-timestamps - Normalize layer timestamps
|
|
88
|
+
normalize-timestamps:ts=N - Use specific timestamp
|
|
89
|
+
search:pattern=GLOB - Search for files
|
|
90
|
+
search:pattern=RE,regex=true - Search with regex
|
|
91
|
+
inspect:file=PATH - Append layer metadata (JSONL)
|
|
92
|
+
exclude:pattern=GLOB - Exclude files from layers
|
|
93
|
+
|
|
94
|
+
\b
|
|
95
|
+
Examples:
|
|
96
|
+
occystrap process registry://docker.io/library/busybox:latest tar://busybox.tar
|
|
97
|
+
occystrap process docker://myimage:v1 dir://./extracted -f normalize-timestamps
|
|
98
|
+
occystrap process tar://image.tar dir://out -f "search:pattern=*.conf"
|
|
99
|
+
occystrap process docker://img:v1 registry://host/img:v1 -f "inspect:file=layers.jsonl"
|
|
100
|
+
"""
|
|
101
|
+
try:
|
|
102
|
+
builder = PipelineBuilder(ctx)
|
|
103
|
+
input_source, output = builder.build_pipeline(
|
|
104
|
+
source, destination, list(filters))
|
|
105
|
+
_fetch(input_source, output)
|
|
106
|
+
|
|
107
|
+
# Handle post-processing for certain outputs
|
|
108
|
+
if hasattr(output, 'write_bundle'):
|
|
109
|
+
# Check if this is an OCI or mounts output that needs write_bundle
|
|
110
|
+
dest_spec = uri.parse_uri(destination)
|
|
111
|
+
if dest_spec.scheme in ('oci', 'mounts'):
|
|
112
|
+
output.write_bundle()
|
|
113
|
+
elif dest_spec.scheme == 'dir' and dest_spec.options.get('expand'):
|
|
114
|
+
output.write_bundle()
|
|
115
|
+
|
|
116
|
+
except (PipelineError, uri.URIParseError) as e:
|
|
117
|
+
click.echo('Error: %s' % e, err=True)
|
|
118
|
+
sys.exit(1)
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
cli.add_command(process_cmd)
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
@click.command('search')
|
|
125
|
+
@click.argument('source')
|
|
126
|
+
@click.argument('pattern')
|
|
127
|
+
@click.option('--regex', is_flag=True, default=False,
|
|
128
|
+
help='Use regex pattern instead of glob pattern')
|
|
129
|
+
@click.option('--script-friendly', is_flag=True, default=False,
|
|
130
|
+
help='Output in script-friendly format: image:tag:layer:path')
|
|
131
|
+
@click.pass_context
|
|
132
|
+
def search_cmd(ctx, source, pattern, regex, script_friendly):
|
|
133
|
+
"""Search for files in container image layers.
|
|
134
|
+
|
|
135
|
+
SOURCE is a URI specifying where to read the image from:
|
|
136
|
+
registry://HOST/IMAGE:TAG - Docker/OCI registry
|
|
137
|
+
docker://IMAGE:TAG - Local Docker daemon
|
|
138
|
+
tar://PATH - Docker-save tarball
|
|
139
|
+
|
|
140
|
+
PATTERN is a glob pattern (or regex with --regex).
|
|
141
|
+
|
|
142
|
+
\b
|
|
143
|
+
Examples:
|
|
144
|
+
occystrap search registry://docker.io/library/busybox:latest "bin/*sh"
|
|
145
|
+
occystrap search docker://myimage:v1 "*.conf"
|
|
146
|
+
occystrap search --regex tar://image.tar ".*\\.py$"
|
|
147
|
+
"""
|
|
148
|
+
try:
|
|
149
|
+
builder = PipelineBuilder(ctx)
|
|
150
|
+
input_source, searcher = builder.build_search_pipeline(
|
|
151
|
+
source, pattern, use_regex=regex, script_friendly=script_friendly)
|
|
152
|
+
_fetch(input_source, searcher)
|
|
153
|
+
|
|
154
|
+
except (PipelineError, uri.URIParseError) as e:
|
|
155
|
+
click.echo('Error: %s' % e, err=True)
|
|
156
|
+
sys.exit(1)
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
cli.add_command(search_cmd)
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
# =============================================================================
|
|
163
|
+
# Legacy commands (kept for backwards compatibility)
|
|
164
|
+
# =============================================================================
|
|
165
|
+
|
|
166
|
+
@click.command(deprecated=True)
|
|
44
167
|
@click.argument('registry')
|
|
45
168
|
@click.argument('image')
|
|
46
169
|
@click.argument('tag')
|
|
47
170
|
@click.argument('path')
|
|
48
171
|
@click.option('--use-unique-names', is_flag=True)
|
|
49
172
|
@click.option('--expand', is_flag=True)
|
|
50
|
-
@click.option('--insecure', is_flag=True, default=False)
|
|
51
173
|
@click.pass_context
|
|
52
174
|
def fetch_to_extracted(ctx, registry, image, tag, path, use_unique_names,
|
|
53
|
-
expand
|
|
175
|
+
expand):
|
|
176
|
+
"""[DEPRECATED] Use: occystrap process registry://... dir://..."""
|
|
54
177
|
d = output_directory.DirWriter(
|
|
55
178
|
image, tag, path, unique_names=use_unique_names, expand=expand)
|
|
56
|
-
|
|
57
|
-
|
|
179
|
+
img = input_registry.Image(
|
|
180
|
+
registry, image, tag, ctx.obj['OS'], ctx.obj['ARCHITECTURE'],
|
|
181
|
+
ctx.obj['VARIANT'], secure=(not ctx.obj['INSECURE']),
|
|
182
|
+
username=ctx.obj['USERNAME'], password=ctx.obj['PASSWORD'])
|
|
183
|
+
_fetch(img, d)
|
|
58
184
|
|
|
59
185
|
if expand:
|
|
60
186
|
d.write_bundle()
|
|
@@ -63,47 +189,58 @@ def fetch_to_extracted(ctx, registry, image, tag, path, use_unique_names,
|
|
|
63
189
|
cli.add_command(fetch_to_extracted)
|
|
64
190
|
|
|
65
191
|
|
|
66
|
-
@click.command()
|
|
192
|
+
@click.command(deprecated=True)
|
|
67
193
|
@click.argument('registry')
|
|
68
194
|
@click.argument('image')
|
|
69
195
|
@click.argument('tag')
|
|
70
196
|
@click.argument('path')
|
|
71
|
-
@click.option('--insecure', is_flag=True, default=False)
|
|
72
197
|
@click.pass_context
|
|
73
|
-
def fetch_to_oci(ctx, registry, image, tag, path
|
|
198
|
+
def fetch_to_oci(ctx, registry, image, tag, path):
|
|
199
|
+
"""[DEPRECATED] Use: occystrap process registry://... oci://..."""
|
|
74
200
|
d = output_ocibundle.OCIBundleWriter(image, tag, path)
|
|
75
|
-
|
|
76
|
-
|
|
201
|
+
img = input_registry.Image(
|
|
202
|
+
registry, image, tag, ctx.obj['OS'], ctx.obj['ARCHITECTURE'],
|
|
203
|
+
ctx.obj['VARIANT'], secure=(not ctx.obj['INSECURE']),
|
|
204
|
+
username=ctx.obj['USERNAME'], password=ctx.obj['PASSWORD'])
|
|
205
|
+
_fetch(img, d)
|
|
77
206
|
d.write_bundle()
|
|
78
207
|
|
|
79
208
|
|
|
80
209
|
cli.add_command(fetch_to_oci)
|
|
81
210
|
|
|
82
211
|
|
|
83
|
-
@click.command()
|
|
212
|
+
@click.command(deprecated=True)
|
|
84
213
|
@click.argument('registry')
|
|
85
214
|
@click.argument('image')
|
|
86
215
|
@click.argument('tag')
|
|
87
216
|
@click.argument('tarfile')
|
|
88
|
-
@click.option('--
|
|
217
|
+
@click.option('--normalize-timestamps', is_flag=True, default=False)
|
|
218
|
+
@click.option('--timestamp', default=0, type=int)
|
|
89
219
|
@click.pass_context
|
|
90
|
-
def fetch_to_tarfile(ctx, registry, image, tag, tarfile,
|
|
220
|
+
def fetch_to_tarfile(ctx, registry, image, tag, tarfile,
|
|
221
|
+
normalize_timestamps, timestamp):
|
|
222
|
+
"""[DEPRECATED] Use: occystrap process registry://... tar://..."""
|
|
91
223
|
tar = output_tarfile.TarWriter(image, tag, tarfile)
|
|
92
|
-
|
|
93
|
-
|
|
224
|
+
if normalize_timestamps:
|
|
225
|
+
tar = TimestampNormalizer(tar, timestamp=timestamp)
|
|
226
|
+
img = input_registry.Image(
|
|
227
|
+
registry, image, tag, ctx.obj['OS'], ctx.obj['ARCHITECTURE'],
|
|
228
|
+
ctx.obj['VARIANT'], secure=(not ctx.obj['INSECURE']),
|
|
229
|
+
username=ctx.obj['USERNAME'], password=ctx.obj['PASSWORD'])
|
|
230
|
+
_fetch(img, tar)
|
|
94
231
|
|
|
95
232
|
|
|
96
233
|
cli.add_command(fetch_to_tarfile)
|
|
97
234
|
|
|
98
235
|
|
|
99
|
-
@click.command()
|
|
236
|
+
@click.command(deprecated=True)
|
|
100
237
|
@click.argument('registry')
|
|
101
238
|
@click.argument('image')
|
|
102
239
|
@click.argument('tag')
|
|
103
240
|
@click.argument('path')
|
|
104
|
-
@click.option('--insecure', is_flag=True, default=False)
|
|
105
241
|
@click.pass_context
|
|
106
|
-
def fetch_to_mounts(ctx, registry, image, tag, path
|
|
242
|
+
def fetch_to_mounts(ctx, registry, image, tag, path):
|
|
243
|
+
"""[DEPRECATED] Use: occystrap process registry://... mounts://..."""
|
|
107
244
|
if not hasattr(os, 'setxattr'):
|
|
108
245
|
print('Sorry, your OS module implementation lacks setxattr')
|
|
109
246
|
sys.exit(1)
|
|
@@ -112,26 +249,188 @@ def fetch_to_mounts(ctx, registry, image, tag, path, insecure):
|
|
|
112
249
|
sys.exit(1)
|
|
113
250
|
|
|
114
251
|
d = output_mounts.MountWriter(image, tag, path)
|
|
115
|
-
|
|
116
|
-
|
|
252
|
+
img = input_registry.Image(
|
|
253
|
+
registry, image, tag, ctx.obj['OS'], ctx.obj['ARCHITECTURE'],
|
|
254
|
+
ctx.obj['VARIANT'], secure=(not ctx.obj['INSECURE']),
|
|
255
|
+
username=ctx.obj['USERNAME'], password=ctx.obj['PASSWORD'])
|
|
256
|
+
_fetch(img, d)
|
|
117
257
|
d.write_bundle()
|
|
118
258
|
|
|
119
259
|
|
|
120
260
|
cli.add_command(fetch_to_mounts)
|
|
121
261
|
|
|
122
262
|
|
|
123
|
-
@click.command()
|
|
263
|
+
@click.command(deprecated=True)
|
|
124
264
|
@click.argument('path')
|
|
125
265
|
@click.argument('image')
|
|
126
266
|
@click.argument('tag')
|
|
127
267
|
@click.argument('tarfile')
|
|
268
|
+
@click.option('--normalize-timestamps', is_flag=True, default=False)
|
|
269
|
+
@click.option('--timestamp', default=0, type=int)
|
|
128
270
|
@click.pass_context
|
|
129
|
-
def recreate_image(ctx, path, image, tag, tarfile
|
|
271
|
+
def recreate_image(ctx, path, image, tag, tarfile, normalize_timestamps,
|
|
272
|
+
timestamp):
|
|
273
|
+
"""[DEPRECATED] Recreate image from shared directory."""
|
|
130
274
|
d = output_directory.DirReader(path, image, tag)
|
|
131
275
|
tar = output_tarfile.TarWriter(image, tag, tarfile)
|
|
276
|
+
if normalize_timestamps:
|
|
277
|
+
tar = TimestampNormalizer(tar, timestamp=timestamp)
|
|
132
278
|
for image_element in d.fetch():
|
|
133
279
|
tar.process_image_element(*image_element)
|
|
134
280
|
tar.finalize()
|
|
135
281
|
|
|
136
282
|
|
|
137
283
|
cli.add_command(recreate_image)
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
@click.command(deprecated=True)
|
|
287
|
+
@click.argument('tarfile')
|
|
288
|
+
@click.argument('path')
|
|
289
|
+
@click.option('--use-unique-names', is_flag=True)
|
|
290
|
+
@click.option('--expand', is_flag=True)
|
|
291
|
+
@click.pass_context
|
|
292
|
+
def tarfile_to_extracted(ctx, tarfile, path, use_unique_names, expand):
|
|
293
|
+
"""[DEPRECATED] Use: occystrap process tar://... dir://..."""
|
|
294
|
+
img = input_tarfile.Image(tarfile)
|
|
295
|
+
d = output_directory.DirWriter(
|
|
296
|
+
img.image, img.tag, path, unique_names=use_unique_names, expand=expand)
|
|
297
|
+
_fetch(img, d)
|
|
298
|
+
|
|
299
|
+
if expand:
|
|
300
|
+
d.write_bundle()
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
cli.add_command(tarfile_to_extracted)
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
@click.command(deprecated=True)
|
|
307
|
+
@click.argument('image')
|
|
308
|
+
@click.argument('tag')
|
|
309
|
+
@click.argument('tarfile')
|
|
310
|
+
@click.option('--socket', default='/var/run/docker.sock',
|
|
311
|
+
help='Path to Docker socket')
|
|
312
|
+
@click.option('--normalize-timestamps', is_flag=True, default=False)
|
|
313
|
+
@click.option('--timestamp', default=0, type=int)
|
|
314
|
+
@click.pass_context
|
|
315
|
+
def docker_to_tarfile(ctx, image, tag, tarfile, socket, normalize_timestamps,
|
|
316
|
+
timestamp):
|
|
317
|
+
"""[DEPRECATED] Use: occystrap process docker://... tar://..."""
|
|
318
|
+
tar = output_tarfile.TarWriter(image, tag, tarfile)
|
|
319
|
+
if normalize_timestamps:
|
|
320
|
+
tar = TimestampNormalizer(tar, timestamp=timestamp)
|
|
321
|
+
img = input_docker.Image(image, tag, socket_path=socket)
|
|
322
|
+
_fetch(img, tar)
|
|
323
|
+
|
|
324
|
+
|
|
325
|
+
cli.add_command(docker_to_tarfile)
|
|
326
|
+
|
|
327
|
+
|
|
328
|
+
@click.command(deprecated=True)
|
|
329
|
+
@click.argument('image')
|
|
330
|
+
@click.argument('tag')
|
|
331
|
+
@click.argument('path')
|
|
332
|
+
@click.option('--socket', default='/var/run/docker.sock',
|
|
333
|
+
help='Path to Docker socket')
|
|
334
|
+
@click.option('--use-unique-names', is_flag=True)
|
|
335
|
+
@click.option('--expand', is_flag=True)
|
|
336
|
+
@click.pass_context
|
|
337
|
+
def docker_to_extracted(ctx, image, tag, path, socket, use_unique_names,
|
|
338
|
+
expand):
|
|
339
|
+
"""[DEPRECATED] Use: occystrap process docker://... dir://..."""
|
|
340
|
+
d = output_directory.DirWriter(
|
|
341
|
+
image, tag, path, unique_names=use_unique_names, expand=expand)
|
|
342
|
+
img = input_docker.Image(image, tag, socket_path=socket)
|
|
343
|
+
_fetch(img, d)
|
|
344
|
+
|
|
345
|
+
if expand:
|
|
346
|
+
d.write_bundle()
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
cli.add_command(docker_to_extracted)
|
|
350
|
+
|
|
351
|
+
|
|
352
|
+
@click.command(deprecated=True)
|
|
353
|
+
@click.argument('image')
|
|
354
|
+
@click.argument('tag')
|
|
355
|
+
@click.argument('path')
|
|
356
|
+
@click.option('--socket', default='/var/run/docker.sock',
|
|
357
|
+
help='Path to Docker socket')
|
|
358
|
+
@click.pass_context
|
|
359
|
+
def docker_to_oci(ctx, image, tag, path, socket):
|
|
360
|
+
"""[DEPRECATED] Use: occystrap process docker://... oci://..."""
|
|
361
|
+
d = output_ocibundle.OCIBundleWriter(image, tag, path)
|
|
362
|
+
img = input_docker.Image(image, tag, socket_path=socket)
|
|
363
|
+
_fetch(img, d)
|
|
364
|
+
d.write_bundle()
|
|
365
|
+
|
|
366
|
+
|
|
367
|
+
cli.add_command(docker_to_oci)
|
|
368
|
+
|
|
369
|
+
|
|
370
|
+
@click.command(deprecated=True)
|
|
371
|
+
@click.argument('image')
|
|
372
|
+
@click.argument('tag')
|
|
373
|
+
@click.argument('pattern')
|
|
374
|
+
@click.option('--socket', default='/var/run/docker.sock',
|
|
375
|
+
help='Path to Docker socket')
|
|
376
|
+
@click.option('--regex', is_flag=True, default=False,
|
|
377
|
+
help='Use regex pattern instead of glob pattern')
|
|
378
|
+
@click.option('--script-friendly', is_flag=True, default=False,
|
|
379
|
+
help='Output in script-friendly format: image:tag:layer:path')
|
|
380
|
+
@click.pass_context
|
|
381
|
+
def search_layers_docker(ctx, image, tag, pattern, socket, regex,
|
|
382
|
+
script_friendly):
|
|
383
|
+
"""[DEPRECATED] Use: occystrap search docker://IMAGE:TAG PATTERN"""
|
|
384
|
+
searcher = SearchFilter(
|
|
385
|
+
None, pattern, use_regex=regex, image=image, tag=tag,
|
|
386
|
+
script_friendly=script_friendly)
|
|
387
|
+
img = input_docker.Image(image, tag, socket_path=socket)
|
|
388
|
+
_fetch(img, searcher)
|
|
389
|
+
|
|
390
|
+
|
|
391
|
+
cli.add_command(search_layers_docker)
|
|
392
|
+
|
|
393
|
+
|
|
394
|
+
@click.command(deprecated=True)
|
|
395
|
+
@click.argument('registry')
|
|
396
|
+
@click.argument('image')
|
|
397
|
+
@click.argument('tag')
|
|
398
|
+
@click.argument('pattern')
|
|
399
|
+
@click.option('--regex', is_flag=True, default=False,
|
|
400
|
+
help='Use regex pattern instead of glob pattern')
|
|
401
|
+
@click.option('--script-friendly', is_flag=True, default=False,
|
|
402
|
+
help='Output in script-friendly format: image:tag:layer:path')
|
|
403
|
+
@click.pass_context
|
|
404
|
+
def search_layers(ctx, registry, image, tag, pattern, regex, script_friendly):
|
|
405
|
+
"""[DEPRECATED] Use: occystrap search registry://HOST/IMAGE:TAG PATTERN"""
|
|
406
|
+
searcher = SearchFilter(
|
|
407
|
+
None, pattern, use_regex=regex, image=image, tag=tag,
|
|
408
|
+
script_friendly=script_friendly)
|
|
409
|
+
img = input_registry.Image(
|
|
410
|
+
registry, image, tag, ctx.obj['OS'], ctx.obj['ARCHITECTURE'],
|
|
411
|
+
ctx.obj['VARIANT'], secure=(not ctx.obj['INSECURE']),
|
|
412
|
+
username=ctx.obj['USERNAME'], password=ctx.obj['PASSWORD'])
|
|
413
|
+
_fetch(img, searcher)
|
|
414
|
+
|
|
415
|
+
|
|
416
|
+
cli.add_command(search_layers)
|
|
417
|
+
|
|
418
|
+
|
|
419
|
+
@click.command(deprecated=True)
|
|
420
|
+
@click.argument('tarfile')
|
|
421
|
+
@click.argument('pattern')
|
|
422
|
+
@click.option('--regex', is_flag=True, default=False,
|
|
423
|
+
help='Use regex pattern instead of glob pattern')
|
|
424
|
+
@click.option('--script-friendly', is_flag=True, default=False,
|
|
425
|
+
help='Output in script-friendly format: image:tag:layer:path')
|
|
426
|
+
@click.pass_context
|
|
427
|
+
def search_layers_tarfile(ctx, tarfile, pattern, regex, script_friendly):
|
|
428
|
+
"""[DEPRECATED] Use: occystrap search tar://PATH PATTERN"""
|
|
429
|
+
img = input_tarfile.Image(tarfile)
|
|
430
|
+
searcher = SearchFilter(
|
|
431
|
+
None, pattern, use_regex=regex, image=img.image, tag=img.tag,
|
|
432
|
+
script_friendly=script_friendly)
|
|
433
|
+
_fetch(img, searcher)
|
|
434
|
+
|
|
435
|
+
|
|
436
|
+
cli.add_command(search_layers_tarfile)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
# Output writers for container images
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
from abc import ABC, abstractmethod
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class ImageOutput(ABC):
|
|
5
|
+
"""Abstract base class for image output writers.
|
|
6
|
+
|
|
7
|
+
Output writers receive image elements (config files and layers) from input
|
|
8
|
+
sources and write them to various destinations (tarballs, directories,
|
|
9
|
+
OCI bundles, etc.).
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
@abstractmethod
|
|
13
|
+
def fetch_callback(self, digest):
|
|
14
|
+
"""Determine whether a layer should be fetched.
|
|
15
|
+
|
|
16
|
+
This is called by input sources before fetching each layer, allowing
|
|
17
|
+
output writers to skip layers that already exist in the destination.
|
|
18
|
+
|
|
19
|
+
Args:
|
|
20
|
+
digest: The layer digest/identifier.
|
|
21
|
+
|
|
22
|
+
Returns:
|
|
23
|
+
True if the layer should be fetched, False to skip.
|
|
24
|
+
"""
|
|
25
|
+
pass
|
|
26
|
+
|
|
27
|
+
@abstractmethod
|
|
28
|
+
def process_image_element(self, element_type, name, data):
|
|
29
|
+
"""Process a single image element (config or layer).
|
|
30
|
+
|
|
31
|
+
Args:
|
|
32
|
+
element_type: constants.CONFIG_FILE or constants.IMAGE_LAYER
|
|
33
|
+
name: The element identifier (config filename or layer digest)
|
|
34
|
+
data: A file-like object containing the element data,
|
|
35
|
+
or None if the layer was skipped by fetch_callback
|
|
36
|
+
"""
|
|
37
|
+
pass
|
|
38
|
+
|
|
39
|
+
@abstractmethod
|
|
40
|
+
def finalize(self):
|
|
41
|
+
"""Complete the output operation.
|
|
42
|
+
|
|
43
|
+
This is called after all image elements have been processed. Use this
|
|
44
|
+
to write manifests, close files, or perform any final cleanup.
|
|
45
|
+
"""
|
|
46
|
+
pass
|
|
@@ -4,6 +4,7 @@ import os
|
|
|
4
4
|
import tarfile
|
|
5
5
|
|
|
6
6
|
from occystrap import constants
|
|
7
|
+
from occystrap.outputs.base import ImageOutput
|
|
7
8
|
|
|
8
9
|
|
|
9
10
|
LOG = logging.getLogger(__name__)
|
|
@@ -94,7 +95,7 @@ TARFILE_TYPE_MAP = {
|
|
|
94
95
|
}
|
|
95
96
|
|
|
96
97
|
|
|
97
|
-
class DirWriter(
|
|
98
|
+
class DirWriter(ImageOutput):
|
|
98
99
|
def __init__(self, image, tag, image_path, unique_names=False, expand=False):
|
|
99
100
|
self.image = image
|
|
100
101
|
self.tag = tag
|
|
@@ -110,9 +111,7 @@ class DirWriter(object):
|
|
|
110
111
|
self.tar_manifest[0]['ImageName'] = self.image
|
|
111
112
|
|
|
112
113
|
self.bundle = {}
|
|
113
|
-
|
|
114
|
-
if not os.path.exists(self.image_path):
|
|
115
|
-
os.makedirs(self.image_path)
|
|
114
|
+
os.makedirs(self.image_path, exist_ok=True)
|
|
116
115
|
|
|
117
116
|
def _manifest_filename(self):
|
|
118
117
|
if not self.unique_names:
|
|
@@ -136,15 +135,18 @@ class DirWriter(object):
|
|
|
136
135
|
|
|
137
136
|
def process_image_element(self, element_type, name, data):
|
|
138
137
|
if element_type == constants.CONFIG_FILE:
|
|
139
|
-
|
|
138
|
+
config_file = os.path.join(self.image_path, name)
|
|
139
|
+
config_dir = os.path.dirname(config_file)
|
|
140
|
+
os.makedirs(config_dir, exist_ok=True)
|
|
141
|
+
|
|
142
|
+
with open(config_file, 'wb') as f:
|
|
140
143
|
d = json.loads(data.read())
|
|
141
144
|
f.write(json.dumps(d, indent=4, sort_keys=True).encode('ascii'))
|
|
142
145
|
self.tar_manifest[0]['Config'] = name
|
|
143
146
|
|
|
144
147
|
elif element_type == constants.IMAGE_LAYER:
|
|
145
148
|
layer_dir = os.path.join(self.image_path, name)
|
|
146
|
-
|
|
147
|
-
os.makedirs(layer_dir)
|
|
149
|
+
os.makedirs(layer_dir, exist_ok=True)
|
|
148
150
|
|
|
149
151
|
layer_file = os.path.join(name, 'layer.tar')
|
|
150
152
|
self.tar_manifest[0]['Layers'].append(layer_file)
|
|
@@ -276,8 +278,7 @@ class DirWriter(object):
|
|
|
276
278
|
def write_bundle(self):
|
|
277
279
|
manifest_filename = self._manifest_filename()
|
|
278
280
|
manifest_path = os.path.join(self.image_path, manifest_filename)
|
|
279
|
-
|
|
280
|
-
os.makedirs(manifest_path)
|
|
281
|
+
os.makedirs(manifest_path, exist_ok=True)
|
|
281
282
|
LOG.info('Writing image bundle to %s' % manifest_path)
|
|
282
283
|
self._extract_rootfs(manifest_path)
|
|
283
284
|
|