occystrap 0.1.1__py3-none-any.whl → 0.3.0__py3-none-any.whl

occystrap/common.py

@@ -0,0 +1,40 @@
+import json
+
+from occystrap.constants import RUNC_SPEC_TEMPLATE
+
+
+def write_container_config(container_config_filename, runtime_config_filename,
+                           container_template=RUNC_SPEC_TEMPLATE,
+                           container_values=None):
+    if not container_values:
+        container_values = {}
+
+    # Read the container config
+    with open(container_config_filename) as f:
+        image_conf = json.loads(f.read())
+
+    # Write a runc specification for the container
+    container_conf = json.loads(container_template)
+
+    container_conf['process']['terminal'] = True
+    cwd = image_conf['config']['WorkingDir']
+    if cwd == '':
+        cwd = '/'
+    container_conf['process']['cwd'] = cwd
+
+    entrypoint = image_conf['config'].get('Entrypoint', [])
+    if not entrypoint:
+        entrypoint = []
+    cmd = image_conf['config'].get('Cmd', [])
+    if cmd:
+        entrypoint.extend(cmd)
+    container_conf['process']['args'] = entrypoint
+
+    # terminal = false means "pass through existing file descriptors"
+    container_conf['process']['terminal'] = False
+
+    container_conf['hostname'] = container_values.get(
+        'hostname', 'occystrap')
+
+    with open(runtime_config_filename, 'w') as f:
+        f.write(json.dumps(container_conf, indent=4, sort_keys=True))

occystrap/constants.py

@@ -0,0 +1,186 @@
+CONFIG_FILE = 'config_file'
+IMAGE_LAYER = 'image_layer'
+
+RUNC_SPEC_TEMPLATE = """{
+    "ociVersion": "1.0.2-dev",
+    "process": {
+        "terminal": false,
+        "user": {
+            "uid": 0,
+            "gid": 0
+        },
+        "args": [
+            "sh"
+        ],
+        "env": [
+            "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
+            "TERM=xterm"
+        ],
+        "cwd": "/",
+        "capabilities": {
+            "bounding": [
+                "CAP_AUDIT_WRITE",
+                "CAP_KILL",
+                "CAP_NET_BIND_SERVICE"
+            ],
+            "effective": [
+                "CAP_AUDIT_WRITE",
+                "CAP_KILL",
+                "CAP_NET_BIND_SERVICE"
+            ],
+            "inheritable": [
+                "CAP_AUDIT_WRITE",
+                "CAP_KILL",
+                "CAP_NET_BIND_SERVICE"
+            ],
+            "permitted": [
+                "CAP_AUDIT_WRITE",
+                "CAP_KILL",
+                "CAP_NET_BIND_SERVICE"
+            ],
+            "ambient": [
+                "CAP_AUDIT_WRITE",
+                "CAP_KILL",
+                "CAP_NET_BIND_SERVICE"
+            ]
+        },
+        "rlimits": [
+            {
+                "type": "RLIMIT_NOFILE",
+                "hard": 1024,
+                "soft": 1024
+            }
+        ],
+        "noNewPrivileges": true
+    },
+    "root": {
+        "path": "rootfs",
+        "readonly": true
+    },
+    "hostname": "runc",
+    "mounts": [
+        {
+            "destination": "/proc",
+            "type": "proc",
+            "source": "proc"
+        },
+        {
+            "destination": "/dev",
+            "type": "tmpfs",
+            "source": "tmpfs",
+            "options": [
+                "nosuid",
+                "strictatime",
+                "mode=755",
+                "size=65536k"
+            ]
+        },
+        {
+            "destination": "/dev/pts",
+            "type": "devpts",
+            "source": "devpts",
+            "options": [
+                "nosuid",
+                "noexec",
+                "newinstance",
+                "ptmxmode=0666",
+                "mode=0620",
+                "gid=5"
+            ]
+        },
+        {
+            "destination": "/dev/shm",
+            "type": "tmpfs",
+            "source": "shm",
+            "options": [
+                "nosuid",
+                "noexec",
+                "nodev",
+                "mode=1777",
+                "size=65536k"
+            ]
+        },
+        {
+            "destination": "/dev/mqueue",
+            "type": "mqueue",
+            "source": "mqueue",
+            "options": [
+                "nosuid",
+                "noexec",
+                "nodev"
+            ]
+        },
+        {
+            "destination": "/sys",
+            "type": "sysfs",
+            "source": "sysfs",
+            "options": [
+                "nosuid",
+                "noexec",
+                "nodev",
+                "ro"
+            ]
+        },
+        {
+            "destination": "/sys/fs/cgroup",
+            "type": "cgroup",
+            "source": "cgroup",
+            "options": [
+                "nosuid",
+                "noexec",
+                "nodev",
+                "relatime",
+                "ro"
+            ]
+        }
+    ],
+    "linux": {
+        "resources": {
+            "devices": [
+                {
+                    "allow": false,
+                    "access": "rwm"
+                }
+            ]
+        },
+        "namespaces": [
+            {
+                "type": "pid"
+            },
+            {
+                "type": "network"
+            },
+            {
+                "type": "ipc"
+            },
+            {
+                "type": "uts"
+            },
+            {
+                "type": "mount"
+            },
+            {
+                "type": "cgroup"
+            }
+        ],
+        "maskedPaths": [
+            "/proc/acpi",
+            "/proc/asound",
+            "/proc/kcore",
+            "/proc/keys",
+            "/proc/latency_stats",
+            "/proc/timer_list",
+            "/proc/timer_stats",
+            "/proc/sched_debug",
+            "/sys/firmware",
+            "/proc/scsi"
+        ],
+        "readonlyPaths": [
+            "/proc/bus",
+            "/proc/fs",
+            "/proc/irq",
+            "/proc/sys",
+            "/proc/sysrq-trigger"
+        ]
+    }
+}"""

occystrap/docker_registry.py

@@ -1,32 +1,43 @@
 # A simple implementation of a docker registry client. Fetches an image to a tarball.
 # With a big nod to https://github.com/NotGlop/docker-drag/blob/master/docker_pull.py
 
-import gzip
+# https://docs.docker.com/registry/spec/manifest-v2-2/ documents the image manifest
+# format, noting that the response format you get back varies based on what you have
+# in your accept header for the request.
+
 import hashlib
 import io
-import json
 import logging
 import os
 import re
 import sys
-import tarfile
 import tempfile
 import zlib
 
+from occystrap import constants
 from occystrap import util
 
 LOG = logging.getLogger(__name__)
 LOG.setLevel(logging.INFO)
 
+DELETED_FILE_RE = re.compile(r'.*/\.wh\.(.*)$')
+
 
-DELETED_FILE_RE = re.compile('.*/\.wh\.(.*)$')
+def always_fetch():
+    return True
 
 
 class Image(object):
-    def __init__(self, registry, image, tag):
+    def __init__(self, registry, image, tag, os='linux', architecture='amd64', variant='',
+                 secure=True):
         self.registry = registry
         self.image = image
         self.tag = tag
+        self.os = os
+        self.architecture = architecture
+        self.variant = variant
+        self.secure = secure
+
         self._cached_auth = None
 
     def request_url(self, method, url, headers=None, data=None, stream=False):
@@ -53,107 +64,129 @@ class Image(object):
             return util.request_url(
                 method, url, headers=headers, data=data, stream=stream)
 
-    def fetch(self, image_path):
+    def fetch(self, fetch_callback=always_fetch):
         LOG.info('Fetching manifest')
+        moniker = 'https'
+        if not self.secure:
+            moniker = 'http'
+
         r = self.request_url(
             'GET',
-            'https://%(registry)s/v2/%(image)s/manifests/%(tag)s'
+            '%(moniker)s://%(registry)s/v2/%(image)s/manifests/%(tag)s'
             % {
+                'moniker': moniker,
                 'registry': self.registry,
                 'image': self.image,
                 'tag': self.tag
             },
-            headers={'Accept': 'application/vnd.docker.distribution.manifest.v2+json'})
-        manifest = r.json()
-        LOG.info('Manifest says: %s' % manifest)
+            headers={'Accept': ('application/vnd.docker.distribution.manifest.v2+json,'
+                                'application/vnd.docker.distribution.manifest.list.v2+json')})
+
+        config_digest = None
+        if r.headers['Content-Type'] == 'application/vnd.docker.distribution.manifest.v2+json':
+            manifest = r.json()
+            config_digest = manifest['config']['digest']
+        elif r.headers['Content-Type'] == 'application/vnd.docker.distribution.manifest.list.v2+json':
+            for m in r.json()['manifests']:
+                if 'variant' in m['platform']:
+                    LOG.info('Found manifest for %s on %s %s'
+                             % (m['platform']['os'], m['platform']['architecture'],
+                                m['platform']['variant']))
+                else:
+                    LOG.info('Found manifest for %s on %s'
+                             % (m['platform']['os'], m['platform']['architecture']))
+
+                if (m['platform']['os'] == self.os and
+                    m['platform']['architecture'] == self.architecture and
+                        m['platform'].get('variant', '') == self.variant):
+                    LOG.info('Fetching matching manifest')
+                    r = self.request_url(
+                        'GET',
+                        '%(moniker)s://%(registry)s/v2/%(image)s/manifests/%(tag)s'
+                        % {
+                            'moniker': moniker,
+                            'registry': self.registry,
+                            'image': self.image,
+                            'tag': m['digest']
+                        },
+                        headers={'Accept': ('application/vnd.docker.distribution.manifest.v2+json')})
+                    manifest = r.json()
+                    config_digest = manifest['config']['digest']
+
+            if not config_digest:
+                raise Exception('Could not find a matching manifest for this '
+                                'os / architecture / variant')
+        else:
+            raise Exception('Unknown manifest content type %s!' %
+                            r.headers['Content-Type'])
 
         LOG.info('Fetching config file')
         r = self.request_url(
             'GET',
-            'https://%(registry)s/v2/%(image)s/blobs/%(config)s'
+            '%(moniker)s://%(registry)s/v2/%(image)s/blobs/%(config)s'
             % {
+                'moniker': moniker,
                 'registry': self.registry,
                 'image': self.image,
-                'config': manifest['config']['digest']
+                'config': config_digest
             })
         config = r.content
         h = hashlib.sha256()
         h.update(config)
-        if h.hexdigest() != manifest['config']['digest'].split(':')[1]:
+        if h.hexdigest() != config_digest.split(':')[1]:
             LOG.error('Hash verification failed for image config blob (%s vs %s)'
-                      % (manifest['config']['digest'].split(':')[1], h.hexdigest()))
+                      % (config_digest.split(':')[1], h.hexdigest()))
             sys.exit(1)
 
-        tar_manifest = [{
-            'Layers': [],
-            'RepoTags': ['%s:%s' % (self.image.split('/')[-1], self.tag)]
-        }]
-
-        with tarfile.open(image_path, 'w') as image_tar:
-            LOG.info('Writing config file to tarball')
-            config_filename = ('%s.json'
-                               % manifest['config']['digest'].split(':')[1])
-            ti = tarfile.TarInfo(config_filename)
-            ti.size = len(config)
-            image_tar.addfile(ti, io.BytesIO(config))
-            tar_manifest[0]['Config'] = config_filename
-
-            LOG.info('There are %d image layers' % len(manifest['layers']))
-            for layer in manifest['layers']:
-                LOG.info('Fetching layer %s (%d bytes)'
-                         % (layer['digest'], layer['size']))
-                r = self.request_url(
-                    'GET',
-                    'https://%(registry)s/v2/%(image)s/blobs/%(layer)s'
-                    % {
-                        'registry': self.registry,
-                        'image': self.image,
-                        'layer': layer['digest']
-                    },
-                    stream=True)
-
-                LOG.info('Writing layer to tarball')
-                layer_filename = layer['digest'].split(':')[1]
-
-                # We can use zlib for streaming decompression, but we need to tell it
-                # to ignore the gzip header which it doesn't understand. Unfortunately
-                # tarfile doesn't do streaming writes (and we need to know the
-                # decompressed size before we can write to the tarfile), so we stream
-                # to a temporary file on disk.
-                try:
-                    h = hashlib.sha256()
-                    d = zlib.decompressobj(16 + zlib.MAX_WBITS)
-
-                    with tempfile.NamedTemporaryFile(delete=False) as tf:
-                        LOG.info('Temporary file for layer is %s' % tf.name)
-                        for chunk in r.iter_content(8192):
-                            tf.write(d.decompress(chunk))
-                            h.update(chunk)
-
-                    if h.hexdigest() != layer_filename:
-                        LOG.error('Hash verification failed for layer (%s vs %s)'
-                                  % (layer_filename, h.hexdigest()))
-                        sys.exit(1)
-
-                    layer_filename += '/layer.tar'
-                    image_tar.add(
-                        tf.name, arcname=layer_filename)
-                    tar_manifest[0]['Layers'].append(layer_filename)
-
-                    with tarfile.open(tf.name) as layer:
-                        for mem in layer.getmembers():
-                            m = DELETED_FILE_RE.match(mem.name)
-                            if m:
-                                LOG.info('Layer tarball contains deleted file: %s'
-                                         % mem.name)
-
-                finally:
-                    os.unlink(tf.name)
-
-            LOG.info('Writing manifest file to tarball')
-            encoded_manifest = json.dumps(tar_manifest).encode('utf-8')
-            ti = tarfile.TarInfo('manifest.json')
-            ti.size = len(encoded_manifest)
-            image_tar.addfile(ti, io.BytesIO(encoded_manifest))
+        config_filename = ('%s.json' % config_digest.split(':')[1])
+        yield (constants.CONFIG_FILE, config_filename,
+               io.BytesIO(config))
+
+        LOG.info('There are %d image layers' % len(manifest['layers']))
+        for layer in manifest['layers']:
+            layer_filename = layer['digest'].split(':')[1]
+            if not fetch_callback(layer_filename):
+                LOG.info('Fetch callback says skip layer %s' % layer['digest'])
+                yield (constants.IMAGE_LAYER, layer_filename, None)
+                continue
+
+            LOG.info('Fetching layer %s (%d bytes)'
+                     % (layer['digest'], layer['size']))
+            r = self.request_url(
+                'GET',
+                '%(moniker)s://%(registry)s/v2/%(image)s/blobs/%(layer)s'
+                % {
+                    'moniker': moniker,
+                    'registry': self.registry,
+                    'image': self.image,
+                    'layer': layer['digest']
+                },
+                stream=True)
+
+            # We can use zlib for streaming decompression, but we need to tell it
+            # to ignore the gzip header which it doesn't understand. Unfortunately
+            # tarfile doesn't do streaming writes (and we need to know the
+            # decompressed size before we can write to the tarfile), so we stream
+            # to a temporary file on disk.
+            try:
+                h = hashlib.sha256()
+                d = zlib.decompressobj(16 + zlib.MAX_WBITS)
+
+                with tempfile.NamedTemporaryFile(delete=False) as tf:
+                    LOG.info('Temporary file for layer is %s' % tf.name)
+                    for chunk in r.iter_content(8192):
+                        tf.write(d.decompress(chunk))
+                        h.update(chunk)
+
+                if h.hexdigest() != layer_filename:
+                    LOG.error('Hash verification failed for layer (%s vs %s)'
+                              % (layer_filename, h.hexdigest()))
+                    sys.exit(1)
+
+                with open(tf.name, 'rb') as f:
+                    yield (constants.IMAGE_LAYER, layer_filename, f)
+
+            finally:
+                os.unlink(tf.name)
 
         LOG.info('Done')

occystrap/main.py

@@ -1,32 +1,137 @@
 import click
 import logging
+import os
+from shakenfist_utilities import logs
+import sys
 
 from occystrap import docker_registry
+from occystrap import output_directory
+from occystrap import output_mounts
+from occystrap import output_ocibundle
+from occystrap import output_tarfile
 
-logging.basicConfig(level=logging.INFO)
 
-LOG = logging.getLogger(__name__)
-LOG.setLevel(logging.INFO)
+LOG = logs.setup_console(__name__)
 
 
 @click.group()
-@click.option('--verbose/--no-verbose', default=False)
+@click.option('--verbose', is_flag=True)
+@click.option('--os', default='linux')
+@click.option('--architecture', default='amd64')
+@click.option('--variant', default='')
 @click.pass_context
-def cli(ctx, verbose=None):
+def cli(ctx, verbose=None, os=None, architecture=None, variant=None):
     if verbose:
         logging.basicConfig(level=logging.DEBUG)
         LOG.setLevel(logging.DEBUG)
 
+    if not ctx.obj:
+        ctx.obj = {}
+    ctx.obj['OS'] = os
+    ctx.obj['ARCHITECTURE'] = architecture
+    ctx.obj['VARIANT'] = variant
+
+
+def _fetch(registry, image, tag, output, os, architecture, variant, secure=True):
+    img = docker_registry.Image(
+        registry, image, tag, os, architecture, variant, secure=secure)
+    for image_element in img.fetch(fetch_callback=output.fetch_callback):
+        output.process_image_element(*image_element)
+    output.finalize()
+
 
 @click.command()
 @click.argument('registry')
 @click.argument('image')
 @click.argument('tag')
+@click.argument('path')
+@click.option('--use-unique-names', is_flag=True)
+@click.option('--expand', is_flag=True)
+@click.option('--insecure', is_flag=True, default=False)
+@click.pass_context
+def fetch_to_extracted(ctx, registry, image, tag, path, use_unique_names,
+                       expand, insecure):
+    d = output_directory.DirWriter(
+        image, tag, path, unique_names=use_unique_names, expand=expand)
+    _fetch(registry, image, tag, d, ctx.obj['OS'], ctx.obj['ARCHITECTURE'],
+           ctx.obj['VARIANT'], secure=(not insecure))
+
+    if expand:
+        d.write_bundle()
+
+
+cli.add_command(fetch_to_extracted)
+
+
+@click.command()
+@click.argument('registry')
+@click.argument('image')
+@click.argument('tag')
+@click.argument('path')
+@click.option('--insecure', is_flag=True, default=False)
+@click.pass_context
+def fetch_to_oci(ctx, registry, image, tag, path, insecure):
+    d = output_ocibundle.OCIBundleWriter(image, tag, path)
+    _fetch(registry, image, tag, d, ctx.obj['OS'], ctx.obj['ARCHITECTURE'],
+           ctx.obj['VARIANT'], secure=(not insecure))
+    d.write_bundle()
+
+
+cli.add_command(fetch_to_oci)
+
+
+@click.command()
+@click.argument('registry')
+@click.argument('image')
+@click.argument('tag')
+@click.argument('tarfile')
+@click.option('--insecure', is_flag=True, default=False)
+@click.pass_context
+def fetch_to_tarfile(ctx, registry, image, tag, tarfile, insecure):
+    tar = output_tarfile.TarWriter(image, tag, tarfile)
+    _fetch(registry, image, tag, tar, ctx.obj['OS'], ctx.obj['ARCHITECTURE'],
+           ctx.obj['VARIANT'], secure=(not insecure))
+
+
+cli.add_command(fetch_to_tarfile)
+
+
+@click.command()
+@click.argument('registry')
+@click.argument('image')
+@click.argument('tag')
+@click.argument('path')
+@click.option('--insecure', is_flag=True, default=False)
+@click.pass_context
+def fetch_to_mounts(ctx, registry, image, tag, path, insecure):
+    if not hasattr(os, 'setxattr'):
+        print('Sorry, your OS module implementation lacks setxattr')
+        sys.exit(1)
+    if not hasattr(os, 'mknod'):
+        print('Sorry, your OS module implementation lacks mknod')
+        sys.exit(1)
+
+    d = output_mounts.MountWriter(image, tag, path)
+    _fetch(registry, image, tag, d, ctx.obj['OS'], ctx.obj['ARCHITECTURE'],
+           ctx.obj['VARIANT'], secure=(not insecure))
+    d.write_bundle()
+
+
+cli.add_command(fetch_to_mounts)
+
+
+@click.command()
+@click.argument('path')
+@click.argument('image')
+@click.argument('tag')
 @click.argument('tarfile')
 @click.pass_context
-def fetch(ctx, registry, image, tag, tarfile):
-    img = docker_registry.Image(registry, image, tag)
-    img.fetch(tarfile)
+def recreate_image(ctx, path, image, tag, tarfile):
+    d = output_directory.DirReader(path, image, tag)
+    tar = output_tarfile.TarWriter(image, tag, tarfile)
+    for image_element in d.fetch():
+        tar.process_image_element(*image_element)
+    tar.finalize()
 
 
-cli.add_command(fetch)
+cli.add_command(recreate_image)