occystrap 0.2.0__py3-none-any.whl → 0.3.0__py3-none-any.whl

occystrap/common.py

@@ -0,0 +1,40 @@
+import json
+
+from occystrap.constants import RUNC_SPEC_TEMPLATE
+
+
+def write_container_config(container_config_filename, runtime_config_filename,
+                           container_template=RUNC_SPEC_TEMPLATE,
+                           container_values=None):
+    if not container_values:
+        container_values = {}
+
+    # Read the container config
+    with open(container_config_filename) as f:
+        image_conf = json.loads(f.read())
+
+    # Write a runc specification for the container
+    container_conf = json.loads(container_template)
+
+    container_conf['process']['terminal'] = True
+    cwd = image_conf['config']['WorkingDir']
+    if cwd == '':
+        cwd = '/'
+    container_conf['process']['cwd'] = cwd
+
+    entrypoint = image_conf['config'].get('Entrypoint', [])
+    if not entrypoint:
+        entrypoint = []
+    cmd = image_conf['config'].get('Cmd', [])
+    if cmd:
+        entrypoint.extend(cmd)
+    container_conf['process']['args'] = entrypoint
+
+    # terminal = false means "pass through existing file descriptors"
+    container_conf['process']['terminal'] = False
+
+    container_conf['hostname'] = container_values.get(
+        'hostname', 'occystrap')
+
+    with open(runtime_config_filename, 'w') as f:
+        f.write(json.dumps(container_conf, indent=4, sort_keys=True))

occystrap/constants.py

@@ -1,2 +1,186 @@
 CONFIG_FILE = 'config_file'
 IMAGE_LAYER = 'image_layer'
+
+RUNC_SPEC_TEMPLATE = """{
+    "ociVersion": "1.0.2-dev",
+    "process": {
+        "terminal": false,
+        "user": {
+            "uid": 0,
+            "gid": 0
+        },
+        "args": [
+            "sh"
+        ],
+        "env": [
+            "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
+            "TERM=xterm"
+        ],
+        "cwd": "/",
+        "capabilities": {
+            "bounding": [
+                "CAP_AUDIT_WRITE",
+                "CAP_KILL",
+                "CAP_NET_BIND_SERVICE"
+            ],
+            "effective": [
+                "CAP_AUDIT_WRITE",
+                "CAP_KILL",
+                "CAP_NET_BIND_SERVICE"
+            ],
+            "inheritable": [
+                "CAP_AUDIT_WRITE",
+                "CAP_KILL",
+                "CAP_NET_BIND_SERVICE"
+            ],
+            "permitted": [
+                "CAP_AUDIT_WRITE",
+                "CAP_KILL",
+                "CAP_NET_BIND_SERVICE"
+            ],
+            "ambient": [
+                "CAP_AUDIT_WRITE",
+                "CAP_KILL",
+                "CAP_NET_BIND_SERVICE"
+            ]
+        },
+        "rlimits": [
+            {
+                "type": "RLIMIT_NOFILE",
+                "hard": 1024,
+                "soft": 1024
+            }
+        ],
+        "noNewPrivileges": true
+    },
+    "root": {
+        "path": "rootfs",
+        "readonly": true
+    },
+    "hostname": "runc",
+    "mounts": [
+        {
+            "destination": "/proc",
+            "type": "proc",
+            "source": "proc"
+        },
+        {
+            "destination": "/dev",
+            "type": "tmpfs",
+            "source": "tmpfs",
+            "options": [
+                "nosuid",
+                "strictatime",
+                "mode=755",
+                "size=65536k"
+            ]
+        },
+        {
+            "destination": "/dev/pts",
+            "type": "devpts",
+            "source": "devpts",
+            "options": [
+                "nosuid",
+                "noexec",
+                "newinstance",
+                "ptmxmode=0666",
+                "mode=0620",
+                "gid=5"
+            ]
+        },
+        {
+            "destination": "/dev/shm",
+            "type": "tmpfs",
+            "source": "shm",
+            "options": [
+                "nosuid",
+                "noexec",
+                "nodev",
+                "mode=1777",
+                "size=65536k"
+            ]
+        },
+        {
+            "destination": "/dev/mqueue",
+            "type": "mqueue",
+            "source": "mqueue",
+            "options": [
+                "nosuid",
+                "noexec",
+                "nodev"
+            ]
+        },
+        {
+            "destination": "/sys",
+            "type": "sysfs",
+            "source": "sysfs",
+            "options": [
+                "nosuid",
+                "noexec",
+                "nodev",
+                "ro"
+            ]
+        },
+        {
+            "destination": "/sys/fs/cgroup",
+            "type": "cgroup",
+            "source": "cgroup",
+            "options": [
+                "nosuid",
+                "noexec",
+                "nodev",
+                "relatime",
+                "ro"
+            ]
+        }
+    ],
+    "linux": {
+        "resources": {
+            "devices": [
+                {
+                    "allow": false,
+                    "access": "rwm"
+                }
+            ]
+        },
+        "namespaces": [
+            {
+                "type": "pid"
+            },
+            {
+                "type": "network"
+            },
+            {
+                "type": "ipc"
+            },
+            {
+                "type": "uts"
+            },
+            {
+                "type": "mount"
+            },
+            {
+                "type": "cgroup"
+            }
+        ],
+        "maskedPaths": [
+            "/proc/acpi",
+            "/proc/asound",
+            "/proc/kcore",
+            "/proc/keys",
+            "/proc/latency_stats",
+            "/proc/timer_list",
+            "/proc/timer_stats",
+            "/proc/sched_debug",
+            "/sys/firmware",
+            "/proc/scsi"
+        ],
+        "readonlyPaths": [
+            "/proc/bus",
+            "/proc/fs",
+            "/proc/irq",
+            "/proc/sys",
+            "/proc/sysrq-trigger"
+        ]
+    }
+}"""

occystrap/docker_registry.py

@@ -1,14 +1,16 @@
 # A simple implementation of a docker registry client. Fetches an image to a tarball.
 # With a big nod to https://github.com/NotGlop/docker-drag/blob/master/docker_pull.py
 
+# https://docs.docker.com/registry/spec/manifest-v2-2/ documents the image manifest
+# format, noting that the response format you get back varies based on what you have
+# in your accept header for the request.
+
 import hashlib
 import io
-import json
 import logging
 import os
 import re
 import sys
-import tarfile
 import tempfile
 import zlib
 
@@ -18,7 +20,7 @@ from occystrap import util
 LOG = logging.getLogger(__name__)
 LOG.setLevel(logging.INFO)
 
-DELETED_FILE_RE = re.compile('.*/\.wh\.(.*)$')
+DELETED_FILE_RE = re.compile(r'.*/\.wh\.(.*)$')
 
 
 def always_fetch():
@@ -26,10 +28,16 @@ def always_fetch():
 
 
 class Image(object):
-    def __init__(self, registry, image, tag):
+    def __init__(self, registry, image, tag, os='linux', architecture='amd64', variant='',
+                 secure=True):
         self.registry = registry
         self.image = image
         self.tag = tag
+        self.os = os
+        self.architecture = architecture
+        self.variant = variant
+        self.secure = secure
+
         self._cached_auth = None
 
     def request_url(self, method, url, headers=None, data=None, stream=False):
@@ -58,36 +66,79 @@ class Image(object):
 
     def fetch(self, fetch_callback=always_fetch):
         LOG.info('Fetching manifest')
+        moniker = 'https'
+        if not self.secure:
+            moniker = 'http'
+
         r = self.request_url(
             'GET',
-            'https://%(registry)s/v2/%(image)s/manifests/%(tag)s'
+            '%(moniker)s://%(registry)s/v2/%(image)s/manifests/%(tag)s'
             % {
+                'moniker': moniker,
                 'registry': self.registry,
                 'image': self.image,
                 'tag': self.tag
             },
-            headers={'Accept': 'application/vnd.docker.distribution.manifest.v2+json'})
-        manifest = r.json()
+            headers={'Accept': ('application/vnd.docker.distribution.manifest.v2+json,'
+                                'application/vnd.docker.distribution.manifest.list.v2+json')})
+
+        config_digest = None
+        if r.headers['Content-Type'] == 'application/vnd.docker.distribution.manifest.v2+json':
+            manifest = r.json()
+            config_digest = manifest['config']['digest']
+        elif r.headers['Content-Type'] == 'application/vnd.docker.distribution.manifest.list.v2+json':
+            for m in r.json()['manifests']:
+                if 'variant' in m['platform']:
+                    LOG.info('Found manifest for %s on %s %s'
+                             % (m['platform']['os'], m['platform']['architecture'],
+                                m['platform']['variant']))
+                else:
+                    LOG.info('Found manifest for %s on %s'
+                             % (m['platform']['os'], m['platform']['architecture']))
+
+                if (m['platform']['os'] == self.os and
+                    m['platform']['architecture'] == self.architecture and
+                        m['platform'].get('variant', '') == self.variant):
+                    LOG.info('Fetching matching manifest')
+                    r = self.request_url(
+                        'GET',
+                        '%(moniker)s://%(registry)s/v2/%(image)s/manifests/%(tag)s'
+                        % {
+                            'moniker': moniker,
+                            'registry': self.registry,
+                            'image': self.image,
+                            'tag': m['digest']
+                        },
+                        headers={'Accept': ('application/vnd.docker.distribution.manifest.v2+json')})
+                    manifest = r.json()
+                    config_digest = manifest['config']['digest']
+
+            if not config_digest:
+                raise Exception('Could not find a matching manifest for this '
+                                'os / architecture / variant')
+        else:
+            raise Exception('Unknown manifest content type %s!' %
+                            r.headers['Content-Type'])
 
         LOG.info('Fetching config file')
         r = self.request_url(
             'GET',
-            'https://%(registry)s/v2/%(image)s/blobs/%(config)s'
+            '%(moniker)s://%(registry)s/v2/%(image)s/blobs/%(config)s'
             % {
+                'moniker': moniker,
                 'registry': self.registry,
                 'image': self.image,
-                'config': manifest['config']['digest']
+                'config': config_digest
             })
         config = r.content
         h = hashlib.sha256()
         h.update(config)
-        if h.hexdigest() != manifest['config']['digest'].split(':')[1]:
+        if h.hexdigest() != config_digest.split(':')[1]:
             LOG.error('Hash verification failed for image config blob (%s vs %s)'
-                      % (manifest['config']['digest'].split(':')[1], h.hexdigest()))
+                      % (config_digest.split(':')[1], h.hexdigest()))
             sys.exit(1)
 
-        config_filename = ('%s.json'
-                           % manifest['config']['digest'].split(':')[1])
+        config_filename = ('%s.json' % config_digest.split(':')[1])
         yield (constants.CONFIG_FILE, config_filename,
                io.BytesIO(config))
 
@@ -96,15 +147,16 @@ class Image(object):
             layer_filename = layer['digest'].split(':')[1]
             if not fetch_callback(layer_filename):
                 LOG.info('Fetch callback says skip layer %s' % layer['digest'])
-                yield(constants.IMAGE_LAYER, layer_filename, None)
+                yield (constants.IMAGE_LAYER, layer_filename, None)
                 continue
 
             LOG.info('Fetching layer %s (%d bytes)'
                      % (layer['digest'], layer['size']))
             r = self.request_url(
                 'GET',
-                'https://%(registry)s/v2/%(image)s/blobs/%(layer)s'
+                '%(moniker)s://%(registry)s/v2/%(image)s/blobs/%(layer)s'
                 % {
+                    'moniker': moniker,
                     'registry': self.registry,
                     'image': self.image,
                     'layer': layer['digest']
@@ -128,18 +180,11 @@ class Image(object):
 
                 if h.hexdigest() != layer_filename:
                     LOG.error('Hash verification failed for layer (%s vs %s)'
-                              % (name, h.hexdigest()))
+                              % (layer_filename, h.hexdigest()))
                     sys.exit(1)
 
-                with tarfile.open(tf.name) as layer:
-                    for mem in layer.getmembers():
-                        m = DELETED_FILE_RE.match(mem.name)
-                        if m:
-                            LOG.info('Layer tarball contains deleted file: %s'
-                                     % mem.name)
-
                 with open(tf.name, 'rb') as f:
-                    yield(constants.IMAGE_LAYER, layer_filename, f)
+                    yield (constants.IMAGE_LAYER, layer_filename, f)
 
             finally:
                 os.unlink(tf.name)

occystrap/main.py

@@ -1,27 +1,40 @@
 import click
 import logging
+import os
+from shakenfist_utilities import logs
+import sys
 
 from occystrap import docker_registry
 from occystrap import output_directory
+from occystrap import output_mounts
+from occystrap import output_ocibundle
 from occystrap import output_tarfile
 
-logging.basicConfig(level=logging.INFO)
 
-LOG = logging.getLogger(__name__)
-LOG.setLevel(logging.INFO)
+LOG = logs.setup_console(__name__)
 
 
 @click.group()
 @click.option('--verbose', is_flag=True)
+@click.option('--os', default='linux')
+@click.option('--architecture', default='amd64')
+@click.option('--variant', default='')
 @click.pass_context
-def cli(ctx, verbose=None):
+def cli(ctx, verbose=None, os=None, architecture=None, variant=None):
     if verbose:
         logging.basicConfig(level=logging.DEBUG)
         LOG.setLevel(logging.DEBUG)
 
+    if not ctx.obj:
+        ctx.obj = {}
+    ctx.obj['OS'] = os
+    ctx.obj['ARCHITECTURE'] = architecture
+    ctx.obj['VARIANT'] = variant
 
-def _fetch(registry, image, tag, output):
-    img = docker_registry.Image(registry, image, tag)
+
+def _fetch(registry, image, tag, output, os, architecture, variant, secure=True):
+    img = docker_registry.Image(
+        registry, image, tag, os, architecture, variant, secure=secure)
     for image_element in img.fetch(fetch_callback=output.fetch_callback):
         output.process_image_element(*image_element)
     output.finalize()
@@ -34,11 +47,14 @@ def _fetch(registry, image, tag, output):
 @click.argument('path')
 @click.option('--use-unique-names', is_flag=True)
 @click.option('--expand', is_flag=True)
+@click.option('--insecure', is_flag=True, default=False)
 @click.pass_context
-def fetch_to_extracted(ctx, registry, image, tag, path, use_unique_names, expand):
+def fetch_to_extracted(ctx, registry, image, tag, path, use_unique_names,
+                       expand, insecure):
     d = output_directory.DirWriter(
         image, tag, path, unique_names=use_unique_names, expand=expand)
-    _fetch(registry, image, tag, d)
+    _fetch(registry, image, tag, d, ctx.obj['OS'], ctx.obj['ARCHITECTURE'],
+           ctx.obj['VARIANT'], secure=(not insecure))
 
     if expand:
         d.write_bundle()
@@ -47,20 +63,63 @@ def fetch_to_extracted(ctx, registry, image, tag, path, use_unique_names, expand
 cli.add_command(fetch_to_extracted)
 
 
+@click.command()
+@click.argument('registry')
+@click.argument('image')
+@click.argument('tag')
+@click.argument('path')
+@click.option('--insecure', is_flag=True, default=False)
+@click.pass_context
+def fetch_to_oci(ctx, registry, image, tag, path, insecure):
+    d = output_ocibundle.OCIBundleWriter(image, tag, path)
+    _fetch(registry, image, tag, d, ctx.obj['OS'], ctx.obj['ARCHITECTURE'],
+           ctx.obj['VARIANT'], secure=(not insecure))
+    d.write_bundle()
+
+
+cli.add_command(fetch_to_oci)
+
+
 @click.command()
 @click.argument('registry')
 @click.argument('image')
 @click.argument('tag')
 @click.argument('tarfile')
+@click.option('--insecure', is_flag=True, default=False)
 @click.pass_context
-def fetch_to_tarfile(ctx, registry, image, tag, tarfile):
+def fetch_to_tarfile(ctx, registry, image, tag, tarfile, insecure):
     tar = output_tarfile.TarWriter(image, tag, tarfile)
-    _fetch(registry, image, tag, tar)
+    _fetch(registry, image, tag, tar, ctx.obj['OS'], ctx.obj['ARCHITECTURE'],
+           ctx.obj['VARIANT'], secure=(not insecure))
 
 
 cli.add_command(fetch_to_tarfile)
 
 
+@click.command()
+@click.argument('registry')
+@click.argument('image')
+@click.argument('tag')
+@click.argument('path')
+@click.option('--insecure', is_flag=True, default=False)
+@click.pass_context
+def fetch_to_mounts(ctx, registry, image, tag, path, insecure):
+    if not hasattr(os, 'setxattr'):
+        print('Sorry, your OS module implementation lacks setxattr')
+        sys.exit(1)
+    if not hasattr(os, 'mknod'):
+        print('Sorry, your OS module implementation lacks mknod')
+        sys.exit(1)
+
+    d = output_mounts.MountWriter(image, tag, path)
+    _fetch(registry, image, tag, d, ctx.obj['OS'], ctx.obj['ARCHITECTURE'],
+           ctx.obj['VARIANT'], secure=(not insecure))
+    d.write_bundle()
+
+
+cli.add_command(fetch_to_mounts)
+
+
 @click.command()
 @click.argument('path')
 @click.argument('image')

occystrap/output_directory.py

@@ -1,11 +1,7 @@
-import hashlib
 import json
 import logging
 import os
-import prettytable
-import sys
 import tarfile
-import zlib
 
 from occystrap import constants
 
@@ -128,14 +124,13 @@ class DirWriter(object):
     def _create_bundle_path(self, path):
         d = self.bundle
         for elem in path.split('/'):
-            if not elem in d:
+            if elem not in d:
                 d[elem] = {}
             d = d[elem]
         return d
 
     def fetch_callback(self, digest):
-        layer_file_in_dir = os.path.join(self.image_path,
-                                         digest, 'layer.tar')
+        layer_file_in_dir = os.path.join(self.image_path, digest, 'layer.tar')
         LOG.info('Layer file is %s' % layer_file_in_dir)
         return not os.path.exists(layer_file_in_dir)
 
@@ -168,16 +163,39 @@ class DirWriter(object):
                 # Build a in-memory map of the layout of the final image bundle
                 with tarfile.open(layer_file_in_dir) as layer:
                     for mem in layer.getmembers():
-                        filename = os.path.split(mem.name)[1]
-                        if filename.startswith('.wh.'):
+                        path = mem.name
+                        dirname, filename = os.path.split(mem.name)
+
+                        # Some light reading on how this works...
+                        # https://github.com/opencontainers/image-spec/blob/main/layer.md#opaque-whiteout
+                        if filename == '.wh..wh..opq':
+                            # A deleted directory, but only for layers below
+                            # this one.
+                            for ent in self.bundle:
+                                if (ent.startswith(dirname) and
+                                        self.bundle[ent][-1].tarpath != layer_file):
+                                    self.bundle[ent].append(
+                                        BundleDeletedFile(ent, layer_file, mem))
+                            continue
+
+                        elif filename.startswith('.wh.'):
+                            # A single deleted element, which might not be a
+                            # file.
+                            path = os.path.join(dirname, filename[4:])
+                            if type(self.bundle[path][-1]) is BundleDirectory:
+                                for ent in self.bundle:
+                                    if ent.startswith(path):
+                                        self.bundle[ent].append(
+                                            BundleDeletedFile(ent, layer_file, mem))
+
                             serialized = BundleDeletedFile(
-                                mem.name, layer_file, mem)
+                                path, layer_file, mem)
                         else:
                             serialized = TARFILE_TYPE_MAP[mem.type](
                                 mem.name, layer_file, mem)
 
-                        self.bundle.setdefault(mem.name, [])
-                        self.bundle[mem.name].append(serialized)
+                        self.bundle.setdefault(path, [])
+                        self.bundle[path].append(serialized)
 
     def _log_bundle(self):
         savings = 0
@@ -185,13 +203,17 @@ class DirWriter(object):
         for path in self.bundle:
             versions = len(self.bundle[path])
             if versions > 1:
-                LOG.info('Bundle path %s has %d versions'
+                path_savings = 0
+                LOG.info('Bundle path "%s" has %d versions'
                          % (path, versions))
                 for ver in self.bundle[path][:-1]:
-                    savings += ver.size
+                    path_savings += ver.size
+                if type(self.bundle[path][-1]) is BundleDeletedFile:
+                    LOG.info('Bundle path "%s" final version is a deleted file, '
+                             'which wasted %d bytes.' % (path, path_savings))
+                savings += path_savings
 
-        LOG.info('Flattening image would save %d bytes'
-                 % savings)
+        LOG.info('Flattening image would save %d bytes' % savings)
 
     def finalize(self):
         if self.expand:
@@ -214,13 +236,7 @@ class DirWriter(object):
         with open(catalog_path, 'w') as f:
             f.write(json.dumps(c, indent=4, sort_keys=True))
 
-    def write_bundle(self):
-        LOG.info('Writing image bundle')
-        manifest_filename = self._manifest_filename()
-        manifest_path = os.path.join(self.image_path, manifest_filename)
-        if not os.path.exists(manifest_path):
-            os.makedirs(manifest_path)
-
+    def _extract_rootfs(self, rootfs_path):
         # Reading tarfiles is expensive, as tarfile needs to scan the
         # entire file to find the right entry. It builds a cache while
         # doing this however, so performance improves if you access a
@@ -250,14 +266,20 @@ class DirWriter(object):
         for tarpath in entities_by_layer:
             with tarfile.open(os.path.join(self.image_path, tarpath)) as layer:
                 for ent in entities_by_layer[tarpath]:
-                    entdest = os.path.join(manifest_path, ent.name)
-                    layer.extract(ent.name, path=entdest)
+                    layer.extract(ent.name, path=rootfs_path)
 
         for tarpath in deferred_by_layer:
             with tarfile.open(os.path.join(self.image_path, tarpath)) as layer:
                 for ent in deferred_by_layer[tarpath]:
-                    entdest = os.path.join(manifest_path, ent.name)
-                    layer.extract(ent.name, path=entdest)
+                    layer.extract(ent.name, path=rootfs_path)
+
+    def write_bundle(self):
+        manifest_filename = self._manifest_filename()
+        manifest_path = os.path.join(self.image_path, manifest_filename)
+        if not os.path.exists(manifest_path):
+            os.makedirs(manifest_path)
+        LOG.info('Writing image bundle to %s' % manifest_path)
+        self._extract_rootfs(manifest_path)
 
 
 class NoSuchImageException(Exception):
@@ -276,9 +298,9 @@ class DirReader(object):
             with open(catalog_path, 'r') as f:
                 c = json.loads(f.read())
 
-        if not self.image in c:
+        if self.image not in c:
             raise NoSuchImageException(self.image)
-        if not self.tag in c[self.image]:
+        if self.tag not in c[self.image]:
             raise NoSuchImageException(self.image)
 
         self.manifest_filename = c[self.image][self.tag]
@@ -289,7 +311,7 @@ class DirReader(object):
 
         config_filename = manifest[0]['Config']
         with open(os.path.join(self.path, config_filename), 'rb') as f:
-            yield(constants.CONFIG_FILE, config_filename, f)
+            yield (constants.CONFIG_FILE, config_filename, f)
 
         for layer in manifest[0]['Layers']:
             with open(os.path.join(self.path, layer), 'rb') as f:

occystrap/output_mounts.py

@@ -0,0 +1,155 @@
+import json
+import logging
+import os
+import stat
+import tarfile
+
+from occystrap import common
+from occystrap import constants
+from occystrap import util
+
+
+LOG = logging.getLogger(__name__)
+LOG.setLevel(logging.INFO)
+
+
+class MountWriter(object):
+    def __init__(self, image, tag, image_path):
+        self.image = image
+        self.tag = tag
+        self.image_path = image_path
+
+        self.tar_manifest = [{
+            'Layers': [],
+            'RepoTags': ['%s:%s' % (self.image.split('/')[-1], self.tag)]
+        }]
+
+        self.bundle = {}
+
+        if not os.path.exists(self.image_path):
+            os.makedirs(self.image_path)
+
+    def _manifest_filename(self):
+        return 'manifest'
+
+    def fetch_callback(self, digest):
+        layer_file_in_dir = os.path.join(self.image_path, digest, 'layer.tar')
+        LOG.info('Layer file is %s' % layer_file_in_dir)
+        return not os.path.exists(layer_file_in_dir)
+
+    def process_image_element(self, element_type, name, data):
+        if element_type == constants.CONFIG_FILE:
+            with open(os.path.join(self.image_path, name), 'wb') as f:
+                d = json.loads(data.read())
+                f.write(json.dumps(d, indent=4, sort_keys=True).encode('ascii'))
+            self.tar_manifest[0]['Config'] = name
+
+        elif element_type == constants.IMAGE_LAYER:
+            layer_dir = os.path.join(self.image_path, name)
+            if not os.path.exists(layer_dir):
+                os.makedirs(layer_dir)
+
+            layer_file = os.path.join(name, 'layer.tar')
+            self.tar_manifest[0]['Layers'].append(layer_file)
+
+            layer_file_in_dir = os.path.join(self.image_path, layer_file)
+            if os.path.exists(layer_file_in_dir):
+                LOG.info('Skipping layer already in output directory')
+            else:
+                with open(layer_file_in_dir, 'wb') as f:
+                    d = data.read(102400)
+                    while d:
+                        f.write(d)
+                        d = data.read(102400)
+
+                layer_dir_in_dir = os.path.join(self.image_path, name, 'layer')
+                os.makedirs(layer_dir_in_dir)
+                with tarfile.open(layer_file_in_dir) as layer:
+                    for mem in layer.getmembers():
+                        dirname, filename = os.path.split(mem.name)
+
+                        # Some light reading on how this works...
+                        # https://www.madebymikal.com/interpreting-whiteout-files-in-docker-image-layers/
+                        # https://github.com/opencontainers/image-spec/blob/main/layer.md#opaque-whiteout
+                        if filename == '.wh..wh..opq':
+                            # A deleted directory, but only for layers below
+                            # this one.
+                            os.setxattr(os.path.join(layer_dir_in_dir, dirname),
+                                        'trusted.overlay.opaque', b'y')
+
+                        elif filename.startswith('.wh.'):
+                            # A single deleted element, which might not be a
+                            # file.
+                            os.mknod(os.path.join(layer_dir_in_dir,
+                                     mem.name[4:]),
+                                     mode=stat.S_IFCHR, device=0)
+
+                        else:
+                            path = mem.name
+                            layer.extract(path, path=layer_dir_in_dir)
+
+    def finalize(self):
+        manifest_filename = self._manifest_filename() + '.json'
+        manifest_path = os.path.join(self.image_path, manifest_filename)
+        with open(manifest_path, 'wb') as f:
+            f.write(json.dumps(self.tar_manifest, indent=4,
+                               sort_keys=True).encode('ascii'))
+
+        c = {}
+        catalog_path = os.path.join(self.image_path, 'catalog.json')
+        if os.path.exists(catalog_path):
+            with open(catalog_path, 'r') as f:
+                c = json.loads(f.read())
+
+        c.setdefault(self.image, {})
+        c[self.image][self.tag] = manifest_filename
+        with open(catalog_path, 'w') as f:
+            f.write(json.dumps(c, indent=4, sort_keys=True))
+
+    def write_bundle(self, container_template=constants.RUNC_SPEC_TEMPLATE,
+                     container_values=None):
+        if not container_values:
+            container_values = {}
+
+        rootfs_path = os.path.join(self.image_path, 'rootfs')
+        if not os.path.exists(rootfs_path):
+            os.makedirs(rootfs_path)
+        LOG.info('Writing image bundle to %s' % rootfs_path)
+
+        working_path = os.path.join(self.image_path, 'working')
+        if not os.path.exists(working_path):
+            os.makedirs(working_path)
+
+        delta_path = os.path.join(self.image_path, 'delta')
+        if not os.path.exists(delta_path):
+            os.makedirs(delta_path)
+
+        # The newest layer is listed first in the mount command
+        layer_dirs = []
+        self.tar_manifest[0]['Layers'].reverse()
+        for layer in self.tar_manifest[0]['Layers']:
+            layer_dirs.append(os.path.join(
+                self.image_path, layer.replace('.tar', '')))
+
+        # Extract the rootfs as overlay mounts
+        util.execute('mount -t overlay overlay -o lowerdir=%(layers)s,'
+                     'upperdir=%(upper)s,workdir=%(working)s %(rootfs)s'
+                     % {
+                         'layers': ':'.join(layer_dirs),
+                         'upper': delta_path,
+                         'working': working_path,
+                         'rootfs': rootfs_path
+                     })
+
+        # Rename the container configuration to a well known location. This is
+        # not part of the OCI specification, but is convenient for now.
+        container_config_filename = os.path.join(self.image_path,
+                                                 'container-config.json')
+        runtime_config_filename = os.path.join(self.image_path, 'config.json')
+        os.rename(os.path.join(self.image_path, self.tar_manifest[0]['Config']),
+                  container_config_filename)
+
+        common.write_container_config(container_config_filename,
+                                      runtime_config_filename,
+                                      container_template=container_template,
+                                      container_values=container_values)

occystrap/output_ocibundle.py

@@ -0,0 +1,53 @@
+# OCI bundles are a special case of our directory output -- they don't contain
+# all of the data that a directory output does, and they place the data they
+# do contain into different locations within the directory structure.
+
+import logging
+import os
+import shutil
+
+from occystrap.constants import RUNC_SPEC_TEMPLATE
+from occystrap import common
+from occystrap.output_directory import DirWriter
+
+
+LOG = logging.getLogger(__name__)
+LOG.setLevel(logging.INFO)
+
+
+class OCIBundleWriter(DirWriter):
+    def __init__(self, image, tag, image_path):
+        super(OCIBundleWriter, self).__init__(
+            image, tag, image_path, expand=True)
+
+    def finalize(self):
+        self._log_bundle()
+
+    def write_bundle(self, container_template=RUNC_SPEC_TEMPLATE,
+                     container_values=None):
+        if not container_values:
+            container_values = {}
+
+        rootfs_path = os.path.join(self.image_path, 'rootfs')
+        if not os.path.exists(rootfs_path):
+            os.makedirs(rootfs_path)
+        LOG.info('Writing image bundle to %s' % rootfs_path)
+        self._extract_rootfs(rootfs_path)
+
+        # Remove parts of the output directory which are not present in OCI
+        for layer_file in self.tar_manifest[0]['Layers']:
+            shutil.rmtree(os.path.join(self.image_path,
+                                       os.path.split(layer_file)[0]))
+
+        # Rename the container configuration to a well known location. This is
+        # not part of the OCI specification, but is convenient for now.
+        container_config_filename = os.path.join(self.image_path,
+                                                 'container-config.json')
+        runtime_config_filename = os.path.join(self.image_path, 'config.json')
+        os.rename(os.path.join(self.image_path, self.tar_manifest[0]['Config']),
+                  container_config_filename)
+
+        common.write_container_config(container_config_filename,
+                                      runtime_config_filename,
+                                      container_template=container_template,
+                                      container_values=container_values)

occystrap/output_tarfile.py

@@ -1,13 +1,8 @@
-import hashlib
 import io
 import json
 import logging
 import os
-import re
-import sys
 import tarfile
-import tempfile
-import zlib
 
 from occystrap import constants
 

occystrap/util.py

@@ -1,5 +1,6 @@
 import json
 import logging
+from oslo_concurrency import processutils
 from pbr.version import VersionInfo
 import requests
 
@@ -23,7 +24,7 @@ STATUS_CODES_TO_ERRORS = {
 def get_user_agent():
     try:
         version = VersionInfo('occystrap').version_string()
-    except:
+    except Exception:
         version = '0.0.0'
     return 'Mozilla/5.0 (Ubuntu; Linux x86_64) Occy Strap/%s' % version
 
@@ -74,3 +75,10 @@ def request_url(method, url, headers=None, data=None, stream=False):
         raise APIException(
             'API request failed', method, url, r.status_code, r.text, r.headers)
     return r
+
+
+def execute(command, check_exit_code=[0], env_variables=None,
+            cwd=None):
+    return processutils.execute(
+        command, check_exit_code=check_exit_code,
+        env_variables=env_variables, shell=True, cwd=cwd)

{occystrap-0.2.0.dist-info → occystrap-0.3.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: occystrap
-Version: 0.2.0
+Version: 0.3.0
 Summary: occystrap: docker and OCI container tools
 Home-page: https://github.com/shakenfist/occystrap
 Author: Michael Still
@@ -16,9 +16,11 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.7
 Description-Content-Type: text/markdown
 Requires-Dist: click (>=7.1.1)
+Requires-Dist: oslo.concurrency
 Requires-Dist: pbr
 Requires-Dist: prettytable
 Requires-Dist: requests
+Requires-Dist: shakenfist-utilities
 
 # Occy Strap
 
@@ -98,5 +100,32 @@ occystrap fetch-to-extracted --expand quay.io \
 
 Note that layers delete files from previous layers with files named ".wh.$previousfilename". These files are _not_ processed in the expanded layers, so that they are visible to the user. They are however processed in the merged layer named for the manifest file.
 
+## Generating an OCI runtime bundle
+
+This isn't fully supported yet, but you can extract an image to an OCI image bundle
+with the following command:
+
+```
+occystrap fetch-to-oci registry-1.docker.io library/hello-world latest bar
+```
+
+You should then be able to run that container by doing something like:
+
+```
+cd bar
+sudo apt-get install runc
+sudo runc run id-0001
+```
+
+## Supporting non-default architectures
+
+Docker image repositories can store multiple versions of a single image, with each image corresponding to a different (operating system, cpu architecture, cpu variant) tuple. Occy Strap supports letting you specify which to use with global command line flags. Occy Strap defaults to linux amd64 if you don't specify something different. For example, to fetch the linux arm64 v8 image for busybox, you would run:
+
+```
+occystrap --os linux --architecture arm64 --variant v8 \
+    fetch-to-extracted registry-1.docker.io library/busybox \
+    latest busybox
+```
+
 
 

occystrap-0.3.0.dist-info/RECORD

@@ -0,0 +1,20 @@
+occystrap/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+occystrap/common.py,sha256=Zm4hHpn8RgSXp0W86HhZzpyXq19QIsLJgp9SxK_1QQg,1300
+occystrap/constants.py,sha256=kmOt-12settGbDTW1efpT3UENRQouG9f0ZjgOqWdrIA,4399
+occystrap/docker_extract.py,sha256=j2GSIOShZZh0c5gckXeu-SO7201p4S1EJg3fi7WPQHk,1055
+occystrap/docker_registry.py,sha256=C-T1bY1UPJfBZB_Q5gqcZYODikxnrTQU2MfYyoux950,7658
+occystrap/main.py,sha256=sevQkqAqgnZRrt61DMUPz9xzoE4AIcWYx2_ZurO_Lls,4059
+occystrap/output_directory.py,sha256=S-uL8NSHyHsVKeWsvPRT63E7wE1pWaluGHOFVsS09Xg,11408
+occystrap/output_mounts.py,sha256=AH4vouBF9PmhQ5oGfsSZyN1FhatWbs_20IDlI9psn_k,6381
+occystrap/output_ocibundle.py,sha256=lsVW66ltL2Y-Mxh5Hp2KSCdh9bvsME1142CJ_Uh99oo,2154
+occystrap/output_tarfile.py,sha256=Di8N_2TSo-gQz490D3XOsKCYiv5T_bxZzTRFbd4WGBA,1620
+occystrap/util.py,sha256=nrKfAxDUjb_v9ERDXTmMSSNdJSuLgjdtwOMWk46n0a0,2720
+occystrap/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+occystrap-0.3.0.dist-info/AUTHORS,sha256=toKLUaf9c-NkNow00B_akwMGcGtm-S_ihcC_eql9qWc,34
+occystrap-0.3.0.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+occystrap-0.3.0.dist-info/METADATA,sha256=PaCk2BSIcly80vIV4LiPlpXoQ9CbYKtb7S5UpaAI5BQ,6308
+occystrap-0.3.0.dist-info/WHEEL,sha256=g4nMs7d-Xl9-xC9XovUrsDHGXt-FT0E17Yqo92DEfvY,92
+occystrap-0.3.0.dist-info/entry_points.txt,sha256=51kLRjAxFtC6GWCbTFGezSYMNk5t6xrBmS8Pf7gehiU,50
+occystrap-0.3.0.dist-info/pbr.json,sha256=2zD1Bsq8TK1jHn5K3MGWkgXT6TxdPUw9MsWVY-Pe89c,46
+occystrap-0.3.0.dist-info/top_level.txt,sha256=06nN7FHq2z_Jpp2PZNm3rGOGUA1cIGlUr6MEZrqgOlc,10
+occystrap-0.3.0.dist-info/RECORD,,

occystrap-0.3.0.dist-info/pbr.json

@@ -0,0 +1 @@
+{"git_version": "27e37dc", "is_release": true}

occystrap-0.2.0.dist-info/RECORD

@@ -1,17 +0,0 @@
-occystrap/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-occystrap/constants.py,sha256=wZwFTAzrxq8Gb2kT7ZRGa6cAz0nHyvnuzR4KrrY2RI0,56
-occystrap/docker_extract.py,sha256=j2GSIOShZZh0c5gckXeu-SO7201p4S1EJg3fi7WPQHk,1055
-occystrap/docker_registry.py,sha256=CLaUWK9uBHKM3KCGfDWw8myF9nndWdEWIztxGd3Fdeg,5416
-occystrap/main.py,sha256=xNNbPo-cpcUe8hfQgpIyTt0ORsiDtWMF-55ImPb8330,2003
-occystrap/output_directory.py,sha256=vhdX0jQQdzhQoCmsu9MCwl6nl1mon4jvkUXYGUEGmko,9938
-occystrap/output_tarfile.py,sha256=R_kdJd_uitEfl1EyczHZ6n_IvafwzkTVuAIjEbV73Q4,1684
-occystrap/util.py,sha256=Kdzae1pkIRt5uAm1-pgwJFWLI8C7QA7KYkgstPs9yYQ,2440
-occystrap/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-occystrap-0.2.0.dist-info/AUTHORS,sha256=toKLUaf9c-NkNow00B_akwMGcGtm-S_ihcC_eql9qWc,34
-occystrap-0.2.0.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-occystrap-0.2.0.dist-info/METADATA,sha256=3wqUvQETACudMvSBxlgzNWbt1o0_yrFN15CgjT2lotg,5269
-occystrap-0.2.0.dist-info/WHEEL,sha256=g4nMs7d-Xl9-xC9XovUrsDHGXt-FT0E17Yqo92DEfvY,92
-occystrap-0.2.0.dist-info/entry_points.txt,sha256=51kLRjAxFtC6GWCbTFGezSYMNk5t6xrBmS8Pf7gehiU,50
-occystrap-0.2.0.dist-info/pbr.json,sha256=kxHs4Gx_jL71k5Snp4MPDUPUNjY6OSZIcmRk9xnPcPY,46
-occystrap-0.2.0.dist-info/top_level.txt,sha256=06nN7FHq2z_Jpp2PZNm3rGOGUA1cIGlUr6MEZrqgOlc,10
-occystrap-0.2.0.dist-info/RECORD,,

occystrap-0.2.0.dist-info/pbr.json

@@ -1 +0,0 @@
-{"git_version": "0f9c4a4", "is_release": true}