occystrap 0.3.0__py3-none-any.whl → 0.4.1__py3-none-any.whl

occystrap/docker_registry.py

@@ -1,192 +0,0 @@
-# A simple implementation of a docker registry client. Fetches an image to a tarball.
-# With a big nod to https://github.com/NotGlop/docker-drag/blob/master/docker_pull.py
-
-# https://docs.docker.com/registry/spec/manifest-v2-2/ documents the image manifest
-# format, noting that the response format you get back varies based on what you have
-# in your accept header for the request.
-
-import hashlib
-import io
-import logging
-import os
-import re
-import sys
-import tempfile
-import zlib
-
-from occystrap import constants
-from occystrap import util
-
-LOG = logging.getLogger(__name__)
-LOG.setLevel(logging.INFO)
-
-DELETED_FILE_RE = re.compile(r'.*/\.wh\.(.*)$')
-
-
-def always_fetch():
-    return True
-
-
-class Image(object):
-    def __init__(self, registry, image, tag, os='linux', architecture='amd64', variant='',
-                 secure=True):
-        self.registry = registry
-        self.image = image
-        self.tag = tag
-        self.os = os
-        self.architecture = architecture
-        self.variant = variant
-        self.secure = secure
-
-        self._cached_auth = None
-
-    def request_url(self, method, url, headers=None, data=None, stream=False):
-        if not headers:
-            headers = {}
-
-        if self._cached_auth:
-            headers.update({'Authorization': 'Bearer %s' % self._cached_auth})
-
-        try:
-            return util.request_url(method, url, headers=headers, data=data,
-                                    stream=stream)
-        except util.UnauthorizedException as e:
-            auth_re = re.compile('Bearer realm="([^"]*)",service="([^"]*)"')
-            m = auth_re.match(e.args[5].get('Www-Authenticate'))
-            if m:
-                auth_url = ('%s?service=%s&scope=repository:%s:pull'
-                            % (m.group(1), m.group(2), self.image))
-                r = util.request_url('GET', auth_url)
-                token = r.json().get('token')
-                headers.update({'Authorization': 'Bearer %s' % token})
-                self._cached_auth = token
-
-            return util.request_url(
-                method, url, headers=headers, data=data, stream=stream)
-
-    def fetch(self, fetch_callback=always_fetch):
-        LOG.info('Fetching manifest')
-        moniker = 'https'
-        if not self.secure:
-            moniker = 'http'
-
-        r = self.request_url(
-            'GET',
-            '%(moniker)s://%(registry)s/v2/%(image)s/manifests/%(tag)s'
-            % {
-                'moniker': moniker,
-                'registry': self.registry,
-                'image': self.image,
-                'tag': self.tag
-            },
-            headers={'Accept': ('application/vnd.docker.distribution.manifest.v2+json,'
-                                'application/vnd.docker.distribution.manifest.list.v2+json')})
-
-        config_digest = None
-        if r.headers['Content-Type'] == 'application/vnd.docker.distribution.manifest.v2+json':
-            manifest = r.json()
-            config_digest = manifest['config']['digest']
-        elif r.headers['Content-Type'] == 'application/vnd.docker.distribution.manifest.list.v2+json':
-            for m in r.json()['manifests']:
-                if 'variant' in m['platform']:
-                    LOG.info('Found manifest for %s on %s %s'
-                             % (m['platform']['os'], m['platform']['architecture'],
-                                m['platform']['variant']))
-                else:
-                    LOG.info('Found manifest for %s on %s'
-                             % (m['platform']['os'], m['platform']['architecture']))
-
-                if (m['platform']['os'] == self.os and
-                    m['platform']['architecture'] == self.architecture and
-                        m['platform'].get('variant', '') == self.variant):
-                    LOG.info('Fetching matching manifest')
-                    r = self.request_url(
-                        'GET',
-                        '%(moniker)s://%(registry)s/v2/%(image)s/manifests/%(tag)s'
-                        % {
-                            'moniker': moniker,
-                            'registry': self.registry,
-                            'image': self.image,
-                            'tag': m['digest']
-                        },
-                        headers={'Accept': ('application/vnd.docker.distribution.manifest.v2+json')})
-                    manifest = r.json()
-                    config_digest = manifest['config']['digest']
-
-            if not config_digest:
-                raise Exception('Could not find a matching manifest for this '
-                                'os / architecture / variant')
-        else:
-            raise Exception('Unknown manifest content type %s!' %
-                            r.headers['Content-Type'])
-
-        LOG.info('Fetching config file')
-        r = self.request_url(
-            'GET',
-            '%(moniker)s://%(registry)s/v2/%(image)s/blobs/%(config)s'
-            % {
-                'moniker': moniker,
-                'registry': self.registry,
-                'image': self.image,
-                'config': config_digest
-            })
-        config = r.content
-        h = hashlib.sha256()
-        h.update(config)
-        if h.hexdigest() != config_digest.split(':')[1]:
-            LOG.error('Hash verification failed for image config blob (%s vs %s)'
-                      % (config_digest.split(':')[1], h.hexdigest()))
-            sys.exit(1)
-
-        config_filename = ('%s.json' % config_digest.split(':')[1])
-        yield (constants.CONFIG_FILE, config_filename,
-               io.BytesIO(config))
-
-        LOG.info('There are %d image layers' % len(manifest['layers']))
-        for layer in manifest['layers']:
-            layer_filename = layer['digest'].split(':')[1]
-            if not fetch_callback(layer_filename):
-                LOG.info('Fetch callback says skip layer %s' % layer['digest'])
-                yield (constants.IMAGE_LAYER, layer_filename, None)
-                continue
-
-            LOG.info('Fetching layer %s (%d bytes)'
-                     % (layer['digest'], layer['size']))
-            r = self.request_url(
-                'GET',
-                '%(moniker)s://%(registry)s/v2/%(image)s/blobs/%(layer)s'
-                % {
-                    'moniker': moniker,
-                    'registry': self.registry,
-                    'image': self.image,
-                    'layer': layer['digest']
-                },
-                stream=True)
-
-            # We can use zlib for streaming decompression, but we need to tell it
-            # to ignore the gzip header which it doesn't understand. Unfortunately
-            # tarfile doesn't do streaming writes (and we need to know the
-            # decompressed size before we can write to the tarfile), so we stream
-            # to a temporary file on disk.
-            try:
-                h = hashlib.sha256()
-                d = zlib.decompressobj(16 + zlib.MAX_WBITS)
-
-                with tempfile.NamedTemporaryFile(delete=False) as tf:
-                    LOG.info('Temporary file for layer is %s' % tf.name)
-                    for chunk in r.iter_content(8192):
-                        tf.write(d.decompress(chunk))
-                        h.update(chunk)
-
-                if h.hexdigest() != layer_filename:
-                    LOG.error('Hash verification failed for layer (%s vs %s)'
-                              % (layer_filename, h.hexdigest()))
-                    sys.exit(1)
-
-                with open(tf.name, 'rb') as f:
-                    yield (constants.IMAGE_LAYER, layer_filename, f)
-
-            finally:
-                os.unlink(tf.name)
-
-        LOG.info('Done')

occystrap-0.3.0.dist-info/METADATA

@@ -1,131 +0,0 @@
-Metadata-Version: 2.1
-Name: occystrap
-Version: 0.3.0
-Summary: occystrap: docker and OCI container tools
-Home-page: https://github.com/shakenfist/occystrap
-Author: Michael Still
-Author-email: mikal@stillhq.com
-License: Apache2
-Platform: UNKNOWN
-Classifier: Intended Audience :: Information Technology
-Classifier: Intended Audience :: System Administrators
-Classifier: License :: OSI Approved :: Apache Software License
-Classifier: Operating System :: POSIX :: Linux
-Classifier: Programming Language :: Python
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.7
-Description-Content-Type: text/markdown
-Requires-Dist: click (>=7.1.1)
-Requires-Dist: oslo.concurrency
-Requires-Dist: pbr
-Requires-Dist: prettytable
-Requires-Dist: requests
-Requires-Dist: shakenfist-utilities
-
-# Occy Strap
-
-Occy Strap is a simple set of Docker and OCI container tools, which can be used either for container forensics or for implementing an OCI orchestrator, depending on your needs. This is a very early implementation, so be braced for impact.
-
-## Downloading an image from a repository and storing as a tarball
-
-Let's say we want to download an image from a repository and store it as a local tarball. This is a common thing to want to do in airgapped environments for example. You could do this with docker with a `docker pull; docker save`. The Occy Strap equivalent is:
-
-```
-occystrap fetch-to-tarfile registry-1.docker.io library/busybox latest busybox.tar
-```
-
-In this example we're pulling from the Docker Hub (registry-1.docker.io), and are downloading busybox's latest version into a tarball named `busybox-occy.tar`. This tarball can be loaded with `docker load -i busybox.tar` on an airgapped Docker environment.
-
-## Downloading an image from a repository and storing as an extracted tarball
-
-The format of the tarball in the previous example is two JSON configuration files and a series of image layers as tarballs inside the main tarball. You can write these elements to a directory instead of to a tarball if you'd like to inspect them. For example:
-
-```
-occystrap fetch-to-extracted registry-1.docker.io library/centos 7 centos7
-```
-
-This example will pull from the Docker Hub the Centos image with the label "7", and write the content to a directory in the current working directory called "centos7". If you tarred centos7 like this, you'd end up with a tarball equivalent to what `fetch-to-tarfile` produces, which could therefore be loaded with `docker load`:
-
-```
-cd centos7; tar -cf ../centos7.tar *
-```
-
-## Downloading an image from a repository and storing it in a merged directory
-
-In scenarios where image layers are likely to be reused between images (for example many images which share a common base layer), you can save disk space by downloading images to a directory which contains more than one image. To make this work, you need to instruct Occy Strap to use unique names for the JSON elements within the image file:
-
-```
-occystrap fetch-to-extracted --use-unique-names registry-1.docker.io \
-    homeassistant/home-assistant latest merged_images
-occystrap fetch-to-extracted --use-unique-names registry-1.docker.io \
-    homeassistant/home-assistant stable merged_images
-occystrap fetch-to-extracted --use-unique-names registry-1.docker.io \
-    homeassistant/home-assistant 2021.3.0.dev20210219 merged_images
-```
-
-Each of these images include 21 layers, but the merged_images directory at the time of writing this there are 25 unique layers in the directory. You end up with a layout like this:
-
-```
-0465ae924726adc52c0216e78eda5ce2a68c42bf688da3f540b16f541fd3018c
-10556f40181a651a72148d6c643ac9b176501d4947190a8732ec48f2bf1ac4fb
-...
-catalog.json
-cd8d37c8075e8a0195ae12f1b5c96fe4e8fe378664fc8943f2748336a7d2f2f3
-d1862a2c28ec9e23d88c8703096d106e0fe89bc01eae4c461acde9519d97b062
-d1ac3982d662e038e06cc7e1136c6a84c295465c9f5fd382112a6d199c364d20.json
-...
-d81f69adf6d8aeddbaa1421cff10ba47869b19cdc721a2ebe16ede57679850f0.json
-...
-manifest-homeassistant_home-assistant-2021.3.0.dev20210219.json
-manifest-homeassistant_home-assistant-latest.json
-manifest-homeassistant_home-assistant-stable.json
-```
-
-`catalog.json` is an Occy Strap specific artefact which maps which layers are used by which image. Each of the manifest files for the various images have been converted to have a unique name instead of `manifest.json` as well.
-
-To extract a single image from such a shared directory, use the `recreate-image` command:
-
-```
-occystrap recreate-image merged_images homeassistant/home-assistant latest ha-latest.tar
-```
-
-## Exploring the contents of layers and overwritten files
-
-Similarly, if you'd like the layers to be expanded from their tarballs to the filesystem, you can pass the `--expand` argument to `fetch-to-extracted` to have them extracted. This will also create a filesystem at the name of the manifest which is the final state of the image (the layers applied sequential). For example:
-
-```
-occystrap fetch-to-extracted --expand quay.io \
-    ukhomeofficedigital/centos-base latest ukhomeoffice-centos
-```
-
-Note that layers delete files from previous layers with files named ".wh.$previousfilename". These files are _not_ processed in the expanded layers, so that they are visible to the user. They are however processed in the merged layer named for the manifest file.
-
-## Generating an OCI runtime bundle
-
-This isn't fully supported yet, but you can extract an image to an OCI image bundle
-with the following command:
-
-```
-occystrap fetch-to-oci registry-1.docker.io library/hello-world latest bar
-```
-
-You should then be able to run that container by doing something like:
-
-```
-cd bar
-sudo apt-get install runc
-sudo runc run id-0001
-```
-
-## Supporting non-default architectures
-
-Docker image repositories can store multiple versions of a single image, with each image corresponding to a different (operating system, cpu architecture, cpu variant) tuple. Occy Strap supports letting you specify which to use with global command line flags. Occy Strap defaults to linux amd64 if you don't specify something different. For example, to fetch the linux arm64 v8 image for busybox, you would run:
-
-```
-occystrap --os linux --architecture arm64 --variant v8 \
-    fetch-to-extracted registry-1.docker.io library/busybox \
-    latest busybox
-```
-
-
-

occystrap-0.3.0.dist-info/RECORD

@@ -1,20 +0,0 @@
-occystrap/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-occystrap/common.py,sha256=Zm4hHpn8RgSXp0W86HhZzpyXq19QIsLJgp9SxK_1QQg,1300
-occystrap/constants.py,sha256=kmOt-12settGbDTW1efpT3UENRQouG9f0ZjgOqWdrIA,4399
-occystrap/docker_extract.py,sha256=j2GSIOShZZh0c5gckXeu-SO7201p4S1EJg3fi7WPQHk,1055
-occystrap/docker_registry.py,sha256=C-T1bY1UPJfBZB_Q5gqcZYODikxnrTQU2MfYyoux950,7658
-occystrap/main.py,sha256=sevQkqAqgnZRrt61DMUPz9xzoE4AIcWYx2_ZurO_Lls,4059
-occystrap/output_directory.py,sha256=S-uL8NSHyHsVKeWsvPRT63E7wE1pWaluGHOFVsS09Xg,11408
-occystrap/output_mounts.py,sha256=AH4vouBF9PmhQ5oGfsSZyN1FhatWbs_20IDlI9psn_k,6381
-occystrap/output_ocibundle.py,sha256=lsVW66ltL2Y-Mxh5Hp2KSCdh9bvsME1142CJ_Uh99oo,2154
-occystrap/output_tarfile.py,sha256=Di8N_2TSo-gQz490D3XOsKCYiv5T_bxZzTRFbd4WGBA,1620
-occystrap/util.py,sha256=nrKfAxDUjb_v9ERDXTmMSSNdJSuLgjdtwOMWk46n0a0,2720
-occystrap/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-occystrap-0.3.0.dist-info/AUTHORS,sha256=toKLUaf9c-NkNow00B_akwMGcGtm-S_ihcC_eql9qWc,34
-occystrap-0.3.0.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-occystrap-0.3.0.dist-info/METADATA,sha256=PaCk2BSIcly80vIV4LiPlpXoQ9CbYKtb7S5UpaAI5BQ,6308
-occystrap-0.3.0.dist-info/WHEEL,sha256=g4nMs7d-Xl9-xC9XovUrsDHGXt-FT0E17Yqo92DEfvY,92
-occystrap-0.3.0.dist-info/entry_points.txt,sha256=51kLRjAxFtC6GWCbTFGezSYMNk5t6xrBmS8Pf7gehiU,50
-occystrap-0.3.0.dist-info/pbr.json,sha256=2zD1Bsq8TK1jHn5K3MGWkgXT6TxdPUw9MsWVY-Pe89c,46
-occystrap-0.3.0.dist-info/top_level.txt,sha256=06nN7FHq2z_Jpp2PZNm3rGOGUA1cIGlUr6MEZrqgOlc,10
-occystrap-0.3.0.dist-info/RECORD,,

occystrap-0.3.0.dist-info/pbr.json

@@ -1 +0,0 @@
-{"git_version": "27e37dc", "is_release": true}