occystrap 0.2.0__tar.gz → 0.4.0__tar.gz

occystrap-0.4.0/.github/workflows/codeql-analysis.yml

@@ -0,0 +1,54 @@
+name: "CodeQL"
+
+on:
+  push:
+    branches: [master, ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [master]
+  schedule:
+    - cron: '0 17 * * 2'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+      with:
+        # We must fetch at least the immediate parents so that if this is
+        # a pull request then we can checkout the head.
+        fetch-depth: 2
+
+    # If this run was triggered by a pull request event, then checkout
+    # the head of the pull request instead of the merge commit.
+    - run: git checkout HEAD^2
+      if: ${{ github.event_name == 'pull_request' }}
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      # Override language selection by uncommenting this and choosing your languages
+      # with:
+      #   languages: go, javascript, csharp, python, cpp, java
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v1
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

occystrap-0.4.0/.github/workflows/functional-tests.yml

@@ -0,0 +1,79 @@
+name: Functional tests
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  functional:
+    runs-on: self-hosted
+    timeout-minutes: 120
+
+    # NOTE(mikal): git repos are checked out to /srv/github/_work/{repo}/{repo}
+    # which is available as GITHUB_WORKSPACE. You can find other environment
+    # variables at https://docs.github.com/en/actions/learn-github-actions/environment-variables
+
+    steps:
+      - name: Remove previous unfinished runs
+        uses: n1hility/cancel-previous-runs@v2
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Get rid of sudo error messages
+        run: |
+          hostname=$(cat /etc/hostname)
+          sudo chmod ugo+rw /etc/hosts
+          echo "127.0.1.1 $hostname" >> /etc/hosts
+
+      - name: Install minimum dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get dist-upgrade -y
+          sudo apt-get install -y python3 python3-dev python3-pip python3-wheel apparmor docker.io runc
+          sudo systemctl restart apparmor
+
+      - name: Log docker setup
+        run: |
+          sudo chown debian.debian /var/run/docker.sock
+
+          echo "/var/run/docker.sock:"
+          sudo ls -l /var/run/docker.sock
+
+      - name: Checkout occystrap
+        uses: actions/checkout@v3
+        with:
+          path: occystrap
+          fetch-depth: 0
+
+      - name: Build occystrap wheel and install it
+        run: |
+          cd /srv/ci/runner/_work/occystrap/occystrap/occystrap
+          rm -f dist/*
+          python3 setup.py sdist bdist_wheel
+          sudo pip3 install dist/occystrap*.whl
+
+      - name: Run a local docker registry to talk to, and populate it with test data
+        run: |
+          docker run -d -p 5000:5000 --restart=always --name registry registry:2
+          cd /srv/ci/runner/_work/occystrap/occystrap/occystrap/deploy/occystrap_ci/testdata
+
+          for img in deletion_layers; do
+            cd $img
+            docker build -t localhost:5000/occystrap_$img:latest .
+            docker push localhost:5000/occystrap_$img:latest
+            cd /srv/ci/runner/_work/occystrap/occystrap/occystrap/deploy/occystrap_ci/testdata
+          done
+
+      - name: Run functional tests
+        run: |
+          cd /srv/ci/runner/_work/occystrap/occystrap/occystrap/deploy
+          sudo pip3 install -r requirements.txt
+          sudo pip3 install -r test-requirements.txt
+
+          # This needs to run as root because some of the tests require
+          # escalated permissions.
+          sudo stestr run --concurrency=5

occystrap-0.4.0/.github/workflows/python-unit-tests.yml

@@ -0,0 +1,33 @@
+# This workflow will install Python dependencies, run tests and lint with a single version of Python
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: Python application
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code with two commits
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 2
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          sudo apt-get update
+          sudo apt-get dist-upgrade -y
+          sudo apt-get install -y -q tox
+
+      - name: Lint with flake8
+        run: |
+          tox -eflake8

{occystrap-0.2.0/occystrap.egg-info → occystrap-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: occystrap
-Version: 0.2.0
+Version: 0.4.0
 Summary: occystrap: docker and OCI container tools
 Home-page: https://github.com/shakenfist/occystrap
 Author: Michael Still
@@ -84,6 +84,33 @@ Description: # Occy Strap
         
         Note that layers delete files from previous layers with files named ".wh.$previousfilename". These files are _not_ processed in the expanded layers, so that they are visible to the user. They are however processed in the merged layer named for the manifest file.
         
+        ## Generating an OCI runtime bundle
+        
+        This isn't fully supported yet, but you can extract an image to an OCI image bundle
+        with the following command:
+        
+        ```
+        occystrap fetch-to-oci registry-1.docker.io library/hello-world latest bar
+        ```
+        
+        You should then be able to run that container by doing something like:
+        
+        ```
+        cd bar
+        sudo apt-get install runc
+        sudo runc run id-0001
+        ```
+        
+        ## Supporting non-default architectures
+        
+        Docker image repositories can store multiple versions of a single image, with each image corresponding to a different (operating system, cpu architecture, cpu variant) tuple. Occy Strap supports letting you specify which to use with global command line flags. Occy Strap defaults to linux amd64 if you don't specify something different. For example, to fetch the linux arm64 v8 image for busybox, you would run:
+        
+        ```
+        occystrap --os linux --architecture arm64 --variant v8 \
+            fetch-to-extracted registry-1.docker.io library/busybox \
+            latest busybox
+        ```
+        
         
 Platform: UNKNOWN
 Classifier: Intended Audience :: Information Technology

{occystrap-0.2.0 → occystrap-0.4.0}/README.md

@@ -75,3 +75,30 @@ occystrap fetch-to-extracted --expand quay.io \
 ```
 
 Note that layers delete files from previous layers with files named ".wh.$previousfilename". These files are _not_ processed in the expanded layers, so that they are visible to the user. They are however processed in the merged layer named for the manifest file.
+
+## Generating an OCI runtime bundle
+
+This isn't fully supported yet, but you can extract an image to an OCI image bundle
+with the following command:
+
+```
+occystrap fetch-to-oci registry-1.docker.io library/hello-world latest bar
+```
+
+You should then be able to run that container by doing something like:
+
+```
+cd bar
+sudo apt-get install runc
+sudo runc run id-0001
+```
+
+## Supporting non-default architectures
+
+Docker image repositories can store multiple versions of a single image, with each image corresponding to a different (operating system, cpu architecture, cpu variant) tuple. Occy Strap supports letting you specify which to use with global command line flags. Occy Strap defaults to linux amd64 if you don't specify something different. For example, to fetch the linux arm64 v8 image for busybox, you would run:
+
+```
+occystrap --os linux --architecture arm64 --variant v8 \
+    fetch-to-extracted registry-1.docker.io library/busybox \
+    latest busybox
+```

occystrap-0.4.0/deploy/.stestr.conf

@@ -0,0 +1,3 @@
+[DEFAULT]
+test_path=./occystrap_ci/tests
+top_dir=./

occystrap-0.4.0/deploy/ansible/ci.yml

@@ -0,0 +1,211 @@
+- hosts: localhost
+  gather_facts: yes
+  connection: ssh
+  vars:
+    identifier: unknown
+    source_path: "/home/jenkins/src/occystrap/"
+    base_image: "debian:11"
+    base_image_user: "debian"
+
+  tasks:
+    - name: Create the namespace
+      shell: sf-client namespace create "{{identifier}}"
+
+    - name: Create a network for outgoing traffic
+      sf_network:
+        netblock: "10.0.0.0/24"
+        name: "public"
+        namespace: "{{identifier}}"
+      register: publicnetwork
+
+    - name: Log network details
+      debug:
+        msg:
+          - "Public network is {{publicnetwork['meta']['uuid']}}"
+
+    - name: Setup cloud-config to add a password to CI nodes for debugging
+      set_fact:
+        userdata_decoded: |
+          #cloud-config
+          ssh_pwauth: True
+          chpasswd: { expire: False }
+
+          users:
+            - name: ciuser
+              lock-passwd: False
+              sudo: [ "ALL=(ALL) NOPASSWD:ALL" ]
+              shell: /bin/bash
+              # This password hash is generated with mkpasswd --method=SHA-512 --rounds=4096
+              passwd: "$6$rounds=4096$jBqTFFRr$75ehRH5AtiUxWMlFf1Ji.szp1NCjut2WiiD./QHsfWbCfKKjX1bs6MtBcWedlqKwcBKEPP/oLinTbH6gwcwUA1"
+
+    - name: Encode user data
+      set_fact:
+        userdata_encoded: "{{ userdata_decoded | b64encode }}"
+
+    - name: Create a primary instance
+      sf_instance:
+        name: "primary"
+        cpu: 4
+        ram: 4096
+        disks:
+          - "100@{{base_image}}"
+        networkspecs:
+          - network_uuid="{{publicnetwork['meta']['uuid']}}",float=True
+        ssh_key: "{{ lookup('file', '/home/jenkins/id_ci.pub') }}"
+        namespace: "{{identifier}}"
+        user_data: "{{userdata_encoded}}"
+        state: present
+      register: primary
+
+    - name: Add primary node to ansible
+      add_host:
+        hostname: primary
+        egress_ip: "{{primary['meta']['network_interfaces'][0]['ipv4']}}"
+        ansible_host: "{{primary['meta']['network_interfaces'][0]['floating']}}"
+        ansible_ssh_host: "{{primary['meta']['network_interfaces'][0]['floating']}}"
+        ansible_ssh_extra_args: "-o StrictHostKeyChecking=no -o GlobalKnownHostsFile=/dev/null -o UserKnownHostsFile=/dev/null"
+        ansible_ssh_user: "{{base_image_user}}"
+        ansible_ssh_private_key_file: "/home/jenkins/id_ci"
+        groups: occystrapall
+
+    - name: Log instance details
+      debug:
+        msg:
+          - "Primary: is {{primary['meta']['uuid']}} at {{primary['meta']['network_interfaces'][0]['ipv4']}}, {{primary['meta']['network_interfaces'][0]['floating']}}"
+
+    - name: Ensure we have somewhere to write the build details to
+      file:
+        path: "{{ lookup('env', 'WORKSPACE') }}/{{ lookup('env', 'BUILD_TAG') }}"
+        state: directory
+        mode: "0755"
+
+    - name: Write details of instances to workspace
+      copy:
+        content: |
+          {% for svr in groups.occystrapall %}
+          export {{ svr | replace('-', '_') }}={{hostvars[svr]['ansible_ssh_host']}}
+          {% endfor %}
+
+          export identifier={{identifier}}
+          export source_path={{source_path}}
+          export nodes="{{ groups.occystrapall | join(' ') | replace('-', '_') }}"
+        dest: "{{ lookup('env', 'WORKSPACE') }}/{{ lookup('env', 'BUILD_TAG') }}/ci-environment.sh"
+        owner: jenkins
+        group: jenkins
+        mode: u=r,g=r,o=r
+
+    - name: Log environment details path
+      debug:
+        msg: "Environment details written to {{ lookup('env', 'WORKSPACE') }}/{{ lookup('env', 'BUILD_TAG') }}/ci-environment.sh"
+
+    - name: Build Occystrap wheel
+      shell: |
+        rm dist/*
+        python3 setup.py sdist bdist_wheel
+      args:
+        chdir: "{{source_path}}"
+
+    - name: Determine wheel filename
+      shell: ls dist | egrep "occystrap.*\.whl"
+      args:
+        chdir: "{{source_path}}"
+      register: occystrap_wheel_file_complex
+
+    - name: Extract the wheel filename
+      set_fact:
+        occystrap_wheel_file: "{{occystrap_wheel_file_complex.stdout}}"
+
+    - debug:
+        msg: "Occystrap wheel file: {{occystrap_wheel_file}}"
+
+    - name: Wait for all instances to present an "OpenSSH" prompt
+      wait_for:
+        port: 22
+        host: "{{hostvars[item]['ansible_ssh_host']}}"
+        search_regex: OpenSSH
+        delay: 10
+      with_items: "{{ groups['occystrapall'] }}"
+
+- hosts: primary
+  gather_facts: yes
+  become: true
+  vars:
+    source_path: "/home/jenkins/src/occystrap/"
+
+  tasks:
+    - name: Install minimum dependencies
+      apt:
+        name:
+          - git
+          - python3
+          - python3-dev
+          - python3-pip
+          - python3-wheel
+          - apparmor
+          - docker.io
+          - runc
+        state: latest
+
+    - name: Restart apparmor
+      service:
+        name: apparmor
+        enabled: yes
+        state: restarted
+
+    - name: Copy occystrap wheel file
+      copy:
+        src: "{{source_path}}/dist/{{hostvars['localhost']['occystrap_wheel_file']}}"
+        dest: "/root/{{hostvars['localhost']['occystrap_wheel_file']}}"
+
+    - name: Install Occystrap
+      shell: pip3 install "/root/{{hostvars['localhost']['occystrap_wheel_file']}}"
+
+    - name: Run a local docker registry for us to talk to
+      shell: docker run -d -p 5000:5000 --restart=always --name registry registry:2
+
+    - name: Make a /srv/occystrap directory
+      file:
+        path: /srv/occystrap
+        state: directory
+        owner: "{{base_image_user}}"
+        group: "{{base_image_user}}"
+
+- hosts: primary
+  gather_facts: yes
+  become: no
+  vars:
+    source_path: "/home/jenkins/src/occystrap/"
+
+  tasks:
+    - name: Copy occystrap CI tests and data as the default user
+      synchronize:
+        src: "{{source_path}}"
+        dest: "/srv/occystrap/"
+        use_ssh_args: yes
+        recursive: yes
+
+- hosts: primary
+  gather_facts: yes
+  become: true
+  vars:
+    source_path: "/home/jenkins/src/occystrap/"
+
+  tasks:
+    - name: Populate the registry with test images
+      shell: |
+        cd {{item}}
+        docker build -t localhost:5000/occystrap_{{item}}:latest .
+        docker push localhost:5000/occystrap_{{item}}:latest
+      args:
+        chdir: /srv/occystrap/deploy/occystrap_ci/testdata
+      with_items:
+        - deletion_layers
+
+    - name: Replace a localhost entry with our own
+      lineinfile:
+        path: /etc/hosts
+        regexp: '^127\.0\.0\.1'
+        line: 127.0.0.1 localhost primary
+        owner: root
+        group: root
+        mode: "0644"

occystrap-0.4.0/deploy/occystrap_ci/testdata/deletion_layers/Dockerfile

@@ -0,0 +1,10 @@
+FROM ubuntu
+RUN echo "test" > file
+RUN mkdir directory
+RUN echo "test" > directory/directoryfile
+RUN echo "test" > anotherfile
+RUN mkdir anotherdirectory
+RUN echo "test" > anotherdirectory/anotherfile
+
+RUN rm file
+RUN rm -rf directory

occystrap-0.4.0/deploy/occystrap_ci/tests/test_dir_deep_images.py

@@ -0,0 +1,33 @@
+import logging
+import os
+import tempfile
+import testtools
+
+
+from occystrap import docker_registry
+from occystrap import output_directory
+
+
+logging.basicConfig(level=logging.INFO, format='%(message)s')
+LOG = logging.getLogger()
+
+
+class DirDeepImageTestCase(testtools.TestCase):
+    def test_deep_image(self):
+        image = 'library/ubuntu'
+        tag = 'latest'
+
+        with tempfile.TemporaryDirectory() as tempdir:
+            oci = output_directory.DirWriter(
+                image, tag, tempdir, expand=True)
+            img = docker_registry.Image(
+                'registry-1.docker.io', image, tag, 'linux', 'amd64', '')
+            for image_element in img.fetch(fetch_callback=oci.fetch_callback):
+                oci.process_image_element(*image_element)
+            oci.finalize()
+            oci.write_bundle()
+
+            self.assertTrue(
+                os.path.exists(os.path.join(tempdir, 'manifest')))
+            self.assertTrue(
+                os.path.exists(os.path.join(tempdir, 'manifest/usr/bin/dash')))

occystrap-0.4.0/deploy/occystrap_ci/tests/test_oci_hello_world.py

@@ -0,0 +1,52 @@
+import logging
+from oslo_concurrency import processutils
+import sys
+import tempfile
+import testtools
+
+
+from occystrap import docker_registry
+from occystrap import output_ocibundle
+
+
+logging.basicConfig(level=logging.INFO, format='%(message)s')
+LOG = logging.getLogger()
+
+
+def _exec(cmd, cwd=None):
+    sys.stderr.write('\n----- Exec: %s -----\n' % cmd)
+    out, err = processutils.execute(cmd, cwd=cwd, shell=True)
+    for line in out.split('\n'):
+        sys.stderr.write('out: %s\n' % line)
+    sys.stderr.write('\n')
+    for line in err.split('\n'):
+        sys.stderr.write('err: %s\n' % line)
+    sys.stderr.write('\n----- End: %s -----\n' % cmd)
+    return out, err
+
+
+# Make sure we haven't broken sudo entirely in the test suite
+class SudoTestCase(testtools.TestCase):
+    def test_sudo(self):
+        out, err = _exec('sudo echo $(( 4 + 6 ))')
+        self.assertEqual('', err)
+        self.assertTrue('10' in out)
+
+
+class OCIHelloWorldTestCase(testtools.TestCase):
+    def test_hello_world(self):
+        image = 'library/hello-world'
+        tag = 'latest'
+
+        with tempfile.TemporaryDirectory() as tempdir:
+            oci = output_ocibundle.OCIBundleWriter(image, tag, tempdir)
+            img = docker_registry.Image(
+                'registry-1.docker.io', image, tag, 'linux', 'amd64', '')
+            for image_element in img.fetch(fetch_callback=oci.fetch_callback):
+                oci.process_image_element(*image_element)
+            oci.finalize()
+            oci.write_bundle()
+
+            out, err = _exec('sudo runc run OCIHellWorld', cwd=tempdir)
+            self.assertEqual('', err)
+            self.assertTrue('Hello from Docker!' in out)

occystrap-0.4.0/deploy/occystrap_ci/tests/test_whiteout.py

@@ -0,0 +1,80 @@
+
+import logging
+import os
+from oslo_concurrency import processutils
+import tempfile
+import testtools
+
+
+from occystrap import docker_registry
+from occystrap import output_ocibundle
+from occystrap import output_mounts
+
+
+logging.basicConfig(level=logging.INFO, format='%(message)s')
+LOG = logging.getLogger()
+
+
+class WhiteoutsTestCase(testtools.TestCase):
+    def test_whiteouts_ocibundle(self):
+        image = 'occystrap_deletion_layers'
+        tag = 'latest'
+
+        with tempfile.TemporaryDirectory() as tempdir:
+            oci = output_ocibundle.OCIBundleWriter(image, tag, tempdir)
+            img = docker_registry.Image(
+                'localhost:5000', image, tag, 'linux', 'amd64', '',
+                secure=False)
+            for image_element in img.fetch(fetch_callback=oci.fetch_callback):
+                oci.process_image_element(*image_element)
+            oci.finalize()
+            oci.write_bundle()
+
+            self.assertTrue(
+                os.path.exists(os.path.join(tempdir, 'rootfs')))
+            self.assertFalse(
+                os.path.exists(os.path.join(tempdir, 'rootfs/file')))
+            self.assertFalse(
+                os.path.exists(os.path.join(tempdir, 'rootfs/.wh.file')))
+            self.assertFalse(
+                os.path.exists(os.path.join(tempdir, 'rootfs/directory')))
+            self.assertTrue(
+                os.path.exists(os.path.join(tempdir, 'rootfs/anotherfile')))
+            self.assertTrue(
+                os.path.exists(os.path.join(tempdir, 'rootfs/anotherdirectory')))
+            self.assertFalse(
+                os.path.exists(os.path.join(tempdir,
+                                            'rootfs/anotherdirectory/.wh..wh..opq')))
+
+    def test_whiteouts_mounts(self):
+        image = 'occystrap_deletion_layers'
+        tag = 'latest'
+
+        with tempfile.TemporaryDirectory() as tempdir:
+            oci = output_mounts.MountWriter(image, tag, tempdir)
+            img = docker_registry.Image(
+                'localhost:5000', image, tag, 'linux', 'amd64', '',
+                secure=False)
+            for image_element in img.fetch(fetch_callback=oci.fetch_callback):
+                oci.process_image_element(*image_element)
+            oci.finalize()
+            oci.write_bundle()
+
+            self.assertTrue(
+                os.path.exists(os.path.join(tempdir, 'rootfs')))
+            self.assertFalse(
+                os.path.exists(os.path.join(tempdir, 'rootfs/file')))
+            self.assertFalse(
+                os.path.exists(os.path.join(tempdir, 'rootfs/.wh.file')))
+            self.assertFalse(
+                os.path.exists(os.path.join(tempdir, 'rootfs/directory')))
+            self.assertTrue(
+                os.path.exists(os.path.join(tempdir, 'rootfs/anotherfile')))
+            self.assertTrue(
+                os.path.exists(os.path.join(tempdir, 'rootfs/anotherdirectory')))
+            self.assertFalse(
+                os.path.exists(os.path.join(tempdir,
+                                            'rootfs/anotherdirectory/.wh..wh..opq')))
+
+            processutils.execute('umount %s' % os.path.join(tempdir, 'rootfs'),
+                                 shell=True)

occystrap-0.4.0/deploy/requirements.txt

@@ -0,0 +1 @@
+oslo.concurrency            # apache2

occystrap-0.4.0/deploy/setup.cfg

@@ -0,0 +1,24 @@
+[metadata]
+name = occystrap_ci
+summary = Occy Strap CI: test and validation for Occy Strap
+license = Apache2
+author = Michael Still
+author-email = mikal@stillhq.com
+home-page = https://www.madebymikal.com/category/occy-strap/
+classifier =
+    Intended Audience :: Information Technology
+    Intended Audience :: System Administrators
+    License :: OSI Approved :: Apache Software License
+    Operating System :: POSIX :: Linux
+    Programming Language :: Python
+    Programming Language :: Python :: 3
+    Programming Language :: Python :: 3.7
+
+[files]
+packages =
+    occystrap_ci
+
+[entry_points]
+
+[pbr]
+skip_changelog = True

occystrap-0.4.0/deploy/setup.py

@@ -0,0 +1,20 @@
+# Copyright (c) 2013 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import setuptools
+
+setuptools.setup(
+    setup_requires=['pbr'],
+    pbr=True)