teuthology 1.0.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

teuthology/task/internal/__init__.py

@@ -4,22 +4,29 @@ Note that there is no corresponding task defined for this module.  All of
 the calls are made from other modules, most notably teuthology/run.py
 """
 import contextlib
+import functools
+import gzip
 import logging
 import os
+import shutil
 import time
 import yaml
 import subprocess
+import tempfile
+import re
+import humanfriendly
 
 import teuthology.lock.ops
-from teuthology import misc
-from teuthology.packaging import get_builder_project
+from teuthology import misc, packaging
 from teuthology import report
 from teuthology.config import config as teuth_config
-from teuthology.exceptions import VersionNotFoundError
+from teuthology.exceptions import ConfigError, VersionNotFoundError
 from teuthology.job_status import get_status, set_status
 from teuthology.orchestra import cluster, remote, run
 # the below import with noqa is to workaround run.py which does not support multilevel submodule import
-from teuthology.task.internal.redhat import setup_cdn_repo, setup_base_repo, setup_additional_repo, setup_stage_cdn  # noqa
+from teuthology.task.internal.redhat import (setup_cdn_repo, setup_base_repo,            # noqa
+                                             setup_additional_repo,                      # noqa
+                                             setup_stage_cdn, setup_container_registry)  # noqa
 
 log = logging.getLogger(__name__)
 
@@ -81,7 +88,7 @@ def check_packages(ctx, config):
     # We can only do this check if there are a defined sha1 and os_type
     # in the job config.
     if os_type and sha1:
-        package = get_builder_project()("ceph", ctx.config)
+        package = packaging.get_builder_project()("ceph", ctx.config)
         template = "Checking packages for os_type '{os}', " \
             "flavor '{flav}' and ceph hash '{ver}'"
         log.info(
@@ -265,7 +272,7 @@ def check_ceph_data(ctx, config):
         try:
             proc.wait()
         except run.CommandFailedError:
-            log.error('Host %s has stale /var/lib/ceph, check lock and nuke/cleanup.', proc.remote.shortname)
+            log.error('Host %s has stale /var/lib/ceph!', proc.remote.shortname)
             failed = True
     if failed:
         raise RuntimeError('Stale /var/lib/ceph detected, aborting.')
@@ -310,7 +317,29 @@ def fetch_binaries_for_coredumps(path, remote):
             # Parse file output to get program, Example output:
             # 1422917770.7450.core: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, \
             # from 'radosgw --rgw-socket-path /home/ubuntu/cephtest/apache/tmp.client.0/fastcgi_soc'
-            dump_program = dump_out.split("from '")[1].split(' ')[0]
+            log.info(f' core looks like: {dump_out}')
+
+            if 'gzip' in dump_out:
+                try:
+                    log.info("core is compressed, try accessing gzip file ...")
+                    with gzip.open(dump_path, 'rb') as f_in, \
+                         tempfile.NamedTemporaryFile(mode='w+b') as f_out:
+                         shutil.copyfileobj(f_in, f_out)
+                         dump_info = subprocess.Popen(['file', f_out.name],
+                                                        stdout=subprocess.PIPE)
+                         dump_out = dump_info.communicate()[0].decode()
+                         log.info(f' core looks like: {dump_out}')
+                except Exception as e:
+                    log.info('Something went wrong while opening the compressed file')
+                    log.error(e)
+                    continue
+            try:
+                dump_program = re.findall("from '([^']+)'", dump_out)[0]
+                log.info(f' dump_program: {dump_program}')
+            except Exception as e:
+                log.info("core doesn't have the desired format, moving on ...")
+                log.error(e)
+                continue
 
             # Find path on remote server:
             remote_path = remote.sh(['which', dump_program]).rstrip()
@@ -334,6 +363,14 @@ def fetch_binaries_for_coredumps(path, remote):
             remote.get_file(debug_path, coredump_path)
 
 
+def gzip_if_too_large(compress_min_size, src, tarinfo, local_path):
+    if tarinfo.size >= compress_min_size:
+        with gzip.open(local_path + '.gz', 'wb') as dest:
+            shutil.copyfileobj(src, dest)
+    else:
+        misc.copy_fileobj(src, tarinfo, local_path)
+
+
 @contextlib.contextmanager
 def archive(ctx, config):
     """
@@ -348,6 +385,9 @@ def archive(ctx, config):
         )
     )
 
+    # Add logs directory to job's info log file
+    misc.add_remote_path(ctx, 'init', archive_dir)
+
     try:
         yield
     except Exception:
@@ -364,7 +404,18 @@ def archive(ctx, config):
                 os.mkdir(logdir)
             for rem in ctx.cluster.remotes.keys():
                 path = os.path.join(logdir, rem.shortname)
-                misc.pull_directory(rem, archive_dir, path)
+                min_size_option = ctx.config.get('log-compress-min-size',
+                                                 '128MB')
+                try:
+                    compress_min_size_bytes = \
+                        humanfriendly.parse_size(min_size_option)
+                except humanfriendly.InvalidSize:
+                    msg = 'invalid "log-compress-min-size": {}'.format(min_size_option)
+                    log.error(msg)
+                    raise ConfigError(msg)
+                maybe_compress = functools.partial(gzip_if_too_large,
+                                                   compress_min_size_bytes)
+                misc.pull_directory(rem, archive_dir, path, maybe_compress)
                 # Check for coredumps and pull binaries
                 fetch_binaries_for_coredumps(path, rem)
 
@@ -414,9 +465,10 @@ def coredump(ctx, config):
     Stash a coredump of this system if an error occurs.
     """
     log.info('Enabling coredump saving...')
+    cluster = ctx.cluster.filter(lambda r: not r.is_container)
     archive_dir = misc.get_archive_dir(ctx)
     run.wait(
-        ctx.cluster.run(
+        cluster.run(
             args=[
                 'install', '-d', '-m0755', '--',
                 '{adir}/coredump'.format(adir=archive_dir),
@@ -435,11 +487,17 @@ def coredump(ctx, config):
     try:
         yield
     finally:
+        cluster = ctx.cluster.filter(lambda r: not r.is_container)
         run.wait(
-            ctx.cluster.run(
+            cluster.run(
                 args=[
                     'sudo', 'sysctl', '-w', 'kernel.core_pattern=core',
                     run.Raw('&&'),
+                    'sudo', 'bash', '-c',
+                    (f'for f in `find {archive_dir}/coredump -type f`; do '
+                     'file $f | grep -q systemd-sysusers && rm $f || true ; '
+                     'done'),
+                    run.Raw('&&'),
                     # don't litter the archive dir if there were no cores dumped
                     'rmdir',
                     '--ignore-fail-on-non-empty',
@@ -452,7 +510,7 @@ def coredump(ctx, config):
 
         # set status = 'fail' if the dir is still there = coredumps were
         # seen
-        for rem in ctx.cluster.remotes.keys():
+        for rem in cluster.remotes.keys():
             try:
                 rem.sh("test -e " + archive_dir + "/coredump")
             except run.CommandFailedError:

teuthology/task/internal/edit_sudoers.sh

@@ -0,0 +1,10 @@
+#! /bin/sh
+
+sudo vi -e /etc/sudoers <<EOF
+g/  requiretty/s// !requiretty/
+g/ !visiblepw/s//  visiblepw/
+w!
+q
+EOF
+exit
+

teuthology/task/internal/lock_machines.py

@@ -1,17 +1,9 @@
 import contextlib
 import logging
-import time
-import yaml
 
 import teuthology.lock.ops
 import teuthology.lock.query
 import teuthology.lock.util
-from teuthology import misc
-from teuthology import provision
-from teuthology import report
-
-from teuthology.config import config as teuth_config
-from teuthology.job_status import get_status, set_status
 
 log = logging.getLogger(__name__)
 
@@ -23,137 +15,15 @@ def lock_machines(ctx, config):
     new machines.  This is not called if the one has teuthology-locked
     machines and placed those keys in the Targets section of a yaml file.
     """
-    # It's OK for os_type and os_version to be None here.  If we're trying
-    # to lock a bare metal machine, we'll take whatever is available.  If
-    # we want a vps, defaults will be provided by misc.get_distro and
-    # misc.get_distro_version in provision.create_if_vm
-    os_type = ctx.config.get("os_type")
-    os_version = ctx.config.get("os_version")
-    arch = ctx.config.get('arch')
-    log.info('Locking machines...')
     assert isinstance(config[0], int), 'config[0] must be an integer'
     machine_type = config[1]
     total_requested = config[0]
     # We want to make sure there are always this many machines available
-    reserved = teuth_config.reserve_machines
-    assert isinstance(reserved, int), 'reserve_machines must be integer'
-    assert (reserved >= 0), 'reserve_machines should >= 0'
-
-    # change the status during the locking process
-    report.try_push_job_info(ctx.config, dict(status='waiting'))
-
-    all_locked = dict()
-    requested = total_requested
-    while True:
-        # get a candidate list of machines
-        machines = teuthology.lock.query.list_locks(machine_type=machine_type, up=True,
-                                                    locked=False, count=requested + reserved)
-        if machines is None:
-            if ctx.block:
-                log.error('Error listing machines, trying again')
-                time.sleep(20)
-                continue
-            else:
-                raise RuntimeError('Error listing machines')
-
-        # make sure there are machines for non-automated jobs to run
-        if len(machines) < reserved + requested and ctx.owner.startswith('scheduled'):
-            if ctx.block:
-                log.info(
-                    'waiting for more %s machines to be free (need %s + %s, have %s)...',
-                    machine_type,
-                    reserved,
-                    requested,
-                    len(machines),
-                )
-                time.sleep(10)
-                continue
-            else:
-                assert 0, ('not enough machines free; need %s + %s, have %s' %
-                           (reserved, requested, len(machines)))
-
-        try:
-            newly_locked = teuthology.lock.ops.lock_many(ctx, requested, machine_type,
-                                                         ctx.owner, ctx.archive, os_type,
-                                                         os_version, arch)
-        except Exception:
-            # Lock failures should map to the 'dead' status instead of 'fail'
-            set_status(ctx.summary, 'dead')
-            raise
-        all_locked.update(newly_locked)
-        log.info(
-            '{newly_locked} {mtype} machines locked this try, '
-            '{total_locked}/{total_requested} locked so far'.format(
-                newly_locked=len(newly_locked),
-                mtype=machine_type,
-                total_locked=len(all_locked),
-                total_requested=total_requested,
-            )
-        )
-        if len(all_locked) == total_requested:
-            vmlist = []
-            for lmach in all_locked:
-                if teuthology.lock.query.is_vm(lmach):
-                    vmlist.append(lmach)
-            if vmlist:
-                log.info('Waiting for virtual machines to come up')
-                keys_dict = dict()
-                loopcount = 0
-                while len(keys_dict) != len(vmlist):
-                    loopcount += 1
-                    time.sleep(10)
-                    keys_dict = misc.ssh_keyscan(vmlist)
-                    log.info('virtual machine is still unavailable')
-                    if loopcount == 40:
-                        loopcount = 0
-                        log.info('virtual machine(s) still not up, ' +
-                                 'recreating unresponsive ones.')
-                        for guest in vmlist:
-                            if guest not in keys_dict.keys():
-                                log.info('recreating: ' + guest)
-                                full_name = misc.canonicalize_hostname(guest)
-                                provision.destroy_if_vm(ctx, full_name)
-                                provision.create_if_vm(ctx, full_name)
-                if teuthology.lock.ops.do_update_keys(keys_dict)[0]:
-                    log.info("Error in virtual machine keys")
-                newscandict = {}
-                for dkey in all_locked.keys():
-                    stats = teuthology.lock.query.get_status(dkey)
-                    newscandict[dkey] = stats['ssh_pub_key']
-                ctx.config['targets'] = newscandict
-            else:
-                ctx.config['targets'] = all_locked
-            locked_targets = yaml.safe_dump(
-                ctx.config['targets'],
-                default_flow_style=False
-            ).splitlines()
-            log.info('\n  '.join(['Locked targets:', ] + locked_targets))
-            # successfully locked machines, change status back to running
-            report.try_push_job_info(ctx.config, dict(status='running'))
-            break
-        elif not ctx.block:
-            assert 0, 'not enough machines are available'
-        else:
-            requested = requested - len(newly_locked)
-            assert requested > 0, "lock_machines: requested counter went" \
-                                  "negative, this shouldn't happen"
-
-        log.info(
-            "{total} machines locked ({new} new); need {more} more".format(
-                total=len(all_locked), new=len(newly_locked), more=requested)
-        )
-        log.warn('Could not lock enough machines, waiting...')
-        time.sleep(10)
+    teuthology.lock.ops.block_and_lock_machines(ctx, total_requested, machine_type)
     try:
         yield
     finally:
-        # If both unlock_on_failure and nuke-on-error are set, don't unlock now
-        # because we're just going to nuke (and unlock) later.
-        unlock_on_failure = (
-            ctx.config.get('unlock_on_failure', False)
-            and not ctx.config.get('nuke-on-error', False)
-        )
-        if get_status(ctx.summary) == 'pass' or unlock_on_failure:
+        if ctx.config.get("unlock_on_failure", True):
             log.info('Unlocking machines...')
             for machine in ctx.config['targets'].keys():
-                teuthology.lock.ops.unlock_one(ctx, machine, ctx.owner, ctx.archive)
+                teuthology.lock.ops.unlock_one(machine, ctx.owner, ctx.archive)

teuthology/task/internal/redhat.py

@@ -4,6 +4,8 @@ Internal tasks  for redhat downstream builds
 import contextlib
 import logging
 import requests
+import yaml
+import os
 from tempfile import NamedTemporaryFile
 from teuthology.config import config as teuthconfig
 from teuthology.parallel import parallel
@@ -19,10 +21,16 @@ def setup_stage_cdn(ctx, config):
     """
     Configure internal stage cdn
     """
+    suite_path = ctx.config.get('suite_path')
+    if not suite_path:
+        raise ConfigError("suite_path missing")
+    teuthconfig.suite_path = suite_path
+
     rhbuild = ctx.config.get('redhat').get('rhbuild')
     if not rhbuild:
-        raise ConfigError("Provide rhbuild attribute")
+        raise ConfigError('Provide rhbuild attribute')
     teuthconfig.rhbuild = str(rhbuild)
+
     with parallel() as p:
         for remote in ctx.cluster.remotes.keys():
             if remote.os.name == 'rhel':
@@ -41,7 +49,7 @@ def _subscribe_stage_cdn(remote):
     cdn_config = teuthconfig.get('cdn-config', dict())
     server_url = cdn_config.get('server-url', 'subscription.rhsm.stage.redhat.com:443/subscription')
     base_url = cdn_config.get('base-url', 'https://cdn.stage.redhat.com')
-    username = cdn_config.get('username', 'cephuser')
+    username = cdn_config.get('username')
     password = cdn_config.get('password')
     remote.run(
         args=[
@@ -80,6 +88,36 @@ def setup_cdn_repo(ctx, config):
         set_cdn_repo(ctx, config)
     yield
 
+@contextlib.contextmanager
+def setup_container_registry(ctx, config):
+    """
+     setup container registry if setup_container_registry in config
+
+     redhat:
+      setup_container_registry: <registry.io>   # registry-name
+    """
+    if ctx.config.get('redhat').get('setup_container_registry', None):
+        registry = ctx.config['redhat']['setup_container_registry']
+
+        # fetch credentials from teuth_config
+        creds = teuthconfig.get('registries', dict()).get(registry)
+        if not creds:
+            raise ConfigError("Registry not found....")
+
+        # container-tool login
+        for remote in ctx.cluster.remotes.keys():
+            container_tool = "podman"
+            if remote.os.version.startswith('7'):
+                container_tool = "docker"
+
+            remote.run(args=[
+                'sudo', container_tool,
+                'login', registry,
+                '--username', creds['username'],
+                '--password', creds['password'],
+                ]
+            )
+    yield
 
 @contextlib.contextmanager
 def setup_additional_repo(ctx, config):
@@ -101,29 +139,20 @@ def setup_additional_repo(ctx, config):
 
 
 def _enable_rhel_repos(remote):
-    rhel_7_rpms = ['rhel-7-server-rpms',
-                   'rhel-7-server-optional-rpms',
-                   'rhel-7-server-extras-rpms']
-
-    rhel_8_rpms = ['rhel-8-for-x86_64-appstream-rpms',
-                   'rhel-8-for-x86_64-baseos-rpms',
-                   'ansible-2.8-for-rhel-8-x86_64-rpms']
 
-    if teuthconfig.rhbuild.startswith("3"):
-        rhel_7_rpms.append('rhel-7-server-ansible-2.6-rpms')
-    elif teuthconfig.rhbuild.startswith("4"):
-        rhel_7_rpms.append('rhel-7-server-ansible-2.8-rpms')
+    # Look for rh specific repos
+    ds_yaml = os.path.join(
+        teuthconfig.get('ds_yaml_dir'),
+        teuthconfig.rhbuild + ".yaml"
+    )
 
-    repos_to_subscribe = {'7': rhel_7_rpms,
-                          '8': rhel_8_rpms}
+    rhel_repos = yaml.safe_load(open(ds_yaml))
+    repos_to_subscribe = rhel_repos.get('rhel_repos').get(remote.os.version[0])
 
-    for repo in repos_to_subscribe.get(remote.os.version[0]):
+    for repo in repos_to_subscribe:
         remote.run(args=['sudo', 'subscription-manager',
                          'repos', '--enable={r}'.format(r=repo)])
 
-    if remote.os.version.startswith('8'):
-        workaround(remote)
-
 
 @contextlib.contextmanager
 def setup_base_repo(ctx, config):
@@ -236,12 +265,3 @@ def _create_temp_repo_file(repos, repo_file):
         repo_file.write(gpgcheck)
         repo_file.write(enabled)
     repo_file.close()
-
-
-def workaround(remote):
-    log.info('temporary workaround')
-    remote.run(args=['sudo',
-                     'yum',
-                     'install', '-y',
-                     'http://download-ib01.fedoraproject.org/pub/epel/7/x86_64/Packages/d/dbench-4.0-10.el7.x86_64.rpm'])
-

teuthology/task/internal/syslog.py

@@ -22,12 +22,17 @@ def syslog(ctx, config):
         yield
         return
 
+    cluster = ctx.cluster.filter(lambda r: not r.is_container)
+    if not len(cluster.remotes.keys()):
+        yield
+        return
+
     log.info('Starting syslog monitoring...')
 
     archive_dir = misc.get_archive_dir(ctx)
     log_dir = '{adir}/syslog'.format(adir=archive_dir)
     run.wait(
-        ctx.cluster.run(
+        cluster.run(
             args=['mkdir', '-p', '-m0755', '--', log_dir],
             wait=False,
         )
@@ -43,7 +48,7 @@ def syslog(ctx, config):
     ]
     conf_fp = BytesIO('\n'.join(conf_lines).encode())
     try:
-        for rem in ctx.cluster.remotes.keys():
+        for rem in cluster.remotes.keys():
             log_context = 'system_u:object_r:var_log_t:s0'
             for log_path in (kern_log, misc_log):
                 rem.run(args=['install', '-m', '666', '/dev/null', log_path])
@@ -55,7 +60,7 @@ def syslog(ctx, config):
             )
             conf_fp.seek(0)
         run.wait(
-            ctx.cluster.run(
+            cluster.run(
                 args=[
                     'sudo',
                     'service',
@@ -70,10 +75,14 @@ def syslog(ctx, config):
 
         yield
     finally:
+        cluster = ctx.cluster.filter(lambda r: not r.is_container)
+        if not len(cluster.remotes.keys()):
+            return
+        
         log.info('Shutting down syslog monitoring...')
 
         run.wait(
-            ctx.cluster.run(
+            cluster.run(
                 args=[
                     'sudo',
                     'rm',
@@ -93,13 +102,13 @@ def syslog(ctx, config):
         # flush the file fully. oh well.
 
         log.info('Checking logs for errors...')
-        for rem in ctx.cluster.remotes.keys():
+        for rem in cluster.remotes.keys():
             log.debug('Checking %s', rem.name)
             stdout = rem.sh(
                 [
                     'egrep', '--binary-files=text',
                     '\\bBUG\\b|\\bINFO\\b|\\bDEADLOCK\\b',
-                    run.Raw('{adir}/syslog/*.log'.format(adir=archive_dir)),
+                    run.Raw(f'{archive_dir}/syslog/kern.log'),
                     run.Raw('|'),
                     'grep', '-v', 'task .* blocked for more than .* seconds',
                     run.Raw('|'),
@@ -152,7 +161,7 @@ def syslog(ctx, config):
 
         log.info('Compressing syslogs...')
         run.wait(
-            ctx.cluster.run(
+            cluster.run(
                 args=[
                     'find',
                     '{adir}/syslog'.format(adir=archive_dir),
@@ -169,5 +178,19 @@ def syslog(ctx, config):
                     '--',
                 ],
                 wait=False,
-            ),
+            )
+        )
+
+        log.info('Gathering journactl -b0...')
+        run.wait(
+            cluster.run(
+                args=[
+                    'sudo', 'journalctl', '-b0',
+                    run.Raw('|'),
+                    'gzip', '-9',
+                    run.Raw('>'),
+                    f'{archive_dir}/syslog/journalctl-b0.gz',
+                ],
+                wait=False,
+            )
         )