teuthology 1.0.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

teuthology/orchestra/run.py

@@ -9,7 +9,7 @@ from paramiko import ChannelFile
 import gevent
 import gevent.event
 import socket
-import pipes
+import shlex
 import logging
 import shutil
 
@@ -90,7 +90,7 @@ class RemoteProcess(object):
         Execute remote command
         """
         for line in self.command.split('\n'):
-            log.getChild(self.hostname).info('%s> %s' % (self.label or '', line))
+            log.getChild(self.hostname).debug('%s> %s' % (self.label or '', line))
 
         if hasattr(self, 'timeout'):
             (self._stdin_buf, self._stdout_buf, self._stderr_buf) = \
@@ -114,7 +114,7 @@ class RemoteProcess(object):
             # FIXME: Is this actually true?
             raise RuntimeError(self.deadlock_warning % 'stdin')
 
-    def setup_output_stream(self, stream_obj, stream_name):
+    def setup_output_stream(self, stream_obj, stream_name, quiet=False):
         if stream_obj is not PIPE:
             # Log the stream
             host_log = self.logger.getChild(self.hostname)
@@ -125,6 +125,7 @@ class RemoteProcess(object):
                     getattr(self, stream_name),
                     stream_log,
                     stream_obj,
+                    quiet,
                 )
             )
             setattr(self, stream_name, stream_obj)
@@ -251,14 +252,14 @@ def quote(args):
             if isinstance(a, Raw):
                 yield a.value
             else:
-                yield pipes.quote(a)
+                yield shlex.quote(a)
     if isinstance(args, list):
         return ' '.join(_quote(args))
     else:
         return args
 
 
-def copy_to_log(f, logger, loglevel=logging.INFO, capture=None):
+def copy_to_log(f, logger, loglevel=logging.INFO, capture=None, quiet=False):
     """
     Copy line by line from file in f to the log from logger
 
@@ -266,6 +267,8 @@ def copy_to_log(f, logger, loglevel=logging.INFO, capture=None):
     :param logger: the destination logger object
     :param loglevel: the level of logging data
     :param capture: an optional stream object for data copy
+    :param quiet: suppress `logger` usage if True, this is useful only
+                  in combination with `capture`, defaults False
     """
     # Work-around for http://tracker.ceph.com/issues/8313
     if isinstance(f, ChannelFile):
@@ -284,6 +287,8 @@ def copy_to_log(f, logger, loglevel=logging.INFO, capture=None):
                     capture.write(line)
         line = line.rstrip()
         # Second part of work-around for http://tracker.ceph.com/issues/8313
+        if quiet:
+            continue
         try:
             if isinstance(line, bytes):
                 line = line.decode('utf-8', 'replace')
@@ -305,15 +310,17 @@ def copy_and_close(src, fdst):
     fdst.close()
 
 
-def copy_file_to(src, logger, stream=None):
+def copy_file_to(src, logger, stream=None, quiet=False):
     """
     Copy file
     :param src: file to be copied.
     :param logger: the logger object
-    :param stream: an optional file-like object which will receive a copy of
-                   src.
+    :param stream: an optional file-like object which will receive
+                   a copy of src.
+    :param quiet: disable logger usage if True, useful in combination
+                  with `stream` parameter, defaults False.
     """
-    copy_to_log(src, logger, capture=stream)
+    copy_to_log(src, logger, capture=stream, quiet=quiet)
 
 def spawn_asyncresult(fn, *args, **kwargs):
     """
@@ -384,6 +391,7 @@ def run(
     wait=True,
     name=None,
     label=None,
+    quiet=False,
     timeout=None,
     cwd=None,
     # omit_sudo is used by vstart_runner.py
@@ -392,7 +400,7 @@ def run(
     """
     Run a command remotely.  If any of 'args' contains shell metacharacters
     that you want to pass unquoted, pass it as an instance of Raw(); otherwise
-    it will be quoted with pipes.quote() (single quote, and single quotes
+    it will be quoted with shlex.quote() (single quote, and single quotes
     enclosed in double quotes).
 
     :param client: SSHConnection to run the command with
@@ -417,6 +425,7 @@ def run(
     :param name: Human readable name (probably hostname) of the destination
                  host
     :param label: Can be used to label or describe what the command is doing.
+    :param quiet: Do not log command's stdout and stderr, defaults False.
     :param timeout: timeout value for args to complete on remote channel of
                     paramiko
     :param cwd: Directory in which the command should be executed.
@@ -440,8 +449,8 @@ def run(
                       cwd=cwd)
     r.execute()
     r.setup_stdin(stdin)
-    r.setup_output_stream(stderr, 'stderr')
-    r.setup_output_stream(stdout, 'stdout')
+    r.setup_output_stream(stderr, 'stderr', quiet)
+    r.setup_output_stream(stdout, 'stdout', quiet)
     if wait:
         r.wait()
     return r

teuthology/packaging.py

@@ -395,12 +395,12 @@ def _get_config_value_for_remote(ctx, remote, config, key):
 
         config = {
             'all':
-                {'branch': 'master'},
+                {'branch': 'main'},
             'branch': 'next'
         }
         _get_config_value_for_remote(ctx, remote, config, 'branch')
 
-    would return 'master'.
+    would return 'main'.
 
     :param ctx: the argparse.Namespace object
     :param remote: the teuthology.orchestra.remote.Remote object
@@ -479,7 +479,7 @@ class GitbuilderProject(object):
         )
         # when we're initializing with a remote we most likely have
         # a task config, not the entire teuthology job config
-        self.flavor = self.job_config.get("flavor", "basic")
+        self.flavor = self.job_config.get("flavor", "default")
         self.tag = self.job_config.get("tag")
 
     def _init_from_config(self):
@@ -549,10 +549,6 @@ class GitbuilderProject(object):
         """
         The base url that points at this project on gitbuilder.
 
-        For example::
-
-            http://gitbuilder.ceph.com/ceph-deb-raring-x86_64-basic/ref/master
-
         :returns: A string of the base url for this project
         """
         return self._get_base_url()
@@ -656,9 +652,9 @@ class GitbuilderProject(object):
         remote, the sha1 from the config will be used.
 
         If a tag, branch or sha1 can't be found it will default to use the
-        build from the master branch.
+        build from the main branch.
 
-        :returns: A string URI. Ex: ref/master
+        :returns: A string URI. Ex: ref/main
         """
         ref_name, ref_val = next(iter(self._choose_reference().items()))
         if ref_name == 'sha1':
@@ -673,7 +669,7 @@ class GitbuilderProject(object):
         Decide which to use.
 
         :returns: a single-key dict containing the name and value of the
-                  reference to use, e.g. {'branch': 'master'}
+                  reference to use, e.g. {'branch': 'main'}
         """
         tag = branch = sha1 = None
         if self.remote:
@@ -716,8 +712,8 @@ class GitbuilderProject(object):
             warn('sha1')
             return dict(sha1=sha1)
         else:
-            log.warning("defaulting to master branch")
-            return dict(branch='master')
+            log.warning("defaulting to main branch")
+            return dict(branch='main')
 
     def _get_base_url(self):
         """
@@ -853,6 +849,9 @@ class ShamanProject(GitbuilderProject):
         super(ShamanProject, self).__init__(project, job_config, ctx, remote)
         self.query_url = 'https://%s/api/' % config.shaman_host
 
+        # Force to use the "noarch" instead to build the uri.
+        self.force_noarch = self.job_config.get("shaman", {}).get("force_noarch", False)
+
     def _get_base_url(self):
         self.assert_result()
         return self._result.json()[0]['url']
@@ -876,13 +875,12 @@ class ShamanProject(GitbuilderProject):
     @property
     def _search_uri(self):
         flavor = self.flavor
-        if flavor == 'basic':
-            flavor = 'default'
         req_obj = OrderedDict()
         req_obj['status'] = 'ready'
         req_obj['project'] = self.project
         req_obj['flavor'] = flavor
-        req_obj['distros'] = '%s/%s' % (self.distro, self.arch)
+        arch = "noarch" if self.force_noarch else self.arch
+        req_obj['distros'] = '%s/%s' % (self.distro, arch)
         ref_name, ref_val = list(self._choose_reference().items())[0]
         if ref_name == 'tag':
             req_obj['sha1'] = self._sha1 = self._tag_to_sha1()
@@ -970,6 +968,47 @@ class ShamanProject(GitbuilderProject):
             'repo',
         )
 
+    @property
+    def build_complete(self):
+        # use the repo search results to get a ref and a sha1; the
+        # input to teuthology-suite doesn't contain both
+        try:
+            self.assert_result()
+        except VersionNotFoundError:
+            return False
+
+        # self._result has status, project, flavor, distros, arch, and sha1
+        # restrictions, so the only reason for multiples should be "multiple
+        # builds of the same sha1 etc."; the first entry is the newest
+        search_result = self._result.json()[0]
+
+        # now look for the build complete status
+        path = '/'.join(
+            ('builds/ceph', search_result['ref'], search_result['sha1'])
+        )
+        build_url = urljoin(self.query_url, path)
+
+        try:
+            resp = requests.get(build_url)
+            resp.raise_for_status()
+        except requests.HttpError:
+            return False
+        log.debug(f'looking for {self.distro} {self.arch} {self.flavor}')
+        for build in resp.json():
+            log.debug(f'build: {build["distro"]}/{build["distro_version"]} {build["distro_arch"]} {build["flavor"]}')
+            if (
+                # we must compare build arch to self.arch, since shaman's
+                # results can have multiple arches but we're searching
+                # for precisely one here
+                build['distro'] == search_result['distro'] and
+                build['distro_version'] == search_result['distro_version'] and
+                build['flavor'] == search_result['flavor'] and
+                build['distro_arch'] == self.arch and
+                build['status'] == 'completed'
+               ):
+                return True
+        return False
+
     def _get_repo(self):
         resp = requests.get(self.repo_url)
         resp.raise_for_status()

teuthology/provision/__init__.py

@@ -1,5 +1,7 @@
 import logging
+import os
 
+import teuthology.exporter
 import teuthology.lock.query
 from teuthology.misc import decanonicalize_hostname, get_distro, get_distro_version
 
@@ -8,19 +10,19 @@ from teuthology.provision import downburst
 from teuthology.provision import fog
 from teuthology.provision import openstack
 from teuthology.provision import pelagos
-import os
 
 log = logging.getLogger(__name__)
 
 
-def _logfile(ctx, shortname):
-    if hasattr(ctx, 'config') and ctx.config.get('archive_path'):
-        return os.path.join(ctx.config['archive_path'],
-                            shortname + '.downburst.log')
+def _logfile(shortname: str, archive_path: str = ""):
+    if os.path.isfile(archive_path):
+        return f"{archive_path}/{shortname}.downburst.log"
+
 
 def get_reimage_types():
     return pelagos.get_types() + fog.get_types()
 
+
 def reimage(ctx, machine_name, machine_type):
     os_type = get_distro(ctx)
     os_version = get_distro_version(ctx)
@@ -36,7 +38,21 @@ def reimage(ctx, machine_name, machine_type):
     else:
         raise Exception("The machine_type '%s' is not known to any "
                         "of configured provisioners" % machine_type)
-    return obj.create()
+    status = "fail"
+    try:
+        result = obj.create()
+        status = "success"
+    except Exception:
+        # We only need this clause so that we avoid triggering the finally
+        # clause below in cases where the exception raised is KeyboardInterrupt
+        # or SystemExit
+        raise
+    finally:
+        teuthology.exporter.NodeReimagingResults().record(
+            machine_type=machine_type,
+            status=status,
+        )
+    return result
 
 
 def create_if_vm(ctx, machine_name, _downburst=None):
@@ -78,8 +94,12 @@ def create_if_vm(ctx, machine_name, _downburst=None):
     return dbrst.create()
 
 
-def destroy_if_vm(ctx, machine_name, user=None, description=None,
-                  _downburst=None):
+def destroy_if_vm(
+    machine_name: str,
+    user: str = "",
+    description: str = "",
+    _downburst=None
+):
     """
     Use downburst to destroy a virtual machine
 
@@ -99,7 +119,7 @@ def destroy_if_vm(ctx, machine_name, user=None, description=None,
         log.error(msg.format(node=machine_name, as_user=user,
                              locked_by=status_info['locked_by']))
         return False
-    if (description is not None and description !=
+    if (description and description !=
             status_info['description']):
         msg = "Tried to destroy {node} with description {desc_arg} " + \
             "but it is locked with description {desc_lock}"
@@ -117,5 +137,5 @@ def destroy_if_vm(ctx, machine_name, user=None, description=None,
     dbrst = _downburst or \
         downburst.Downburst(name=machine_name, os_type=None,
                             os_version=None, status=status_info,
-                            logfile=_logfile(ctx, shortname))
+                            logfile=_logfile(description, shortname))
     return dbrst.destroy()

teuthology/provision/cloud/openstack.py

@@ -84,7 +84,13 @@ class OpenStackProvider(Provider):
     @property
     def images(self):
         if not hasattr(self, '_images'):
-            self._images = retry(self.driver.list_images)
+            exclude_image = self.conf.get('exclude_image', [])
+            if exclude_image and not isinstance(exclude_image, list):
+                exclude_image = [exclude_image]
+            exclude_re = [re.compile(x) for x in exclude_image]
+            images = retry(self.driver.list_images)
+            self._images = [_ for _ in images
+                if not any(x.match(_.name) for x in exclude_re)]
         return self._images
 
     @property
@@ -126,7 +132,7 @@ class OpenStackProvider(Provider):
                 else:
                     self._networks = list()
             except AttributeError:
-                log.warn("Unable to list networks for %s", self.driver)
+                log.warning("Unable to list networks for %s", self.driver)
                 self._networks = list()
         return self._networks
 
@@ -144,7 +150,7 @@ class OpenStackProvider(Provider):
                     self.driver.ex_list_security_groups
                 )
             except AttributeError:
-                log.warn("Unable to list security groups for %s", self.driver)
+                log.warning("Unable to list security groups for %s", self.driver)
                 self._security_groups = list()
         return self._security_groups
 
@@ -420,7 +426,7 @@ class OpenStackProvisioner(base.Provisioner):
         msg = "Unknown error locating %s"
         if not matches:
             msg = "No nodes found with name '%s'" % self.name
-            log.warn(msg)
+            log.warning(msg)
             return
         elif len(matches) > 1:
             msg = "More than one node found with name '%s'"
@@ -438,9 +444,9 @@ class OpenStackProvisioner(base.Provisioner):
         self._destroy_volumes()
         nodes = self._find_nodes()
         if not nodes:
-            log.warn("Didn't find any nodes named '%s' to destroy!", self.name)
+            log.warning("Didn't find any nodes named '%s' to destroy!", self.name)
             return True
         if len(nodes) > 1:
-            log.warn("Found multiple nodes named '%s' to destroy!", self.name)
+            log.warning("Found multiple nodes named '%s' to destroy!", self.name)
         log.info("Destroying nodes: %s", nodes)
         return all([node.destroy() for node in nodes])

teuthology/provision/cloud/util.py

@@ -1,5 +1,4 @@
 import datetime
-import dateutil.tz
 import dateutil.parser
 import json
 import os
@@ -103,7 +102,7 @@ class AuthToken(object):
     def expired(self):
         if self.expires is None:
             return True
-        utcnow = datetime.datetime.now(dateutil.tz.tzutc())
+        utcnow = datetime.datetime.now(datetime.timezone.utc)
         offset = datetime.timedelta(minutes=30)
         return self.expires < (utcnow + offset)
 

teuthology/provision/downburst.py

@@ -8,6 +8,7 @@ import yaml
 from teuthology.config import config
 from teuthology.contextutil import safe_while
 from teuthology.misc import decanonicalize_hostname
+from teuthology.misc import deep_merge
 from teuthology.lock import query
 
 log = logging.getLogger(__name__)
@@ -20,7 +21,11 @@ def downburst_executable():
     Return '' if no executable downburst is found.
     """
     if config.downburst:
-        return config.downburst
+        if isinstance(config.downburst, dict):
+            if 'path' in config.downburst:
+                return config.downburst['path']
+        else:
+            return config.downburst
     path = os.environ.get('PATH', None)
     if path:
         for p in os.environ.get('PATH', '').split(os.pathsep):
@@ -37,6 +42,18 @@ def downburst_executable():
     return ''
 
 
+def downburst_environment():
+    env = dict()
+    env['PATH'] = os.environ.get('PATH')
+    discover_url = os.environ.get('DOWNBURST_DISCOVER_URL')
+    if config.downburst and not discover_url:
+        if isinstance(config.downburst, dict):
+            discover_url = config.downburst.get('discover_url')
+    if discover_url:
+        env['DOWNBURST_DISCOVER_URL'] = discover_url
+    return env
+
+
 class Downburst(object):
     """
     A class that provides methods for creating and destroying virtual machine
@@ -55,6 +72,7 @@ class Downburst(object):
         self.logfile = logfile
         self.host = decanonicalize_hostname(self.status['vm_host']['name'])
         self.executable = downburst_executable()
+        self.environment = downburst_environment()
 
     def create(self):
         """
@@ -118,7 +136,10 @@ class Downburst(object):
             distro=self.os_type,
             distroversion=self.os_version
         ))
-        proc = subprocess.Popen(args, stdout=subprocess.PIPE,
+        log.debug(args)
+        proc = subprocess.Popen(args, universal_newlines=True,
+                                env=self.environment,
+                                stdout=subprocess.PIPE,
                                 stderr=subprocess.PIPE)
         out, err = proc.communicate()
         return (proc.returncode, out, err)
@@ -135,7 +156,9 @@ class Downburst(object):
         if self.logfile:
             args.extend(['-l', self.logfile])
         args.extend(['destroy', self.shortname])
-        proc = subprocess.Popen(args, stdout=subprocess.PIPE,
+        log.debug(args)
+        proc = subprocess.Popen(args, universal_newlines=True,
+                                stdout=subprocess.PIPE,
                                 stderr=subprocess.PIPE,)
         out, err = proc.communicate()
         log.info(out)
@@ -143,7 +166,7 @@ class Downburst(object):
         if proc.returncode != 0:
             not_found_msg = "no domain with matching name '%s'" % self.shortname
             if not_found_msg in err:
-                log.warn("Ignoring error during destroy: %s", err)
+                log.warning("Ignoring error during destroy: %s", err)
                 return True
             log.error("Error destroying %s: %s", self.name, err)
             return False
@@ -156,24 +179,44 @@ class Downburst(object):
         """
         Assemble a configuration to pass to downburst, and write it to a file.
         """
-        config_fd = tempfile.NamedTemporaryFile(delete=False)
+        config_fd = tempfile.NamedTemporaryFile(delete=False, mode='wt')
 
         os_type = self.os_type.lower()
-        mac_address = self.status['mac_address']
+        os_version = self.os_version.lower()
 
+        mac_address = self.status['mac_address']
+        defaults = dict(
+            downburst=dict(
+                machine=dict(
+                    disk=os.environ.get('DOWNBURST_DISK_SIZE', '100G'),
+                    ram=os.environ.get('DOWNBURST_RAM_SIZE', '3.8G'),
+                    cpus=int(os.environ.get('DOWNBURST_CPUS', 1)),
+                    volumes=dict(
+                        count=int(os.environ.get('DOWNBURST_EXTRA_DISK_NUMBER', 4)),
+                        size=os.environ.get('DOWNBURST_EXTRA_DISK_SIZE', '100G'),
+                    ),
+                ),
+            )
+        )
+        downburst_config = defaults['downburst']
+        if config.downburst and isinstance(config.downburst, dict):
+            deep_merge(downburst_config, config.downburst)
+        log.debug('downburst_config: %s', downburst_config)
+        machine = downburst_config['machine']
+        log.debug('Using machine config: %s', machine)
         file_info = {
-            'disk-size': '100G',
-            'ram': '3.8G',
-            'cpus': 1,
+            'disk-size': machine['disk'],
+            'ram': machine['ram'],
+            'cpus': machine['cpus'],
             'networks': [
                 {'source': 'front', 'mac': mac_address}],
             'distro': os_type,
             'distroversion': self.os_version,
-            'additional-disks': 4,
-            'additional-disks-size': '100G',
+            'additional-disks': machine['volumes']['count'],
+            'additional-disks-size': machine['volumes']['size'],
             'arch': 'x86_64',
         }
-        fqdn = self.name.split('@')[1]
+        fqdn = self.name.split('@')[-1]
         file_out = {
             'downburst': file_info,
             'local-hostname': fqdn,
@@ -188,6 +231,12 @@ class Downburst(object):
                 ['passwd', '-d', self.user],
             ]
         }
+        # for opensuse-15.2 we need to replace systemd-logger with rsyslog for teuthology
+        if os_type == 'opensuse' and os_version == '15.2':
+            user_info['runcmd'].extend([
+                ['zypper', 'rm', '-y', 'systemd-logger'],
+                ['zypper', 'in', '-y', 'rsyslog'],
+            ])
         # Install git on downbursted VMs to clone upstream linux-firmware.
         # Issue #17154
         if 'packages' not in user_info:
@@ -208,7 +257,7 @@ class Downburst(object):
         # to install 'python' to get python2.7, which ansible needs
         if os_type in ('ubuntu', 'fedora'):
             user_info['packages'].append('python')
-        user_fd = tempfile.NamedTemporaryFile(delete=False)
+        user_fd = tempfile.NamedTemporaryFile(delete=False, mode='wt')
         user_str = "#cloud-config\n" + yaml.safe_dump(user_info)
         user_fd.write(user_str)
         self.user_path = user_fd.name
@@ -240,27 +289,32 @@ def get_distro_from_downburst():
     or if downburst is unable to produce a json list, then use a default
     table.
     """
-    default_table = {u'rhel_minimal': [u'6.4', u'6.5'],
-                     u'fedora': [u'17', u'18', u'19', u'20', u'22'],
-                     u'centos': [u'6.3', u'6.4', u'6.5', u'7.0',
-				 u'7.2'],
-                     u'centos_minimal': [u'6.4', u'6.5'],
-                     u'ubuntu': [u'8.04(hardy)', u'9.10(karmic)',
-                                 u'10.04(lucid)', u'10.10(maverick)',
-                                 u'11.04(natty)', u'11.10(oneiric)',
-                                 u'12.04(precise)', u'12.10(quantal)',
-                                 u'13.04(raring)', u'13.10(saucy)',
-                                 u'14.04(trusty)', u'utopic(utopic)',
-                                 u'16.04(xenial)'],
-                     u'sles': [u'11-sp2'],
-                     u'debian': [u'6.0', u'7.0', u'8.0']}
+    default_table = {'rhel_minimal': ['6.4', '6.5'],
+                     'fedora': ['17', '18', '19', '20', '22'],
+                     'centos': ['6.3', '6.4', '6.5', '7.0',
+				 '7.2', '7.4', '8.2'],
+                     'centos_minimal': ['6.4', '6.5'],
+                     'ubuntu': ['8.04(hardy)', '9.10(karmic)',
+                                 '10.04(lucid)', '10.10(maverick)',
+                                 '11.04(natty)', '11.10(oneiric)',
+                                 '12.04(precise)', '12.10(quantal)',
+                                 '13.04(raring)', '13.10(saucy)',
+                                 '14.04(trusty)', 'utopic(utopic)',
+                                 '16.04(xenial)', '18.04(bionic)',
+                                 '20.04(focal)'],
+                     'sles': ['12-sp3', '15-sp1', '15-sp2'],
+                     'opensuse': ['12.3', '15.1', '15.2'],
+                     'debian': ['6.0', '7.0', '8.0']}
     executable_cmd = downburst_executable()
+    environment_dict = downburst_environment()
     if not executable_cmd:
-        log.warn("Downburst not found!")
+        log.warning("Downburst not found!")
         log.info('Using default values for supported os_type/os_version')
         return default_table
     try:
-        output = subprocess.check_output([executable_cmd, 'list-json'])
+        log.debug(executable_cmd)
+        output = subprocess.check_output([executable_cmd, 'list-json'],
+                                                        env=environment_dict)
         downburst_data = json.loads(output)
         return downburst_data
     except (subprocess.CalledProcessError, OSError):