teuthology 1.0.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

teuthology/provision/fog.py

@@ -1,10 +1,10 @@
+import datetime
 import json
 import logging
 import requests
 import socket
 import re
 
-from datetime import datetime
 from paramiko import SSHException
 from paramiko.ssh_exception import NoValidConnectionsError
 
@@ -13,6 +13,7 @@ import teuthology.orchestra
 from teuthology.config import config
 from teuthology.contextutil import safe_while
 from teuthology.exceptions import MaxWhileTries
+from teuthology.orchestra.opsys import OS
 from teuthology import misc
 
 log = logging.getLogger(__name__)
@@ -29,7 +30,7 @@ def enabled(warn=False):
     params = ['endpoint', 'api_token', 'user_token', 'machine_types']
     unset = [param for param in params if not fog_conf.get(param)]
     if unset and warn:
-        log.warn(
+        log.warning(
             "FOG disabled; set the following config options to enable: %s",
             ' '.join(unset),
         )
@@ -89,6 +90,7 @@ class FOG(object):
             raise
         self._wait_for_ready()
         self._fix_hostname()
+        self._verify_installed_os()
         self.log.info("Deploy complete!")
 
     def do_request(self, url_suffix, data=None, method='GET', verify=True):
@@ -147,18 +149,39 @@ class FOG(object):
         represents it
         :returns: A dict describing the image
         """
-        name = '_'.join([
-            self.remote.machine_type, self.os_type.lower(), self.os_version])
-        resp = self.do_request(
-            '/image',
-            data=json.dumps(dict(name=name)),
-        )
-        obj = resp.json()
-        if not obj['count']:
+        def do_get(name):
+            resp = self.do_request(
+                '/image',
+                data=json.dumps(dict(name=name)),
+            )
+            obj = resp.json()
+            if obj['count']:
+                return obj['images'][0]
+
+        os_type = self.os_type.lower()
+        os_version = self.os_version
+        name = f"{self.remote.machine_type}_{os_type}_{os_version}"
+        if image := do_get(name):
+            return image
+        elif os_type == 'centos' and not os_version.endswith('.stream'):
+            image = do_get(f"{name}.stream")
+        if image:
+            return image
+        else:
             raise RuntimeError(
-                "Could not find an image for %s %s" %
-                (self.os_type, self.os_version))
-        return obj['images'][0]
+                "Fog has no %s image. Available %s images: %s" %
+                (name, self.remote.machine_type, self.suggest_image_names()))
+
+    def suggest_image_names(self):
+        """
+        Suggest available image names for this machine type.
+
+        :returns: A list of image names.
+        """
+        resp = self.do_request('/image/search/%s' % self.remote.machine_type)
+        obj = resp.json()
+        images = obj['images']
+        return [image['name'] for image in images]
 
     def set_image(self, host_id):
         """
@@ -167,6 +190,8 @@ class FOG(object):
         """
         image_data = self.get_image_data()
         image_id = int(image_data['id'])
+        image_name = image_data.get("name")
+        self.log.debug(f"Requesting image {image_name} (ID {image_id})")
         self.do_request(
             '/host/%s' % host_id,
             method='PUT',
@@ -202,8 +227,8 @@ class FOG(object):
         for task in host_tasks:
             timestamp = task['createdTime']
             time_delta = (
-                datetime.utcnow() - datetime.strptime(
-                    timestamp, self.timestamp_format)
+                datetime.datetime.now(datetime.timezone.utc) - datetime.datetime.strptime(
+                    timestamp, self.timestamp_format).replace(tzinfo=datetime.timezone.utc)
             ).total_seconds()
             # There should only be one deploy task matching our host. Just in
             # case there are multiple, select a very recent one.
@@ -240,13 +265,14 @@ class FOG(object):
         completed)
         """
         self.log.info("Waiting for deploy to finish")
-        with safe_while(sleep=15, tries=60) as proceed:
+        with safe_while(sleep=15, tries=120, timeout=config.fog_reimage_timeout) as proceed:
             while proceed():
                 if not self.deploy_task_active(task_id):
                     break
 
     def cancel_deploy_task(self,  task_id):
         """ Cancel an active deploy task """
+        self.log.debug(f"Canceling deploy task with ID {task_id}")
         resp = self.do_request(
             '/task/cancel',
             method='DELETE',
@@ -256,7 +282,7 @@ class FOG(object):
 
     def _wait_for_ready(self):
         """ Attempt to connect to the machine via SSH """
-        with safe_while(sleep=6, tries=100) as proceed:
+        with safe_while(sleep=6, timeout=config.fog_wait_for_ssh_timeout) as proceed:
             while proceed():
                 try:
                     self.remote.connect()
@@ -267,12 +293,26 @@ class FOG(object):
                     NoValidConnectionsError,
                     MaxWhileTries,
                     EOFError,
-                ):
-                    pass
+                ) as e:
+                    # log this, because otherwise lots of failures just
+                    # keep retrying without any notification (like, say,
+                    # a mismatched host key in ~/.ssh/known_hosts, or
+                    # something)
+                    self.log.warning(e)
         sentinel_file = config.fog.get('sentinel_file', None)
         if sentinel_file:
             cmd = "while [ ! -e '%s' ]; do sleep 5; done" % sentinel_file
-            self.remote.run(args=cmd, timeout=600)
+            action = f"wait for sentinel on {self.shortname}"
+            with safe_while(action=action, timeout=1800, increment=3) as proceed:
+                while proceed():
+                    try:
+                        self.remote.run(args=cmd, timeout=600)
+                        break
+                    except (
+                        ConnectionError,
+                        EOFError,
+                    ) as e:
+                        log.error(f"{e} on {self.shortname}")
         self.log.info("Node is ready")
 
     def _fix_hostname(self):
@@ -307,6 +347,14 @@ class FOG(object):
             check_status=False,
         )
 
+    def _verify_installed_os(self):
+        wanted_os = OS(name=self.os_type, version=self.os_version)
+        if self.remote.os != wanted_os:
+            raise RuntimeError(
+                f"Expected {self.remote.shortname}'s OS to be {wanted_os} but "
+                f"found {self.remote.os}"
+            )
+
     def destroy(self):
         """A no-op; we just leave idle nodes as-is"""
         pass

teuthology/provision/openstack.py

@@ -27,7 +27,8 @@ class ProvisionOpenStack(OpenStack):
     """
     def __init__(self):
         super(ProvisionOpenStack, self).__init__()
-        self.user_data = tempfile.mktemp()
+        fd, self.user_data = tempfile.mkstemp()
+        os.close(fd)
         log.debug("ProvisionOpenStack: " + str(config.openstack))
         self.basename = 'target'
         self.up_string = 'The system is finally up'
@@ -77,7 +78,7 @@ class ProvisionOpenStack(OpenStack):
             if 'No volume with a name or ID' not in e.output:
                 raise e
         if volume_id:
-            log.warn("Volume {} already exists with ID {}; using it"
+            log.warning("Volume {} already exists with ID {}; using it"
                      .format(volume_name, volume_id))
         volume_id = self._openstack(
             "volume create %s" % config['openstack'].get('volume-create','')
@@ -106,7 +107,7 @@ class ProvisionOpenStack(OpenStack):
                         log.debug("volume %s not in '%s' status yet"
                                   % (volume_id, status))
                 except subprocess.CalledProcessError:
-                        log.warn("volume " + volume_id +
+                        log.warning("volume " + volume_id +
                                  " not information available yet")
 
     def _attach_volume(self, volume_id, name):
@@ -151,7 +152,7 @@ class ProvisionOpenStack(OpenStack):
         """
         return the instance name suffixed with the IP address.
         """
-        digits = map(int, re.findall('(\d+)\.(\d+)\.(\d+)\.(\d+)', ip)[0])
+        digits = map(int, re.findall(r'(\d+)\.(\d+)\.(\d+)\.(\d+)', ip)[0])
         return prefix + "%03d%03d%03d%03d" % tuple(digits)
 
     def create(self, num, os_type, os_version, arch, resources_hint):

teuthology/provision/pelagos.py

@@ -26,7 +26,7 @@ def enabled(warn=False):
     params = ['endpoint', 'machine_types']
     unset = [_ for _ in params if not conf.get(_)]
     if unset and warn:
-        log.warn(
+        log.warning(
             "Pelagos is disabled; set the following config options to enable: %s",
             ' '.join(unset),
         )
@@ -49,8 +49,8 @@ def get_types():
     return [_ for _ in types if _]
 
 def park_node(name):
-        p = Pelagos(name, "maintenance_image")
-        p.create()
+    p = Pelagos(name, "maintenance_image")
+    p.create(wait=False)
 
 
 class Pelagos(object):
@@ -71,18 +71,26 @@ class Pelagos(object):
             self.os_name = os_type
         self.log = log.getChild(self.name)
 
-    def create(self):
+    def create(self, wait=True):
         """
         Initiate deployment via REST requests and wait until completion
+         :param wait: optional, by default is True, if set to False, function
+                      doesn't wait for the end of node provisioning
+         :returns: http response code if operation is successful
+         :raises: :class:`Exception`: if node provision failure reported by
+                             Pelagos or if timeout is reached
+         :raises: :class:`RuntimeError`: if pelagos is not configured
 
         """
         if not enabled():
             raise RuntimeError("Pelagos is not configured!")
         location = None
         try:
-            params=dict(os=self.os_name, node=self.name)
+            params = dict(os=self.os_name, node=self.name)
             response = self.do_request('node/provision',
                                 data=params, method='POST')
+            if not wait:
+                return response
             location = response.headers.get('Location')
             self.log.debug("provision task: '%s'", location)
             # gracefully wait till provision task gets created on pelagos

teuthology/repo_utils.py

@@ -1,3 +1,4 @@
+import functools
 import logging
 import os
 import re
@@ -5,11 +6,13 @@ import shutil
 import subprocess
 import time
 
+import teuthology.exporter as exporter
+
 from teuthology import misc
 from teuthology.util.flock import FileLock
 from teuthology.config import config
 from teuthology.contextutil import MaxWhileTries, safe_while
-from teuthology.exceptions import BootstrapError, BranchNotFoundError, GitError
+from teuthology.exceptions import BootstrapError, BranchNotFoundError, CommitNotFoundError, GitError
 
 log = logging.getLogger(__name__)
 
@@ -50,10 +53,11 @@ def build_git_url(project, project_owner='ceph'):
         base = config.get_ceph_git_url()
     else:
         base = 'https://github.com/{project_owner}/{project}'
-    url_templ = re.sub('\.git$', '', base)
+    url_templ = re.sub(r'\.git$', '', base)
     return url_templ.format(project_owner=project_owner, project=project)
 
 
+@functools.lru_cache()
 def ls_remote(url, ref):
     """
     Return the current sha1 for a given repository and ref
@@ -70,33 +74,61 @@ def ls_remote(url, ref):
     return sha1
 
 
-def enforce_repo_state(repo_url, dest_path, branch, remove_on_error=True):
+def current_branch(path: str) -> str:
+    """
+    Return the current branch for a given on-disk repository.
+
+    :returns: the current branch, or an empty string if none is found.
+    """
+    # git branch --show-current was added in 2.22.0, and we can't assume
+    # our version is new enough.
+    cmd = "git rev-parse --abbrev-ref HEAD"
+    result = subprocess.Popen(
+        cmd,
+        shell=True,
+        cwd=path,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.DEVNULL,
+        ).communicate()[0].strip().decode()
+    if result == "HEAD":
+        return ""
+    return result
+
+
+def enforce_repo_state(repo_url, dest_path, branch, commit=None, remove_on_error=True):
     """
     Use git to either clone or update a given repo, forcing it to switch to the
     specified branch.
 
-    :param repo_url:  The full URL to the repo (not including the branch)
-    :param dest_path: The full path to the destination directory
-    :param branch:    The branch.
-    :param remove:    Whether or not to remove dest_dir when an error occurs
-    :raises:          BranchNotFoundError if the branch is not found;
-                      GitError for other errors
+    :param repo_url:        The full URL to the repo (not including the branch)
+    :param dest_path:       The full path to the destination directory
+    :param branch:          The branch.
+    :param commit:          The sha1 to checkout. Defaults to None, which uses HEAD of the branch.
+    :param remove_on_error: Whether or not to remove dest_dir when an error occurs
+    :raises:                BranchNotFoundError if the branch is not found;
+                            CommitNotFoundError if the commit is not found;
+                            GitError for other errors
     """
     validate_branch(branch)
     sentinel = os.path.join(dest_path, '.fetched')
+    # sentinel to track whether the repo has checked out the intended
+    # version, in addition to being cloned
+    repo_reset = os.path.join(dest_path, '.fetched_and_reset')
     try:
         if not os.path.isdir(dest_path):
-            clone_repo(repo_url, dest_path, branch)
-        elif not is_fresh(sentinel):
+            clone_repo(repo_url, dest_path, branch, shallow=commit is None)
+        elif not commit and not is_fresh(sentinel):
             set_remote(dest_path, repo_url)
             fetch_branch(dest_path, branch)
             touch_file(sentinel)
-        else:
-            log.info("%s was just updated; assuming it is current", dest_path)
 
-        reset_repo(repo_url, dest_path, branch)
+        if commit and os.path.exists(repo_reset):
+            return
+
+        reset_repo(repo_url, dest_path, branch, commit)
+        touch_file(repo_reset)
         # remove_pyc_files(dest_path)
-    except BranchNotFoundError:
+    except (BranchNotFoundError, CommitNotFoundError):
         if remove_on_error:
             shutil.rmtree(dest_path, ignore_errors=True)
         raise
@@ -118,7 +150,7 @@ def clone_repo(repo_url, dest_path, branch, shallow=True):
     if branch.startswith('refs/'):
         clone_repo_ref(repo_url, dest_path, branch)
         return
-    args = ['git', 'clone']
+    args = ['git', 'clone', '--single-branch']
     if shallow:
         args.extend(['--depth', '1'])
     args.extend(['--branch', branch, repo_url, dest_path])
@@ -133,8 +165,9 @@ def clone_repo(repo_url, dest_path, branch, shallow=True):
     result = proc.wait()
     # Newer git versions will bail if the branch is not found, but older ones
     # will not. Fortunately they both output similar text.
-    if not_found_str in out:
+    if result != 0:
         log.error(out)
+    if not_found_str in out:
         if result == 0:
             # Old git left a repo with the wrong branch. Remove it.
             shutil.rmtree(dest_path, ignore_errors=True)
@@ -242,7 +275,7 @@ def fetch_branch(repo_path, branch, shallow=True):
                       GitError for other errors
     """
     validate_branch(branch)
-    log.info("Fetching %s from origin", branch)
+    log.info("Fetching %s from origin", repo_path.split("/")[-1])
     args = ['git', 'fetch']
     if shallow:
         args.extend(['--depth', '1'])
@@ -262,13 +295,15 @@ def fetch_branch(repo_path, branch, shallow=True):
             raise GitError("git fetch failed!")
 
 
-def reset_repo(repo_url, dest_path, branch):
+def reset_repo(repo_url, dest_path, branch, commit=None):
     """
 
     :param repo_url:  The full URL to the repo (not including the branch)
     :param dest_path: The full path to the destination directory
     :param branch:    The branch.
+    :param commit:    The sha1 to checkout. Defaults to None, which uses HEAD of the branch.
     :raises:          BranchNotFoundError if the branch is not found;
+                      CommitNotFoundError if the commit is not found;
                       GitError for other errors
     """
     validate_branch(branch)
@@ -276,15 +311,18 @@ def reset_repo(repo_url, dest_path, branch):
         reset_branch = lsstrip(remote_ref_from_ref(branch), 'refs/remotes/')
     else:
         reset_branch = 'origin/%s' % branch
-    log.info('Resetting repo at %s to branch %s', dest_path, reset_branch)
+    reset_ref = commit or reset_branch
+    log.debug('Resetting repo at %s to %s', dest_path, reset_ref)
     # This try/except block will notice if the requested branch doesn't
     # exist, whether it was cloned or fetched.
     try:
         subprocess.check_output(
-            ('git', 'reset', '--hard', reset_branch),
+            ('git', 'reset', '--hard', reset_ref),
             cwd=dest_path,
         )
     except subprocess.CalledProcessError:
+        if commit:
+            raise CommitNotFoundError(commit, repo_url)
         raise BranchNotFoundError(branch, repo_url)
 
 
@@ -299,7 +337,7 @@ def validate_branch(branch):
         raise ValueError("Illegal branch name: '%s'" % branch)
 
 
-def fetch_repo(url, branch, bootstrap=None, lock=True):
+def fetch_repo(url, branch, commit=None, bootstrap=None, lock=True):
     """
     Make sure we have a given project's repo checked out and up-to-date with
     the current branch requested
@@ -308,24 +346,33 @@ def fetch_repo(url, branch, bootstrap=None, lock=True):
     :param bootstrap:  An optional callback function to execute. Gets passed a
                        dest_dir argument: the path to the repo on-disk.
     :param branch:     The branch we want
+    :param commit:     The sha1 to checkout. Defaults to None, which uses HEAD of the branch.
     :returns:          The destination path
     """
     src_base_path = config.src_base_path
     if not os.path.exists(src_base_path):
         os.mkdir(src_base_path)
-    branch_dir = ref_to_dirname(branch)
-    dirname = '%s_%s' % (url_to_dirname(url), branch_dir)
+    ref_dir = ref_to_dirname(commit or branch)
+    dirname = '%s_%s' % (url_to_dirname(url), ref_dir)
     dest_path = os.path.join(src_base_path, dirname)
     # only let one worker create/update the checkout at a time
     lock_path = dest_path.rstrip('/') + '.lock'
     with FileLock(lock_path, noop=not lock):
-        with safe_while(sleep=10, tries=60) as proceed:
+        with safe_while(sleep=10, tries=6) as proceed:
             try:
                 while proceed():
                     try:
-                        enforce_repo_state(url, dest_path, branch)
+                        enforce_repo_state(url, dest_path, branch, commit)
                         if bootstrap:
+                            sentinel = os.path.join(dest_path, '.bootstrapped')
+                            if commit and os.path.exists(sentinel) or is_fresh(sentinel):
+                                log.info(
+                                    "Skipping bootstrap as it was already done in the last %ss",
+                                    FRESHNESS_INTERVAL,
+                                )
+                                break
                             bootstrap(dest_path)
+                            touch_file(sentinel)
                         break
                     except GitError:
                         log.exception("Git error encountered; retrying")
@@ -368,36 +415,32 @@ def url_to_dirname(url):
     return string
 
 
-def fetch_qa_suite(branch, lock=True):
+def fetch_qa_suite(branch, commit=None, lock=True):
     """
     Make sure ceph-qa-suite is checked out.
 
     :param branch: The branch to fetch
+    :param commit: The sha1 to checkout. Defaults to None, which uses HEAD of the branch.
     :returns:      The destination path
     """
     return fetch_repo(config.get_ceph_qa_suite_git_url(),
-                      branch, lock=lock)
+                      branch, commit, lock=lock)
 
 
-def fetch_teuthology(branch, lock=True):
+def fetch_teuthology(branch, commit=None, lock=True):
     """
     Make sure we have the correct teuthology branch checked out and up-to-date
 
     :param branch: The branch we want
+    :param commit: The sha1 to checkout. Defaults to None, which uses HEAD of the branch.
     :returns:      The destination path
     """
     url = config.ceph_git_base_url + 'teuthology.git'
-    return fetch_repo(url, branch, bootstrap_teuthology, lock)
+    return fetch_repo(url, branch, commit, bootstrap_teuthology, lock)
 
 
 def bootstrap_teuthology(dest_path):
-        sentinel = os.path.join(dest_path, '.bootstrapped')
-        if is_fresh(sentinel):
-            log.info(
-                "Skipping bootstrap as it was already done in the last %ss",
-                FRESHNESS_INTERVAL,
-            )
-            return
+    with exporter.BootstrapTime().time():
         log.info("Bootstrapping %s", dest_path)
         # This magic makes the bootstrap script not attempt to clobber an
         # existing virtualenv. But the branch's bootstrap needs to actually
@@ -405,17 +448,21 @@ def bootstrap_teuthology(dest_path):
         env = os.environ.copy()
         env['NO_CLOBBER'] = '1'
         cmd = './bootstrap'
-        boot_proc = subprocess.Popen(cmd, shell=True, cwd=dest_path, env=env,
-                                     stdout=subprocess.PIPE,
-                                     stderr=subprocess.STDOUT)
-        out, err = boot_proc.communicate()
+        boot_proc = subprocess.Popen(
+            cmd, shell=True,
+            cwd=dest_path,
+            env=env,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            universal_newlines=True
+        )
+        out, _ = boot_proc.communicate()
         returncode = boot_proc.wait()
         log.info("Bootstrap exited with status %s", returncode)
         if returncode != 0:
-            for line in out.split():
-                log.warn(line.strip())
+            for line in out.split("\n"):
+                log.warning(line.strip())
             venv_path = os.path.join(dest_path, 'virtualenv')
             log.info("Removing %s", venv_path)
             shutil.rmtree(venv_path, ignore_errors=True)
             raise BootstrapError("Bootstrap failed!")
-        touch_file(sentinel)