teuthology 1.1.0__py3-none-any.whl → 1.2.1__py3-none-any.whl

teuthology/suite/util.py

@@ -1,10 +1,11 @@
 import copy
+import functools
 import logging
 import os
 import requests
 import smtplib
 import socket
-import subprocess
+from subprocess import Popen, PIPE, DEVNULL
 import sys
 
 from email.mime.text import MIMEText
@@ -17,26 +18,25 @@ from teuthology.config import config
 from teuthology.exceptions import BranchNotFoundError, ScheduleFailError
 from teuthology.misc import deep_merge
 from teuthology.repo_utils import fetch_qa_suite, fetch_teuthology
-from teuthology.orchestra.opsys import OS
-from teuthology.packaging import get_builder_project
+from teuthology.orchestra.opsys import OS, DEFAULT_OS_VERSION
+from teuthology.packaging import get_builder_project, VersionNotFoundError
 from teuthology.repo_utils import build_git_url
-from teuthology.suite.build_matrix import combine_path
 from teuthology.task.install import get_flavor
 
 log = logging.getLogger(__name__)
 
-CONTAINER_DISTRO = 'centos/8'       # the one to check for build_complete
-CONTAINER_FLAVOR = 'basic'          # basic maps to default on shaman
+CONTAINER_DISTRO = 'centos/9'       # the one to check for build_complete
+CONTAINER_FLAVOR = 'default'
 
 
-def fetch_repos(branch, test_name):
+def fetch_repos(branch, test_name, dry_run):
     """
     Fetch the suite repo (and also the teuthology repo) so that we can use it
     to build jobs. Repos are stored in ~/src/.
 
     The reason the teuthology repo is also fetched is that currently we use
     subprocess to call teuthology-schedule to schedule jobs so we need to make
-    sure it is up-to-date. For that reason we always fetch the master branch
+    sure it is up-to-date. For that reason we always fetch the main branch
     for test scheduling, regardless of what teuthology branch is requested for
     testing.
 
@@ -46,22 +46,23 @@ def fetch_repos(branch, test_name):
         # When a user is scheduling a test run from their own copy of
         # teuthology, let's not wreak havoc on it.
         if config.automated_scheduling:
-            # We use teuthology's master branch in all cases right now
+            # We use teuthology's main branch in all cases right now
             if config.teuthology_path is None:
-                fetch_teuthology('master')
+                fetch_teuthology('main')
         suite_repo_path = fetch_qa_suite(branch)
     except BranchNotFoundError as exc:
-        schedule_fail(message=str(exc), name=test_name)
+        schedule_fail(message=str(exc), name=test_name, dry_run=dry_run)
     return suite_repo_path
 
 
-def schedule_fail(message, name=''):
+def schedule_fail(message, name='', dry_run=None):
     """
     If an email address has been specified anywhere, send an alert there. Then
     raise a ScheduleFailError.
+    Don't send the mail if --dry-run has been passed.
     """
     email = config.results_email
-    if email:
+    if email and not dry_run:
         subject = "Failed to schedule {name}".format(name=name)
         msg = MIMEText(message)
         msg['Subject'] = subject
@@ -100,7 +101,7 @@ def get_gitbuilder_hash(project=None, branch=None, flavor=None,
     # Alternate method for github-hosted projects - left here for informational
     # purposes
     # resp = requests.get(
-    #     'https://api.github.com/repos/ceph/ceph/git/refs/heads/master')
+    #     'https://api.github.com/repos/ceph/ceph/git/refs/heads/main')
     # hash = .json()['object']['sha']
     (arch, release, _os) = get_distro_defaults(distro, machine_type)
     if distro is None:
@@ -122,38 +123,15 @@ def get_distro_defaults(distro, machine_type):
     """
     Given a distro (e.g. 'ubuntu') and machine type, return:
         (arch, release, pkg_type)
-
-    This is used to default to:
-        ('x86_64', 'trusty', 'deb') when passed 'ubuntu' and 'plana'
-        ('armv7l', 'saucy', 'deb') when passed 'ubuntu' and 'saya'
-        ('x86_64', 'wheezy', 'deb') when passed 'debian'
-        ('x86_64', 'fedora20', 'rpm') when passed 'fedora'
-    And ('x86_64', 'centos7', 'rpm') when passed anything else
     """
     arch = 'x86_64'
-    if distro in (None, 'None'):
-        os_type = 'centos'
-        os_version = '7'
-    elif distro in ('rhel', 'centos'):
-        os_type = 'centos'
-        os_version = '7'
-    elif distro == 'ubuntu':
-        os_type = distro
-        if machine_type == 'saya':
-            os_version = '13.10'
-            arch = 'armv7l'
-        else:
-            os_version = '16.04'
-    elif distro == 'debian':
-        os_type = distro
-        os_version = '7'
-    elif distro == 'fedora':
-        os_type = distro
-        os_version = '20'
-    elif distro == 'opensuse':
+    if distro in (None, 'None', 'rhel'):
+        distro = 'centos'
+
+    try:
+        os_version = DEFAULT_OS_VERSION[distro]
         os_type = distro
-        os_version = '15.1'
-    else:
+    except IndexError:
         raise ValueError("Invalid distro value passed: %s", distro)
     _os = OS(name=os_type, version=os_version)
     release = get_builder_project()._get_distro(
@@ -161,15 +139,6 @@ def get_distro_defaults(distro, machine_type):
         _os.version,
         _os.codename,
     )
-    template = "Defaults for machine_type {mtype} distro {distro}: " \
-        "arch={arch}, release={release}, pkg_type={pkg}"
-    log.debug(template.format(
-        mtype=machine_type,
-        distro=_os.name,
-        arch=arch,
-        release=release,
-        pkg=_os.package_type)
-    )
     return (
         arch,
         release,
@@ -187,7 +156,7 @@ def git_ls_remote(project_or_url, branch, project_owner='ceph'):
                            name is passed; not when a URL is passed
     :returns: The sha1 if found; else None
     """
-    if '://' in project_or_url:
+    if '://' in project_or_url or project_or_url.startswith('git@'):
         url = project_or_url
     else:
         url = build_git_url(project_or_url, project_owner)
@@ -254,7 +223,8 @@ def get_branch_info(project, branch, project_owner='ceph'):
         return resp.json()
 
 
-def package_version_for_hash(hash, kernel_flavor='basic', distro='rhel',
+@functools.lru_cache()
+def package_version_for_hash(hash, flavor='default', distro='rhel',
                              distro_version='8.0', machine_type='smithi'):
     """
     Does what it says on the tin. Uses gitbuilder repos.
@@ -267,7 +237,7 @@ def package_version_for_hash(hash, kernel_flavor='basic', distro='rhel',
     bp = get_builder_project()(
         'ceph',
         dict(
-            flavor=kernel_flavor,
+            flavor=flavor,
             os_type=distro,
             os_version=distro_version,
             arch=arch,
@@ -275,13 +245,15 @@ def package_version_for_hash(hash, kernel_flavor='basic', distro='rhel',
         ),
     )
 
-    if bp.distro == CONTAINER_DISTRO and bp.flavor == CONTAINER_FLAVOR:
-        log.info('container build %s, checking for build_complete' % bp.distro)
-        if not bp.build_complete:
-            log.info('build not complete')
-            return None
+    if (bp.distro == CONTAINER_DISTRO and bp.flavor == CONTAINER_FLAVOR and 
+            not bp.build_complete):
+        log.info("Container build incomplete")
+        return None
 
-    return bp.version
+    try:
+        return bp.version
+    except VersionNotFoundError:
+        return None
 
 
 def get_arch(machine_type):
@@ -291,9 +263,9 @@ def get_arch(machine_type):
 
     :returns: A string or None
     """
-    result = teuthology.lock.query.list_locks(machine_type=machine_type, count=1)
+    result = teuthology.lock.query.list_locks(machine_type=machine_type, count=1, tries=1)
     if not result:
-        log.warn("No machines found with machine_type %s!", machine_type)
+        log.warning("No machines found with machine_type %s!", machine_type)
     else:
         return result[0]['arch']
 
@@ -333,102 +305,7 @@ def get_install_task_flavor(job_config):
     return get_flavor(first_install_config)
 
 
-def get_package_versions(sha1, os_type, os_version, flavor,
-                         package_versions=None):
-    """
-    Will retrieve the package versions for the given sha1, os_type/version,
-    and flavor from gitbuilder.
-
-    Optionally, a package_versions dict can be provided
-    from previous calls to this function to avoid calling gitbuilder for
-    information we've already retrieved.
-
-    The package_versions dict will be in the following format::
-
-        {
-            "sha1": {
-                "ubuntu": {
-                    "14.04": {
-                        "basic": "version",
-                    }
-                    "15.04": {
-                        "notcmalloc": "version",
-                    }
-                }
-                "rhel": {
-                    "basic": "version",
-                    }
-            },
-            "another-sha1": {
-                "ubuntu": {
-                    "basic": "version",
-                    }
-            }
-        }
-
-    :param sha1:             The sha1 hash of the ceph version.
-    :param os_type:          The distro we want to get packages for, given
-                             the ceph sha1. Ex. 'ubuntu', 'rhel', etc.
-    :param os_version:       The distro's version, e.g. '14.04', '7.0'
-    :param flavor:           Package flavor ('testing', 'notcmalloc', etc.)
-    :param package_versions: Use this optionally to use cached results of
-                             previous calls to gitbuilder.
-    :returns:                A dict of package versions. Will return versions
-                             for all hashes/distros/vers, not just for the given
-                             hash/distro/ver.
-    """
-    if package_versions is None:
-        package_versions = dict()
-
-    os_type = str(os_type)
-
-    os_types = package_versions.get(sha1, dict())
-    os_versions = os_types.get(os_type, dict())
-    flavors = os_versions.get(os_version, dict())
-    if flavor not in flavors:
-        package_version = package_version_for_hash(
-            sha1,
-            flavor,
-            distro=os_type,
-            distro_version=os_version,
-        )
-        flavors[flavor] = package_version
-        os_versions[os_version] = flavors
-        os_types[os_type] = os_versions
-        package_versions[sha1] = os_types
-
-    return package_versions
-
-
-def has_packages_for_distro(sha1, os_type, os_version, flavor,
-                            package_versions=None):
-    """
-    Checks to see if gitbuilder has packages for the given sha1, os_type and
-    kernel_flavor.
-
-    See above for package_versions description.
-
-    :param sha1:             The sha1 hash of the ceph version.
-    :param os_type:          The distro we want to get packages for, given
-                             the ceph sha1. Ex. 'ubuntu', 'rhel', etc.
-    :param kernel_flavor:    The kernel flavor
-    :param package_versions: Use this optionally to use cached results of
-                             previous calls to gitbuilder.
-    :returns:                True, if packages are found. False otherwise.
-    """
-    os_type = str(os_type)
-    if package_versions is None:
-        package_versions = get_package_versions(
-            sha1, os_type, os_version, flavor)
-
-    flavors = package_versions.get(sha1, dict()).get(
-            os_type, dict()).get(
-            os_version, dict())
-    # we want to return a boolean here, not the actual package versions
-    return bool(flavors.get(flavor, None))
-
-
-def teuthology_schedule(args, verbose, dry_run, log_prefix=''):
+def teuthology_schedule(args, verbose, dry_run, log_prefix='', stdin=None):
     """
     Run teuthology-schedule to schedule individual jobs.
 
@@ -451,84 +328,53 @@ def teuthology_schedule(args, verbose, dry_run, log_prefix=''):
                 printable_args.append("'%s'" % item)
             else:
                 printable_args.append(item)
-        log.info('{0}{1}'.format(
+        log.debug('{0} command: {1}'.format(
             log_prefix,
             ' '.join(printable_args),
         ))
     if not dry_run or (dry_run and verbose > 1):
-        subprocess.check_call(args=args)
-
+        astdin = DEVNULL if stdin is None else PIPE
+        p = Popen(args, stdin=astdin)
+        if stdin is not None:
+            p.communicate(input=stdin.encode('utf-8'))
+        else:
+            p.communicate()
 
-def find_git_parent(project, sha1):
+def find_git_parents(project: str, sha1: str, count=1):
 
     base_url = config.githelper_base_url
     if not base_url:
         log.warning('githelper_base_url not set, --newest disabled')
-        return None
+        return []
 
-    def refresh(project):
-        url = '%s/%s.git/refresh' % (base_url, project)
+    def refresh():
+        url = f"{base_url}/{project}.git/refresh"
+        log.info(f"Forcing refresh of git mirror: {url}")
         resp = requests.get(url)
         if not resp.ok:
             log.error('git refresh failed for %s: %s',
                       project, resp.content.decode())
 
     def get_sha1s(project, committish, count):
-        url = '/'.join((base_url, '%s.git' % project,
-                       'history/?committish=%s&count=%d' % (committish, count)))
+        url = f"{base_url}/{project}.git/history?committish={committish}&count={count}"
+        log.info(f"Looking for parent commits: {url}")
         resp = requests.get(url)
         resp.raise_for_status()
         sha1s = resp.json()['sha1s']
         if len(sha1s) != count:
-            log.debug('got response: %s', resp.json())
-            log.error('can''t find %d parents of %s in %s: %s',
-                       int(count), sha1, project, resp.json()['error'])
+            resp_json = resp.json()
+            err_msg = resp_json.get("error") or resp_json.get("err")
+            log.debug(f"Got {resp.status_code} response: {resp_json}")
+            log.error(f"Can't find {count} parents of {sha1} in {project}: {err_msg}")
         return sha1s
 
-    # XXX don't do this every time?..
-    refresh(project)
-    # we want the one just before sha1; list two, return the second
-    sha1s = get_sha1s(project, sha1, 2)
-    if len(sha1s) == 2:
-        return sha1s[1]
-    else:
-        return None
-
-
-def filter_configs(configs, suite_name=None,
-                            filter_in=None,
-                            filter_out=None,
-                            filter_all=None,
-                            filter_fragments=True):
-    """
-    Returns a generator for pairs of description and fragment paths.
-
-    Usage:
-
-        configs = build_matrix(path, subset, seed)
-        for description, fragments in filter_configs(configs):
-            pass
-    """
-    for item in configs:
-        fragment_paths = item[1]
-        description = combine_path(suite_name, item[0]) \
-                                        if suite_name else item[0]
-        base_frag_paths = [strip_fragment_path(x)
-                                        for x in fragment_paths]
-        def matches(f):
-            if f in description:
-                return True
-            if filter_fragments and \
-                    any(f in path for path in base_frag_paths):
-                return True
-            return False
-        if filter_all:
-            if not all(matches(f) for f in filter_all):
-                continue
-        if filter_in:
-            if not any(matches(f) for f in filter_in):
-                continue
-        if filter_out:
-            if any(matches(f) for f in filter_out):
-                continue
-        yield([description, fragment_paths])
+    # index 0 will be the commit whose parents we want to find.
+    # So we will query for count+1, and strip index 0 from the result.
+    sha1s = get_sha1s(project, sha1, count + 1)
+    if not sha1s:
+        log.error("Will try to refresh git mirror and try again")
+        refresh()
+        sha1s = get_sha1s(project, sha1, count + 1)
+    if sha1s:
+        return sha1s[1:]
+    return []

teuthology/task/__init__.py

@@ -24,7 +24,7 @@ class Task(object):
                   name = 'mytask.mysubtask'
     """
 
-    def __init__(self, ctx=None, config=None):
+    def __init__(self, ctx, config=None):
         if not hasattr(self, 'name'):
             self.name = self.__class__.__name__.lower()
         self.log = log

teuthology/task/ansible.py

@@ -1,39 +1,96 @@
 import json
 import logging
+import re
 import requests
 import os
+import pathlib
 import pexpect
 import yaml
 import shutil
 
 from tempfile import mkdtemp, NamedTemporaryFile
 
+from teuthology import repo_utils
 from teuthology.config import config as teuth_config
 from teuthology.exceptions import CommandFailedError, AnsibleFailedError
 from teuthology.job_status import set_status
-from teuthology.repo_utils import fetch_repo
-
 from teuthology.task import Task
+from teuthology.util.loggerfile import LoggerFile
 
 log = logging.getLogger(__name__)
 
 
-class LoggerFile(object):
-    """
-    A thin wrapper around a logging.Logger instance that provides a file-like
-    interface.
-
-    Used by Ansible.execute_playbook() when it calls pexpect.run()
-    """
-    def __init__(self, logger, level):
-        self.logger = logger
-        self.level = level
-
-    def write(self, string):
-        self.logger.log(self.level, string.decode('utf-8', 'ignore'))
+class FailureAnalyzer:
+    def analyze(self, failure_log):
+        failure_obj = yaml.safe_load(failure_log)
+        lines = set()
+        if failure_obj is None:
+            return lines
+        for host_obj in failure_obj.values():
+            if not isinstance(host_obj, dict):
+                continue
+            lines = lines.union(self.analyze_host_record(host_obj))
+        return sorted(lines)
+
+    def analyze_host_record(self, record):
+        lines = set()
+        for result in record.get("results", [record]):
+            cmd = result.get("cmd", "")
+            # When a CPAN task fails, we get _lots_ of stderr_lines, and they
+            # aren't practical to reduce meaningfully. Instead of analyzing lines,
+            # just report the command that failed.
+            if "cpan" in cmd:
+                lines.add(f"CPAN command failed: {cmd}")
+                continue
+            lines_to_analyze = []
+            if "stderr_lines" in result:
+                lines_to_analyze = result["stderr_lines"]
+            elif "msg" in result:
+                lines_to_analyze = result["msg"].split("\n")
+            lines_to_analyze.extend(result.get("err", "").split("\n"))
+            for line in lines_to_analyze:
+                line = self.analyze_line(line.strip())
+                if line:
+                    lines.add(line)
+        return list(lines)
+
+    def analyze_line(self, line):
+        if line.startswith("W: ") or line.endswith("?"):
+            return ""
+        drop_phrases = [
+            # apt output sometimes contains warnings or suggestions. Those won't be
+            # helpful, so throw them out.
+            r"^W: ",
+            r"\?$",
+            # some output from SSH is not useful
+            r"Warning: Permanently added .+ to the list of known hosts.",
+            r"^@+$",
+        ]
+        for phrase in drop_phrases:
+            match = re.search(rf"({phrase})", line, flags=re.IGNORECASE)
+            if match:
+                return ""
+
+        # Next, we can normalize some common phrases.
+        phrases = [
+            "connection timed out",
+            r"(unable to|could not) connect to [^ ]+",
+            r"temporary failure resolving [^ ]+",
+            r"Permissions \d+ for '.+' are too open.",
+        ]
+        for phrase in phrases:
+            match = re.search(rf"({phrase})", line, flags=re.IGNORECASE)
+            if match:
+                line = match.groups()[0]
+                break
 
-    def flush(self):
-        pass
+        # Strip out URLs for specific packages
+        package_re = re.compile(r"https?://.*\.(deb|rpm)")
+        line = package_re.sub("<package>", line)
+        # Strip out IP addresses
+        ip_re = re.compile(r"\[IP: \d+\.\d+\.\d+\.\d+( \d+)?\]")
+        line = ip_re.sub("", line)
+        return line
 
 
 class Ansible(Task):
@@ -50,7 +107,7 @@ class Ansible(Task):
         repo:       A path or URL to a repo (defaults to '.'). Given a repo
                     value of 'foo', ANSIBLE_ROLES_PATH is set to 'foo/roles'
         branch:     If pointing to a remote git repo, use this branch. Defaults
-                    to 'master'.
+                    to 'main'.
         hosts:      A list of teuthology roles or partial hostnames (or a
                     combination of the two). ansible-playbook will only be run
                     against hosts that match.
@@ -112,9 +169,12 @@ class Ansible(Task):
 
     def __init__(self, ctx, config):
         super(Ansible, self).__init__(ctx, config)
-        self.log = log
         self.generated_inventory = False
         self.generated_playbook = False
+        self.log = logging.Logger(__name__)
+        if ctx.archive:
+            self.log.addHandler(logging.FileHandler(
+                os.path.join(ctx.archive, "ansible.log")))
 
     def setup(self):
         super(Ansible, self).setup()
@@ -139,9 +199,9 @@ class Ansible(Task):
         """
         repo = self.config.get('repo', '.')
         if repo.startswith(('http://', 'https://', 'git@', 'git://')):
-            repo_path = fetch_repo(
+            repo_path = repo_utils.fetch_repo(
                 repo,
-                self.config.get('branch', 'master'),
+                self.config.get('branch', 'main'),
             )
         else:
             repo_path = os.path.abspath(os.path.expanduser(repo))
@@ -260,7 +320,10 @@ class Ansible(Task):
 
     def begin(self):
         super(Ansible, self).begin()
-        self.execute_playbook()
+        if len(self.cluster.remotes) > 0:
+            self.execute_playbook()
+        else:
+            log.info("There are no remotes; skipping playbook execution")
 
     def execute_playbook(self, _logfile=None):
         """
@@ -274,15 +337,18 @@ class Ansible(Task):
         environ['ANSIBLE_FAILURE_LOG'] = self.failure_log.name
         environ['ANSIBLE_ROLES_PATH'] = "%s/roles" % self.repo_path
         environ['ANSIBLE_NOCOLOR'] = "1"
+        # Store collections in <repo root>/.ansible/
+        # This is the same path used in <repo root>/ansible.cfg
+        environ['ANSIBLE_COLLECTIONS_PATH'] = str(
+            pathlib.Path(__file__).parents[2] / ".ansible")
         args = self._build_args()
         command = ' '.join(args)
         log.debug("Running %s", command)
 
-        out_log = self.log.getChild('out')
         out, status = pexpect.run(
             command,
             cwd=self.repo_path,
-            logfile=_logfile or LoggerFile(out_log, logging.INFO),
+            logfile=_logfile or LoggerFile(self.log, logging.INFO),
             withexitstatus=True,
             timeout=None,
         )
@@ -298,17 +364,20 @@ class Ansible(Task):
     def _handle_failure(self, command, status):
         self._set_status('dead')
         failures = None
-        with open(self.failure_log.name, 'r') as fail_log:
+        with open(self.failure_log.name, 'r') as fail_log_file:
+            fail_log = fail_log_file.read()
             try:
-                failures = yaml.safe_load(fail_log)
+                analyzer = FailureAnalyzer()
+                failures = analyzer.analyze(fail_log)
             except yaml.YAMLError as e:
                 log.error(
-                    "Failed to parse ansible failure log: {0} ({1})".format(
-                        self.failure_log.name, e
-                    )
+                    f"Failed to parse ansible failure log: {self.failure_log.name} ({e})"
                 )
-                fail_log.seek(0)
-                failures = fail_log.read().replace('\n', '')
+            except Exception:
+                log.exception(f"Failed to analyze ansible failure log: {self.failure_log.name}")
+            # If we hit an exception, or if analyze() returned nothing, use the log as-is
+            if not failures:
+                failures = fail_log.replace('\n', '')
 
         if failures:
             self._archive_failures()
@@ -390,7 +459,7 @@ class CephLab(Ansible):
 
     - ansible.cephlab:
         repo: {git_base}ceph-cm-ansible.git
-        branch: master
+        branch: main
         playbook: cephlab.yml
 
     If a dynamic inventory is used, all hosts will be assigned to the

teuthology/task/buildpackages.py

@@ -53,7 +53,7 @@ def apply_overrides(ctx, config):
 def get_config_install(ctx, config):
     config = apply_overrides(ctx, config)
     log.debug('install config %s' % config)
-    return [(config.get('flavor', 'basic'),
+    return [(config.get('flavor', 'default'),
              config.get('tag', ''),
              config.get('branch', ''),
              config.get('sha1'))]
@@ -69,7 +69,7 @@ def get_config_install_upgrade(ctx, config):
         log.debug('install.upgrade config ' + str(role_config) +
                   ' and with overrides ' + str(o))
         # for install.upgrade overrides are actually defaults
-        configs.append((o.get('flavor', 'basic'),
+        configs.append((o.get('flavor', 'default'),
                         role_config.get('tag', o.get('tag', '')),
                         role_config.get('branch', o.get('branch', '')),
                         role_config.get('sha1', o.get('sha1'))))