teuthology 1.0.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

teuthology/suite/__init__.py

@@ -5,6 +5,7 @@
 import logging
 import os
 import random
+import sys
 import time
 from distutils.util import strtobool
 
@@ -43,8 +44,6 @@ def process_args(args):
         'ceph': 'ceph_branch',
         'sha1': 'ceph_sha1',
         'kernel': 'kernel_branch',
-        # FIXME: ceph flavor and kernel flavor are separate things
-        'flavor': 'kernel_flavor',
         '<config_yaml>': 'base_yaml_paths',
         'filter': 'filter_in',
     }
@@ -59,11 +58,14 @@ def process_args(args):
             value = normalize_suite_name(value)
         if key == 'suite_relpath' and value is None:
             value = ''
-        elif key in ('limit', 'priority', 'num', 'newest', 'seed'):
+        elif key in ('limit', 'priority', 'num', 'newest', 'seed', 'job_threshold'):
             value = int(value)
         elif key == 'subset' and value is not None:
             # take input string '2/3' and turn into (2, 3)
             value = tuple(map(int, value.split('/')))
+        elif key == 'expire' and value is None:
+            # Skip empty 'expire' values
+            continue
         elif key in ('filter_all', 'filter_in', 'filter_out', 'rerun_statuses'):
             if not value:
                 value = []
@@ -77,7 +79,7 @@ def process_args(args):
             value = expand_short_repo_name(
                 value,
                 config.get_ceph_qa_suite_git_url())
-        elif key in ('validate_sha1', 'filter_fragments'):
+        elif key in ('validate_sha1', 'filter_fragments', 'kdb'):
             value = strtobool(value)
         conf[key] = value
     return conf
@@ -111,11 +113,15 @@ def main(args):
     if conf.verbose:
         teuthology.log.setLevel(logging.DEBUG)
 
+    dry_run = conf.dry_run
     if not conf.machine_type or conf.machine_type == 'None':
-        schedule_fail("Must specify a machine_type")
+        if not config.default_machine_type or config.default_machine_type == 'None':
+            schedule_fail("Must specify a machine_type", dry_run=dry_run)
+        else:
+           conf.machine_type = config.default_machine_type
     elif 'multi' in conf.machine_type:
         schedule_fail("'multi' is not a valid machine_type. " +
-                      "Maybe you want 'plana,mira,burnupi' or similar")
+                      "Maybe you want 'gibba,smithi,mira' or similar", dry_run=dry_run)
 
     if conf.email:
         config.results_email = conf.email
@@ -124,16 +130,7 @@ def main(args):
         log.info('Will upload archives to ' + conf.archive_upload)
 
     if conf.rerun:
-        rerun_filters = get_rerun_filters(conf.rerun, conf.rerun_statuses)
-        if len(rerun_filters['descriptions']) == 0:
-            log.warn(
-                "No jobs matched the status filters: %s",
-                conf.rerun_statuses,
-            )
-            return
-        conf.filter_in.extend(rerun_filters['descriptions'])
-        conf.suite = normalize_suite_name(rerun_filters['suite'])
-        conf.subset, conf.seed = get_rerun_conf(conf)
+        get_rerun_conf_overrides(conf)
     if conf.seed < 0:
         conf.seed = random.randint(0, 9999)
         log.info('Using random seed=%s', conf.seed)
@@ -146,11 +143,68 @@ def main(args):
                     conf.archive_upload_url)
 
 
-def get_rerun_filters(name, statuses):
+def get_rerun_conf_overrides(conf):
     reporter = ResultsReporter()
-    run = reporter.get_run(name)
+    run = reporter.get_run(conf.rerun)
+
+    conf.suite = normalize_suite_name(run['suite'])
+
+    try:
+        job0 = run['jobs'][0]
+    except IndexError:
+        job0 = None
+
+    seed = None if job0 is None else job0.get('seed')
+    if conf.seed >= 0 and conf.seed != seed:
+        log.error('--seed %s does not match with rerun seed: %s',
+                  conf.seed, seed)
+        sys.exit(1)
+    else:
+        log.info('Using rerun seed=%s', seed)
+        conf.seed = seed
+
+    if job0 is not None:
+        subset = job0.get('subset', '1/1')
+        if subset is None:
+            subset = '1/1'
+        subset =  tuple(map(int, subset.split('/')))
+    else:
+        subset = None
+    if conf.subset is not None and conf.subset != subset:
+        log.error('--subset %s does not match with '
+                  'rerun subset: %s',
+                  conf.subset, subset)
+        sys.exit(1)
+    else:
+        if subset == (1, 1):
+            conf.subset = None
+        else:
+            log.info('Using rerun subset=%s', subset)
+            conf.subset = subset
+
+    no_nested_subset = False if job0 is None else job0.get('no_nested_subset', False)
+    if conf.no_nested_subset is not None and conf.no_nested_subset != no_nested_subset:
+        log.error('--no-nested-subset specified but does not match with '
+                  'rerun --no-nested-subset: %s',
+                  no_nested_subset)
+        sys.exit(1)
+    else:
+        log.info('Using rerun no_nested_subset=%s', no_nested_subset)
+        conf.no_nested_subset = no_nested_subset
+
+    rerun_filters = get_rerun_filters(run, conf.rerun_statuses)
+    if len(rerun_filters['descriptions']) == 0:
+        log.warning(
+            "No jobs matched the status filters: %s",
+            conf.rerun_statuses,
+        )
+        return
+
+    conf.filter_in.extend(rerun_filters['descriptions'])
+
+
+def get_rerun_filters(run, statuses):
     filters = dict()
-    filters['suite'] = run['suite']
     jobs = []
     for job in run['jobs']:
         if job['status'] in statuses:
@@ -159,31 +213,6 @@ def get_rerun_filters(name, statuses):
     return filters
 
 
-def get_rerun_conf(conf):
-    reporter = ResultsReporter()
-    try:
-        subset, seed = reporter.get_rerun_conf(conf.rerun)
-    except IOError:
-        return conf.subset, conf.seed
-    if seed is None:
-        return conf.subset, conf.seed
-    if conf.seed < 0:
-        log.info('Using stored seed=%s', seed)
-    elif conf.seed != seed:
-        log.error('--seed {conf_seed} does not match with ' +
-                  'stored seed: {stored_seed}',
-                  conf_seed=conf.seed,
-                  stored_seed=seed)
-    if conf.subset is None:
-        log.info('Using stored subset=%s', subset)
-    elif conf.subset != subset:
-        log.error('--subset {conf_subset} does not match with ' +
-                  'stored subset: {stored_subset}',
-                  conf_subset=conf.subset,
-                  stored_subset=subset)
-    return subset, seed
-
-
 class WaitException(Exception):
     pass
 
@@ -193,7 +222,7 @@ def wait(name, max_job_time, upload_url):
     reporter = ResultsReporter()
     past_unfinished_jobs = []
     progress = time.time()
-    log.info("waiting for the suite to complete")
+    log.info(f"waiting for the run {name} to complete")
     log.debug("the list of unfinished jobs will be displayed "
               "every " + str(Run.WAIT_PAUSE / 60) + " minutes")
     exit_code = 0
@@ -228,5 +257,5 @@ def wait(name, max_job_time, upload_url):
             url = os.path.join(upload_url, name, job['job_id'])
         else:
             url = job['log_href']
-        log.info(job['status'] + " " + url + " " + job['description'])
+        log.info(f"{job['status']} {url} {job['description']}")
     return exit_code

teuthology/suite/build_matrix.py

@@ -7,7 +7,7 @@ from teuthology.suite import matrix
 log = logging.getLogger(__name__)
 
 
-def build_matrix(path, subset=None, seed=None):
+def build_matrix(path, subset=None, no_nested_subset=False, seed=None):
     """
     Return a list of items descibed by path such that if the list of
     items is chunked into mincyclicity pieces, each piece is still a
@@ -33,7 +33,9 @@ def build_matrix(path, subset=None, seed=None):
 
     For a directory with a magic '%' file, we generate a result set
     for each item in the directory, and then do a product to generate
-    a result list with all combinations (A Product).
+    a result list with all combinations (A Product). If the file
+    contains an integer, it is used as the divisor for a random
+    subset.
 
     For a directory with a magic '$' file, or for a directory whose name
     ends in '$', we generate a list of all items that we will randomly
@@ -46,6 +48,7 @@ def build_matrix(path, subset=None, seed=None):
 
     :param path:        The path to search for yaml fragments
     :param subset:	(index, outof)
+    :param no_nested_subset:	disable nested subsets
     :param seed:        The seed for repeatable random test
     """
     if subset:
@@ -53,31 +56,24 @@ def build_matrix(path, subset=None, seed=None):
             'Subset=%s/%s' %
             (str(subset[0]), str(subset[1]))
         )
+    if no_nested_subset:
+        log.info("no_nested_subset")
     random.seed(seed)
-    mat, first, matlimit = _get_matrix(path, subset)
+    mat, first, matlimit = _get_matrix(path, subset, no_nested_subset)
     return generate_combinations(path, mat, first, matlimit)
 
 
-def _get_matrix(path, subset=None):
-    mat = None
-    first = None
-    matlimit = None
-    if subset:
-        (index, outof) = subset
-        mat = _build_matrix(path, mincyclicity=outof)
-        first = (mat.size() // outof) * index
-        if index == outof or index == outof - 1:
-            matlimit = mat.size()
-        else:
-            matlimit = (mat.size() // outof) * (index + 1)
+def _get_matrix(path, subset=None, no_nested_subset=False):
+    (which, divisions) = (0,1) if subset is None else subset
+    if divisions > 1:
+        mat = _build_matrix(path, mincyclicity=divisions, no_nested_subset=no_nested_subset)
+        mat = matrix.Subset(mat, divisions, which=which)
     else:
-        first = 0
-        mat = _build_matrix(path)
-        matlimit = mat.size()
-    return mat, first, matlimit
+        mat = _build_matrix(path, no_nested_subset=no_nested_subset)
+    return mat, 0, mat.size()
 
 
-def _build_matrix(path, mincyclicity=0, item=''):
+def _build_matrix(path, mincyclicity=0, no_nested_subset=False, item=''):
     if os.path.basename(path)[0] == '.':
         return None
     if not os.path.exists(path):
@@ -100,6 +96,7 @@ def _build_matrix(path, mincyclicity=0, item=''):
                 submat = _build_matrix(
                     os.path.join(path, fn),
                     mincyclicity,
+                    no_nested_subset,
                     fn)
                 if submat is not None:
                     submats.append(submat)
@@ -115,6 +112,7 @@ def _build_matrix(path, mincyclicity=0, item=''):
                 submat = _build_matrix(
                     os.path.join(path, fn),
                     mincyclicity,
+                    no_nested_subset,
                     fn)
                 if submat is not None:
                     submats.append(submat)
@@ -122,19 +120,30 @@ def _build_matrix(path, mincyclicity=0, item=''):
         elif '%' in files:
             # convolve items
             files.remove('%')
+            with open(os.path.join(path, '%')) as f:
+                divisions = f.read()
+                if no_nested_subset or len(divisions) == 0:
+                    divisions = 1
+                else:
+                    divisions = int(divisions)
+                    assert divisions > 0
             submats = []
             for fn in sorted(files):
                 submat = _build_matrix(
                     os.path.join(path, fn),
-                    mincyclicity=0,
-                    item=fn)
+                    0,
+                    no_nested_subset,
+                    fn)
                 if submat is not None:
                     submats.append(submat)
             mat = matrix.Product(item, submats)
-            if mat and mat.cyclicity() < mincyclicity:
+            minc = mincyclicity * divisions
+            if mat and mat.cyclicity() < minc:
                 mat = matrix.Cycle(
-                    (mincyclicity + mat.cyclicity() - 1) // mat.cyclicity(), mat
+                    (minc + mat.cyclicity() - 1) // mat.cyclicity(), mat
                 )
+            if divisions > 1:
+                mat = matrix.Subset(mat, divisions)
             return mat
         else:
             # list items
@@ -143,6 +152,7 @@ def _build_matrix(path, mincyclicity=0, item=''):
                 submat = _build_matrix(
                     os.path.join(path, fn),
                     mincyclicity,
+                    no_nested_subset,
                     fn)
                 if submat is None:
                     continue

teuthology/suite/fragment-merge.lua

@@ -0,0 +1,105 @@
+-- allow only some Lua (and lunatic) builtins for use by scripts
+local lua_allowlist = {
+  assert = assert,
+  error = error,
+  ipairs = ipairs,
+  next = next,
+  pairs = pairs,
+  tonumber = tonumber,
+  tostring = tostring,
+  py_attrgetter = python.as_attrgetter,
+  py_dict = python.builtins.dict,
+  py_len = python.builtins.len,
+  py_list = python.builtins.list,
+  py_tuple = python.builtins.tuple,
+  py_enumerate = python.enumerate,
+  py_iterex = python.iterex,
+  py_itemgetter = python.as_itemgetter,
+  math = math,
+}
+lua_allowlist.__index = lua_allowlist
+
+-- accept a fragment/config (or just return true from the script!)
+local function accept()
+  coroutine.yield(true)
+end
+-- reject a fragment/config (or just return false from the script!)
+local function reject()
+  coroutine.yield(false)
+end
+-- this implements logic for filtering (via teuthology-suite CLI flags)
+local function matches(_ENV, f)
+  if description:find(f, 1, true) then
+    return true
+  end
+  if filter_fragments then
+    for i,path in py_enumerate(base_frag_paths) do
+      if path:find(f) then
+        return true
+      end
+    end
+  end
+end
+
+local function check_filters(_ENV)
+  if filter_all then
+    for i,f in py_enumerate(filter_all) do
+      if not matches(_ENV, f) then
+        reject()
+      end
+    end
+  end
+  if filter_in then
+    local found, tried = false, false
+    for i,f in py_enumerate(filter_in) do
+      tried = true
+      if matches(_ENV, f) then
+        found = true
+        break
+      end
+    end
+    if tried and not found then
+      reject()
+    end
+  end
+  if filter_out then
+    for i,f in py_enumerate(filter_out) do
+      if matches(_ENV, f) then
+        reject()
+      end
+    end
+  end
+end
+
+function new_script(script, log, deep_merge, yaml_load)
+  -- create a restricted sandbox for the script:
+  local env = setmetatable({
+    accept = accept,
+    deep_merge = deep_merge,
+    log = log,
+    reject = reject,
+    yaml_load = yaml_load,
+  }, lua_allowlist)
+
+  -- avoid putting check_filters in _ENV
+  -- try to keep line numbers correct:
+  local header = [[do local check_filters = ...; accept(); check_filters(_ENV) end local function main() do ]]
+  local footer = [[ end return true end return main()]]
+  local function chunks()
+    coroutine.yield(header)
+    if #script > 0 then
+      coroutine.yield(script)
+    end
+    coroutine.yield(footer)
+  end
+
+  -- put the script in a coroutine so we can yield success/failure from
+  -- anywhere in the script, including in nested function calls.
+  local f, err = load(coroutine.wrap(chunks), 'teuthology', 't', env)
+  if f == nil then
+    error("failure to load script: "..err)
+  end
+  f = coroutine.wrap(f)
+  f(check_filters)
+  return env, f
+end

teuthology/suite/matrix.py

@@ -1,7 +1,7 @@
 import os
 import random
 import heapq
-from fractions import gcd
+from math import gcd
 from functools import reduce
 
 def lcm(a, b):
@@ -75,6 +75,35 @@ class Cycle(Matrix):
     def tostr(self, depth):
         return '\t'*depth + "Cycle({num}):\n".format(num=self.num) + self.mat.tostr(depth + 1)
 
+# Logically, inverse of Cycle
+class Subset(Matrix):
+    """
+    Run a matrix subset.
+    """
+    def __init__(self, mat, divisions, which=None):
+        self.mat = mat
+        self.divisions = divisions
+        if which is None:
+            self.which = random.randint(0, divisions-1)
+        else:
+            assert which < divisions
+            self.which = which
+
+    def size(self):
+        return self.mat.size() // self.divisions
+
+    def index(self, i):
+        i += self.which * self.size()
+        assert i < self.mat.size()
+        return self.mat.index(i)
+
+    def minscanlen(self):
+        return self.mat.minscanlen()
+
+    def tostr(self, depth):
+        return '\t'*depth + "Subset({num}, {index}):\n".format(num=self.num, index=self.index) + self.mat.tostr(depth + 1)
+
+
 class Base(Matrix):
     """
     Just a single item.
@@ -248,7 +277,7 @@ class Sum(Matrix):
     """
     def __init__(self, item, _submats):
         assert len(_submats) > 0, \
-            "Sum requires non-empty _submats"
+            f"Sum requires non-empty _submats: {item}"
         self.item = item
 
         self._pseudo_size = lcml((i.size() for i in _submats)) * len(_submats)

teuthology/suite/merge.py

@@ -0,0 +1,175 @@
+import copy
+import logging
+import lupa
+import os
+from types import MappingProxyType
+import yaml
+
+from teuthology.suite.build_matrix import combine_path
+from teuthology.suite.util import strip_fragment_path
+from teuthology.misc import deep_merge
+
+log = logging.getLogger(__name__)
+
+TEUTHOLOGY_TEMPLATE = MappingProxyType({
+  "teuthology": {
+    "fragments_dropped": [],
+    "meta": {},
+    "postmerge": [],
+  }
+})
+
+L = lupa.LuaRuntime()
+FRAGMENT_MERGE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fragment-merge.lua")
+with open(FRAGMENT_MERGE) as f:
+    L.execute(f.read())
+
+def config_merge(configs, suite_name=None, **kwargs):
+    """
+    This procedure selects and merges YAML fragments for each job in the
+    configs array generated for the matrix of jobs.
+
+    The primary task here is to run premerge and postmerge scripts specified
+    with the YAML fragments as part of filtering out jobs or individual YAML
+    fragments. This is done with Lua scripting (via "lupa", a "lunatic"
+    derivative).
+
+    A premerge script looks like:
+
+    <foo.yaml>
+    teuthology:
+      premerge: |
+                if yaml.os_type == 'ubuntu' then reject() end
+    </foo.yaml>
+
+    This script runs prior to a YAML fragment merging into the complete YAML
+    specification for a job.  The script has access to the complete YAML
+    description generated so far as part of merging earlier fragments
+    (remember: fragments are ordered lexicographically). In the above case, the
+    os_type is checked with the foo.yaml fragment dropped if the job is
+    configured to run on Ubuntu (note: this does not account for a jobs'
+    default os_type which is not yet known).
+
+    The postmerge scripts look like:
+
+    <bar.yaml>
+    teuthology:
+      postmerge:
+        - if yaml.os_type == "ubuntu" then reject() end
+    </bar.yaml>
+
+    This script is the same but has a different effect: if, after combining all
+    the YAML fragments for a job, the os_type is "ubuntu", then the entire job
+    is dropped (filtered out / rejected). postmerge scripts are also specified
+    as a list of strings in the teuthology.postmerge array. All of these
+    strings are concatenated and then executed as a single script. So,
+    postmerge scripts from multiple fragments are all combined. You may use
+    this to define variables, functions, or anything else you need.
+
+    Scripts have access to the entire yaml object and may do any desired advanced
+    checks. It is also possible to programatically change the YAML definition:
+
+    <foo.yaml>
+    teuthology:
+      postmerge:
+        - |
+          local attr = py_attrgetter
+          local tasks = py_list()
+          for i = 1, 3 do
+            local task = py_dict(
+              exec = py_dict(py_list(
+                py_tuple("mon.a", py_list(
+                  "echo "..i
+                )
+              ))
+            )
+            attr(tasks).append(task)
+          end
+          deep_merge(yaml.tasks, tasks)
+    </foo.yaml>
+
+    This will be as if the yaml file contained:
+
+    <foo.yaml>
+    tasks:
+      exec:
+        mon.a:
+          - echo 1
+      exec:
+        mon.a:
+          - echo 2
+      exec:
+        mon.a:
+          - echo 3
+    </foo.yaml>
+
+    Which will be merged normally (via deep_merge) after the script is run.
+
+    Scripts are well sandboxed with access to a small selection of the Lua
+    builtin libraries. There is also access to some python/lupa specific
+    functions which are prefixed with "py_". No I/O or other system functions
+    permitted.
+
+    The teuthology-suite filtering options are now implemented via builtin
+    postmerge scripts. Logically, if a filter matches then reject will drop
+    the entire job (config) from the list.
+    """
+    seed = kwargs.setdefault('seed', 1)
+    if not isinstance(seed, int):
+        log.debug("no valid seed input: using 1")
+        seed = 1
+    log.debug("configuring Lua randomseed to %d", seed)
+    L.execute(f'local math = require"math"; math.randomseed({seed});')
+    new_script = L.eval('new_script')
+    yaml_cache = {}
+    for desc, paths in configs:
+        log.debug("merging config %s", desc)
+
+        if suite_name is not None:
+            desc = combine_path(suite_name, desc)
+
+        yaml_complete_obj = {}
+        deep_merge(yaml_complete_obj, dict(TEUTHOLOGY_TEMPLATE))
+        for path in paths:
+            if path not in yaml_cache:
+                with open(path) as f:
+                    txt = f.read()
+                    yaml_cache[path] = (txt, yaml.safe_load(txt))
+
+            yaml_fragment_txt, yaml_fragment_obj = yaml_cache[path]
+            if yaml_fragment_obj is None:
+                continue
+            yaml_fragment_obj = copy.deepcopy(yaml_fragment_obj)
+            premerge = yaml_fragment_obj.get('teuthology', {}).pop('premerge', '')
+            if premerge:
+                log.debug("premerge script running:\n%s", premerge)
+                env, script = new_script(premerge, log, deep_merge, yaml.safe_load)
+                env['base_frag_paths'] = [strip_fragment_path(x) for x in paths]
+                env['description'] = desc
+                env['frag_paths'] = paths
+                env['suite_name'] = suite_name
+                env['yaml'] = yaml_complete_obj
+                env['yaml_fragment'] = yaml_fragment_obj
+                for k,v in kwargs.items():
+                    env[k] = v
+                if not script():
+                    log.debug("skipping merge of fragment %s due to premerge filter", path)
+                    yaml_complete_obj['teuthology']['fragments_dropped'].append(path)
+                    continue
+            deep_merge(yaml_complete_obj, yaml_fragment_obj)
+
+        postmerge = yaml_complete_obj.get('teuthology', {}).get('postmerge', [])
+        postmerge = "\n".join(postmerge)
+        log.debug("postmerge script running:\n%s", postmerge)
+        env, script = new_script(postmerge, log, deep_merge, yaml.safe_load)
+        env['base_frag_paths'] = [strip_fragment_path(x) for x in paths]
+        env['description'] = desc
+        env['frag_paths'] = paths
+        env['suite_name'] = suite_name
+        env['yaml'] = yaml_complete_obj
+        for k,v in kwargs.items():
+            env[k] = v
+        if not script():
+            log.debug("skipping config %s due to postmerge filter", desc)
+            continue
+        yield desc, paths, yaml_complete_obj