teuthology 1.0.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

teuthology/lock/ops.py

@@ -1,19 +1,26 @@
 import logging
 import json
 import os
-
+import random
+import time
+import yaml
 import requests
 
+from typing import List, Union
+
 import teuthology.orchestra.remote
 import teuthology.parallel
 import teuthology.provision
-from teuthology import misc
+
+from teuthology import misc, report, provision
 from teuthology.config import config
 from teuthology.contextutil import safe_while
 from teuthology.task import console_log
 from teuthology.misc import canonicalize_hostname
+from teuthology.job_status import set_status
 
 from teuthology.lock import util, query
+from teuthology.orchestra import remote
 
 log = logging.getLogger(__name__)
 
@@ -52,7 +59,7 @@ def lock_many_openstack(ctx, num, machine_type, user=None, description=None,
 
 
 def lock_many(ctx, num, machine_type, user=None, description=None,
-              os_type=None, os_version=None, arch=None):
+              os_type=None, os_version=None, arch=None, reimage=True):
     if user is None:
         user = misc.get_user()
 
@@ -111,8 +118,13 @@ def lock_many(ctx, num, machine_type, user=None, description=None,
             headers={'content-type': 'application/json'},
         )
         if response.ok:
-            machines = {misc.canonicalize_hostname(machine['name']):
-                        machine['ssh_pub_key'] for machine in response.json()}
+            machines = dict()
+            for machine in response.json():
+                key = misc.canonicalize_hostname(
+                    machine['name'],
+                    user=machine.get('user'),
+                )
+                machines[key] = machine['ssh_pub_key']
             log.debug('locked {machines}'.format(
                 machines=', '.join(machines.keys())))
             if machine_type in vm_types:
@@ -124,28 +136,12 @@ def lock_many(ctx, num, machine_type, user=None, description=None,
                     else:
                         log.error('Unable to create virtual machine: %s',
                                   machine)
-                        unlock_one(ctx, machine, user)
+                        unlock_one(machine, user)
                     ok_machs = do_update_keys(list(ok_machs.keys()))[1]
                 update_nodes(ok_machs)
                 return ok_machs
-            elif machine_type in reimage_types:
-                reimaged = dict()
-                console_log_conf = dict(
-                    logfile_name='{shortname}_reimage.log',
-                    remotes=[teuthology.orchestra.remote.Remote(machine)
-                             for machine in machines],
-                )
-                with console_log.task(
-                        ctx, console_log_conf):
-                    update_nodes(reimaged, True)
-                    with teuthology.parallel.parallel() as p:
-                        for machine in machines:
-                            p.spawn(teuthology.provision.reimage, ctx,
-                                    machine, machine_type)
-                            reimaged[machine] = machines[machine]
-                reimaged = do_update_keys(list(reimaged.keys()))[1]
-                update_nodes(reimaged)
-                return reimaged
+            elif reimage and machine_type in reimage_types:
+                return reimage_machines(ctx, machines, machine_type)
             return machines
         elif response.status_code == 503:
             log.error('Insufficient nodes available to lock %d %s nodes.',
@@ -178,6 +174,28 @@ def lock_one(name, user=None, description=None):
     return response
 
 
+def unlock_safe(names: List[str], owner: str, run_name: str = "", job_id: str = ""):
+    with teuthology.parallel.parallel() as p:
+        for name in names:
+            p.spawn(unlock_one_safe, name, owner, run_name, job_id)
+        return all(p)
+
+
+def unlock_one_safe(name: str, owner: str, run_name: str = "", job_id: str = "") -> bool:
+    node_status = query.get_status(name)
+    if node_status.get("locked", False) is False:
+        log.warn(f"Refusing to unlock {name} since it is already unlocked")
+        return False
+    maybe_job = query.node_active_job(name, node_status)
+    if not maybe_job:
+        return unlock_one(name, owner, node_status["description"], node_status)
+    if run_name and job_id and maybe_job.endswith(f"{run_name}/{job_id}"):
+            log.error(f"Refusing to unlock {name} since it has an active job: {run_name}/{job_id}")
+            return False
+    log.warning(f"Refusing to unlock {name} since it has an active job: {maybe_job}")
+    return False
+
+
 def unlock_many(names, user):
     fixed_names = [misc.canonicalize_hostname(name, user=None) for name in
                    names]
@@ -187,23 +205,35 @@ def unlock_many(names, user):
         locked_by=user,
         names=names,
     )
-    response = requests.post(
-        uri,
-        data=json.dumps(data),
-        headers={'content-type': 'application/json'},
-    )
-    if response.ok:
-        log.debug("Unlocked: %s", ', '.join(names))
-    else:
-        log.error("Failed to unlock: %s", ', '.join(names))
-    return response.ok
+    with safe_while(
+            sleep=1, increment=0.5, action=f'unlock_many {names}') as proceed:
+        while proceed():
+            response = requests.post(
+                uri,
+                data=json.dumps(data),
+                headers={'content-type': 'application/json'},
+            )
+            if response.ok:
+                log.debug("Unlocked: %s", ', '.join(names))
+                return True
+    log.error("Failed to unlock: %s", ', '.join(names))
+    return False
 
 
-def unlock_one(ctx, name, user, description=None):
+def unlock_one(name, user, description=None, status: Union[dict, None] = None) -> bool:
     name = misc.canonicalize_hostname(name, user=None)
-    if not teuthology.provision.destroy_if_vm(ctx, name, user, description):
+    if not description and status:
+        description = status["description"]
+    if not teuthology.provision.destroy_if_vm(name, user, description or ""):
         log.error('destroy failed for %s', name)
         return False
+    # we're trying to stop node before actual unlocking
+    status_info = teuthology.lock.query.get_status(name)
+    try:
+        if not teuthology.lock.query.is_vm(status=status_info):
+            stop_node(name, status)
+    except Exception:
+        log.exception(f"Failed to stop {name}!")
     request = dict(name=name, locked=False, locked_by=user,
                    description=description)
     uri = os.path.join(config.lock_server, 'nodes', name, 'lock', '')
@@ -212,21 +242,21 @@ def unlock_one(ctx, name, user, description=None):
         while proceed():
             try:
                 response = requests.put(uri, json.dumps(request))
-                break
+                if response.ok:
+                    log.info('unlocked: %s', name)
+                    return response.ok
+                if response.status_code == 403:
+                    break
             # Work around https://github.com/kennethreitz/requests/issues/2364
             except requests.ConnectionError as e:
-                log.warn("Saw %s while unlocking; retrying...", str(e))
-    success = response.ok
-    if success:
-        log.info('unlocked %s', name)
-    else:
-        try:
-            reason = response.json().get('message')
-        except ValueError:
-            reason = str(response.status_code)
-        log.error('failed to unlock {node}. reason: {reason}'.format(
-            node=name, reason=reason))
-    return success
+                log.warning("Saw %s while unlocking; retrying...", str(e))
+    try:
+        reason = response.json().get('message')
+    except ValueError:
+        reason = str(response.status_code)
+    log.error('failed to unlock {node}. reason: {reason}'.format(
+        node=name, reason=reason))
+    return False
 
 
 def update_lock(name, description=None, status=None, ssh_pub_key=None):
@@ -241,9 +271,15 @@ def update_lock(name, description=None, status=None, ssh_pub_key=None):
 
     if updated:
         uri = os.path.join(config.lock_server, 'nodes', name, '')
-        response = requests.put(
-            uri,
-            json.dumps(updated))
+        inc = random.uniform(0, 1)
+        with safe_while(
+                sleep=1, increment=inc, action=f'update lock {name}') as proceed:
+            while proceed():
+                response = requests.put(
+                    uri,
+                    json.dumps(updated))
+                if response.ok:
+                    return True
         return response.ok
     return True
 
@@ -260,24 +296,25 @@ def update_inventory(node_dict):
         return
     uri = os.path.join(config.lock_server, 'nodes', name, '')
     log.info("Updating %s on lock server", name)
-    response = requests.put(
-        uri,
-        json.dumps(node_dict),
-        headers={'content-type': 'application/json'},
-        )
-    if response.status_code == 404:
-        log.info("Creating new node %s on lock server", name)
-        uri = os.path.join(config.lock_server, 'nodes', '')
-        response = requests.post(
-            uri,
-            json.dumps(node_dict),
-            headers={'content-type': 'application/json'},
-        )
-    if not response.ok:
-        log.error("Node update/creation failed for %s: %s",
-                  name, response.text)
-    return response.ok
-
+    inc = random.uniform(0, 1)
+    with safe_while(
+            sleep=1, increment=inc, action=f'update inventory {name}') as proceed:
+        while proceed():
+            response = requests.put(
+                uri,
+                json.dumps(node_dict),
+                headers={'content-type': 'application/json'},
+            )
+            if response.status_code == 404:
+                log.info("Creating new node %s on lock server", name)
+                uri = os.path.join(config.lock_server, 'nodes', '')
+                response = requests.post(
+                    uri,
+                    json.dumps(node_dict),
+                    headers={'content-type': 'application/json'},
+                )
+            if response.ok:
+                return
 
 def do_update_keys(machines, all_=False, _raise=True):
     reference = query.list_locks(keyed_by_name=True)
@@ -297,3 +334,172 @@ def push_new_keys(keys_dict, reference):
                 log.error('failed to update %s!', hostname)
                 ret = 1
     return ret
+
+
+def reimage_machines(ctx, machines, machine_type):
+    reimage_types = teuthology.provision.get_reimage_types()
+    if machine_type not in reimage_types:
+        log.info(f"Skipping reimage of {machines.keys()} because {machine_type} is not in {reimage_types}")
+        return machines
+    # Setup log file, reimage machines and update their keys
+    reimaged = dict()
+    console_log_conf = dict(
+        logfile_name='{shortname}_reimage.log',
+        remotes=[teuthology.orchestra.remote.Remote(machine)
+                 for machine in machines],
+    )
+    with console_log.task(ctx, console_log_conf):
+        with teuthology.parallel.parallel() as p:
+            for machine in machines:
+                log.info("Start node '%s' reimaging", machine)
+                update_nodes([machine], True)
+                p.spawn(teuthology.provision.reimage, ctx,
+                        machine, machine_type)
+                reimaged[machine] = machines[machine]
+    reimaged = do_update_keys(list(reimaged.keys()))[1]
+    update_nodes(reimaged)
+    return reimaged
+
+
+def block_and_lock_machines(ctx, total_requested, machine_type, reimage=True, tries=10):
+    # It's OK for os_type and os_version to be None here.  If we're trying
+    # to lock a bare metal machine, we'll take whatever is available.  If
+    # we want a vps, defaults will be provided by misc.get_distro and
+    # misc.get_distro_version in provision.create_if_vm
+    os_type = ctx.config.get("os_type")
+    os_version = ctx.config.get("os_version")
+    arch = ctx.config.get('arch')
+    reserved = config.reserve_machines
+    assert isinstance(reserved, int), 'reserve_machines must be integer'
+    assert (reserved >= 0), 'reserve_machines should >= 0'
+
+    log.info('Locking machines...')
+    # change the status during the locking process
+    report.try_push_job_info(ctx.config, dict(status='waiting'))
+
+    all_locked = dict()
+    requested = total_requested
+    while True:
+        # get a candidate list of machines
+        machines = query.list_locks(
+            machine_type=machine_type,
+            up=True,
+            locked=False,
+            count=requested + reserved,
+            tries=tries,
+        )
+        if machines is None:
+            if ctx.block:
+                log.error('Error listing machines, trying again')
+                time.sleep(20)
+                continue
+            else:
+                raise RuntimeError('Error listing machines')
+
+        # make sure there are machines for non-automated jobs to run
+        if len(machines) < reserved + requested \
+                and ctx.owner.startswith('scheduled'):
+            if ctx.block:
+                log.info(
+                    'waiting for more %s machines to be free (need %s + %s, have %s)...',
+                    machine_type,
+                    reserved,
+                    requested,
+                    len(machines),
+                )
+                time.sleep(10)
+                continue
+            else:
+                assert 0, ('not enough machines free; need %s + %s, have %s' %
+                           (reserved, requested, len(machines)))
+
+        try:
+            newly_locked = lock_many(ctx, requested, machine_type,
+                                     ctx.owner, ctx.archive, os_type,
+                                     os_version, arch, reimage=reimage)
+        except Exception:
+            # Lock failures should map to the 'dead' status instead of 'fail'
+            if 'summary' in ctx:
+                set_status(ctx.summary, 'dead')
+            raise
+        all_locked.update(newly_locked)
+        log.info(
+            '{newly_locked} {mtype} machines locked this try, '
+            '{total_locked}/{total_requested} locked so far'.format(
+                newly_locked=len(newly_locked),
+                mtype=machine_type,
+                total_locked=len(all_locked),
+                total_requested=total_requested,
+            )
+        )
+        if len(all_locked) == total_requested:
+            vmlist = []
+            for lmach in all_locked:
+                if query.is_vm(lmach):
+                    vmlist.append(lmach)
+            if vmlist:
+                log.info('Waiting for virtual machines to come up')
+                keys_dict = dict()
+                loopcount = 0
+                while len(keys_dict) != len(vmlist):
+                    loopcount += 1
+                    time.sleep(10)
+                    keys_dict = misc.ssh_keyscan(vmlist)
+                    log.info('virtual machine is still unavailable')
+                    if loopcount == 40:
+                        loopcount = 0
+                        log.info('virtual machine(s) still not up, ' +
+                                 'recreating unresponsive ones.')
+                        for guest in vmlist:
+                            if guest not in keys_dict.keys():
+                                log.info('recreating: ' + guest)
+                                full_name = misc.canonicalize_hostname(guest)
+                                teuthology.provision.destroy_if_vm(full_name)
+                                teuthology.provision.create_if_vm(ctx, full_name)
+                if do_update_keys(keys_dict)[0]:
+                    log.info("Error in virtual machine keys")
+                newscandict = {}
+                for dkey in all_locked.keys():
+                    stats = query.get_status(dkey)
+                    newscandict[dkey] = stats['ssh_pub_key']
+                ctx.config['targets'] = newscandict
+            else:
+                ctx.config['targets'] = all_locked
+            locked_targets = yaml.safe_dump(
+                ctx.config['targets'],
+                default_flow_style=False
+            ).splitlines()
+            log.info('\n  '.join(['Locked targets:', ] + locked_targets))
+            # successfully locked machines, change status back to running
+            report.try_push_job_info(ctx.config, dict(status='running'))
+            break
+        elif not ctx.block:
+            assert 0, 'not enough machines are available'
+        else:
+            requested = requested - len(newly_locked)
+            assert requested > 0, "lock_machines: requested counter went" \
+                                  "negative, this shouldn't happen"
+
+        log.info(
+            "{total} machines locked ({new} new); need {more} more".format(
+                total=len(all_locked), new=len(newly_locked), more=requested)
+        )
+        log.warning('Could not lock enough machines, waiting...')
+        time.sleep(10)
+
+
+def stop_node(name: str, status: Union[dict, None]):
+    status = status or query.get_status(name)
+    remote_ = remote.Remote(name)
+    if status['machine_type'] in provision.fog.get_types():
+        remote_.console.power_off()
+        return
+    elif status['machine_type'] in provision.pelagos.get_types():
+        provision.pelagos.park_node(name)
+        return
+    elif remote_.is_container:
+        remote_.run(
+            args=['sudo', '/testnode_stop.sh'],
+            check_status=False,
+        )
+        return

teuthology/lock/query.py

@@ -1,26 +1,32 @@
 import logging
 import os
-
 import requests
 
+from typing import Dict, List, Union
+
 from teuthology import misc
 from teuthology.config import config
+from teuthology.contextutil import safe_while
 from teuthology.util.compat import urlencode
 
 
 log = logging.getLogger(__name__)
 
 
-def get_status(name):
+def get_status(name) -> dict:
     name = misc.canonicalize_hostname(name, user=None)
     uri = os.path.join(config.lock_server, 'nodes', name, '')
-    response = requests.get(uri)
-    success = response.ok
-    if success:
-        return response.json()
+    with safe_while(
+            sleep=1, increment=0.5, action=f'get_status {name}') as proceed:
+        while proceed():
+            response = requests.get(uri)
+            if response.ok:
+                return response.json()
+            elif response.status_code == 404:
+                return dict()
     log.warning(
         "Failed to query lock server for status of {name}".format(name=name))
-    return None
+    return dict()
 
 
 def get_statuses(machines):
@@ -48,7 +54,7 @@ def is_vm(name=None, status=None):
     return status.get('is_vm', False)
 
 
-def list_locks(keyed_by_name=False, **kwargs):
+def list_locks(keyed_by_name=False, tries=10, **kwargs):
     uri = os.path.join(config.lock_server, 'nodes', '')
     for key, value in kwargs.items():
         if kwargs[key] is False:
@@ -59,14 +65,20 @@ def list_locks(keyed_by_name=False, **kwargs):
         if 'machine_type' in kwargs:
             kwargs['machine_type'] = kwargs['machine_type'].replace(',','|')
         uri += '?' + urlencode(kwargs)
-    try:
-        response = requests.get(uri)
-    except requests.ConnectionError:
-        success = False
-        log.exception("Could not contact lock server: %s", config.lock_server)
-    else:
-        success = response.ok
-    if success:
+    with safe_while(
+            sleep=1,
+            increment=0.5,
+            tries=tries,
+            action='list_locks'
+    ) as proceed:
+        while proceed():
+            try:
+                response = requests.get(uri)
+                if response.ok:
+                    break
+            except requests.ConnectionError:
+                log.exception("Could not contact lock server: %s, retrying...", config.lock_server)
+    if response.ok:
         if not keyed_by_name:
             return response.json()
         else:
@@ -75,11 +87,11 @@ def list_locks(keyed_by_name=False, **kwargs):
     return dict()
 
 
-def find_stale_locks(owner=None):
+def find_stale_locks(owner=None) -> List[Dict]:
     """
     Return a list of node dicts corresponding to nodes that were locked to run
     a job, but the job is no longer running. The purpose of this is to enable
-    us to nuke nodes that were left locked due to e.g. infrastructure failures
+    us to find nodes that were left locked due to e.g. infrastructure failures
     and return them to the pool.
 
     :param owner: If non-None, return nodes locked by owner. Default is None.
@@ -108,36 +120,41 @@ def find_stale_locks(owner=None):
         nodes = [node for node in nodes if node['locked_by'] == owner]
     nodes = filter(might_be_stale, nodes)
 
-    def node_job_is_active(node, cache):
-        """
-        Is this node's job active (e.g. running or waiting)?
-
-        :param node:  The node dict as returned from the lock server
-        :param cache: A set() used for caching results
-        :returns:     True or False
-        """
-        description = node['description']
-        if description in cache:
-            return True
-        (name, job_id) = description.split('/')[-2:]
-        url = os.path.join(config.results_server, 'runs', name, 'jobs', job_id,
-                           '')
-        resp = requests.get(url)
-        if not resp.ok:
-            return False
-        job_info = resp.json()
-        if job_info['status'] in ('running', 'waiting'):
-            cache.add(description)
-            return True
-        return False
-
-    result = list()
     # Here we build the list of of nodes that are locked, for a job (as opposed
     # to being locked manually for random monkeying), where the job is not
     # running
-    active_jobs = set()
+    result = list()
     for node in nodes:
-        if node_job_is_active(node, active_jobs):
+        if node_active_job(node["name"]):
             continue
         result.append(node)
     return result
+
+def node_active_job(name: str, status: Union[dict, None] = None) -> Union[str, None]:
+    """
+    Is this node's job active (e.g. running or waiting)?
+
+    :param node:  The node dict as returned from the lock server
+    :param cache: A set() used for caching results
+    :returns:     A string if the node has an active job, or None if not
+    """
+    status = status or get_status(name)
+    if not status:
+        # This should never happen with a normal node
+        return "node had no status"
+    description = status['description']
+    (run_name, job_id) = description.split('/')[-2:]
+    if not run_name or job_id == '':
+        # We thought this node might have a stale job, but no.
+        return "node description does not contained scheduled job info"
+    url = f"{config.results_server}/runs/{run_name}/jobs/{job_id}/"
+    job_status = ""
+    with safe_while(
+            sleep=1, increment=0.5, action='node_is_active') as proceed:
+        while proceed():
+            resp = requests.get(url)
+            if resp.ok:
+                job_status = resp.json()["status"]
+                break
+    if job_status and job_status not in ('pass', 'fail', 'dead'):
+        return description

teuthology/ls.py

@@ -43,7 +43,7 @@ def get_jobs(archive_dir):
     dir_contents = os.listdir(archive_dir)
 
     def is_job_dir(parent, subdir):
-        if (os.path.isdir(os.path.join(parent, subdir)) and re.match('\d+$',
+        if (os.path.isdir(os.path.join(parent, subdir)) and re.match(r'\d+$',
                                                                      subdir)):
             return True
         return False
@@ -56,23 +56,14 @@ def print_debug_info(job, job_dir, archive_dir):
     print('%s      ' % job, end='')
 
     try:
-        pidfile = os.path.join(job_dir, 'pid')
-        found = False
-        if os.path.isfile(pidfile):
-            pid = open(pidfile, 'r').read()
-            if os.path.isdir("/proc/%s" % pid):
-                cmdline = open('/proc/%s/cmdline' % pid,
-                               'r').read()
-                if cmdline.find(archive_dir) >= 0:
-                    print('(pid %s)' % pid, end='')
-                    found = True
-        if not found:
-            print('(no process or summary.yaml)', end='')
-        # tail
-        tail = os.popen(
-            'tail -1 %s/%s/teuthology.log' % (archive_dir, job)
-        ).read().rstrip()
-        print(tail, end='')
+        log_path = os.path.join(archive_dir, job, 'teuthology.log')
+        if os.path.exists(log_path):
+            tail = os.popen(
+                'tail -1 %s' % log_path
+            ).read().rstrip()
+            print(tail, end='')
+        else:
+            print('<no teuthology.log yet>', end='')
     except IOError:
         pass
     print('')