teuthology 1.1.0__py3-none-any.whl → 1.2.1__py3-none-any.whl

teuthology/lock/ops.py

@@ -1,16 +1,18 @@
 import logging
 import json
 import os
+import random
 import time
 import yaml
-
 import requests
 
+from typing import List, Union
+
 import teuthology.orchestra.remote
 import teuthology.parallel
 import teuthology.provision
-from teuthology import misc
-from teuthology import report
+
+from teuthology import misc, report, provision
 from teuthology.config import config
 from teuthology.contextutil import safe_while
 from teuthology.task import console_log
@@ -18,6 +20,7 @@ from teuthology.misc import canonicalize_hostname
 from teuthology.job_status import set_status
 
 from teuthology.lock import util, query
+from teuthology.orchestra import remote
 
 log = logging.getLogger(__name__)
 
@@ -115,8 +118,13 @@ def lock_many(ctx, num, machine_type, user=None, description=None,
             headers={'content-type': 'application/json'},
         )
         if response.ok:
-            machines = {misc.canonicalize_hostname(machine['name']):
-                        machine['ssh_pub_key'] for machine in response.json()}
+            machines = dict()
+            for machine in response.json():
+                key = misc.canonicalize_hostname(
+                    machine['name'],
+                    user=machine.get('user'),
+                )
+                machines[key] = machine['ssh_pub_key']
             log.debug('locked {machines}'.format(
                 machines=', '.join(machines.keys())))
             if machine_type in vm_types:
@@ -128,7 +136,7 @@ def lock_many(ctx, num, machine_type, user=None, description=None,
                     else:
                         log.error('Unable to create virtual machine: %s',
                                   machine)
-                        unlock_one(ctx, machine, user)
+                        unlock_one(machine, user)
                     ok_machs = do_update_keys(list(ok_machs.keys()))[1]
                 update_nodes(ok_machs)
                 return ok_machs
@@ -166,6 +174,28 @@ def lock_one(name, user=None, description=None):
     return response
 
 
+def unlock_safe(names: List[str], owner: str, run_name: str = "", job_id: str = ""):
+    with teuthology.parallel.parallel() as p:
+        for name in names:
+            p.spawn(unlock_one_safe, name, owner, run_name, job_id)
+        return all(p)
+
+
+def unlock_one_safe(name: str, owner: str, run_name: str = "", job_id: str = "") -> bool:
+    node_status = query.get_status(name)
+    if node_status.get("locked", False) is False:
+        log.warn(f"Refusing to unlock {name} since it is already unlocked")
+        return False
+    maybe_job = query.node_active_job(name, node_status)
+    if not maybe_job:
+        return unlock_one(name, owner, node_status["description"], node_status)
+    if run_name and job_id and maybe_job.endswith(f"{run_name}/{job_id}"):
+            log.error(f"Refusing to unlock {name} since it has an active job: {run_name}/{job_id}")
+            return False
+    log.warning(f"Refusing to unlock {name} since it has an active job: {maybe_job}")
+    return False
+
+
 def unlock_many(names, user):
     fixed_names = [misc.canonicalize_hostname(name, user=None) for name in
                    names]
@@ -175,23 +205,35 @@ def unlock_many(names, user):
         locked_by=user,
         names=names,
     )
-    response = requests.post(
-        uri,
-        data=json.dumps(data),
-        headers={'content-type': 'application/json'},
-    )
-    if response.ok:
-        log.debug("Unlocked: %s", ', '.join(names))
-    else:
-        log.error("Failed to unlock: %s", ', '.join(names))
-    return response.ok
+    with safe_while(
+            sleep=1, increment=0.5, action=f'unlock_many {names}') as proceed:
+        while proceed():
+            response = requests.post(
+                uri,
+                data=json.dumps(data),
+                headers={'content-type': 'application/json'},
+            )
+            if response.ok:
+                log.debug("Unlocked: %s", ', '.join(names))
+                return True
+    log.error("Failed to unlock: %s", ', '.join(names))
+    return False
 
 
-def unlock_one(ctx, name, user, description=None):
+def unlock_one(name, user, description=None, status: Union[dict, None] = None) -> bool:
     name = misc.canonicalize_hostname(name, user=None)
-    if not teuthology.provision.destroy_if_vm(ctx, name, user, description):
+    if not description and status:
+        description = status["description"]
+    if not teuthology.provision.destroy_if_vm(name, user, description or ""):
         log.error('destroy failed for %s', name)
         return False
+    # we're trying to stop node before actual unlocking
+    status_info = teuthology.lock.query.get_status(name)
+    try:
+        if not teuthology.lock.query.is_vm(status=status_info):
+            stop_node(name, status)
+    except Exception:
+        log.exception(f"Failed to stop {name}!")
     request = dict(name=name, locked=False, locked_by=user,
                    description=description)
     uri = os.path.join(config.lock_server, 'nodes', name, 'lock', '')
@@ -200,21 +242,21 @@ def unlock_one(ctx, name, user, description=None):
         while proceed():
             try:
                 response = requests.put(uri, json.dumps(request))
-                break
+                if response.ok:
+                    log.info('unlocked: %s', name)
+                    return response.ok
+                if response.status_code == 403:
+                    break
             # Work around https://github.com/kennethreitz/requests/issues/2364
             except requests.ConnectionError as e:
-                log.warn("Saw %s while unlocking; retrying...", str(e))
-    success = response.ok
-    if success:
-        log.info('unlocked %s', name)
-    else:
-        try:
-            reason = response.json().get('message')
-        except ValueError:
-            reason = str(response.status_code)
-        log.error('failed to unlock {node}. reason: {reason}'.format(
-            node=name, reason=reason))
-    return success
+                log.warning("Saw %s while unlocking; retrying...", str(e))
+    try:
+        reason = response.json().get('message')
+    except ValueError:
+        reason = str(response.status_code)
+    log.error('failed to unlock {node}. reason: {reason}'.format(
+        node=name, reason=reason))
+    return False
 
 
 def update_lock(name, description=None, status=None, ssh_pub_key=None):
@@ -229,9 +271,15 @@ def update_lock(name, description=None, status=None, ssh_pub_key=None):
 
     if updated:
         uri = os.path.join(config.lock_server, 'nodes', name, '')
-        response = requests.put(
-            uri,
-            json.dumps(updated))
+        inc = random.uniform(0, 1)
+        with safe_while(
+                sleep=1, increment=inc, action=f'update lock {name}') as proceed:
+            while proceed():
+                response = requests.put(
+                    uri,
+                    json.dumps(updated))
+                if response.ok:
+                    return True
         return response.ok
     return True
 
@@ -248,24 +296,25 @@ def update_inventory(node_dict):
         return
     uri = os.path.join(config.lock_server, 'nodes', name, '')
     log.info("Updating %s on lock server", name)
-    response = requests.put(
-        uri,
-        json.dumps(node_dict),
-        headers={'content-type': 'application/json'},
-        )
-    if response.status_code == 404:
-        log.info("Creating new node %s on lock server", name)
-        uri = os.path.join(config.lock_server, 'nodes', '')
-        response = requests.post(
-            uri,
-            json.dumps(node_dict),
-            headers={'content-type': 'application/json'},
-        )
-    if not response.ok:
-        log.error("Node update/creation failed for %s: %s",
-                  name, response.text)
-    return response.ok
-
+    inc = random.uniform(0, 1)
+    with safe_while(
+            sleep=1, increment=inc, action=f'update inventory {name}') as proceed:
+        while proceed():
+            response = requests.put(
+                uri,
+                json.dumps(node_dict),
+                headers={'content-type': 'application/json'},
+            )
+            if response.status_code == 404:
+                log.info("Creating new node %s on lock server", name)
+                uri = os.path.join(config.lock_server, 'nodes', '')
+                response = requests.post(
+                    uri,
+                    json.dumps(node_dict),
+                    headers={'content-type': 'application/json'},
+                )
+            if response.ok:
+                return
 
 def do_update_keys(machines, all_=False, _raise=True):
     reference = query.list_locks(keyed_by_name=True)
@@ -288,6 +337,10 @@ def push_new_keys(keys_dict, reference):
 
 
 def reimage_machines(ctx, machines, machine_type):
+    reimage_types = teuthology.provision.get_reimage_types()
+    if machine_type not in reimage_types:
+        log.info(f"Skipping reimage of {machines.keys()} because {machine_type} is not in {reimage_types}")
+        return machines
     # Setup log file, reimage machines and update their keys
     reimaged = dict()
     console_log_conf = dict(
@@ -303,13 +356,12 @@ def reimage_machines(ctx, machines, machine_type):
                 p.spawn(teuthology.provision.reimage, ctx,
                         machine, machine_type)
                 reimaged[machine] = machines[machine]
-                log.info("Node '%s' reimaging is complete", machine)
     reimaged = do_update_keys(list(reimaged.keys()))[1]
     update_nodes(reimaged)
     return reimaged
 
 
-def block_and_lock_machines(ctx, total_requested, machine_type, reimage=True):
+def block_and_lock_machines(ctx, total_requested, machine_type, reimage=True, tries=10):
     # It's OK for os_type and os_version to be None here.  If we're trying
     # to lock a bare metal machine, we'll take whatever is available.  If
     # we want a vps, defaults will be provided by misc.get_distro and
@@ -329,8 +381,13 @@ def block_and_lock_machines(ctx, total_requested, machine_type, reimage=True):
     requested = total_requested
     while True:
         # get a candidate list of machines
-        machines = query.list_locks(machine_type=machine_type, up=True,
-                                    locked=False, count=requested + reserved)
+        machines = query.list_locks(
+            machine_type=machine_type,
+            up=True,
+            locked=False,
+            count=requested + reserved,
+            tries=tries,
+        )
         if machines is None:
             if ctx.block:
                 log.error('Error listing machines, trying again')
@@ -378,7 +435,7 @@ def block_and_lock_machines(ctx, total_requested, machine_type, reimage=True):
         if len(all_locked) == total_requested:
             vmlist = []
             for lmach in all_locked:
-                if teuthology.lock.query.is_vm(lmach):
+                if query.is_vm(lmach):
                     vmlist.append(lmach)
             if vmlist:
                 log.info('Waiting for virtual machines to come up')
@@ -397,13 +454,13 @@ def block_and_lock_machines(ctx, total_requested, machine_type, reimage=True):
                             if guest not in keys_dict.keys():
                                 log.info('recreating: ' + guest)
                                 full_name = misc.canonicalize_hostname(guest)
-                                teuthology.provision.destroy_if_vm(ctx, full_name)
+                                teuthology.provision.destroy_if_vm(full_name)
                                 teuthology.provision.create_if_vm(ctx, full_name)
-                if teuthology.lock.ops.do_update_keys(keys_dict)[0]:
+                if do_update_keys(keys_dict)[0]:
                     log.info("Error in virtual machine keys")
                 newscandict = {}
                 for dkey in all_locked.keys():
-                    stats = teuthology.lock.query.get_status(dkey)
+                    stats = query.get_status(dkey)
                     newscandict[dkey] = stats['ssh_pub_key']
                 ctx.config['targets'] = newscandict
             else:
@@ -427,5 +484,22 @@ def block_and_lock_machines(ctx, total_requested, machine_type, reimage=True):
             "{total} machines locked ({new} new); need {more} more".format(
                 total=len(all_locked), new=len(newly_locked), more=requested)
         )
-        log.warn('Could not lock enough machines, waiting...')
+        log.warning('Could not lock enough machines, waiting...')
         time.sleep(10)
+
+
+def stop_node(name: str, status: Union[dict, None]):
+    status = status or query.get_status(name)
+    remote_ = remote.Remote(name)
+    if status['machine_type'] in provision.fog.get_types():
+        remote_.console.power_off()
+        return
+    elif status['machine_type'] in provision.pelagos.get_types():
+        provision.pelagos.park_node(name)
+        return
+    elif remote_.is_container:
+        remote_.run(
+            args=['sudo', '/testnode_stop.sh'],
+            check_status=False,
+        )
+        return

teuthology/lock/query.py

@@ -1,26 +1,32 @@
 import logging
 import os
-
 import requests
 
+from typing import Dict, List, Union
+
 from teuthology import misc
 from teuthology.config import config
+from teuthology.contextutil import safe_while
 from teuthology.util.compat import urlencode
 
 
 log = logging.getLogger(__name__)
 
 
-def get_status(name):
+def get_status(name) -> dict:
     name = misc.canonicalize_hostname(name, user=None)
     uri = os.path.join(config.lock_server, 'nodes', name, '')
-    response = requests.get(uri)
-    success = response.ok
-    if success:
-        return response.json()
+    with safe_while(
+            sleep=1, increment=0.5, action=f'get_status {name}') as proceed:
+        while proceed():
+            response = requests.get(uri)
+            if response.ok:
+                return response.json()
+            elif response.status_code == 404:
+                return dict()
     log.warning(
         "Failed to query lock server for status of {name}".format(name=name))
-    return None
+    return dict()
 
 
 def get_statuses(machines):
@@ -48,7 +54,7 @@ def is_vm(name=None, status=None):
     return status.get('is_vm', False)
 
 
-def list_locks(keyed_by_name=False, **kwargs):
+def list_locks(keyed_by_name=False, tries=10, **kwargs):
     uri = os.path.join(config.lock_server, 'nodes', '')
     for key, value in kwargs.items():
         if kwargs[key] is False:
@@ -59,14 +65,20 @@ def list_locks(keyed_by_name=False, **kwargs):
         if 'machine_type' in kwargs:
             kwargs['machine_type'] = kwargs['machine_type'].replace(',','|')
         uri += '?' + urlencode(kwargs)
-    try:
-        response = requests.get(uri)
-    except requests.ConnectionError:
-        success = False
-        log.exception("Could not contact lock server: %s", config.lock_server)
-    else:
-        success = response.ok
-    if success:
+    with safe_while(
+            sleep=1,
+            increment=0.5,
+            tries=tries,
+            action='list_locks'
+    ) as proceed:
+        while proceed():
+            try:
+                response = requests.get(uri)
+                if response.ok:
+                    break
+            except requests.ConnectionError:
+                log.exception("Could not contact lock server: %s, retrying...", config.lock_server)
+    if response.ok:
         if not keyed_by_name:
             return response.json()
         else:
@@ -75,11 +87,11 @@ def list_locks(keyed_by_name=False, **kwargs):
     return dict()
 
 
-def find_stale_locks(owner=None):
+def find_stale_locks(owner=None) -> List[Dict]:
     """
     Return a list of node dicts corresponding to nodes that were locked to run
     a job, but the job is no longer running. The purpose of this is to enable
-    us to nuke nodes that were left locked due to e.g. infrastructure failures
+    us to find nodes that were left locked due to e.g. infrastructure failures
     and return them to the pool.
 
     :param owner: If non-None, return nodes locked by owner. Default is None.
@@ -108,36 +120,41 @@ def find_stale_locks(owner=None):
         nodes = [node for node in nodes if node['locked_by'] == owner]
     nodes = filter(might_be_stale, nodes)
 
-    def node_job_is_active(node, cache):
-        """
-        Is this node's job active (e.g. running or waiting)?
-
-        :param node:  The node dict as returned from the lock server
-        :param cache: A set() used for caching results
-        :returns:     True or False
-        """
-        description = node['description']
-        if description in cache:
-            return True
-        (name, job_id) = description.split('/')[-2:]
-        url = os.path.join(config.results_server, 'runs', name, 'jobs', job_id,
-                           '')
-        resp = requests.get(url)
-        if not resp.ok:
-            return False
-        job_info = resp.json()
-        if job_info['status'] in ('running', 'waiting'):
-            cache.add(description)
-            return True
-        return False
-
-    result = list()
     # Here we build the list of of nodes that are locked, for a job (as opposed
     # to being locked manually for random monkeying), where the job is not
     # running
-    active_jobs = set()
+    result = list()
     for node in nodes:
-        if node_job_is_active(node, active_jobs):
+        if node_active_job(node["name"]):
             continue
         result.append(node)
     return result
+
+def node_active_job(name: str, status: Union[dict, None] = None) -> Union[str, None]:
+    """
+    Is this node's job active (e.g. running or waiting)?
+
+    :param node:  The node dict as returned from the lock server
+    :param cache: A set() used for caching results
+    :returns:     A string if the node has an active job, or None if not
+    """
+    status = status or get_status(name)
+    if not status:
+        # This should never happen with a normal node
+        return "node had no status"
+    description = status['description']
+    (run_name, job_id) = description.split('/')[-2:]
+    if not run_name or job_id == '':
+        # We thought this node might have a stale job, but no.
+        return "node description does not contained scheduled job info"
+    url = f"{config.results_server}/runs/{run_name}/jobs/{job_id}/"
+    job_status = ""
+    with safe_while(
+            sleep=1, increment=0.5, action='node_is_active') as proceed:
+        while proceed():
+            resp = requests.get(url)
+            if resp.ok:
+                job_status = resp.json()["status"]
+                break
+    if job_status and job_status not in ('pass', 'fail', 'dead'):
+        return description

teuthology/ls.py

@@ -43,7 +43,7 @@ def get_jobs(archive_dir):
     dir_contents = os.listdir(archive_dir)
 
     def is_job_dir(parent, subdir):
-        if (os.path.isdir(os.path.join(parent, subdir)) and re.match('\d+$',
+        if (os.path.isdir(os.path.join(parent, subdir)) and re.match(r'\d+$',
                                                                      subdir)):
             return True
         return False