teuthology 1.1.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

teuthology/exporter.py

@@ -0,0 +1,347 @@
+import contextlib
+import itertools
+import logging
+import os
+import psutil
+import time
+
+from pathlib import Path
+
+import teuthology.beanstalk as beanstalk
+import teuthology.dispatcher
+from teuthology.config import config
+from teuthology.lock.query import list_locks
+
+log = logging.getLogger(__name__)
+
+
+PROMETHEUS_MULTIPROC_DIR = Path("~/.cache/teuthology-exporter").expanduser()
+os.environ["PROMETHEUS_MULTIPROC_DIR"] = str(PROMETHEUS_MULTIPROC_DIR)
+
+# We can't import prometheus_client until after we set PROMETHEUS_MULTIPROC_DIR
+from prometheus_client import (  # noqa: E402
+    start_http_server,
+    Gauge,
+    Counter,
+    Summary,
+    multiprocess,
+    CollectorRegistry,
+)
+
+MACHINE_TYPES = list(config.active_machine_types)
+REGISTRY = None
+
+
+class TeuthologyExporter:
+    port = 61764  # int(''.join([str((ord(c) - 100) % 10) for c in "teuth"]))
+
+    def __init__(self, interval=60):
+        if REGISTRY:
+            for file in PROMETHEUS_MULTIPROC_DIR.iterdir():
+                file.unlink()
+        self.interval = interval
+        self.metrics = [
+            Dispatchers(),
+            BeanstalkQueue(),
+            JobProcesses(),
+            Nodes(),
+        ]
+        self._created_time = time.perf_counter()
+
+    def start(self):
+        if REGISTRY:
+            start_http_server(self.port, registry=REGISTRY)
+        self.loop()
+
+    def update(self):
+        log.info("Updating...")
+        for metric in self.metrics:
+            metric.update()
+        log.info("Update finished.")
+
+    def loop(self):
+        log.info("Starting teuthology-exporter...")
+        while True:
+            try:
+                before = time.perf_counter()
+                if before - self._created_time > 24 * 60 * 60:
+                    self.restart()
+                try:
+                    self.update()
+                except Exception:
+                    log.exception("Failed to update metrics")
+                interval = self.interval
+                # try to deliver metrics _at_ $interval, as opposed to sleeping
+                # for $interval between updates
+                elapsed: float = time.perf_counter() - before
+                if elapsed < 0:
+                    interval *= 2
+                interval -= elapsed
+                time.sleep(interval)
+            except KeyboardInterrupt:
+                log.info("Stopping.")
+                raise SystemExit
+
+    def restart(self):
+        # Use the dispatcher's restart function - note that by using this here,
+        # it restarts the exporter, *not* the dispatcher.
+        if REGISTRY:
+            return teuthology.dispatcher.restart(log=log)
+
+
+class SingletonMeta(type):
+    _instances = {}
+    def __call__(cls, *args, **kwargs):
+        if cls not in cls._instances:
+            instance = super().__call__(*args, **kwargs)
+            cls._instances[cls] = instance
+        return cls._instances[cls]
+
+
+class TeuthologyMetric(metaclass=SingletonMeta):
+    def __init__(self):
+        if REGISTRY:
+            self._init()
+
+    def _init(self):
+        raise NotImplementedError
+
+    def update(self):
+        if REGISTRY:
+            self._update()
+
+    def _update(self):
+        raise NotImplementedError
+
+    def record(self, **kwargs):
+        if REGISTRY:
+            self._record(**kwargs)
+
+    def _record(self, **_):
+        raise NotImplementedError
+
+    @contextlib.contextmanager
+    def time(self, **labels):
+        if REGISTRY:
+            yield self._time(**labels)
+        else:
+            yield
+
+    def _time(self):
+        raise NotImplementedError
+
+
+class Dispatchers(TeuthologyMetric):
+    def _init(self):
+        self.metric = Gauge(
+            "teuthology_dispatchers",
+            "Teuthology Dispatchers",
+            ["machine_type"],
+        )
+
+    def _update(self):
+        dispatcher_procs = teuthology.dispatcher.find_dispatcher_processes()
+        for machine_type in MACHINE_TYPES:
+            self.metric.labels(machine_type).set(
+                len(dispatcher_procs.get(machine_type, []))
+            )
+
+
+class BeanstalkQueue(TeuthologyMetric):
+    def _init(self):
+        self.length = Gauge(
+            "beanstalk_queue_length",
+            "Beanstalk Queue Length",
+            ["machine_type"],
+        )
+        self.paused = Gauge(
+            "beanstalk_queue_paused", "Beanstalk Queue is Paused", ["machine_type"]
+        )
+
+    def _update(self):
+        for machine_type in MACHINE_TYPES:
+            queue_stats = beanstalk.stats_tube(beanstalk.connect(), machine_type)
+            self.length.labels(machine_type).set(queue_stats["count"])
+            self.paused.labels(machine_type).set(1 if queue_stats["paused"] else 0)
+
+
+class JobProcesses(TeuthologyMetric):
+    def _init(self):
+        self.metric = Gauge(
+            "teuthology_job_processes",
+            "Teuthology Job Processes",
+        )
+
+    def _update(self):
+        attrs = ["pid", "cmdline"]
+        total = 0
+        for proc in psutil.process_iter(attrs=attrs):
+            if self._match(proc):
+                total += 1
+        self.metric.set(total)
+
+    @staticmethod
+    def _match(proc):
+        try:
+            cmdline = proc.cmdline()
+        except psutil.ZombieProcess:
+            return False
+        except psutil.AccessDenied:
+            return False
+        if not len(cmdline) > 1:
+            return False
+        if not cmdline[1].endswith("teuthology"):
+            return False
+        if "--archive" not in cmdline:
+            return False
+        if "--name" not in cmdline:
+            return False
+        try:
+            owner_index = cmdline.index("--owner") + 1
+            if not cmdline[owner_index].startswith("scheduled_"):
+                return False
+        except ValueError:
+            return False
+        return True
+
+
+class Nodes(TeuthologyMetric):
+    def _init(self):
+        self.metric = Gauge(
+            "teuthology_nodes",
+            "Teuthology Nodes",
+            ["machine_type", "locked", "up"],
+        )
+
+    def _update(self):
+        for machine_type in MACHINE_TYPES:
+            nodes = list_locks(machine_type=machine_type)
+            for up, locked in itertools.product([True, False], [True, False]):
+                self.metric.labels(machine_type=machine_type, up=up, locked=locked).set(
+                    len([n for n in nodes if n["up"] is up and n["locked"] is locked])
+                )
+
+
+class JobResults(TeuthologyMetric):
+    def _init(self):
+        self.metric = Counter(
+            "teuthology_job_results",
+            "Teuthology Job Results",
+            ["machine_type", "status"],
+        )
+
+    # As this is to be used within job processes, we implement record() rather than update()
+    def _record(self, **labels):
+        self.metric.labels(**labels).inc()
+
+
+class NodeReimagingResults(TeuthologyMetric):
+    def _init(self):
+        self.metric = Counter(
+            "teuthology_reimaging_results",
+            "Teuthology Reimaging Results",
+            ["machine_type", "status"],
+        )
+
+    # As this is to be used within job processes, we implement record() rather than update()
+    def _record(self, **labels):
+        if REGISTRY:
+            self.metric.labels(**labels).inc()
+
+
+class NodeLockingTime(TeuthologyMetric):
+    def _init(self):
+        self.metric = Summary(
+            "teuthology_node_locking_duration_seconds",
+            "Time spent waiting to lock nodes",
+            ["machine_type", "count"],
+        )
+
+    def _time(self, **labels):
+        yield self.metric.labels(**labels).time()
+
+
+class NodeReimagingTime(TeuthologyMetric):
+    def _init(self):
+        self.metric = Summary(
+            "teuthology_node_reimaging_duration_seconds",
+            "Time spent reimaging nodes",
+            ["machine_type", "count"],
+        )
+
+    def _time(self, **labels):
+        yield self.metric.labels(**labels).time()
+
+
+class JobTime(TeuthologyMetric):
+    def _init(self):
+        self.metric = Summary(
+            "teuthology_job_duration_seconds",
+            "Time spent executing a job",
+            ["suite"],
+        )
+
+    def _time(self, **labels):
+        yield self.metric.labels(**labels).time()
+
+
+class TaskTime(TeuthologyMetric):
+    def _init(self):
+        self.metric = Summary(
+            "teuthology_task_duration_seconds",
+            "Time spent executing a task",
+            ["name", "phase"],
+        )
+
+    def _time(self, **labels):
+        yield self.metric.labels(**labels).time()
+
+
+class BootstrapTime(TeuthologyMetric):
+    def _init(self):
+        self.metric = Summary(
+            "teuthology_bootstrap_duration_seconds",
+            "Time spent running teuthology's bootstrap script",
+        )
+
+    def _time(self, **labels):
+        yield self.metric.labels(**labels).time()
+
+
+def find_exporter_process() -> int | None:
+    attrs = ['pid', 'uids', 'cmdline']
+    for proc in psutil.process_iter(attrs=attrs):
+        try:
+            cmdline = proc.info['cmdline']
+        except psutil.AccessDenied:
+            continue
+        pid = proc.info['pid']
+        if not cmdline:
+            continue
+        if not [i for i in cmdline if i.split('/')[-1] == 'teuthology-exporter']:
+            continue
+        if os.getuid() not in proc.info['uids']:
+            continue
+        return pid
+
+
+def main(args) -> int:
+    if pid := find_exporter_process():
+        if os.getpid() != pid:
+            log.error(f"teuthology-exporter is already running as PID {pid}")
+            return 2
+    exporter = TeuthologyExporter(interval=int(args["--interval"]))
+    try:
+        exporter.start()
+    except Exception:
+        log.exception("Exporter failed")
+        return 1
+    else:
+        return 0
+
+
+pid = find_exporter_process()
+if pid:
+    PROMETHEUS_MULTIPROC_DIR.mkdir(parents=True, exist_ok=True)
+    REGISTRY = CollectorRegistry()
+    multiprocess.MultiProcessCollector(REGISTRY)

teuthology/kill.py

@@ -4,15 +4,17 @@ import sys
 import yaml
 import psutil
 import subprocess
-import tempfile
 import logging
 import getpass
 
+from typing import Union
+
+import teuthology.exporter
 
 from teuthology import beanstalk
 from teuthology import report
 from teuthology.config import config
-from teuthology import misc
+from teuthology.lock import ops as lock_ops
 
 log = logging.getLogger(__name__)
 
@@ -47,36 +49,61 @@ def kill_run(run_name, archive_base=None, owner=None, machine_type=None,
         run_archive_dir = os.path.join(archive_base, run_name)
         if os.path.isdir(run_archive_dir):
             run_info = find_run_info(serializer, run_name)
-            machine_type = run_info['machine_type']
-            owner = run_info['owner']
+            if 'machine_type' in run_info:
+                machine_type = run_info['machine_type']
+                owner = run_info['owner']
+            else:
+                log.warning("The run info does not have machine type: %s" % run_info)
+                log.warning("Run archive used: %s" % run_archive_dir)
+                log.info("Using machine type '%s' and owner '%s'" % (machine_type, owner))
         elif machine_type is None:
-            raise RuntimeError("The run is still entirely enqueued; " +
-                               "you must also pass --machine-type")
+            # no jobs found in archive and no machine type specified,
+            # so we try paddles to see if there is anything scheduled
+            run_info = report.ResultsReporter().get_run(run_name)
+            machine_type = run_info.get('machine_type', None)
+            if machine_type:
+                log.info(f"Using machine type '{machine_type}' received from paddles.")
+            else:
+                raise RuntimeError(f"Cannot find machine type for the run {run_name}; " +
+                                    "you must also pass --machine-type")
 
     if not preserve_queue:
         remove_beanstalk_jobs(run_name, machine_type)
         remove_paddles_jobs(run_name)
-    kill_processes(run_name, run_info.get('pids'))
+    if kill_processes(run_name, run_info.get('pids')):
+        return
     if owner is not None:
-        targets = find_targets(run_name, owner)
-        nuke_targets(targets, owner)
+        targets = find_targets(run_name)
+        names = list(targets.keys())
+        lock_ops.unlock_safe(names, owner, run_name)
+    report.try_mark_run_dead(run_name)
 
 
-def kill_job(run_name, job_id, archive_base=None, owner=None, save_logs=False):
+def kill_job(run_name, job_id, archive_base=None, owner=None, skip_unlock=False):
     serializer = report.ResultsSerializer(archive_base)
     job_info = serializer.job_info(run_name, job_id)
+    # If we can't read the filesystem, job_info will be nearly empty. Ask paddles:
+    if 'name' not in job_info:
+        job_info = report.ResultsReporter().get_jobs(run_name, job_id)
     if not owner:
         if 'owner' not in job_info:
             raise RuntimeError(
                 "I could not figure out the owner of the requested job. "
                 "Please pass --owner <owner>.")
         owner = job_info['owner']
-    kill_processes(run_name, [job_info.get('pid')])
-    # Because targets can be missing for some cases, for example, when all
-    # the necessary nodes ain't locked yet, we do not use job_info to get them,
-    # but use find_targets():
-    targets = find_targets(run_name, owner, job_id)
-    nuke_targets(targets, owner, save_logs)
+    if kill_processes(run_name, [job_info.get('pid')]):
+        return
+    report.try_push_job_info(job_info, dict(status="dead"))
+    if 'machine_type' in job_info:
+        teuthology.exporter.JobResults().record(
+            machine_type=job_info["machine_type"],
+            status=job_info.get("status", "dead")
+        )
+    else:
+        log.warn(f"Job {job_id} has no machine_type; cannot report via Prometheus")
+    if not skip_unlock:
+        targets = find_targets(run_name, job_id)
+        lock_ops.unlock_safe(list(targets.keys()), owner, run_name, job_id)
 
 
 def find_run_info(serializer, run_name):
@@ -156,23 +183,36 @@ def kill_processes(run_name, pids=None):
     else:
         to_kill = find_pids(run_name)
 
-    # Remove processes that don't match run-name from the set
-    to_check = set(to_kill)
-    for pid in to_check:
+    pids_need_sudo = set()
+    for pid in set(to_kill):
         if not process_matches_run(pid, run_name):
             to_kill.remove(pid)
+        elif psutil.Process(int(pid)).username() != getpass.getuser():
+            pids_need_sudo.add(pid)
 
+    survivors = []
     if len(to_kill) == 0:
         log.info("No teuthology processes running")
     else:
         log.info("Killing Pids: " + str(to_kill))
+        sudo_works = False
+        if pids_need_sudo:
+            sudo_works = subprocess.Popen(['sudo', '-n', '-l']).wait() == 0
+            if not sudo_works:
+                log.debug("Passwordless sudo not configured; not using sudo")
         for pid in to_kill:
+            use_sudo = pid in pids_need_sudo and sudo_works
             args = ['kill', str(pid)]
             # Don't attempt to use sudo if it's not necessary
-            proc_user = psutil.Process(int(pid)).username()
-            if proc_user != getpass.getuser():
-                args.insert(0, 'sudo')
-            subprocess.call(args)
+            if use_sudo:
+                args = ['sudo', '-n'] + args
+            try:
+                subprocess.check_call(args)
+            except subprocess.CalledProcessError:
+                survivors.append(pid)
+    if survivors:
+        log.error(f"Failed to kill PIDs: {survivors}")
+    return survivors
 
 
 def process_matches_run(pid, run_name):
@@ -183,6 +223,8 @@ def process_matches_run(pid, run_name):
             return True
     except psutil.NoSuchProcess:
         pass
+    except psutil.AccessDenied:
+        pass
     return False
 
 
@@ -193,61 +235,14 @@ def find_pids(run_name):
             run_pids.append(pid)
     return run_pids
 
-
-def find_targets(run_name, owner, job_id=None):
-    lock_args = [
-        'teuthology-lock',
-        '--list-targets',
-        '--desc-pattern',
-        '/' + run_name + '/' + str(job_id or ''),
-        '--status',
-        'up',
-        '--owner',
-        owner
-    ]
-    proc = subprocess.Popen(lock_args, stdout=subprocess.PIPE)
-    stdout, stderr = proc.communicate()
-    out_obj = yaml.safe_load(stdout)
-    if not out_obj or 'targets' not in out_obj:
-        return {}
-
-    return out_obj
-
-
-def nuke_targets(targets_dict, owner, save_logs=False):
-    targets = targets_dict.get('targets')
-    if not targets:
-        log.info("No locked machines. Not nuking anything")
-        return
-
-    to_nuke = []
-    for target in targets:
-        to_nuke.append(misc.decanonicalize_hostname(target))
-
-    target_file = tempfile.NamedTemporaryFile(delete=False, mode='w+t')
-    target_file.write(yaml.safe_dump(targets_dict))
-    target_file.close()
-
-    log.info("Nuking machines: " + str(to_nuke))
-    nuke_args = [
-        'teuthology-nuke',
-        '-t',
-        target_file.name,
-        '--owner',
-        owner
-    ]
-    if save_logs:
-        nuke_args.extend(['--no-reboot', '--keep-logs'])
-    else:
-        nuke_args.extend(['--reboot-all', '--unlock'])
-
-    proc = subprocess.Popen(
-        nuke_args,
-        stdout=subprocess.PIPE,
-        stderr=subprocess.STDOUT)
-    for line in proc.stdout:
-        line = line.replace(b'\r', b'').replace(b'\n', b'')
-        log.info(line.decode())
-        sys.stdout.flush()
-
-    os.unlink(target_file.name)
+def find_targets(run_name: str, job_id: Union[str, int, None] = None) -> dict:
+    if job_id is not None:
+        job_info = report.ResultsReporter().get_jobs(run_name, str(job_id))
+        return job_info.get("targets") or dict()
+    result = dict()
+    run_info = report.ResultsReporter().get_jobs(run_name)
+    for job_info in run_info:
+        if job_info.get("status") not in ("running", "waiting"):
+            continue
+        result.update(job_info.get("targets") or dict())
+    return result

teuthology/lock/cli.py

@@ -133,7 +133,7 @@ def main(ctx):
                 for s in sorted(statuses, key=lambda s: s.get('name')):
                     locked = 'unlocked' if s['locked'] == 0 else 'locked'
                     up = 'up' if s['up'] else 'down'
-                    mo = re.match('\w+@(\w+?)\..*', s['name'])
+                    mo = re.match(r'\w+@(\w+?)\..*', s['name'])
                     host = mo.group(1) if mo else s['name']
                     print(node_status_template.format(
                         up=up, locked=locked, host=host,
@@ -200,7 +200,7 @@ def main(ctx):
             res = ops.unlock_many(machines, user)
             return 0 if res else 1
         for machine in machines:
-            if not ops.unlock_one(ctx, machine, user):
+            if not ops.unlock_one(machine, user):
                 ret = 1
                 if not ctx.f:
                     return ret
@@ -221,7 +221,7 @@ def main(ctx):
                 if len(result) < ctx.num_to_lock:
                     log.error("Locking failed.")
                     for machine in result:
-                        ops.unlock_one(ctx, machine, user)
+                        ops.unlock_one(machine, user)
                     ret = 1
                 else:
                     log.info("Successfully Locked:\n%s\n" % shortnames)