atex 0.7__py3-none-any.whl → 0.9__py3-none-any.whl

atex/orchestrator/__init__.py

@@ -1,59 +1,3 @@
-import importlib as _importlib
-import pkgutil as _pkgutil
-#import threading as _threading
-
-
-class Orchestrator:
-    """
-    A scheduler for parallel execution on multiple resources (machines/systems).
-
-    Given a list of Provisioner-derived class instances, it attempts to reserve
-    resources and uses them on-demand as they become available, calling run()
-    on each.
-
-    Note that run() and report() always run in a separate threads (are allowed
-    to block), and may access instance attributes, which are transparently
-    guarded by a thread-aware mutex.
-
-    """
-
-    def __init__(self):
-        pass
-        # TODO: configure via args, max workers, etc.
-
-#    def reserve(self, provisioner):
-#        # call provisioner.reserve(), return its return
-#        ...
-
-    def add_provisioner(self, provisioner):
-        # add to a self.* list of provisioners to be used for getting machines
-        ...
-
-    def run(self, provisioner):
-        # run tests, if destructive, call provisioner.release()
-        # returns anything
-        ...
-
-    def report(self):
-        # gets return from run
-        # writes it out to somewhere else
-        ...
-
-
-_submodules = [
-    info.name for info in _pkgutil.iter_modules(__spec__.submodule_search_locations)
-]
-
-__all__ = [*_submodules, Orchestrator.__name__]  # noqa: PLE0604
-
-
-def __dir__():
-    return __all__
-
-
-# lazily import submodules
-def __getattr__(attr):
-    if attr in _submodules:
-        return _importlib.import_module(f".{attr}", __name__)
-    else:
-        raise AttributeError(f"module '{__name__}' has no attribute '{attr}'")
+#from .aggregator import CSVAggregator, JSONAggregator  # noqa: F401
+from .aggregator import JSONAggregator  # noqa: F401
+from .orchestrator import Orchestrator, OrchestratorError, FailedSetupError  # noqa: F401

atex/orchestrator/aggregator.py

@@ -1,163 +1,111 @@
-"""
-Functions and utilities for persistently storing test results and files (logs).
-
-There is a global aggregator (ie. CSVAggregator) that handles all the results
-from all platforms (arches and distros), and several  per-platform aggregators
-that are used by test execution logic.
-
-    with CSVAggregator("results.csv.gz", "file/storage/dir") as global_aggr:
-        reporter = global_aggr.for_platform("rhel-9@x86_64")
-        reporter.report({"name": "/some/test", "status": "pass"})
-        with reporter.open_tmpfile() as fd:
-            os.write(fd, "some contents")
-            reporter.link_tmpfile_to("/some/test", "test.log", fd)
-"""
-
-import os
-import csv
 import gzip
-import ctypes
-import ctypes.util
+import json
+import shutil
 import threading
-import contextlib
 from pathlib import Path
 
 
-libc = ctypes.CDLL(ctypes.util.find_library("c"), use_errno=True)
-
-# int linkat(int olddirfd, const char *oldpath, int newdirfd, const char *newpath, int flags)
-libc.linkat.argtypes = (
-    ctypes.c_int,
-    ctypes.c_char_p,
-    ctypes.c_int,
-    ctypes.c_char_p,
-    ctypes.c_int,
-)
-libc.linkat.restype = ctypes.c_int
-
-# fcntl.h:#define AT_EMPTY_PATH                0x1000  /* Allow empty relative pathname */
-AT_EMPTY_PATH = 0x1000
-
-# fcntl.h:#define AT_FDCWD             -100    /* Special value used to indicate
-AT_FDCWD = -100
-
+class JSONAggregator:
+    """
+    Collects reported results as a GZIP-ed line-JSON and files (logs) from
+    multiple test runs under a shared directory.
 
-def linkat(*args):
-    if (ret := libc.linkat(*args)) == -1:
-        errno = ctypes.get_errno()
-        raise OSError(errno, os.strerror(errno))
-    return ret
+    Note that the aggregated JSON file *does not* use the test-based JSON format
+    described by executor/RESULTS.md - both use JSON, but are very different.
 
+    This aggergated format uses a top-level array (on each line) with a fixed
+    field order:
 
-def _normalize_path(path):
-    # the magic here is to treat any dangerous path as starting at /
-    # and resolve any weird constructs relative to /, and then simply
-    # strip off the leading / and use it as a relative path
-    path = path.lstrip("/")
-    path = os.path.normpath(f"/{path}")
-    return path[1:]
+        platform, status, test name, subtest name, files, note
 
+    All these are strings except 'files', which is another (nested) array
+    of strings.
 
-class CSVAggregator:
-    """
-    Collects reported results as a GZIP-ed CSV and files (logs) under a related
-    directory.
+    If a field is missing in the source result, it is translated to a null
+    value.
     """
 
-    class _ExcelWithUnixNewline(csv.excel):
-        lineterminator = "\n"
+    def __init__(self, json_file, storage_dir):
+        """
+        'json_file' is a string/Path to a .json.gz file with aggregated results.
 
-    def __init__(self, results_file, storage_dir):
+        'storage_dir' is a string/Path of the top-level parent for all
+        per-platform / per-test files uploaded by tests.
+        """
         self.lock = threading.RLock()
         self.storage_dir = Path(storage_dir)
-        self.results_file = Path(results_file)
-        self.csv_writer = None
-        self.results_gzip_handle = None
+        self.json_file = Path(json_file)
+        self.json_gzip_fobj = None
 
-    def __enter__(self):
-        if self.results_file.exists():
-            raise FileExistsError(f"{self.results_file} already exists")
-        f = gzip.open(self.results_file, "wt", newline="")
-        try:
-            self.csv_writer = csv.writer(f, dialect=self._ExcelWithUnixNewline)
-        except:
-            f.close()
-            raise
-        self.results_gzip_handle = f
+    def open(self):
+        if self.json_file.exists():
+            raise FileExistsError(f"{self.json_file} already exists")
+        self.json_gzip_fobj = gzip.open(self.json_file, "wt", newline="\n")
 
         if self.storage_dir.exists():
             raise FileExistsError(f"{self.storage_dir} already exists")
         self.storage_dir.mkdir()
 
-        return self
-
-    def __exit__(self, exc_type, exc_value, traceback):
-        self.results_gzip_handle.close()
-        self.results_gzip_handle = None
-        self.csv_writer = None
-
-    def report(self, platform, status, name, note, *files):
-        with self.lock:
-            self.csv_writer.writerow((platform, status, name, note, *files))
-
-    def for_platform(self, platform_string):
-        """
-        Return a ResultAggregator instance that writes results into this
-        CSVAgreggator instance.
-        """
-        def report(result_line):
-            file_names = []
-            if "testout" in result_line:
-                file_names.append(result_line["testout"])
-            if "files" in result_line:
-                file_names += (f["name"] for f in result_line["files"])
-            self.report(
-                platform_string, result_line["status"], result_line["name"],
-                result_line.get("note", ""), *file_names,
-            )
-        platform_dir = self.storage_dir / platform_string
-        platform_dir.mkdir(exist_ok=True)
-        return ResultAggregator(report, platform_dir)
+    def close(self):
+        if self.json_gzip_fobj:
+            self.json_gzip_fobj.close()
+            self.json_gzip_fobj = None
 
+    def __enter__(self):
+        try:
+            self.open()
+            return self
+        except Exception:
+            self.close()
+            raise
 
-class ResultAggregator:
-    """
-    Collects reported results (in a format specified by RESULTS.md) for
-    a specific platform, storing them persistently.
-    """
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.close()
 
-    def __init__(self, callback, storage_dir):
+    def ingest(self, platform, test_name, results_file, files_dir):
         """
-        'callback' is a function to call to record a result, with the
-        result dict passed as an argument.
-
-        'storage_dir' is a directory for storing uploaded files.
+        Process 'results_file' (string/Path) for reported results and append
+        them to the overall aggregated line-JSON file, recursively copying over
+        the dir structure under 'files_dir' (string/Path) under the respective
+        platform and test name in the aggregated storage dir.
         """
-        self.report = callback
-        self.storage_dir = storage_dir
+        platform_dir = self.storage_dir / platform
+        test_dir = platform_dir / test_name.lstrip("/")
+        if test_dir.exists():
+            raise FileExistsError(f"{test_dir} already exists for {test_name}")
+
+        # parse the results separately, before writing any aggregated output,
+        # to ensure that either all results from the test are ingested, or none
+        # at all (ie. if one of the result lines contains JSON errors)
+        output_lines = []
+        with open(results_file) as results_fobj:
+            for raw_line in results_fobj:
+                result_line = json.loads(raw_line)
+
+                file_names = []
+                if "testout" in result_line:
+                    file_names.append(result_line["testout"])
+                if "files" in result_line:
+                    file_names += (f["name"] for f in result_line["files"])
+
+                output_line = (
+                    platform,
+                    result_line["status"],
+                    test_name,
+                    result_line.get("name"),
+                    file_names,
+                    result_line.get("note"),
+                )
+                encoded = json.dumps(output_line, indent=None)
+                output_lines.append(encoded)
+
+        output_str = "\n".join(output_lines) + "\n"
 
-    @contextlib.contextmanager
-    def open_tmpfile(self, open_mode=os.O_WRONLY):
-        """
-        Open an anonymous (name-less) file for writing and yield its file
-        descriptor (int) as context, closing it when the context is exited.
-        """
-        flags = open_mode | os.O_TMPFILE
-        fd = os.open(self.storage_dir, flags, 0o644)
-        try:
-            yield fd
-        finally:
-            os.close(fd)
+        with self.lock:
+            self.json_gzip_fobj.write(output_str)
+            self.json_gzip_fobj.flush()
 
-    def link_tmpfile_to(self, result_name, file_name, fd):
-        """
-        Store a file named 'file_name' in a directory relevant to 'result_name'
-        whose 'fd' (a file descriptor) was created by .open_tmpfile().
+        Path(results_file).unlink()
 
-        This function can be called multiple times with the same 'fd', and
-        does not close or otherwise alter the descriptor.
-        """
-        # /path/to/all/logs / some/test/name / path/to/file.log
-        file_path = self.storage_dir / result_name.lstrip("/") / _normalize_path(file_name)
-        file_path.parent.mkdir(parents=True, exist_ok=True)
-        linkat(fd, b"", AT_FDCWD, bytes(file_path), AT_EMPTY_PATH)
+        platform_dir.mkdir(exist_ok=True)
+        shutil.move(files_dir, test_dir)

atex/orchestrator/orchestrator.py

@@ -0,0 +1,385 @@
+import time
+import tempfile
+import traceback
+import concurrent
+import collections
+from pathlib import Path
+
+from .. import util, executor
+
+
+class OrchestratorError(Exception):
+    pass
+
+
+class FailedSetupError(OrchestratorError):
+    pass
+
+
+class Orchestrator:
+    """
+    A scheduler for parallel execution on multiple resources (machines/systems).
+    """
+
+    class SetupInfo(
+        util.NamedMapping,
+        required=(
+            # class Provisioner instance this machine is provided by
+            # (for logging purposes)
+            "provisioner",
+            # class Remote instance returned by the Provisioner
+            "remote",
+            # class Executor instance uploading tests / running setup or tests
+            "executor",
+        ),
+    ):
+        pass
+
+    class RunningInfo(
+        SetupInfo,
+        required=(
+            # string with /test/name
+            "test_name",
+            # class tempfile.TemporaryDirectory instance passed to Executor
+            "tmp_dir",
+        ),
+    ):
+        pass
+
+    class FinishedInfo(
+        RunningInfo,
+        required=(
+            # integer with exit code of the test
+            # (None if exception happened)
+            "exit_code",
+            # exception class instance if running the test failed
+            # (None if no exception happened (exit_code is defined))
+            "exception",
+        ),
+    ):
+        pass
+
+    def __init__(
+        self, platform, fmf_tests, provisioners, aggregator, tmp_dir, *,
+        max_reruns=2, max_failed_setups=10, env=None,
+    ):
+        """
+        'platform' is a string with platform name.
+
+        'fmf_tests' is a class FMFTests instance of the tests to run.
+
+        'provisioners' is an iterable of class Provisioner instances.
+
+        'aggregator' is a class CSVAggregator instance.
+
+        'tmp_dir' is a string/Path to a temporary directory, to be used for
+        storing per-test results and uploaded files before being ingested
+        by the aggregator. Can be safely shared by Orchestrator instances.
+
+        'max_reruns' is an integer of how many times to re-try running a failed
+        test (which exited with non-0 or caused an Executor exception).
+
+        'max_failed_setups' is an integer of how many times an Executor's
+        plan setup (uploading tests, running prepare scripts, etc.) may fail
+        before FailedSetupError is raised.
+
+        'env' is a dict of extra environment variables to pass to Executor.
+        """
+        self.platform = platform
+        self.fmf_tests = fmf_tests
+        self.provisioners = tuple(provisioners)
+        self.aggregator = aggregator
+        self.tmp_dir = tmp_dir
+        self.failed_setups_left = max_failed_setups
+        # indexed by test name, value being integer of how many times
+        self.reruns = collections.defaultdict(lambda: max_reruns)
+        self.env = env
+        # tests still waiting to be run
+        self.to_run = set(fmf_tests.tests)
+        # running setup functions, as a list of SetupInfo items
+        self.running_setups = []
+        # running tests as a dict, indexed by test name, with RunningInfo values
+        self.running_tests = {}
+        # thread queue for actively running tests
+        self.test_queue = util.ThreadQueue(daemon=False)
+        # thread queue for remotes being set up (uploading tests, etc.)
+        self.setup_queue = util.ThreadQueue(daemon=True)
+        # NOTE: running_setups and test_running are just for debugging and
+        #       cancellation, the execution flow itself uses ThreadQueues
+
+    @staticmethod
+    def run_setup(sinfo):
+        """
+        Set up a newly acquired class Remote instance for test execution.
+
+        'sinfo' is a SetupInfo instance with the (fully connected) remote.
+        """
+        sinfo.executor.setup()
+        sinfo.executor.upload_tests()
+        sinfo.executor.plan_prepare()
+        # NOTE: we never run executor.plan_finish() or even executor.cleanup()
+        #       anywhere - instead, we assume the remote (and its connection)
+        #       was invalidated by the test, so we just rely on remote.release()
+        #       destroying the system
+
+    def _run_new_test(self, info):
+        """
+        'info' can be either
+          - SetupInfo instance with Remote/Executor to run the new test.
+          - FinishedInfo instance of a previously executed test
+            (reusing Remote/Executor for a new test).
+        """
+        next_test_name = self.next_test(self.to_run, self.fmf_tests.tests, info)
+        assert next_test_name in self.to_run, "next_test() returned valid test name"
+
+        util.info(f"starting '{next_test_name}' on {info.remote}")
+
+        self.to_run.remove(next_test_name)
+
+        rinfo = self.RunningInfo._from(
+            info,
+            test_name=next_test_name,
+            tmp_dir=tempfile.TemporaryDirectory(
+                prefix=next_test_name.strip("/").replace("/","-") + "-",
+                dir=self.tmp_dir,
+                delete=False,
+            ),
+        )
+
+        tmp_dir_path = Path(rinfo.tmp_dir.name)
+        self.test_queue.start_thread(
+            target=info.executor.run_test,
+            target_args=(
+                next_test_name,
+                tmp_dir_path,
+            ),
+            rinfo=rinfo,
+        )
+
+        self.running_tests[next_test_name] = rinfo
+
+    def _process_finished_test(self, finfo):
+        """
+        'finfo' is a FinishedInfo instance.
+        """
+        remote_with_test = f"{finfo.remote}: '{finfo.test_name}'"
+
+        def ingest_result():
+            tmp_dir_path = Path(finfo.tmp_dir.name)
+            results_file = tmp_dir_path / "results"
+            files_dir = tmp_dir_path / "files"
+            # in case Executor code itself threw an unrecoverable exception
+            # and didn't even report the fallback 'infra' result
+            if results_file.exists() and files_dir.exists():
+                self.aggregator.ingest(self.platform, finfo.test_name, results_file, files_dir)
+                finfo.tmp_dir.cleanup()
+
+        # if executor (or test) threw exception, schedule a re-run
+        if finfo.exception:
+            exc_name = type(finfo.exception).__name__
+            exc_tb = "".join(traceback.format_exception(finfo.exception)).rstrip("\n")
+            msg = f"{remote_with_test} threw {exc_name} during test runtime"
+            #finfo.remote.release()
+            if (reruns_left := self.reruns[finfo.test_name]) > 0:
+                util.info(f"{msg}, re-running ({reruns_left} reruns left):\n{exc_tb}")
+                self.reruns[finfo.test_name] -= 1
+                self.to_run.add(finfo.test_name)
+            else:
+                util.info(f"{msg}, reruns exceeded, giving up:\n{exc_tb}")
+                # record the final result anyway
+                ingest_result()
+
+        # if the test exited as non-0, try a re-run
+        elif finfo.exit_code != 0:
+            msg = f"{remote_with_test} exited with non-zero: {finfo.exit_code}"
+            #finfo.remote.release()
+            if (reruns_left := self.reruns[finfo.test_name]) > 0:
+                util.info(f"{msg}, re-running ({reruns_left} reruns left)")
+                self.reruns[finfo.test_name] -= 1
+                self.to_run.add(finfo.test_name)
+            else:
+                util.info(f"{msg}, reruns exceeded, giving up")
+                # record the final result anyway
+                ingest_result()
+
+        # test finished successfully - ingest its results
+        else:
+            util.info(f"{remote_with_test} finished successfully")
+            ingest_result()
+
+        # if destroyed, release the remote
+        # (Executor exception is always considered destructive)
+        test_data = self.fmf_tests.tests[finfo.test_name]
+        if finfo.exception or self.destructive(finfo, test_data):
+            util.debug(f"{remote_with_test} was destructive, releasing remote")
+            finfo.remote.release()
+
+        # if still not destroyed, run another test on it
+        # (without running plan setup, re-using already set up remote)
+        elif self.to_run:
+            util.debug(f"{remote_with_test} was non-destructive, running next test")
+            self._run_new_test(finfo)
+
+    def serve_once(self):
+        """
+        Run the orchestration logic, processing any outstanding requests
+        (for provisioning, new test execution, etc.) and returning once these
+        are taken care of.
+
+        Returns True to indicate that it should be called again by the user
+        (more work to be done), False once all testing is concluded.
+        """
+        util.debug(
+            f"to_run: {len(self.to_run)} tests / "
+            f"running: {len(self.running_tests)} tests, {len(self.running_setups)} setups",
+        )
+        # all done
+        if not self.to_run and not self.running_tests:
+            return False
+
+        # process all finished tests, potentially reusing remotes for executing
+        # further tests
+        while True:
+            try:
+                treturn = self.test_queue.get_raw(block=False)
+            except util.ThreadQueue.Empty:
+                break
+
+            rinfo = treturn.rinfo
+            del self.running_tests[rinfo.test_name]
+
+            finfo = self.FinishedInfo(
+                **rinfo,
+                exit_code=treturn.returned,
+                exception=treturn.exception,
+            )
+            self._process_finished_test(finfo)
+
+        # process any remotes with finished plan setup (uploaded tests,
+        # plan-defined pkgs / prepare scripts), start executing tests on them
+        while self.to_run:
+            try:
+                treturn = self.setup_queue.get_raw(block=False)
+            except util.ThreadQueue.Empty:
+                break
+
+            sinfo = treturn.sinfo
+            self.running_setups.remove(sinfo)
+
+            if treturn.exception:
+                exc_name = type(treturn.exception).__name__
+                exc_tb = "".join(traceback.format_exception(treturn.exception)).rstrip("\n")
+                msg = f"{sinfo.remote}: setup failed with {exc_name}"
+                sinfo.remote.release()
+                if (reruns_left := self.failed_setups_left) > 0:
+                    util.warning(f"{msg}, re-trying ({reruns_left} setup retries left):\n{exc_tb}")
+                    self.failed_setups_left -= 1
+                else:
+                    util.warning(f"{msg}, setup retries exceeded, giving up:\n{exc_tb}")
+                    raise FailedSetupError("setup retries limit exceeded, broken infra?")
+            else:
+                self._run_new_test(sinfo)
+
+        # try to get new remotes from Provisioners - if we get some, start
+        # running setup on them
+        for provisioner in self.provisioners:
+            while (remote := provisioner.get_remote(block=False)) is not None:
+                ex = executor.Executor(self.fmf_tests, remote, env=self.env)
+                sinfo = self.SetupInfo(
+                    provisioner=provisioner,
+                    remote=remote,
+                    executor=ex,
+                )
+                self.setup_queue.start_thread(
+                    target=self.run_setup,
+                    target_args=(sinfo,),
+                    sinfo=sinfo,
+                )
+                self.running_setups.append(sinfo)
+                util.info(f"{provisioner}: running setup on new {remote}")
+
+        return True
+
+    def serve_forever(self):
+        """
+        Run the orchestration logic, blocking until all testing is concluded.
+        """
+        while self.serve_once():
+            time.sleep(1)
+
+    def start(self):
+        # start all provisioners
+        for prov in self.provisioners:
+            prov.start()
+        return self
+
+    def stop(self):
+        # cancel all running tests and wait for them to clean up (up to 0.1sec)
+        for rinfo in self.running_tests.values():
+            rinfo.executor.cancel()
+        self.test_queue.join()  # also ignore any exceptions raised
+
+        # stop all provisioners, also releasing all remotes
+        if self.provisioners:
+            workers = min(len(self.provisioners), 20)
+            with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as ex:
+                for provisioner in self.provisioners:
+                    for func in provisioner.stop_defer():
+                        ex.submit(func)
+
+    def __enter__(self):
+        try:
+            self.start()
+            return self
+        except Exception:
+            self.stop()
+            raise
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.stop()
+
+    @staticmethod
+    def next_test(to_run, all_tests, previous):  # noqa: ARG004
+        """
+        Return a test name (string) to be executed next.
+
+        'to_run' is a set of test names to pick from. The returned test name
+        must be chosen from this set.
+
+        'tests' is a dict indexed by test name (string), with values being
+        fully resolved fmf test metadata (dicts) of all possible tests.
+
+        'previous' can be either
+          - Orchestrator.SetupInfo instance (first test to be run)
+          - Orchestrator.FinishedInfo instance (previous executed test)
+
+        This method must not modify any of its arguments, it must treat them
+        as read-only, eg. don't remove the returned test name from 'to_run'.
+        """
+        # default to simply picking any available test
+        return next(iter(to_run))
+
+    @staticmethod
+    def destructive(info, test_data):  # noqa: ARG004
+        """
+        Return a boolean result whether a finished test was destructive
+        to a class Remote instance, indicating that the Remote instance
+        should not be used for further test execution.
+
+        'info' is Orchestrator.FinishedInfo namedtuple of the test.
+
+        'test_data' is a dict of fully resolved fmf test metadata of that test.
+        """
+        # if Executor ended with an exception (ie. duration exceeded),
+        # consider the test destructive
+        if info.exception:
+            return True
+        # if the test returned non-0 exit code, it could have thrown
+        # a python exception of its own, or (if bash) aborted abruptly
+        # due to 'set -e', don't trust the remote, consider it destroyed
+        if info.exit_code != 0:
+            return True
+        # otherwise we good
+        return False
+        # TODO: override with additional 'extra-contest: destructive: True' fmf metadata
+        # destructive = test_data.get("extra-contest", {}).get("destructive", False)