atex 0.9__py3-none-any.whl → 0.11__py3-none-any.whl

atex/orchestrator/{orchestrator.py → adhoc.py}

@@ -1,24 +1,19 @@
-import time
 import tempfile
-import traceback
-import concurrent
 import collections
+import concurrent.futures
 from pathlib import Path
 
 from .. import util, executor
-
-
-class OrchestratorError(Exception):
-    pass
+from . import Orchestrator, OrchestratorError
 
 
 class FailedSetupError(OrchestratorError):
     pass
 
 
-class Orchestrator:
+class AdHocOrchestrator(Orchestrator):
     """
-    A scheduler for parallel execution on multiple resources (machines/systems).
+    TODO: document function specific to this reference, ie. run_setup(), etc.
     """
 
     class SetupInfo(
@@ -55,13 +50,17 @@ class Orchestrator:
             # exception class instance if running the test failed
             # (None if no exception happened (exit_code is defined))
             "exception",
+            # Path of a 'results' JSON file with test-reported results
+            "results",
+            # Path of a 'files' directory with test-uploaded files
+            "files",
         ),
     ):
         pass
 
     def __init__(
         self, platform, fmf_tests, provisioners, aggregator, tmp_dir, *,
-        max_reruns=2, max_failed_setups=10, env=None,
+        max_remotes=1, max_spares=0, max_reruns=2, max_failed_setups=10, env=None,
     ):
         """
         'platform' is a string with platform name.
@@ -76,6 +75,15 @@ class Orchestrator:
         storing per-test results and uploaded files before being ingested
         by the aggregator. Can be safely shared by Orchestrator instances.
 
+        'max_remotes' is how many Remotes to hold reserved at any given time,
+        eg. how many tests to run in parallel. Clamped to the number of
+        to-be-run tests given as 'fmf_tests'.
+
+        'max_spares' is how many set-up Remotes to hold reserved and unused,
+        ready to replace a Remote destroyed by test. Values above 0 greatly
+        speed up test reruns as Remote reservation happens asynchronously
+        to test execution. Spares are reserved on top of 'max_remotes'.
+
         'max_reruns' is an integer of how many times to re-try running a failed
         test (which exited with non-0 or caused an Executor exception).
 
@@ -91,36 +99,23 @@ class Orchestrator:
         self.aggregator = aggregator
         self.tmp_dir = tmp_dir
         self.failed_setups_left = max_failed_setups
+        self.max_remotes = max_remotes
+        self.max_spares = max_spares
         # indexed by test name, value being integer of how many times
         self.reruns = collections.defaultdict(lambda: max_reruns)
         self.env = env
         # tests still waiting to be run
         self.to_run = set(fmf_tests.tests)
-        # running setup functions, as a list of SetupInfo items
-        self.running_setups = []
         # running tests as a dict, indexed by test name, with RunningInfo values
         self.running_tests = {}
         # thread queue for actively running tests
         self.test_queue = util.ThreadQueue(daemon=False)
         # thread queue for remotes being set up (uploading tests, etc.)
         self.setup_queue = util.ThreadQueue(daemon=True)
-        # NOTE: running_setups and test_running are just for debugging and
-        #       cancellation, the execution flow itself uses ThreadQueues
-
-    @staticmethod
-    def run_setup(sinfo):
-        """
-        Set up a newly acquired class Remote instance for test execution.
-
-        'sinfo' is a SetupInfo instance with the (fully connected) remote.
-        """
-        sinfo.executor.setup()
-        sinfo.executor.upload_tests()
-        sinfo.executor.plan_prepare()
-        # NOTE: we never run executor.plan_finish() or even executor.cleanup()
-        #       anywhere - instead, we assume the remote (and its connection)
-        #       was invalidated by the test, so we just rely on remote.release()
-        #       destroying the system
+        # thread queue for remotes being released
+        self.release_queue = util.ThreadQueue(daemon=True)
+        # thread queue for results being ingested
+        self.ingest_queue = util.ThreadQueue(daemon=False)
 
     def _run_new_test(self, info):
         """
@@ -132,7 +127,7 @@ class Orchestrator:
         next_test_name = self.next_test(self.to_run, self.fmf_tests.tests, info)
         assert next_test_name in self.to_run, "next_test() returned valid test name"
 
-        util.info(f"starting '{next_test_name}' on {info.remote}")
+        util.info(f"{info.remote}: starting '{next_test_name}'")
 
         self.to_run.remove(next_test_name)
 
@@ -147,6 +142,7 @@ class Orchestrator:
         )
 
         tmp_dir_path = Path(rinfo.tmp_dir.name)
+        tmp_dir_path.chmod(0o755)
         self.test_queue.start_thread(
             target=info.executor.run_test,
             target_args=(
@@ -162,57 +158,74 @@ class Orchestrator:
         """
         'finfo' is a FinishedInfo instance.
         """
+        test_data = self.fmf_tests.tests[finfo.test_name]
+
+        # TODO: somehow move logging from was_successful and should_be_rerun here,
+        #       probably print just some generic info from those functions that doesn't
+        #       imply any outcome, ie.
+        #           {remote_with_test} threw {exception}
+        #           {remote_with_test} exited with {code}
+        #           {remote_with_test} has {N} reruns left
+        #           {remote_with_test} has 0 reruns left
+        #       and then log the decision separately, here below, such as
+        #           {remote_with_test} failed, re-running
+        #           {remote_with_test} completed, ingesting result
+        #           {remote_with_test} was destructive, releasing remote
+        #           {remote_with_test} ...., running next test
+        #       That allows the user to override the functions, while keeping critical
+        #       flow reliably logged here.
+
         remote_with_test = f"{finfo.remote}: '{finfo.test_name}'"
 
-        def ingest_result():
-            tmp_dir_path = Path(finfo.tmp_dir.name)
-            results_file = tmp_dir_path / "results"
-            files_dir = tmp_dir_path / "files"
-            # in case Executor code itself threw an unrecoverable exception
+        if not self.was_successful(finfo, test_data) and self.should_be_rerun(finfo, test_data):
+            # re-run the test
+            util.info(f"{remote_with_test} failed, re-running")
+            self.to_run.add(finfo.test_name)
+        else:
+            # ingest the result
+            #
+            # a condition just in case Executor code itself threw an exception
             # and didn't even report the fallback 'infra' result
-            if results_file.exists() and files_dir.exists():
-                self.aggregator.ingest(self.platform, finfo.test_name, results_file, files_dir)
-                finfo.tmp_dir.cleanup()
-
-        # if executor (or test) threw exception, schedule a re-run
-        if finfo.exception:
-            exc_name = type(finfo.exception).__name__
-            exc_tb = "".join(traceback.format_exception(finfo.exception)).rstrip("\n")
-            msg = f"{remote_with_test} threw {exc_name} during test runtime"
-            #finfo.remote.release()
-            if (reruns_left := self.reruns[finfo.test_name]) > 0:
-                util.info(f"{msg}, re-running ({reruns_left} reruns left):\n{exc_tb}")
-                self.reruns[finfo.test_name] -= 1
-                self.to_run.add(finfo.test_name)
-            else:
-                util.info(f"{msg}, reruns exceeded, giving up:\n{exc_tb}")
-                # record the final result anyway
-                ingest_result()
-
-        # if the test exited as non-0, try a re-run
-        elif finfo.exit_code != 0:
-            msg = f"{remote_with_test} exited with non-zero: {finfo.exit_code}"
-            #finfo.remote.release()
-            if (reruns_left := self.reruns[finfo.test_name]) > 0:
-                util.info(f"{msg}, re-running ({reruns_left} reruns left)")
-                self.reruns[finfo.test_name] -= 1
-                self.to_run.add(finfo.test_name)
-            else:
-                util.info(f"{msg}, reruns exceeded, giving up")
-                # record the final result anyway
-                ingest_result()
+            if finfo.results is not None and finfo.files is not None:
+                util.info(f"{remote_with_test} completed, ingesting result")
+
+                def ingest_and_cleanup(ingest, args, cleanup):
+                    ingest(*args)
+                    # also delete the tmpdir housing these
+                    cleanup()
+
+                self.ingest_queue.start_thread(
+                    ingest_and_cleanup,
+                    target_args=(
+                        # ingest func itself
+                        self.aggregator.ingest,
+                        # args for ingest
+                        (self.platform, finfo.test_name, finfo.results, finfo.files),
+                        # cleanup func itself
+                        finfo.tmp_dir.cleanup,
+                    ),
+                    test_name=finfo.test_name,
+                )
 
-        # test finished successfully - ingest its results
-        else:
-            util.info(f"{remote_with_test} finished successfully")
-            ingest_result()
+                # ingesting destroys these
+                finfo = self.FinishedInfo._from(
+                    finfo,
+                    results=None,
+                    files=None,
+                    tmp_dir=None,
+                )
 
-        # if destroyed, release the remote
+        # if destroyed, release the remote and request a replacement
         # (Executor exception is always considered destructive)
-        test_data = self.fmf_tests.tests[finfo.test_name]
         if finfo.exception or self.destructive(finfo, test_data):
             util.debug(f"{remote_with_test} was destructive, releasing remote")
-            finfo.remote.release()
+            self.release_queue.start_thread(
+                finfo.remote.release,
+                remote=finfo.remote,
+            )
+            # TODO: should this be conditioned by 'self.to_run:' ? to not uselessly fall
+            #       into setup spares and get immediately released after setup?
+            finfo.provisioner.provision(1)
 
         # if still not destroyed, run another test on it
         # (without running plan setup, re-using already set up remote)
@@ -220,6 +233,14 @@ class Orchestrator:
             util.debug(f"{remote_with_test} was non-destructive, running next test")
             self._run_new_test(finfo)
 
+        # no more tests to run, release the remote
+        else:
+            util.debug(f"{finfo.remote} no longer useful, releasing it")
+            self.release_queue.start_thread(
+                finfo.remote.release,
+                remote=finfo.remote,
+            )
+
     def serve_once(self):
         """
         Run the orchestration logic, processing any outstanding requests
@@ -229,10 +250,6 @@ class Orchestrator:
         Returns True to indicate that it should be called again by the user
         (more work to be done), False once all testing is concluded.
         """
-        util.debug(
-            f"to_run: {len(self.to_run)} tests / "
-            f"running: {len(self.running_tests)} tests, {len(self.running_setups)} setups",
-        )
         # all done
         if not self.to_run and not self.running_tests:
             return False
@@ -248,10 +265,16 @@ class Orchestrator:
             rinfo = treturn.rinfo
             del self.running_tests[rinfo.test_name]
 
+            tmp_dir_path = Path(rinfo.tmp_dir.name)
+            results_path = tmp_dir_path / "results"
+            files_path = tmp_dir_path / "files"
+
             finfo = self.FinishedInfo(
                 **rinfo,
                 exit_code=treturn.returned,
                 exception=treturn.exception,
+                results=results_path if results_path.exists() else None,
+                files=files_path if files_path.exists() else None,
             )
             self._process_finished_test(finfo)
 
@@ -264,22 +287,38 @@ class Orchestrator:
                 break
 
             sinfo = treturn.sinfo
-            self.running_setups.remove(sinfo)
 
             if treturn.exception:
-                exc_name = type(treturn.exception).__name__
-                exc_tb = "".join(traceback.format_exception(treturn.exception)).rstrip("\n")
-                msg = f"{sinfo.remote}: setup failed with {exc_name}"
-                sinfo.remote.release()
+                exc_str = f"{type(treturn.exception).__name__}({treturn.exception})"
+                msg = f"{sinfo.remote}: setup failed with {exc_str}"
+                self.release_queue.start_thread(
+                    sinfo.remote.release,
+                    remote=sinfo.remote,
+                )
                 if (reruns_left := self.failed_setups_left) > 0:
-                    util.warning(f"{msg}, re-trying ({reruns_left} setup retries left):\n{exc_tb}")
+                    util.warning(f"{msg}, re-trying ({reruns_left} setup retries left)")
                     self.failed_setups_left -= 1
+                    sinfo.provisioner.provision(1)
                 else:
-                    util.warning(f"{msg}, setup retries exceeded, giving up:\n{exc_tb}")
+                    util.warning(f"{msg}, setup retries exceeded, giving up")
                     raise FailedSetupError("setup retries limit exceeded, broken infra?")
             else:
                 self._run_new_test(sinfo)
 
+        # release any extra Remotes being held as set-up when we know we won't
+        # use them for any tests (because to_run is empty)
+        else:
+            while self.setup_queue.qsize() > self.max_spares:
+                try:
+                    treturn = self.setup_queue.get_raw(block=False)
+                except util.ThreadQueue.Empty:
+                    break
+                util.debug(f"releasing extraneous set-up {treturn.sinfo.remote}")
+                self.release_queue.start_thread(
+                    treturn.sinfo.remote.release,
+                    remote=treturn.sinfo.remote,
+                )
+
         # try to get new remotes from Provisioners - if we get some, start
         # running setup on them
         for provisioner in self.provisioners:
@@ -295,48 +334,96 @@ class Orchestrator:
                     target_args=(sinfo,),
                     sinfo=sinfo,
                 )
-                self.running_setups.append(sinfo)
                 util.info(f"{provisioner}: running setup on new {remote}")
 
-        return True
+        # gather returns from Remote.release() functions - check for exceptions
+        # thrown, re-report them as warnings as they are not typically critical
+        # for operation
+        while True:
+            try:
+                treturn = self.release_queue.get_raw(block=False)
+            except util.ThreadQueue.Empty:
+                break
+            else:
+                if treturn.exception:
+                    exc_str = f"{type(treturn.exception).__name__}({treturn.exception})"
+                    util.warning(f"{treturn.remote} release failed: {exc_str}")
+                else:
+                    util.debug(f"{treturn.remote} release completed")
 
-    def serve_forever(self):
-        """
-        Run the orchestration logic, blocking until all testing is concluded.
-        """
-        while self.serve_once():
-            time.sleep(1)
+        # gather returns from Aggregator.ingest() calls - check for exceptions
+        while True:
+            try:
+                treturn = self.ingest_queue.get_raw(block=False)
+            except util.ThreadQueue.Empty:
+                break
+            else:
+                if treturn.exception:
+                    exc_str = f"{type(treturn.exception).__name__}({treturn.exception})"
+                    util.warning(f"'{treturn.test_name}' ingesting failed: {exc_str}")
+                else:
+                    util.debug(f"'{treturn.test_name}' ingesting completed")
+
+        return True
 
     def start(self):
         # start all provisioners
         for prov in self.provisioners:
             prov.start()
-        return self
+
+        # start up initial reservations, balanced evenly across all available
+        # provisioner instances
+        count = min(self.max_remotes, len(self.fmf_tests.tests)) + self.max_spares
+        provisioners = self.provisioners[:count]
+        for idx, prov in enumerate(provisioners):
+            if count % len(provisioners) > idx:
+                prov.provision((count // len(provisioners)) + 1)
+            else:
+                prov.provision(count // len(provisioners))
 
     def stop(self):
         # cancel all running tests and wait for them to clean up (up to 0.1sec)
         for rinfo in self.running_tests.values():
             rinfo.executor.cancel()
-        self.test_queue.join()  # also ignore any exceptions raised
+        self.test_queue.join()    # also ignore any exceptions raised
+
+        # wait for all running ingestions to finish, print exceptions
+        # (we would rather stop provisioners further below than raise here)
+        while True:
+            try:
+                treturn = self.ingest_queue.get_raw(block=False)
+            except util.ThreadQueue.Empty:
+                break
+            else:
+                if treturn.exception:
+                    exc_str = f"{type(treturn.exception).__name__}({treturn.exception})"
+                    util.warning(f"'{treturn.test_name}' ingesting failed: {exc_str}")
+                else:
+                    util.debug(f"'{treturn.test_name}' ingesting completed")
+        self.ingest_queue.join()
 
         # stop all provisioners, also releasing all remotes
+        # - parallelize up to 10 provisioners at a time
         if self.provisioners:
-            workers = min(len(self.provisioners), 20)
+            workers = min(len(self.provisioners), 10)
             with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as ex:
                 for provisioner in self.provisioners:
-                    for func in provisioner.stop_defer():
-                        ex.submit(func)
+                    ex.submit(provisioner.stop)
 
-    def __enter__(self):
-        try:
-            self.start()
-            return self
-        except Exception:
-            self.stop()
-            raise
+    @staticmethod
+    def run_setup(sinfo):
+        """
+        Set up a newly acquired class Remote instance for test execution.
 
-    def __exit__(self, exc_type, exc_value, traceback):
-        self.stop()
+        'sinfo' is a SetupInfo instance with the (fully connected) remote.
+        """
+        sinfo.executor.start()
+        sinfo.executor.upload_tests()
+        sinfo.executor.plan_prepare()
+        # NOTE: we never run executor.plan_finish() or even executor.stop()
+        #       anywhere - instead, we assume the remote (and its connection)
+        #       was invalidated by the test, so we just rely on remote.release()
+        #       destroying the system
 
     @staticmethod
     def next_test(to_run, all_tests, previous):  # noqa: ARG004
@@ -381,5 +468,56 @@ class Orchestrator:
             return True
         # otherwise we good
         return False
-        # TODO: override with additional 'extra-contest: destructive: True' fmf metadata
-        # destructive = test_data.get("extra-contest", {}).get("destructive", False)
+
+    @staticmethod
+    def was_successful(info, test_data):  # noqa: ARG004
+        """
+        Return a boolean result whether a finished test was successful.
+        Returning False might cause it to be re-run (per should_be_rerun()).
+
+        'info' is Orchestrator.FinishedInfo namedtuple of the test.
+
+        'test_data' is a dict of fully resolved fmf test metadata of that test.
+        """
+        remote_with_test = f"{info.remote}: '{info.test_name}'"
+
+        # executor (or test) threw exception
+        if info.exception:
+            exc_str = f"{type(info.exception).__name__}({info.exception})"
+            util.info(f"{remote_with_test} threw {exc_str} during test runtime")
+            return False
+
+        # the test exited as non-0
+        if info.exit_code != 0:
+            util.info(f"{remote_with_test} exited with non-zero: {info.exit_code}")
+            return False
+
+        # otherwise we good
+        return True
+
+    # TODO: @staticmethod and remove ARG002
+    #@staticmethod
+    def should_be_rerun(self, info, test_data):  # noqa: ARG004, ARG002
+        """
+        Return a boolean result whether a finished test failed in a way
+        that another execution attempt might succeed, due to race conditions
+        in the test or other non-deterministic factors.
+
+        'info' is Orchestrator.FinishedInfo namedtuple of the test.
+
+        'test_data' is a dict of fully resolved fmf test metadata of that test.
+        """
+        remote_with_test = f"{info.remote}: '{info.test_name}'"
+
+        # TODO: remove self.reruns and the whole X-reruns logic from AdHocOrchestrator,
+        #       leave it up to the user to wrap should_be_rerun() with an external dict
+        #       of tests, counting reruns for each
+        #        - allows the user to adjust counts per-test (ie. test_data metadata)
+        #        - allows this template to be @staticmethod
+        reruns_left = self.reruns[info.test_name]
+        util.info(f"{remote_with_test}: {reruns_left} reruns left")
+        if reruns_left > 0:
+            self.reruns[info.test_name] -= 1
+            return True
+        else:
+            return False

atex/orchestrator/contest.py

@@ -0,0 +1,94 @@
+from .. import util
+from .adhoc import AdHocOrchestrator
+
+
+# copy/pasted from the Contest repo, lib/virt.py
+def calculate_guest_tag(tags):
+    if "snapshottable" not in tags:
+        return None
+    name = "default"
+    if "with-gui" in tags:
+        name += "_gui"
+    if "uefi" in tags:
+        name += "_uefi"
+    if "fips" in tags:
+        name += "_fips"
+    return name
+
+
+class ContestOrchestrator(AdHocOrchestrator):
+    """
+    Orchestrator for the Contest test suite:
+        https://github.com/RHSecurityCompliance/contest
+
+    Includes SCAP content upload via rsync and other Contest-specific
+    optimizations (around VM snapshots and scheduling).
+    """
+    content_dir_on_remote = "/root/upstream-content"
+
+    def __init__(self, *args, content_dir, **kwargs):
+        self.content_dir = content_dir
+        super().__init__(*args, **kwargs)
+
+    def run_setup(self, sinfo):
+        super().run_setup(sinfo)
+        # upload pre-built content
+        sinfo.remote.rsync(
+            "-r", "--delete", "--exclude=.git/",
+            f"{self.content_dir}/",
+            f"remote:{self.content_dir_on_remote}",
+            func=util.subprocess_log,
+        )
+
+    @classmethod
+    def next_test(cls, to_run, all_tests, previous):
+        # fresh remote, prefer running destructive tests (which likely need
+        # clean OS) to get them out of the way and prevent them from running
+        # on a tainted OS later
+        if type(previous) is AdHocOrchestrator.SetupInfo:
+            for next_name in to_run:
+                next_tags = all_tests[next_name].get("tag", ())
+                util.debug(f"considering next_test for destructivity: {next_name}")
+                if "destructive" in next_tags:
+                    util.debug(f"chosen next_test: {next_name}")
+                    return next_name
+
+        # previous test was run and finished non-destructively,
+        # try to find a next test with the same Contest lib.virt guest tags
+        # as the previous one, allowing snapshot reuse by Contest
+        elif type(previous) is AdHocOrchestrator.FinishedInfo:
+            finished_tags = all_tests[previous.test_name].get("tag", ())
+            util.debug(f"previous finished test on {previous.remote}: {previous.test_name}")
+            # if Guest tag is None, don't bother searching
+            if finished_guest_tag := calculate_guest_tag(finished_tags):
+                for next_name in to_run:
+                    util.debug(f"considering next_test with tags {finished_tags}: {next_name}")
+                    next_tags = all_tests[next_name].get("tag", ())
+                    next_guest_tag = calculate_guest_tag(next_tags)
+                    if next_guest_tag and finished_guest_tag == next_guest_tag:
+                        util.debug(f"chosen next_test: {next_name}")
+                        return next_name
+
+        # fallback to the default next_test()
+        return super().next_test(to_run, all_tests, previous)
+
+    @classmethod
+    def destructive(cls, info, test_data):
+        # if Executor ended with an exception (ie. duration exceeded),
+        # consider the test destructive
+        if info.exception:
+            return True
+
+        # if the test returned non-0 exit code, it could have thrown
+        # a python exception of its own, or (if bash) aborted abruptly
+        # due to 'set -e', don't trust the remote, consider it destroyed
+        # (0 = pass, 2 = fail, anything else = bad)
+        if info.exit_code not in [0,2]:
+            return True
+
+        # if the test was destructive, assume the remote is destroyed
+        tags = test_data.get("tag", ())
+        if "destructive" in tags:
+            return True
+
+        return False