atex 0.10__py3-none-any.whl → 0.12__py3-none-any.whl

atex/executor/executor.py

@@ -8,18 +8,11 @@ import subprocess
 from pathlib import Path
 
 from .. import util, fmf
-from . import testcontrol, scripts
+from . import TestSetupError, TestAbortedError, testcontrol, scripts
 from .duration import Duration
 from .reporter import Reporter
 
 
-class TestAbortedError(Exception):
-    """
-    Raised when an infrastructure-related issue happened while running a test.
-    """
-    pass
-
-
 class Executor:
     """
     Logic for running tests on a remote system and processing results
@@ -153,11 +146,11 @@ class Executor:
             **self.env,
             "TMT_PLAN_ENVIRONMENT_FILE": self.plan_env_file,
         }
-        env_args = (f"{k}={v}" for k, v in env.items())
+        env_args = tuple(f"{k}={v}" for k, v in env.items())
         # run the scripts
         for script in scripts:
             self.conn.cmd(
-                ("env", *env_args, "bash"),
+                ("env", "-C", self.tests_dir, *env_args, "bash"),
                 func=util.subprocess_log,
                 stderr=subprocess.STDOUT,
                 input=script,
@@ -222,16 +215,6 @@ class Executor:
         output_dir = Path(output_dir)
         test_data = self.fmf_tests.tests[test_name]
 
-        # run a setup script, preparing wrapper + test scripts
-        setup_script = scripts.test_setup(
-            test=scripts.Test(test_name, test_data, self.fmf_tests.test_dirs[test_name]),
-            tests_dir=self.tests_dir,
-            wrapper_exec=f"{self.work_dir}/wrapper.sh",
-            test_exec=f"{self.work_dir}/test.sh",
-            test_yaml=f"{self.work_dir}/metadata.yaml",
-        )
-        self.conn.cmd(("bash",), input=setup_script, text=True, check=True)
-
         # start with fmf-plan-defined environment
         env_vars = {
             **self.fmf_tests.plan_env,
@@ -253,6 +236,28 @@ class Executor:
             duration = Duration(test_data.get("duration", "5m"))
             control = testcontrol.TestControl(reporter=reporter, duration=duration)
 
+            # run a setup script, preparing wrapper + test scripts
+            setup_script = scripts.test_setup(
+                test=scripts.Test(test_name, test_data, self.fmf_tests.test_dirs[test_name]),
+                tests_dir=self.tests_dir,
+                wrapper_exec=f"{self.work_dir}/wrapper.sh",
+                test_exec=f"{self.work_dir}/test.sh",
+                test_yaml=f"{self.work_dir}/metadata.yaml",
+            )
+            setup_proc = self.conn.cmd(
+                ("bash",),
+                input=setup_script,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,
+                text=True,
+            )
+            if setup_proc.returncode != 0:
+                reporter.report({
+                    "status": "infra",
+                    "note": f"TestSetupError({setup_proc.stdout})",
+                })
+                raise TestSetupError(setup_proc.stdout)
+
             test_proc = None
             control_fd = None
             stack.callback(lambda: os.close(control_fd) if control_fd else None)
@@ -387,7 +392,7 @@ class Executor:
                         pass
                     reporter.report({
                         "status": "infra",
-                        "note": repr(exception),
+                        "note": f"{type(exception).__name__}({exception})",
                         "testout": "output.txt",
                     })
 

atex/executor/reporter.py

@@ -85,16 +85,15 @@ class Reporter:
         file_path.parent.mkdir(parents=True, exist_ok=True)
         return file_path
 
-    def open_file(self, file_name, result_name=None, mode="wb"):
+    def open_fd(self, file_name, mode, result_name=None):
         """
         Open a file named 'file_name' in a directory relevant to 'result_name'.
-        Returns an opened file-like object that can be used in a context manager
-        just like with regular open().
+        Returns an opened file descriptor that can be closed with os.close().
 
         If 'result_name' (typically a subtest) is not given, open the file
         for the test (name) itself.
         """
-        return open(self._dest_path(file_name, result_name), mode)
+        return os.open(self._dest_path(file_name, result_name), mode)
 
     def link_testout(self, file_name, result_name=None):
         # TODO: docstring

atex/executor/scripts.py

@@ -86,7 +86,8 @@ def test_wrapper(*, test, tests_dir, test_exec):
     out += ")\n"
 
     # write test exitcode to test control stream
-    out += "echo exitcode $? >&$orig_stdout\n"
+    if os.environ.get("ATEX_DEBUG_NO_EXITCODE") != "1":
+        out += "echo exitcode $? >&$orig_stdout\n"
 
     # always exit the wrapper with 0 if test execution was normal
     out += "exit 0\n"
@@ -94,17 +95,6 @@ def test_wrapper(*, test, tests_dir, test_exec):
     return out
 
 
-def _install_packages(pkgs, extra_opts=None):
-    pkgs_str = " ".join(pkgs)
-    extra_opts = extra_opts or ()
-    dnf = ["dnf", "-y", "--setopt=install_weak_deps=False", "install", *extra_opts]
-    dnf_str = " ".join(dnf)
-    return util.dedent(fr"""
-        not_installed=$(rpm -q --qf '' {pkgs_str} | sed -nr 's/^package ([^ ]+) is not installed$/\1/p')
-        [[ $not_installed ]] && {dnf_str} $not_installed
-    """)  # noqa: E501
-
-
 def test_setup(*, test, wrapper_exec, test_exec, test_yaml, **kwargs):
     """
     Generate a bash script that should prepare the remote end for test
@@ -133,9 +123,19 @@ def test_setup(*, test, wrapper_exec, test_exec, test_yaml, **kwargs):
     # install test dependencies
     # - only strings (package names) in require/recommend are supported
     if require := list(fmf.test_pkg_requires(test.data, "require")):
-        out += _install_packages(require) + "\n"
+        pkgs_str = " ".join(require)
+        out += util.dedent(fr"""
+            not_installed=$(rpm -q --qf '' {pkgs_str} | sed -nr 's/^package ([^ ]+) is not installed$/\1/p')
+            [[ $not_installed ]] && dnf -y --setopt=install_weak_deps=False install $not_installed
+        """) + "\n"  # noqa: E501
     if recommend := list(fmf.test_pkg_requires(test.data, "recommend")):
-        out += _install_packages(recommend, ("--skip-broken",)) + "\n"
+        pkgs_str = " ".join(recommend)
+        out += util.dedent(fr"""
+            have_dnf5=$(command -v dnf5) || true
+            skip_bad="--skip-broken${{have_dnf5:+ --skip-unavailable}}"
+            not_installed=$(rpm -q --qf '' {pkgs_str} | sed -nr 's/^package ([^ ]+) is not installed$/\1/p')
+            [[ $not_installed ]] && dnf -y --setopt=install_weak_deps=False install $skip_bad $not_installed
+        """) + "\n"  # noqa: E501
 
     # write out test data
     out += f"cat > '{test_yaml}' <<'ATEX_SETUP_EOF'\n"

atex/executor/testcontrol.py

@@ -267,40 +267,45 @@ class TestControl:
             except ValueError as e:
                 raise BadReportJSONError(f"file entry {file_name} length: {str(e)}") from None
 
+            fd = self.reporter.open_fd(file_name, os.O_WRONLY | os.O_CREAT, name)
             try:
-                with self.reporter.open_file(file_name, name) as f:
-                    fd = f.fileno()
-                    while file_length > 0:
+                # Linux can't do splice(2) on O_APPEND fds, so we open it above
+                # as O_WRONLY and just seek to the end, simulating append
+                os.lseek(fd, 0, os.SEEK_END)
+
+                while file_length > 0:
+                    try:
+                        # try a more universal sendfile first, fall back to splice
                         try:
-                            # try a more universal sendfile first, fall back to splice
-                            try:
-                                written = os.sendfile(fd, self.control_fd, None, file_length)
-                            except OSError as e:
-                                if e.errno == 22:  # EINVAL
-                                    written = os.splice(self.control_fd, fd, file_length)
-                                else:
-                                    raise
-                        except BlockingIOError:
-                            yield
-                            continue
-                        if written == 0:
-                            raise BadControlError("EOF when reading data")
-                        file_length -= written
+                            written = os.sendfile(fd, self.control_fd, None, file_length)
+                        except OSError as e:
+                            if e.errno == 22:  # EINVAL
+                                written = os.splice(self.control_fd, fd, file_length)
+                            else:
+                                raise
+                    except BlockingIOError:
                         yield
-            except FileExistsError:
-                raise BadReportJSONError(f"file '{file_name}' already exists") from None
+                        continue
+                    if written == 0:
+                        raise BadControlError("EOF when reading data")
+                    file_length -= written
+                    yield
+            finally:
+                os.close(fd)
 
         # either store partial result + return,
         # or load previous partial result and merge into it
-        partial = result.get("partial", False)
-        if partial:
-            # do not store the 'partial' key in the result
+        partial = result.get("partial")
+        if partial is not None:
+            # do not store the 'partial' key in the result, even if False
             del result["partial"]
-            # note that nameless result will get None as dict key,
-            # which is perfectly fine
-            self._merge(self.partial_results[name], result)
-            # partial = do nothing
-            return
+            # if it exists and is True
+            if partial:
+                # note that nameless result will get None as dict key,
+                # which is perfectly fine
+                self._merge(self.partial_results[name], result)
+                # partial = do nothing
+                return
 
         # if previously-stored partial result exist, merge the current one
         # into it, but then use the merged result

atex/orchestrator/adhoc.py

@@ -1,6 +1,5 @@
 import tempfile
-import concurrent
-import collections
+import concurrent.futures
 from pathlib import Path
 
 from .. import util, executor
@@ -60,7 +59,7 @@ class AdHocOrchestrator(Orchestrator):
 
     def __init__(
         self, platform, fmf_tests, provisioners, aggregator, tmp_dir, *,
-        max_remotes=1, max_spares=0, max_reruns=2, max_failed_setups=10, env=None,
+        max_remotes=1, max_spares=0, max_failed_setups=10, env=None,
     ):
         """
         'platform' is a string with platform name.
@@ -84,15 +83,15 @@ class AdHocOrchestrator(Orchestrator):
         speed up test reruns as Remote reservation happens asynchronously
         to test execution. Spares are reserved on top of 'max_remotes'.
 
-        'max_reruns' is an integer of how many times to re-try running a failed
-        test (which exited with non-0 or caused an Executor exception).
-
         'max_failed_setups' is an integer of how many times an Executor's
         plan setup (uploading tests, running prepare scripts, etc.) may fail
         before FailedSetupError is raised.
 
         'env' is a dict of extra environment variables to pass to Executor.
         """
+        if not fmf_tests.tests:
+            raise ValueError("'fmf_tests' has no tests (bad discover params?)")
+
         self.platform = platform
         self.fmf_tests = fmf_tests
         self.provisioners = tuple(provisioners)
@@ -101,11 +100,11 @@ class AdHocOrchestrator(Orchestrator):
         self.failed_setups_left = max_failed_setups
         self.max_remotes = max_remotes
         self.max_spares = max_spares
-        # indexed by test name, value being integer of how many times
-        self.reruns = collections.defaultdict(lambda: max_reruns)
         self.env = env
         # tests still waiting to be run
         self.to_run = set(fmf_tests.tests)
+        # number of Remotes being provisioned + set up (not running tests)
+        self.remotes_requested = 0
         # running tests as a dict, indexed by test name, with RunningInfo values
         self.running_tests = {}
         # thread queue for actively running tests
@@ -114,6 +113,8 @@ class AdHocOrchestrator(Orchestrator):
         self.setup_queue = util.ThreadQueue(daemon=True)
         # thread queue for remotes being released
         self.release_queue = util.ThreadQueue(daemon=True)
+        # thread queue for results being ingested
+        self.ingest_queue = util.ThreadQueue(daemon=False)
 
     def _run_new_test(self, info):
         """
@@ -125,7 +126,7 @@ class AdHocOrchestrator(Orchestrator):
         next_test_name = self.next_test(self.to_run, self.fmf_tests.tests, info)
         assert next_test_name in self.to_run, "next_test() returned valid test name"
 
-        util.info(f"starting '{next_test_name}' on {info.remote}")
+        util.info(f"{info.remote}: starting '{next_test_name}'")
 
         self.to_run.remove(next_test_name)
 
@@ -140,6 +141,7 @@ class AdHocOrchestrator(Orchestrator):
         )
 
         tmp_dir_path = Path(rinfo.tmp_dir.name)
+        tmp_dir_path.chmod(0o755)
         self.test_queue.start_thread(
             target=info.executor.run_test,
             target_args=(
@@ -156,26 +158,11 @@ class AdHocOrchestrator(Orchestrator):
         'finfo' is a FinishedInfo instance.
         """
         test_data = self.fmf_tests.tests[finfo.test_name]
-
-        # TODO: somehow move logging from was_successful and should_be_rerun here,
-        #       probably print just some generic info from those functions that doesn't
-        #       imply any outcome, ie.
-        #           {remote_with_test} threw {exception}
-        #           {remote_with_test} exited with {code}
-        #           {remote_with_test} has {N} reruns left
-        #           {remote_with_test} has 0 reruns left
-        #       and then log the decision separately, here below, such as
-        #           {remote_with_test} failed, re-running
-        #           {remote_with_test} completed, ingesting result
-        #           {remote_with_test} was destructive, releasing remote
-        #           {remote_with_test} ...., running next test
-        #       That allows the user to override the functions, while keeping critical
-        #       flow reliably logged here.
-
         remote_with_test = f"{finfo.remote}: '{finfo.test_name}'"
 
         if not self.was_successful(finfo, test_data) and self.should_be_rerun(finfo, test_data):
             # re-run the test
+            util.info(f"{remote_with_test} failed, re-running")
             self.to_run.add(finfo.test_name)
         else:
             # ingest the result
@@ -183,15 +170,27 @@ class AdHocOrchestrator(Orchestrator):
             # a condition just in case Executor code itself threw an exception
             # and didn't even report the fallback 'infra' result
             if finfo.results is not None and finfo.files is not None:
-                self.aggregator.ingest(
-                    self.platform,
-                    finfo.test_name,
-                    finfo.results,
-                    finfo.files,
+                util.info(f"{remote_with_test} completed, ingesting result")
+
+                def ingest_and_cleanup(ingest, args, cleanup):
+                    ingest(*args)
+                    # also delete the tmpdir housing these
+                    cleanup()
+
+                self.ingest_queue.start_thread(
+                    ingest_and_cleanup,
+                    target_args=(
+                        # ingest func itself
+                        self.aggregator.ingest,
+                        # args for ingest
+                        (self.platform, finfo.test_name, finfo.results, finfo.files),
+                        # cleanup func itself
+                        finfo.tmp_dir.cleanup,
+                    ),
+                    test_name=finfo.test_name,
                 )
-                # also delete the tmpdir housing these
-                finfo.tmp_dir.cleanup()
-                # ingesting destroyed these
+
+                # ingesting destroys these
                 finfo = self.FinishedInfo._from(
                     finfo,
                     results=None,
@@ -199,22 +198,34 @@ class AdHocOrchestrator(Orchestrator):
                     tmp_dir=None,
                 )
 
-        # if destroyed, release the remote and request a replacement
-        # (Executor exception is always considered destructive)
-        if finfo.exception or self.destructive(finfo, test_data):
-            util.debug(f"{remote_with_test} was destructive, releasing remote")
+        # if there are still tests to be run and the last test was not
+        # destructive, just run a new test on it
+        if self.to_run and not (finfo.exception or self.destructive(finfo, test_data)):
+            util.debug(f"{remote_with_test} was non-destructive, running next test")
+            self._run_new_test(finfo)
+            return
+
+        # we are not running a new test right now, serve_once() might run it
+        # some time later, just decide what to do with the current remote
+
+        if self.remotes_requested >= len(self.to_run):
+            # we have enough remotes in the pipe to run every test,
+            # we don't need a new one - just release the current one
+            util.debug(f"{finfo.remote} no longer useful, releasing it")
+            self.release_queue.start_thread(
+                finfo.remote.release,
+                remote=finfo.remote,
+            )
+        else:
+            # we need more remotes and the last test was destructive,
+            # get a new one and let serve_once() run a test later
+            util.debug(f"{remote_with_test} was destructive, getting a new Remote")
             self.release_queue.start_thread(
                 finfo.remote.release,
                 remote=finfo.remote,
             )
             finfo.provisioner.provision(1)
 
-        # if still not destroyed, run another test on it
-        # (without running plan setup, re-using already set up remote)
-        elif self.to_run:
-            util.debug(f"{remote_with_test} was non-destructive, running next test")
-            self._run_new_test(finfo)
-
     def serve_once(self):
         """
         Run the orchestration logic, processing any outstanding requests
@@ -225,7 +236,7 @@ class AdHocOrchestrator(Orchestrator):
         (more work to be done), False once all testing is concluded.
         """
         # all done
-        if not self.to_run and not self.running_tests and self.release_queue.qsize() == 0:
+        if not self.to_run and not self.running_tests:
             return False
 
         # process all finished tests, potentially reusing remotes for executing
@@ -260,18 +271,21 @@ class AdHocOrchestrator(Orchestrator):
             except util.ThreadQueue.Empty:
                 break
 
+            self.remotes_requested -= 1
             sinfo = treturn.sinfo
 
             if treturn.exception:
-                msg = f"{sinfo.remote}: setup failed with {repr(treturn.exception)}"
+                exc_str = f"{type(treturn.exception).__name__}({treturn.exception})"
+                msg = f"{sinfo.remote}: setup failed with {exc_str}"
                 self.release_queue.start_thread(
                     sinfo.remote.release,
                     remote=sinfo.remote,
                 )
-                if (reruns_left := self.failed_setups_left) > 0:
-                    util.warning(f"{msg}, re-trying ({reruns_left} setup retries left)")
+                if (retries_left := self.failed_setups_left) > 0:
+                    util.warning(f"{msg}, re-trying ({retries_left} setup retries left)")
                     self.failed_setups_left -= 1
                     sinfo.provisioner.provision(1)
+                    self.remotes_requested += 1
                 else:
                     util.warning(f"{msg}, setup retries exceeded, giving up")
                     raise FailedSetupError("setup retries limit exceeded, broken infra?")
@@ -286,12 +300,14 @@ class AdHocOrchestrator(Orchestrator):
                     treturn = self.setup_queue.get_raw(block=False)
                 except util.ThreadQueue.Empty:
                     break
+                util.debug(f"releasing extraneous set-up {treturn.sinfo.remote}")
                 self.release_queue.start_thread(
                     treturn.sinfo.remote.release,
                     remote=treturn.sinfo.remote,
                 )
+                self.remotes_requested -= 1
 
-        # try to get new remotes from Provisioners - if we get some, start
+        # try to get new Remotes from Provisioners - if we get some, start
         # running setup on them
         for provisioner in self.provisioners:
             while (remote := provisioner.get_remote(block=False)) is not None:
@@ -311,15 +327,30 @@ class AdHocOrchestrator(Orchestrator):
         # gather returns from Remote.release() functions - check for exceptions
         # thrown, re-report them as warnings as they are not typically critical
         # for operation
-        try:
-            treturn = self.release_queue.get_raw(block=False)
-        except util.ThreadQueue.Empty:
-            pass
-        else:
-            if treturn.exception:
-                util.warning(f"{treturn.remote} release failed: {repr(treturn.exception)}")
+        while True:
+            try:
+                treturn = self.release_queue.get_raw(block=False)
+            except util.ThreadQueue.Empty:
+                break
             else:
-                util.debug(f"{treturn.remote}: completed .release()")
+                if treturn.exception:
+                    exc_str = f"{type(treturn.exception).__name__}({treturn.exception})"
+                    util.warning(f"{treturn.remote} release failed: {exc_str}")
+                else:
+                    util.debug(f"{treturn.remote} release completed")
+
+        # gather returns from Aggregator.ingest() calls - check for exceptions
+        while True:
+            try:
+                treturn = self.ingest_queue.get_raw(block=False)
+            except util.ThreadQueue.Empty:
+                break
+            else:
+                if treturn.exception:
+                    exc_str = f"{type(treturn.exception).__name__}({treturn.exception})"
+                    util.warning(f"'{treturn.test_name}' ingesting failed: {exc_str}")
+                else:
+                    util.debug(f"'{treturn.test_name}' ingesting completed")
 
         return True
 
@@ -328,9 +359,12 @@ class AdHocOrchestrator(Orchestrator):
         for prov in self.provisioners:
             prov.start()
 
+        # just the base remotes, no spares
+        self.remotes_requested = min(self.max_remotes, len(self.fmf_tests.tests))
+
         # start up initial reservations, balanced evenly across all available
         # provisioner instances
-        count = min(self.max_remotes, len(self.fmf_tests.tests)) + self.max_spares
+        count = self.remotes_requested + self.max_spares
         provisioners = self.provisioners[:count]
         for idx, prov in enumerate(provisioners):
             if count % len(provisioners) > idx:
@@ -342,16 +376,30 @@ class AdHocOrchestrator(Orchestrator):
         # cancel all running tests and wait for them to clean up (up to 0.1sec)
         for rinfo in self.running_tests.values():
             rinfo.executor.cancel()
-        self.test_queue.join()  # also ignore any exceptions raised
+        self.test_queue.join()    # also ignore any exceptions raised
+
+        # wait for all running ingestions to finish, print exceptions
+        # (we would rather stop provisioners further below than raise here)
+        while True:
+            try:
+                treturn = self.ingest_queue.get_raw(block=False)
+            except util.ThreadQueue.Empty:
+                break
+            else:
+                if treturn.exception:
+                    exc_str = f"{type(treturn.exception).__name__}({treturn.exception})"
+                    util.warning(f"'{treturn.test_name}' ingesting failed: {exc_str}")
+                else:
+                    util.debug(f"'{treturn.test_name}' ingesting completed")
+        self.ingest_queue.join()
 
         # stop all provisioners, also releasing all remotes
-        # TODO: don't parallelize here, remove .stop_defer() and parallelize in provisioners
+        # - parallelize up to 10 provisioners at a time
         if self.provisioners:
-            workers = min(len(self.provisioners), 20)
+            workers = min(len(self.provisioners), 10)
             with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as ex:
                 for provisioner in self.provisioners:
-                    for func in provisioner.stop_defer():
-                        ex.submit(func)
+                    ex.submit(provisioner.stop)
 
     @staticmethod
     def run_setup(sinfo):
@@ -423,23 +471,20 @@ class AdHocOrchestrator(Orchestrator):
         'test_data' is a dict of fully resolved fmf test metadata of that test.
         """
         remote_with_test = f"{info.remote}: '{info.test_name}'"
-
         # executor (or test) threw exception
         if info.exception:
-            util.info(f"{remote_with_test} threw {repr(info.exception)} during test runtime")
+            exc_str = f"{type(info.exception).__name__}({info.exception})"
+            util.info(f"{remote_with_test} threw {exc_str} during test runtime")
             return False
-
         # the test exited as non-0
         if info.exit_code != 0:
             util.info(f"{remote_with_test} exited with non-zero: {info.exit_code}")
             return False
-
         # otherwise we good
         return True
 
-    # TODO: @staticmethod and remove ARG002
-    #@staticmethod
-    def should_be_rerun(self, info, test_data):  # noqa: ARG004, ARG002
+    @staticmethod
+    def should_be_rerun(info, test_data):  # noqa: ARG004
         """
         Return a boolean result whether a finished test failed in a way
         that another execution attempt might succeed, due to race conditions
@@ -449,17 +494,5 @@ class AdHocOrchestrator(Orchestrator):
 
         'test_data' is a dict of fully resolved fmf test metadata of that test.
         """
-        remote_with_test = f"{info.remote}: '{info.test_name}'"
-
-        # TODO: remove self.reruns and the whole X-reruns logic from AdHocOrchestrator,
-        #       leave it up to the user to wrap should_be_rerun() with an external dict
-        #       of tests, counting reruns for each
-        #        - allows the user to adjust counts per-test (ie. test_data metadata)
-        #        - allows this template to be @staticmethod
-        if (reruns_left := self.reruns[info.test_name]) > 0:
-            util.info(f"{remote_with_test}: re-running ({reruns_left} reruns left)")
-            self.reruns[info.test_name] -= 1
-            return True
-        else:
-            util.info(f"{remote_with_test}: reruns exceeded, giving up")
-            return False
+        # never rerun by default
+        return False