atex 0.5__py3-none-any.whl → 0.8__py3-none-any.whl

atex/executor/executor.py

@@ -0,0 +1,378 @@
+import os
+import select
+import threading
+import contextlib
+import subprocess
+from pathlib import Path
+
+from .. import util, fmf
+from . import testcontrol, scripts
+from .duration import Duration
+from .reporter import Reporter
+
+
+class TestAbortedError(Exception):
+    """
+    Raised when an infrastructure-related issue happened while running a test.
+    """
+    pass
+
+
+class Executor:
+    """
+    Logic for running tests on a remote system and processing results
+    and uploaded files by those tests.
+
+        tests_repo = "path/to/cloned/tests"
+        tests_data = atex.fmf.FMFTests(tests_repo, "/plans/default")
+
+        with Executor(tests_data, conn) as e:
+            e.upload_tests(tests_repo)
+            e.setup_plan()
+            e.run_test("/some/test", "results/here.json", "uploaded/files/here")
+            e.run_test(...)
+
+    One Executor instance may be used to run multiple tests sequentially.
+    In addition, multiple Executor instances can run in parallel on the same
+    host, provided each receives a unique class Connection instance to it.
+
+        conn.cmd(["mkdir", "-p", "/shared"])
+
+        with Executor(tests_data, conn, state_dir="/shared") as e:
+            e.upload_tests(tests_repo)
+            e.setup_plan()
+
+        # in parallel (ie. threading or multiprocessing)
+        with Executor(tests_data, unique_conn, state_dir="/shared") as e:
+            e.run_test(...)
+    """
+
+    def __init__(self, fmf_tests, connection, *, state_dir=None):
+        """
+        'fmf_tests' is a class FMFTests instance with (discovered) tests.
+
+        'connection' is a class Connection instance, already fully connected.
+
+        'state_dir' is a string or Path specifying path on the remote system for
+        storing additional data, such as tests, execution wrappers, temporary
+        plan-exported variables, etc. If left as None, a tmpdir is used.
+        """
+        self.lock = threading.RLock()
+        self.conn = connection
+        self.fmf_tests = fmf_tests
+        self.state_dir = state_dir
+        self.work_dir = None
+        self.tests_dir = None
+        self.plan_env_file = None
+        self.cancelled = False
+
+    def setup(self):
+        with self.lock:
+            state_dir = self.state_dir
+
+        # if user defined a state dir, have shared tests, but use per-instance
+        # work_dir for test wrappers, etc., identified by this instance's id(),
+        # which should be unique as long as this instance exists
+        if state_dir:
+            state_dir = Path(state_dir)
+            work_dir = state_dir / f"atex-{id(self)}"
+            self.conn.cmd(("mkdir", work_dir), check=True)
+            with self.lock:
+                self.tests_dir = state_dir / "tests"
+                self.plan_env_file = state_dir / "plan_env"
+                self.work_dir = work_dir
+
+        # else just create a tmpdir
+        else:
+            tmp_dir = self.conn.cmd(
+                # /var is not cleaned up by bootc, /var/tmp is
+                ("mktemp", "-d", "-p", "/var", "atex-XXXXXXXXXX"),
+                func=util.subprocess_output,
+            )
+            tmp_dir = Path(tmp_dir)
+            with self.lock:
+                self.tests_dir = tmp_dir / "tests"
+                self.plan_env_file = tmp_dir / "plan_env"
+                # use the tmpdir as work_dir, avoid extra mkdir over conn
+                self.work_dir = tmp_dir
+
+    def cleanup(self):
+        with self.lock:
+            work_dir = self.work_dir
+
+        if work_dir:
+            self.conn.cmd(("rm", "-rf", work_dir), check=True)
+
+        with self.lock:
+            self.work_dir = None
+            self.tests_dir = None
+            self.plan_env_file = None
+
+    def __enter__(self):
+        self.setup()
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.cleanup()
+
+    def cancel(self):
+        with self.lock:
+            self.cancelled = True
+
+    def upload_tests(self):
+        """
+        Upload a directory of all tests, the location of which was provided to
+        __init__() inside 'fmf_tests', to the remote host.
+        """
+        self.conn.rsync(
+            "-rv" if util.in_debug_mode() else "-rq",
+            "--delete", "--exclude=.git/",
+            f"{self.fmf_tests.root}/",
+            f"remote:{self.tests_dir}",
+        )
+
+    def setup_plan(self):
+        """
+        Install packages and run scripts extracted from a TMT plan by a FMFTests
+        instance given during class initialization.
+
+        Also prepare additional environment for tests, ie. create and export
+        a path to TMT_PLAN_ENVIRONMENT_FILE.
+        """
+        # install packages from the plan
+        if self.fmf_tests.prepare_pkgs:
+            self.conn.cmd(
+                (
+                    "dnf", "-y", "--setopt=install_weak_deps=False",
+                    "install", *self.fmf_tests.prepare_pkgs,
+                ),
+                check=True,
+                stdout=None if util.in_debug_mode() else subprocess.DEVNULL,
+                stderr=subprocess.STDOUT,
+            )
+
+        # make envionment for 'prepare' scripts
+        self.conn.cmd(("truncate", "-s", "0", self.plan_env_file), check=True)
+        env = self.fmf_tests.plan_env.copy()
+        env["TMT_PLAN_ENVIRONMENT_FILE"] = self.plan_env_file
+        env_args = (f"{k}={v}" for k, v in env.items())
+
+        # run the prepare scripts
+        for script in self.fmf_tests.prepare_scripts:
+            self.conn.cmd(
+                ("env", *env_args, "bash"),
+                input=script,
+                text=True,
+                check=True,
+                stdout=None if util.in_debug_mode() else subprocess.DEVNULL,
+                stderr=subprocess.STDOUT,
+            )
+
+    def run_test(self, test_name, json_file, files_dir, *, env=None):
+        """
+        Run one test on the remote system.
+
+        'test_name' is a string with test name.
+
+        'json_file' is a destination file (string or Path) for results.
+
+        'files_dir' is a destination dir (string or Path) for uploaded files.
+
+        'env' is a dict of extra environment variables to pass to the test.
+
+        Returns an integer exit code of the test script.
+        """
+        test_data = self.fmf_tests.tests[test_name]
+
+        # start with fmf-plan-defined environment
+        env_vars = self.fmf_tests.plan_env.copy()
+        # append fmf-test-defined environment into it
+        for item in fmf.listlike(test_data, "environment"):
+            env_vars.update(item)
+        # append additional variables typically exported by tmt
+        env_vars["TMT_PLAN_ENVIRONMENT_FILE"] = self.plan_env_file
+        env_vars["TMT_TEST_NAME"] = test_name
+        env_vars["ATEX_TEST_NAME"] = test_name
+        # append variables given to this function call
+        if env:
+            env_vars.update(env)
+
+        # run a setup script, preparing wrapper + test scripts
+        setup_script = scripts.test_setup(
+            test=scripts.Test(test_name, test_data, self.fmf_tests.test_dirs[test_name]),
+            tests_dir=self.tests_dir,
+            wrapper_exec=f"{self.work_dir}/wrapper.sh",
+            test_exec=f"{self.work_dir}/test.sh",
+        )
+        self.conn.cmd(("bash",), input=setup_script, text=True, check=True)
+
+        with contextlib.ExitStack() as stack:
+            reporter = stack.enter_context(Reporter(json_file, files_dir))
+            testout_fd = stack.enter_context(reporter.open_tmpfile())
+            duration = Duration(test_data.get("duration", "5m"))
+
+            test_proc = None
+            control_fd = None
+            stack.callback(lambda: os.close(control_fd) if control_fd else None)
+
+            reconnects = 0
+
+            def abort(msg):
+                if test_proc:
+                    test_proc.kill()
+                    test_proc.wait()
+                raise TestAbortedError(msg) from None
+
+            try:
+                # TODO: probably enum
+                state = "starting_test"
+                while not duration.out_of_time():
+                    with self.lock:
+                        if self.cancelled:
+                            abort("cancel requested")
+
+                    if state == "starting_test":
+                        control_fd, pipe_w = os.pipe()
+                        os.set_blocking(control_fd, False)
+                        control = testcontrol.TestControl(
+                            control_fd=control_fd,
+                            reporter=reporter,
+                            duration=duration,
+                            testout_fd=testout_fd,
+                        )
+                        # reconnect/reboot count (for compatibility)
+                        env_vars["TMT_REBOOT_COUNT"] = str(reconnects)
+                        env_vars["TMT_TEST_RESTART_COUNT"] = str(reconnects)
+                        # run the test in the background, letting it log output directly to
+                        # an opened file (we don't handle it, cmd client sends it to kernel)
+                        env_args = (f"{k}={v}" for k, v in env_vars.items())
+                        test_proc = self.conn.cmd(
+                            ("env", *env_args, f"{self.work_dir}/wrapper.sh"),
+                            stdout=pipe_w,
+                            stderr=testout_fd,
+                            func=util.subprocess_Popen,
+                        )
+                        os.close(pipe_w)
+                        state = "reading_control"
+
+                    elif state == "reading_control":
+                        rlist, _, xlist = select.select((control_fd,), (), (control_fd,), 0.1)
+                        if xlist:
+                            abort(f"got exceptional condition on control_fd {control_fd}")
+                        elif rlist:
+                            control.process()
+                            if control.eof:
+                                os.close(control_fd)
+                                control_fd = None
+                                state = "waiting_for_exit"
+
+                    elif state == "waiting_for_exit":
+                        # control stream is EOF and it has nothing for us to read,
+                        # we're now just waiting for proc to cleanly terminate
+                        try:
+                            code = test_proc.wait(0.1)
+                            if code == 0:
+                                # wrapper exited cleanly, testing is done
+                                break
+                            else:
+                                # unexpected error happened (crash, disconnect, etc.)
+                                self.conn.disconnect()
+                                # if reconnect was requested, do so, otherwise abort
+                                if control.reconnect:
+                                    state = "reconnecting"
+                                    if control.reconnect != "always":
+                                        control.reconnect = None
+                                else:
+                                    abort(
+                                        f"test wrapper unexpectedly exited with {code} and "
+                                        "reconnect was not sent via test control",
+                                    )
+                            test_proc = None
+                        except subprocess.TimeoutExpired:
+                            pass
+
+                    elif state == "reconnecting":
+                        try:
+                            self.conn.connect(block=False)
+                            reconnects += 1
+                            state = "starting_test"
+                        except BlockingIOError:
+                            pass
+
+                    else:
+                        raise AssertionError("reached unexpected state")
+
+                else:
+                    abort("test duration timeout reached")
+
+                # testing successful, do post-testing tasks
+
+                # test wrapper hasn't provided exitcode
+                if control.exit_code is None:
+                    abort("exitcode not reported, wrapper bug?")
+
+                # partial results that were never reported
+                if control.partial_results:
+                    for result in control.partial_results.values():
+                        name = result.get("name")
+                        if not name:
+                            # partial result is also a result
+                            control.nameless_result_seen = True
+                        if testout := result.get("testout"):
+                            try:
+                                reporter.link_tmpfile_to(testout_fd, testout, name)
+                            except FileExistsError:
+                                raise testcontrol.BadReportJSONError(
+                                    f"file '{testout}' already exists",
+                                ) from None
+                        reporter.report(result)
+
+                # test hasn't reported a result for itself, add an automatic one
+                # as specified in RESULTS.md
+                # {"status": "pass", "testout": "output.txt"}
+                if not control.nameless_result_seen:
+                    reporter.link_tmpfile_to(testout_fd, "output.txt")
+                    reporter.report({
+                        "status": "pass" if control.exit_code == 0 else "fail",
+                        "testout": "output.txt",
+                    })
+
+                return control.exit_code
+
+            except Exception:
+                # if the test hasn't reported a result for itself, but still
+                # managed to break something, provide at least the default log
+                # for manual investigation - otherwise test output disappears
+                if not control.nameless_result_seen:
+                    try:
+                        reporter.link_tmpfile_to(testout_fd, "output.txt")
+                        reporter.report({
+                            "status": "infra",
+                            "testout": "output.txt",
+                        })
+                    # in case outout.txt exists as a directory
+                    except FileExistsError:
+                        pass
+                raise
+
+
+#__all__ = [
+#    info.name for info in _pkgutil.iter_modules(__spec__.submodule_search_locations)
+#]
+#
+#
+#import importlib as _importlib
+#import pkgutil as _pkgutil
+#
+#
+#def __dir__():
+#    return __all__
+#
+#
+## lazily import submodules
+#def __getattr__(attr):
+#    # importing a module known to exist
+#    if attr in __all__:
+#        return _importlib.import_module(f".{attr}", __name__)
+#    else:
+#        raise AttributeError(f"module '{__name__}' has no attribute '{attr}'")

atex/executor/reporter.py

@@ -0,0 +1,106 @@
+import os
+import json
+import ctypes
+import ctypes.util
+import contextlib
+from pathlib import Path
+
+from .. import util
+
+
+libc = ctypes.CDLL(ctypes.util.find_library("c"), use_errno=True)
+
+# int linkat(int olddirfd, const char *oldpath, int newdirfd, const char *newpath, int flags)
+libc.linkat.argtypes = (
+    ctypes.c_int,
+    ctypes.c_char_p,
+    ctypes.c_int,
+    ctypes.c_char_p,
+    ctypes.c_int,
+)
+libc.linkat.restype = ctypes.c_int
+
+# fcntl.h:#define AT_EMPTY_PATH                0x1000  /* Allow empty relative pathname */
+AT_EMPTY_PATH = 0x1000
+
+# fcntl.h:#define AT_FDCWD             -100    /* Special value used to indicate
+AT_FDCWD = -100
+
+
+def linkat(*args):
+    if (ret := libc.linkat(*args)) == -1:
+        errno = ctypes.get_errno()
+        raise OSError(errno, os.strerror(errno))
+    return ret
+
+
+class Reporter:
+    """
+    Collects reported results (in a format specified by RESULTS.md) for
+    a specific test, storing them persistently.
+    """
+
+    def __init__(self, json_file, files_dir):
+        """
+        'json_file' is a destination file (string or Path) for results.
+
+        'files_dir' is a destination dir (string or Path) for uploaded files.
+        """
+        self.json_file = json_file
+        self.files_dir = Path(files_dir)
+        self.json_fobj = None
+
+    def __enter__(self):
+        if self.json_file.exists():
+            raise FileExistsError(f"{self.json_file} already exists")
+        self.json_fobj = open(self.json_file, "w")
+
+        if self.files_dir.exists():
+            raise FileExistsError(f"{self.files_dir} already exists")
+        self.files_dir.mkdir()
+
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        if self.json_fobj:
+            self.json_fobj.close()
+            self.json_fobj = None
+
+    def report(self, result_line):
+        """
+        Persistently record a test result.
+
+        'result_line' is a dict in the format specified by RESULTS.md.
+        """
+        json.dump(result_line, self.json_fobj, indent=None)
+        self.json_fobj.write("\n")
+        self.json_fobj.flush()
+
+    @contextlib.contextmanager
+    def open_tmpfile(self, open_mode=os.O_WRONLY):
+        """
+        Open an anonymous (name-less) file for writing and yield its file
+        descriptor (int) as context, closing it when the context is exited.
+        """
+        flags = open_mode | os.O_TMPFILE
+        fd = os.open(self.files_dir, flags, 0o644)
+        try:
+            yield fd
+        finally:
+            os.close(fd)
+
+    def link_tmpfile_to(self, fd, file_name, result_name=None):
+        """
+        Store a file named 'file_name' in a directory relevant to 'result_name'
+        whose 'fd' (a file descriptor) was created by .open_tmpfile().
+
+        This function can be called multiple times with the same 'fd', and
+        does not close or otherwise alter the descriptor.
+
+        If 'result_name' is not given, link files to the test (name) itself.
+        """
+        result_name = util.normalize_path(result_name) if result_name else "."
+        # /path/to/files_dir / path/to/subresult / path/to/file.log
+        file_path = self.files_dir / result_name / util.normalize_path(file_name)
+        file_path.parent.mkdir(parents=True, exist_ok=True)
+        linkat(fd, b"", AT_FDCWD, bytes(file_path), AT_EMPTY_PATH)

atex/executor/scripts.py

@@ -0,0 +1,155 @@
+import collections
+from pathlib import Path
+
+from .. import util, fmf
+
+# name: fmf path to the test as string, ie. /some/test
+# data: dict of the parsed fmf metadata (ie. {'tag': ... , 'environment': ...})
+# dir:  relative pathlib.Path of the test .fmf to repo root, ie. some/test
+#       (may be different from name for "virtual" tests that share the same dir)
+Test = collections.namedtuple("Test", ["name", "data", "dir"])
+
+
+# NOTE that we split test execution into 3 scripts:
+#      - "setup script" (package installs, etc.)
+#      - "wrapper script" (runs test script)
+#      - "test script" (exact contents of the 'test:' FMF metadata key)
+#
+#      this is to allow interactive test execution - the setup script
+#      can run in 'bash' via stdin pipe into 'ssh', creating the wrapper
+#      script somewhere on the disk, making it executable,
+#
+#      then the "wrapper" script can run via a separate 'ssh' execution,
+#      passed by an argument to 'ssh', leaving stdin/out/err untouched,
+#      allowing the user to interact with it (if run interactively)
+
+def test_wrapper(*, test, tests_dir, test_exec):
+    """
+    Generate a bash script that runs a user-specified test, preparing
+    a test control channel for it, and reporting its exit code.
+    The script must be as "transparent" as possible, since any output
+    is considered as test output and any unintended environment changes
+    will impact the test itself.
+
+    'test' is a class Test instance.
+
+    'test_dir' is a remote directory (repository) of all the tests,
+    a.k.a. FMF metadata root.
+
+    'test_exec' is a remote path to the actual test to run.
+    """
+    out = "#!/bin/bash\n"
+
+    # stdout-over-ssh is used as Test Control (see TEST_CONTROL.md),
+    # so duplicate stderr to stdout, and then open a new fd pointing to the
+    # original stdout
+    out += "exec {orig_stdout}>&1 1>&2\n"
+
+    # TODO: if interactive, keep original stdin, else exec 0</dev/null ,
+    #       doing it here avoids unnecessary traffic (reading stdin) via ssh,
+    #       even if it is fed from subprocess.DEVNULL on the runner
+
+    if util.in_debug_mode():
+        out += "set -x\n"
+
+    # use a subshell to limit the scope of the CWD change
+    out += "(\n"
+
+    # if TMT_PLAN_ENVIRONMENT_FILE exists, export everything from it
+    # (limited by the subshell, so it doesn't leak)
+    out += util.dedent("""
+        if [[ -f $TMT_PLAN_ENVIRONMENT_FILE ]]; then
+            set -o allexport
+            . "$TMT_PLAN_ENVIRONMENT_FILE"
+            set +o allexport
+        fi
+    """) + "\n"
+
+    # TODO: custom PATH with tmt-* style commands?
+
+    # join the directory with all tests and nested path of our test inside it
+    test_cwd = Path(tests_dir) / test.dir
+    out += f"cd '{test_cwd}' || exit 1\n"
+
+    # run the test script
+    # - the '-e -o pipefail' is to mimic what full fat tmt uses
+    out += (
+        "ATEX_TEST_CONTROL=$orig_stdout"
+        f" exec -a 'bash: atex running {test.name}'"
+        f" bash -e -o pipefail '{test_exec}'\n"
+    )
+
+    # subshell end
+    out += ")\n"
+
+    # write test exitcode to test control stream
+    out += "echo exitcode $? >&$orig_stdout\n"
+
+    # always exit the wrapper with 0 if test execution was normal
+    out += "exit 0\n"
+
+    return out
+
+
+def _install_packages(pkgs, extra_opts=None):
+    pkgs_str = " ".join(pkgs)
+    extra_opts = extra_opts or ()
+    dnf = ["dnf", "-y", "--setopt=install_weak_deps=False", "install", *extra_opts]
+    dnf_str = " ".join(dnf)
+    return util.dedent(fr"""
+        not_installed=$(rpm -q --qf '' {pkgs_str} | sed -nr 's/^package ([^ ]+) is not installed$/\1/p')
+        [[ $not_installed ]] && {dnf_str} $not_installed
+    """)  # noqa: E501
+
+
+def test_setup(*, test, wrapper_exec, test_exec, **kwargs):
+    """
+    Generate a bash script that should prepare the remote end for test
+    execution.
+
+    The bash script itself will (among other things) generate two more bash
+    scripts: a test script (contents of 'test' from FMF) and a wrapper script
+    to run the test script.
+
+    'wrapper_exec' is the remote path where the wrapper script should be put.
+
+    'test_exec' is the remote path where the test script should be put.
+
+    'test' is a class Test instance.
+
+    Any 'kwargs' are passed to test_wrapper().
+    """
+    out = "#!/bin/bash\n"
+
+    if util.in_debug_mode():
+        out += "set -xe\n"
+    else:
+        out += "exec 1>/dev/null\n"
+        out += "set -e\n"
+
+    # install test dependencies
+    # - only strings (package names) in require/recommend are supported
+    if require := list(fmf.test_pkg_requires(test.data, "require")):
+        out += _install_packages(require) + "\n"
+    if recommend := list(fmf.test_pkg_requires(test.data, "recommend")):
+        out += _install_packages(recommend, ("--skip-broken",)) + "\n"
+
+    # make the wrapper script
+    out += f"cat > '{wrapper_exec}' <<'ATEX_SETUP_EOF'\n"
+    out += test_wrapper(
+        test=test,
+        test_exec=test_exec,
+        **kwargs,
+    )
+    out += "ATEX_SETUP_EOF\n"
+    # make the test script
+    out += f"cat > '{test_exec}' <<'ATEX_SETUP_EOF'\n"
+    out += test.data["test"]
+    out += "\n"  # for safety, in case 'test' doesn't have a newline
+    out += "ATEX_SETUP_EOF\n"
+    # make both executable
+    out += f"chmod 0755 '{wrapper_exec}' '{test_exec}'\n"
+
+    out += "exit 0\n"
+
+    return out