atex 0.7__py3-none-any.whl → 0.9__py3-none-any.whl

atex/executor/reporter.py

@@ -0,0 +1,101 @@
+import os
+import json
+from pathlib import Path
+
+from .. import util
+
+
+class Reporter:
+    """
+    Collects reported results (in a format specified by RESULTS.md) for
+    a specific test, storing them persistently.
+    """
+
+    # internal name, stored inside 'output_dir' and hardlinked to
+    # 'testout'-JSON-key-specified result entries; deleted on exit
+    TESTOUT = "testout.temp"
+
+    def __init__(self, output_dir, results_file, files_dir):
+        """
+        'output_dir' is a destination dir (string or Path) for results reported
+        and files uploaded.
+
+        'results_file' is a file name inside 'output_dir' the results will be
+        reported into.
+
+        'files_dir' is a dir name inside 'output_dir' any files will be
+        uploaded to.
+        """
+        output_dir = Path(output_dir)
+        self.testout_file = output_dir / self.TESTOUT
+        self.results_file = output_dir / results_file
+        self.files_dir = output_dir / files_dir
+        self.output_dir = output_dir
+        self.results_fobj = None
+        self.testout_fobj = None
+
+    def start(self):
+        if self.testout_file.exists():
+            raise FileExistsError(f"{self.testout_file} already exists")
+        self.testout_fobj = open(self.testout_file, "wb")
+
+        if self.results_file.exists():
+            raise FileExistsError(f"{self.results_file} already exists")
+        self.results_fobj = open(self.results_file, "w", newline="\n")
+
+        if self.files_dir.exists():
+            raise FileExistsError(f"{self.files_dir} already exists")
+        self.files_dir.mkdir()
+
+    def stop(self):
+        if self.results_fobj:
+            self.results_fobj.close()
+            self.results_fobj = None
+
+        if self.testout_fobj:
+            self.testout_fobj.close()
+            self.testout_fobj = None
+            Path(self.testout_file).unlink()
+
+    def __enter__(self):
+        try:
+            self.start()
+            return self
+        except Exception:
+            self.stop()
+            raise
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.stop()
+
+    def report(self, result_line):
+        """
+        Persistently record a test result.
+
+        'result_line' is a dict in the format specified by RESULTS.md.
+        """
+        json.dump(result_line, self.results_fobj, indent=None)
+        self.results_fobj.write("\n")
+        self.results_fobj.flush()
+
+    def _dest_path(self, file_name, result_name=None):
+        result_name = util.normalize_path(result_name) if result_name else "."
+        # /path/to/files_dir / path/to/subtest / path/to/file.log
+        file_path = self.files_dir / result_name / util.normalize_path(file_name)
+        file_path.parent.mkdir(parents=True, exist_ok=True)
+        return file_path
+
+    def open_file(self, file_name, result_name=None, mode="wb"):
+        """
+        Open a file named 'file_name' in a directory relevant to 'result_name'.
+        Returns an opened file-like object that can be used in a context manager
+        just like with regular open().
+
+        If 'result_name' (typically a subtest) is not given, open the file
+        for the test (name) itself.
+        """
+        return open(self._dest_path(file_name, result_name), mode)
+
+    def link_testout(self, file_name, result_name=None):
+        # TODO: docstring
+        os.link(self.testout_file, self._dest_path(file_name, result_name))

atex/{minitmt → executor}/scripts.py

@@ -1,8 +1,15 @@
+import collections
+import yaml
 from pathlib import Path
 
-from .. import util
+from .. import util, fmf
+
+# name: fmf path to the test as string, ie. /some/test
+# data: dict of the parsed fmf metadata (ie. {'tag': ... , 'environment': ...})
+# dir:  relative pathlib.Path of the test .fmf to repo root, ie. some/test
+#       (may be different from name for "virtual" tests that share the same dir)
+Test = collections.namedtuple("Test", ["name", "data", "dir"])
 
-from . import fmf
 
 # NOTE that we split test execution into 3 scripts:
 #      - "setup script" (package installs, etc.)
@@ -17,8 +24,7 @@ from . import fmf
 #      passed by an argument to 'ssh', leaving stdin/out/err untouched,
 #      allowing the user to interact with it (if run interactively)
 
-
-def test_wrapper(*, test, tests_dir, test_exec, debug=False):
+def test_wrapper(*, test, tests_dir, test_exec):
     """
     Generate a bash script that runs a user-specified test, preparing
     a test control channel for it, and reporting its exit code.
@@ -26,14 +32,12 @@ def test_wrapper(*, test, tests_dir, test_exec, debug=False):
     is considered as test output and any unintended environment changes
     will impact the test itself.
 
-    'test' is a atex.minitmt.fmf.FMFTest instance.
+    'test' is a class Test instance.
 
     'test_dir' is a remote directory (repository) of all the tests,
     a.k.a. FMF metadata root.
 
     'test_exec' is a remote path to the actual test to run.
-
-    'debug' specifies whether to include wrapper output inside test output.
     """
     out = "#!/bin/bash\n"
 
@@ -46,7 +50,7 @@ def test_wrapper(*, test, tests_dir, test_exec, debug=False):
     #       doing it here avoids unnecessary traffic (reading stdin) via ssh,
     #       even if it is fed from subprocess.DEVNULL on the runner
 
-    if debug:
+    if util.in_debug_mode():
         out += "set -x\n"
 
     # use a subshell to limit the scope of the CWD change
@@ -88,7 +92,18 @@ def test_wrapper(*, test, tests_dir, test_exec, debug=False):
     return out
 
 
-def test_setup(*, test, wrapper_exec, test_exec, debug=False, **kwargs):
+def _install_packages(pkgs, extra_opts=None):
+    pkgs_str = " ".join(pkgs)
+    extra_opts = extra_opts or ()
+    dnf = ["dnf", "-y", "--setopt=install_weak_deps=False", "install", *extra_opts]
+    dnf_str = " ".join(dnf)
+    return util.dedent(fr"""
+        not_installed=$(rpm -q --qf '' {pkgs_str} | sed -nr 's/^package ([^ ]+) is not installed$/\1/p')
+        [[ $not_installed ]] && {dnf_str} $not_installed
+    """)  # noqa: E501
+
+
+def test_setup(*, test, wrapper_exec, test_exec, test_yaml, **kwargs):
     """
     Generate a bash script that should prepare the remote end for test
     execution.
@@ -97,42 +112,39 @@ def test_setup(*, test, wrapper_exec, test_exec, debug=False, **kwargs):
     scripts: a test script (contents of 'test' from FMF) and a wrapper script
     to run the test script.
 
+    'test' is a class Test instance.
+
     'wrapper_exec' is the remote path where the wrapper script should be put.
 
     'test_exec' is the remote path where the test script should be put.
 
-    'test' is a atex.minitmt.fmf.FMFTest instance.
-
-    'debug' specifies whether to make the setup script extra verbose.
-
     Any 'kwargs' are passed to test_wrapper().
     """
     out = "#!/bin/bash\n"
 
-    # have deterministic stdin, avoid leaking parent console
-    # also avoid any accidental stdout output, we use it for wrapper path
-    if debug:
-        out += "exec {orig_stdout}>&1 1>&2\n"
+    if util.in_debug_mode():
         out += "set -xe\n"
     else:
-        out += "exec {orig_stdout}>&1 2>/dev/null 1>&2\n"
+        out += "exec 1>/dev/null\n"
         out += "set -e\n"
 
     # install test dependencies
     # - only strings (package names) in require/recommend are supported
-    if require := [x for x in fmf.listlike(test.data, "require") if isinstance(x, str)]:
-        out += "dnf -y --setopt=install_weak_deps=False install "
-        out += " ".join(f"'{pkg}'" for pkg in require) + "\n"
-    if recommend := [x for x in fmf.listlike(test.data, "recommend") if isinstance(x, str)]:
-        out += "dnf -y --setopt=install_weak_deps=False install --skip-broken "
-        out += " ".join(f"'{pkg}'" for pkg in recommend) + "\n"
+    if require := list(fmf.test_pkg_requires(test.data, "require")):
+        out += _install_packages(require) + "\n"
+    if recommend := list(fmf.test_pkg_requires(test.data, "recommend")):
+        out += _install_packages(recommend, ("--skip-broken",)) + "\n"
+
+    # write out test data
+    out += f"cat > '{test_yaml}' <<'ATEX_SETUP_EOF'\n"
+    out += yaml.dump(test.data).rstrip("\n")  # don't rely on trailing \n
+    out += "\nATEX_SETUP_EOF\n"
 
     # make the wrapper script
     out += f"cat > '{wrapper_exec}' <<'ATEX_SETUP_EOF'\n"
     out += test_wrapper(
         test=test,
         test_exec=test_exec,
-        debug=debug,
         **kwargs,
     )
     out += "ATEX_SETUP_EOF\n"

atex/{minitmt → executor}/testcontrol.py

@@ -95,30 +95,43 @@ class TestControl:
     processing test-issued commands, results and uploaded files.
     """
 
-    def __init__(self, *, control_fd, aggregator, duration, testout_fd):
+    def __init__(self, *, reporter, duration, control_fd=None):
         """
         'control_fd' is a non-blocking file descriptor to be read.
 
-        'aggregator' is an instance of a result aggregator (ie. CSVAggregator)
-        all the results and uploaded files will be written to.
+        'reporter' is an instance of class Reporter all the results
+        and uploaded files will be written to.
 
         'duration' is a class Duration instance.
-
-        'testout_fd' is an optional file descriptor handle which the test uses
-        to write its output to - useful here for the 'result' control word and
-        its protocol, which allows "hardlinking" the fd to a real file name.
         """
-        self.control_fd = control_fd
-        self.stream = NonblockLineReader(control_fd)
-        self.aggregator = aggregator
+        self.reporter = reporter
         self.duration = duration
-        self.testout_fd = testout_fd
+        if control_fd:
+            self.control_fd = control_fd
+            self.stream = NonblockLineReader(control_fd)
+        else:
+            self.control_fd = None
+            self.stream = None
         self.eof = False
         self.in_progress = None
         self.partial_results = collections.defaultdict(dict)
-        self.result_seen = False
         self.exit_code = None
         self.reconnect = None
+        self.nameless_result_seen = False
+
+    def reassign(self, new_fd):
+        """
+        Assign a new control file descriptor to read test control from,
+        replacing a previous one. Useful on test reconnect.
+        """
+        err = "tried to assign new control fd while"
+        if self.in_progress:
+            raise BadControlError(f"{err} old one is reading non-control binary data")
+        elif self.stream and self.stream.bytes_read != 0:
+            raise BadControlError(f"{err} old one is in the middle of reading a control line")
+        self.eof = False
+        self.control_fd = new_fd
+        self.stream = NonblockLineReader(new_fd)
 
     def process(self):
         """
@@ -143,7 +156,7 @@ class TestControl:
         except BufferFullError as e:
             raise BadControlError(str(e)) from None
 
-        util.debug(f"got control line: {line}")
+        util.debug(f"got control line: {line} // eof: {self.stream.eof}")
 
         if self.stream.eof:
             self.eof = True
@@ -238,9 +251,10 @@ class TestControl:
         except json.decoder.JSONDecodeError as e:
             raise BadReportJSONError(f"JSON decode: {str(e)} caused by: {json_data}") from None
 
+        # note that this may be None (result for the test itself)
         name = result.get("name")
         if not name:
-            raise BadReportJSONError("'name' not specified, but mandatory")
+            self.nameless_result_seen = True
 
         # upload files
         for entry in result.get("files", ()):
@@ -253,30 +267,28 @@ class TestControl:
             except ValueError as e:
                 raise BadReportJSONError(f"file entry {file_name} length: {str(e)}") from None
 
-            with self.aggregator.open_tmpfile() as fd:
-                while file_length > 0:
-                    try:
-                        # try a more universal sendfile first, fall back to splice
+            try:
+                with self.reporter.open_file(file_name, name) as f:
+                    fd = f.fileno()
+                    while file_length > 0:
                         try:
-                            written = os.sendfile(fd, self.control_fd, None, file_length)
-                        except OSError as e:
-                            if e.errno == 22:  # EINVAL
-                                written = os.splice(self.control_fd, fd, file_length)
-                            else:
-                                raise
-                    except BlockingIOError:
+                            # try a more universal sendfile first, fall back to splice
+                            try:
+                                written = os.sendfile(fd, self.control_fd, None, file_length)
+                            except OSError as e:
+                                if e.errno == 22:  # EINVAL
+                                    written = os.splice(self.control_fd, fd, file_length)
+                                else:
+                                    raise
+                        except BlockingIOError:
+                            yield
+                            continue
+                        if written == 0:
+                            raise BadControlError("EOF when reading data")
+                        file_length -= written
                         yield
-                        continue
-                    if written == 0:
-                        raise BadControlError("EOF when reading data")
-                    file_length -= written
-                    yield
-                try:
-                    self.aggregator.link_tmpfile_to(name, file_name, fd)
-                except FileExistsError:
-                    raise BadReportJSONError(
-                        f"file '{file_name}' for '{name}' already exists",
-                    ) from None
+            except FileExistsError:
+                raise BadReportJSONError(f"file '{file_name}' already exists") from None
 
         # either store partial result + return,
         # or load previous partial result and merge into it
@@ -284,6 +296,8 @@ class TestControl:
         if partial:
             # do not store the 'partial' key in the result
             del result["partial"]
+            # note that nameless result will get None as dict key,
+            # which is perfectly fine
             self._merge(self.partial_results[name], result)
             # partial = do nothing
             return
@@ -295,7 +309,7 @@ class TestControl:
         if name in self.partial_results:
             partial_result = self.partial_results[name]
             del self.partial_results[name]
-            self._nested_merge(partial_result, result)
+            self._merge(partial_result, result)
             result = partial_result
 
         if "testout" in result:
@@ -303,13 +317,11 @@ class TestControl:
             if not testout:
                 raise BadReportJSONError("'testout' specified, but empty")
             try:
-                self.aggregator.link_tmpfile_to(name, testout, self.testout_fd)
+                self.reporter.link_testout(testout, name)
             except FileExistsError:
-                raise BadReportJSONError(f"file '{testout}' for '{name}' already exists") from None
-
-        self.aggregator.report(result)
+                raise BadReportJSONError(f"file '{testout}' already exists") from None
 
-        self.result_seen = True
+        self.reporter.report(result)
 
     def _parser_duration(self, arg):
         if not arg:

atex/fmf.py

@@ -0,0 +1,237 @@
+import re
+import collections
+from pathlib import Path
+
+# from system-wide sys.path
+import fmf
+
+
+def listlike(data, key):
+    """
+    Get a piece of fmf metadata as an iterable regardless of whether it was
+    defined as a dict or a list.
+
+    This is needed because many fmf metadata keys can be used either as
+        some_key: 123
+    or as lists via YAML syntax
+        some_key:
+          - 123
+          - 456
+    and, for simplicity, we want to always deal with lists (iterables).
+    """
+    if value := data.get(key):
+        return value if isinstance(value, list) else (value,)
+    else:
+        return ()
+
+
+class FMFTests:
+    """
+    FMF test metadata parsed from on-disk metadata using a specific plan name,
+    with all metadata dictionaries for all nodes being adjusted by that plan
+    and (optionally) a specified context.
+    """
+    # TODO: usage example ^^^^
+
+    def __init__(
+        self, fmf_tree, plan_name=None, *,
+        names=None, filters=None, conditions=None, excludes=None,
+        context=None,
+    ):
+        """
+        'fmf_tree' is filesystem path somewhere inside fmf metadata tree,
+        or a root fmf.Tree instance.
+
+        'plan_name' is fmf identifier (like /some/thing) of a tmt plan
+        to use for discovering tests. If None, a dummy (empty) plan is used.
+
+        'names', 'filters', 'conditions' and 'exclude' (all tuple/list)
+        are fmf tree filters (resolved by the fmf module), overriding any
+        existing tree filters in the plan's discover phase specifies, where:
+
+            'names' are test regexes like ["/some/test", "/another/test"]
+
+            'filters' are fmf-style filter expressions, as documented on
+            https://fmf.readthedocs.io/en/stable/modules.html#fmf.filter
+
+            'conditions' are python expressions whose namespace locals()
+            are set up to be a dictionary of the fmf tree. When any of the
+            expressions returns True, the tree is returned, ie.
+                ["environment['FOO'] == 'BAR'"]
+                ["'enabled' not in locals() or enabled"]
+            Note that KeyError is silently ignored and treated as False.
+
+            'excludes' are test regexes to exclude, format same as 'names'
+
+        'context' is a dict like {'distro': 'rhel-9.6'} used for additional
+        adjustment of the discovered fmf metadata.
+        """
+        # list of packages to install, as extracted from plan
+        self.prepare_pkgs = []
+        # list of scripts to run, as extracted from plan
+        self.prepare_scripts = []
+        self.finish_scripts = []
+        # dict of environment, as extracted from plan
+        self.plan_env = {}
+        # dict indexed by test name, value is dict with fmf-parsed metadata
+        self.tests = {}
+        # dict indexed by test name, value is pathlib.Path of relative path
+        # of the fmf metadata root towards the test metadata location
+        self.test_dirs = {}
+
+        # fmf.Context instance, as used for test discovery
+        context = fmf.Context(**context) if context else fmf.Context()
+        # allow the user to pass fmf.Tree directly, greatly speeding up the
+        # instantiation of multiple FMFTests instances
+        tree = fmf_tree.copy() if isinstance(fmf_tree, fmf.Tree) else fmf.Tree(fmf_tree)
+        tree.adjust(context=context)
+
+        # Path of the metadata root
+        self.root = Path(tree.root)
+
+        # lookup the plan first
+        if plan_name:
+            plan = tree.find(plan_name)
+            if not plan:
+                raise ValueError(f"plan {plan_name} not found in {tree.root}")
+            if "test" in plan.data:
+                raise ValueError(f"plan {plan_name} appears to be a test")
+        # fall back to a dummy plan
+        else:
+            class plan:  # noqa: N801
+                data = {}
+
+        # gather and merge plan-defined environment variables
+        #
+        # environment:
+        #  - FOO: BAR
+        #    BAR: BAZ
+        for entry in listlike(plan.data, "environment"):
+            self.plan_env.update(entry)
+
+        # gather all prepare scripts / packages
+        #
+        # prepare:
+        #   - how: install
+        #     package:
+        #       - some-rpm-name
+        #   - how: shell
+        #     script:
+        #       - some-command
+        for entry in listlike(plan.data, "prepare"):
+            if entry.get("how") == "install":
+                self.prepare_pkgs += listlike(entry, "package")
+            elif entry.get("how") == "shell":
+                self.prepare_scripts += listlike(entry, "script")
+
+        # gather all finish scripts, same as prepare scripts
+        for entry in listlike(plan.data, "finish"):
+            if entry.get("how") == "shell":
+                self.finish_scripts += listlike(entry, "script")
+
+        # gather all tests selected by the plan
+        #
+        # discover:
+        #   - how: fmf
+        #     filter:
+        #       - tag:some_tag
+        #     test:
+        #       - some-test-regex
+        #     exclude:
+        #       - some-test-regex
+        plan_filters = collections.defaultdict(list)
+        for entry in listlike(plan.data, "discover"):
+            if entry.get("how") != "fmf":
+                continue
+            for meta_name in ("filter", "test", "exclude"):
+                if value := listlike(entry, meta_name):
+                    plan_filters[meta_name] += value
+
+        prune_kwargs = {}
+        if names:
+            prune_kwargs["names"] = names
+        elif "test" in plan_filters:
+            prune_kwargs["names"] = plan_filters["test"]
+        if filters:
+            prune_kwargs["filters"] = filters
+        elif "filter" in plan_filters:
+            prune_kwargs["filters"] = plan_filters["filter"]
+        if conditions:
+            prune_kwargs["conditions"] = conditions
+        if not excludes:
+            excludes = plan_filters.get("exclude")
+
+        # actually discover the tests
+        for child in tree.prune(**prune_kwargs):
+            # excludes not supported by .prune(), we have to do it here
+            if excludes and any(re.match(x, child.name) for x in excludes):
+                continue
+            # only tests
+            if "test" not in child.data:
+                continue
+            # only enabled tests
+            if "enabled" in child.data and not child.data["enabled"]:
+                continue
+            # no manual tests and no stories
+            if child.data.get("manual") or child.data.get("story"):
+                continue
+            # after adjusting above, any adjusts are useless, free some space
+            if "adjust" in child.data:
+                del child.data["adjust"]
+
+            self.tests[child.name] = child.data
+            # child.sources ie. ['/abs/path/to/some.fmf', '/abs/path/to/some/node.fmf']
+            self.test_dirs[child.name] = \
+                Path(child.sources[-1]).parent.relative_to(self.root)
+
+
+def test_pkg_requires(data, key="require"):
+    """
+    Yield RPM package names specified by test 'data' (fmf metadata dict)
+    in the metadata 'key' (require or recommend), ignoring any non-RPM-package
+    requires/recommends.
+    """
+    for entry in listlike(data, key):
+        # skip type:library and type:path
+        if not isinstance(entry, str):
+            continue
+        # skip "fake RPMs" that begin with 'library('
+        if entry.startswith("library("):
+            continue
+        yield entry
+
+
+def all_pkg_requires(fmf_tests, key="require"):
+    """
+    Yield RPM package names from the plan and all tests discovered by
+    a class FMFTests instance 'fmf_tests', ignoring any non-RPM-package
+    requires/recommends.
+    """
+    # use a set to avoid duplicates
+    pkgs = set()
+    pkgs.update(fmf_tests.prepare_pkgs)
+    for data in fmf_tests.tests.values():
+        pkgs.update(test_pkg_requires(data, key))
+    yield from pkgs
+
+
+# Some extra notes for fmf.prune() arguments:
+#
+# Set 'names' to filter by a list of fmf node names, ie.
+#     ['/some/test', '/another/test']
+#
+# Set 'filters' to filter by a list of fmf-style filter expressions, see
+#     https://fmf.readthedocs.io/en/stable/modules.html#fmf.filter
+#
+# Set 'conditions' to filter by a list of python expressions whose namespace
+# locals() are set up to be a dictionary of the tree. When any of the
+# expressions returns True, the tree is returned, ie.
+#     ['environment["FOO"] == "BAR"']
+#     ['"enabled" not in locals() or enabled']
+# Note that KeyError is silently ignored and treated as False.
+#
+# Set 'context' to a dictionary to post-process the tree metadata with
+# adjust expressions (that may be present in a tree) using the specified
+# context. Any other filters are applied afterwards to allow modification
+# of tree metadata by the adjust expressions. Ie.
+#     {'distro': 'rhel-9.6.0', 'arch': 'x86_64'}