atex 0.7__py3-none-any.whl → 0.8__py3-none-any.whl

atex/executor/reporter.py

@@ -0,0 +1,106 @@
+import os
+import json
+import ctypes
+import ctypes.util
+import contextlib
+from pathlib import Path
+
+from .. import util
+
+
+libc = ctypes.CDLL(ctypes.util.find_library("c"), use_errno=True)
+
+# int linkat(int olddirfd, const char *oldpath, int newdirfd, const char *newpath, int flags)
+libc.linkat.argtypes = (
+    ctypes.c_int,
+    ctypes.c_char_p,
+    ctypes.c_int,
+    ctypes.c_char_p,
+    ctypes.c_int,
+)
+libc.linkat.restype = ctypes.c_int
+
+# fcntl.h:#define AT_EMPTY_PATH                0x1000  /* Allow empty relative pathname */
+AT_EMPTY_PATH = 0x1000
+
+# fcntl.h:#define AT_FDCWD             -100    /* Special value used to indicate
+AT_FDCWD = -100
+
+
+def linkat(*args):
+    if (ret := libc.linkat(*args)) == -1:
+        errno = ctypes.get_errno()
+        raise OSError(errno, os.strerror(errno))
+    return ret
+
+
+class Reporter:
+    """
+    Collects reported results (in a format specified by RESULTS.md) for
+    a specific test, storing them persistently.
+    """
+
+    def __init__(self, json_file, files_dir):
+        """
+        'json_file' is a destination file (string or Path) for results.
+
+        'files_dir' is a destination dir (string or Path) for uploaded files.
+        """
+        self.json_file = json_file
+        self.files_dir = Path(files_dir)
+        self.json_fobj = None
+
+    def __enter__(self):
+        if self.json_file.exists():
+            raise FileExistsError(f"{self.json_file} already exists")
+        self.json_fobj = open(self.json_file, "w")
+
+        if self.files_dir.exists():
+            raise FileExistsError(f"{self.files_dir} already exists")
+        self.files_dir.mkdir()
+
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        if self.json_fobj:
+            self.json_fobj.close()
+            self.json_fobj = None
+
+    def report(self, result_line):
+        """
+        Persistently record a test result.
+
+        'result_line' is a dict in the format specified by RESULTS.md.
+        """
+        json.dump(result_line, self.json_fobj, indent=None)
+        self.json_fobj.write("\n")
+        self.json_fobj.flush()
+
+    @contextlib.contextmanager
+    def open_tmpfile(self, open_mode=os.O_WRONLY):
+        """
+        Open an anonymous (name-less) file for writing and yield its file
+        descriptor (int) as context, closing it when the context is exited.
+        """
+        flags = open_mode | os.O_TMPFILE
+        fd = os.open(self.files_dir, flags, 0o644)
+        try:
+            yield fd
+        finally:
+            os.close(fd)
+
+    def link_tmpfile_to(self, fd, file_name, result_name=None):
+        """
+        Store a file named 'file_name' in a directory relevant to 'result_name'
+        whose 'fd' (a file descriptor) was created by .open_tmpfile().
+
+        This function can be called multiple times with the same 'fd', and
+        does not close or otherwise alter the descriptor.
+
+        If 'result_name' is not given, link files to the test (name) itself.
+        """
+        result_name = util.normalize_path(result_name) if result_name else "."
+        # /path/to/files_dir / path/to/subresult / path/to/file.log
+        file_path = self.files_dir / result_name / util.normalize_path(file_name)
+        file_path.parent.mkdir(parents=True, exist_ok=True)
+        linkat(fd, b"", AT_FDCWD, bytes(file_path), AT_EMPTY_PATH)

atex/{minitmt → executor}/scripts.py

@@ -1,8 +1,14 @@
+import collections
 from pathlib import Path
 
-from .. import util
+from .. import util, fmf
+
+# name: fmf path to the test as string, ie. /some/test
+# data: dict of the parsed fmf metadata (ie. {'tag': ... , 'environment': ...})
+# dir:  relative pathlib.Path of the test .fmf to repo root, ie. some/test
+#       (may be different from name for "virtual" tests that share the same dir)
+Test = collections.namedtuple("Test", ["name", "data", "dir"])
 
-from . import fmf
 
 # NOTE that we split test execution into 3 scripts:
 #      - "setup script" (package installs, etc.)
@@ -17,8 +23,7 @@ from . import fmf
 #      passed by an argument to 'ssh', leaving stdin/out/err untouched,
 #      allowing the user to interact with it (if run interactively)
 
-
-def test_wrapper(*, test, tests_dir, test_exec, debug=False):
+def test_wrapper(*, test, tests_dir, test_exec):
     """
     Generate a bash script that runs a user-specified test, preparing
     a test control channel for it, and reporting its exit code.
@@ -26,14 +31,12 @@ def test_wrapper(*, test, tests_dir, test_exec, debug=False):
     is considered as test output and any unintended environment changes
     will impact the test itself.
 
-    'test' is a atex.minitmt.fmf.FMFTest instance.
+    'test' is a class Test instance.
 
     'test_dir' is a remote directory (repository) of all the tests,
     a.k.a. FMF metadata root.
 
     'test_exec' is a remote path to the actual test to run.
-
-    'debug' specifies whether to include wrapper output inside test output.
     """
     out = "#!/bin/bash\n"
 
@@ -46,7 +49,7 @@ def test_wrapper(*, test, tests_dir, test_exec, debug=False):
     #       doing it here avoids unnecessary traffic (reading stdin) via ssh,
     #       even if it is fed from subprocess.DEVNULL on the runner
 
-    if debug:
+    if util.in_debug_mode():
         out += "set -x\n"
 
     # use a subshell to limit the scope of the CWD change
@@ -88,7 +91,18 @@ def test_wrapper(*, test, tests_dir, test_exec, debug=False):
     return out
 
 
-def test_setup(*, test, wrapper_exec, test_exec, debug=False, **kwargs):
+def _install_packages(pkgs, extra_opts=None):
+    pkgs_str = " ".join(pkgs)
+    extra_opts = extra_opts or ()
+    dnf = ["dnf", "-y", "--setopt=install_weak_deps=False", "install", *extra_opts]
+    dnf_str = " ".join(dnf)
+    return util.dedent(fr"""
+        not_installed=$(rpm -q --qf '' {pkgs_str} | sed -nr 's/^package ([^ ]+) is not installed$/\1/p')
+        [[ $not_installed ]] && {dnf_str} $not_installed
+    """)  # noqa: E501
+
+
+def test_setup(*, test, wrapper_exec, test_exec, **kwargs):
     """
     Generate a bash script that should prepare the remote end for test
     execution.
@@ -101,38 +115,30 @@ def test_setup(*, test, wrapper_exec, test_exec, debug=False, **kwargs):
 
     'test_exec' is the remote path where the test script should be put.
 
-    'test' is a atex.minitmt.fmf.FMFTest instance.
-
-    'debug' specifies whether to make the setup script extra verbose.
+    'test' is a class Test instance.
 
     Any 'kwargs' are passed to test_wrapper().
     """
     out = "#!/bin/bash\n"
 
-    # have deterministic stdin, avoid leaking parent console
-    # also avoid any accidental stdout output, we use it for wrapper path
-    if debug:
-        out += "exec {orig_stdout}>&1 1>&2\n"
+    if util.in_debug_mode():
         out += "set -xe\n"
     else:
-        out += "exec {orig_stdout}>&1 2>/dev/null 1>&2\n"
+        out += "exec 1>/dev/null\n"
         out += "set -e\n"
 
     # install test dependencies
     # - only strings (package names) in require/recommend are supported
-    if require := [x for x in fmf.listlike(test.data, "require") if isinstance(x, str)]:
-        out += "dnf -y --setopt=install_weak_deps=False install "
-        out += " ".join(f"'{pkg}'" for pkg in require) + "\n"
-    if recommend := [x for x in fmf.listlike(test.data, "recommend") if isinstance(x, str)]:
-        out += "dnf -y --setopt=install_weak_deps=False install --skip-broken "
-        out += " ".join(f"'{pkg}'" for pkg in recommend) + "\n"
+    if require := list(fmf.test_pkg_requires(test.data, "require")):
+        out += _install_packages(require) + "\n"
+    if recommend := list(fmf.test_pkg_requires(test.data, "recommend")):
+        out += _install_packages(recommend, ("--skip-broken",)) + "\n"
 
     # make the wrapper script
     out += f"cat > '{wrapper_exec}' <<'ATEX_SETUP_EOF'\n"
     out += test_wrapper(
         test=test,
         test_exec=test_exec,
-        debug=debug,
         **kwargs,
     )
     out += "ATEX_SETUP_EOF\n"

atex/{minitmt → executor}/testcontrol.py

@@ -95,12 +95,12 @@ class TestControl:
     processing test-issued commands, results and uploaded files.
     """
 
-    def __init__(self, *, control_fd, aggregator, duration, testout_fd):
+    def __init__(self, *, control_fd, reporter, duration, testout_fd):
         """
         'control_fd' is a non-blocking file descriptor to be read.
 
-        'aggregator' is an instance of a result aggregator (ie. CSVAggregator)
-        all the results and uploaded files will be written to.
+        'reporter' is an instance of class Reporter all the results
+        and uploaded files will be written to.
 
         'duration' is a class Duration instance.
 
@@ -110,15 +110,15 @@ class TestControl:
         """
         self.control_fd = control_fd
         self.stream = NonblockLineReader(control_fd)
-        self.aggregator = aggregator
+        self.reporter = reporter
         self.duration = duration
         self.testout_fd = testout_fd
         self.eof = False
         self.in_progress = None
         self.partial_results = collections.defaultdict(dict)
-        self.result_seen = False
         self.exit_code = None
         self.reconnect = None
+        self.nameless_result_seen = False
 
     def process(self):
         """
@@ -238,9 +238,10 @@ class TestControl:
         except json.decoder.JSONDecodeError as e:
             raise BadReportJSONError(f"JSON decode: {str(e)} caused by: {json_data}") from None
 
+        # note that this may be None (result for the test itself)
         name = result.get("name")
         if not name:
-            raise BadReportJSONError("'name' not specified, but mandatory")
+            self.nameless_result_seen = True
 
         # upload files
         for entry in result.get("files", ()):
@@ -253,7 +254,7 @@ class TestControl:
             except ValueError as e:
                 raise BadReportJSONError(f"file entry {file_name} length: {str(e)}") from None
 
-            with self.aggregator.open_tmpfile() as fd:
+            with self.reporter.open_tmpfile() as fd:
                 while file_length > 0:
                     try:
                         # try a more universal sendfile first, fall back to splice
@@ -272,11 +273,9 @@ class TestControl:
                     file_length -= written
                     yield
                 try:
-                    self.aggregator.link_tmpfile_to(name, file_name, fd)
+                    self.reporter.link_tmpfile_to(fd, file_name, name)
                 except FileExistsError:
-                    raise BadReportJSONError(
-                        f"file '{file_name}' for '{name}' already exists",
-                    ) from None
+                    raise BadReportJSONError(f"file '{file_name}' already exists") from None
 
         # either store partial result + return,
         # or load previous partial result and merge into it
@@ -284,6 +283,8 @@ class TestControl:
         if partial:
             # do not store the 'partial' key in the result
             del result["partial"]
+            # note that nameless result will get None as dict key,
+            # which is perfectly fine
             self._merge(self.partial_results[name], result)
             # partial = do nothing
             return
@@ -295,7 +296,7 @@ class TestControl:
         if name in self.partial_results:
             partial_result = self.partial_results[name]
             del self.partial_results[name]
-            self._nested_merge(partial_result, result)
+            self._merge(partial_result, result)
             result = partial_result
 
         if "testout" in result:
@@ -303,13 +304,11 @@ class TestControl:
             if not testout:
                 raise BadReportJSONError("'testout' specified, but empty")
             try:
-                self.aggregator.link_tmpfile_to(name, testout, self.testout_fd)
+                self.reporter.link_tmpfile_to(self.testout_fd, testout, name)
             except FileExistsError:
-                raise BadReportJSONError(f"file '{testout}' for '{name}' already exists") from None
+                raise BadReportJSONError(f"file '{testout}' already exists") from None
 
-        self.aggregator.report(result)
-
-        self.result_seen = True
+        self.reporter.report(result)
 
     def _parser_duration(self, arg):
         if not arg:

atex/{minitmt/fmf.py → fmf.py}

@@ -1,16 +1,9 @@
 import re
-import collections
 from pathlib import Path
 
 # from system-wide sys.path
 import fmf
 
-# name: fmf path to the test as string, ie. /some/test
-# data: dict of the parsed fmf metadata (ie. {'tag': ... , 'environment': ...})
-# dir:  relative pathlib.Path of the test .fmf to repo root, ie. some/test
-#       (may be different from name for "virtual" tests that share the same dir)
-FMFTest = collections.namedtuple("FMFTest", ["name", "data", "dir"])
-
 
 def listlike(data, key):
     """
@@ -61,12 +54,14 @@ class FMFTests:
         # dict indexed by test name, value is pathlib.Path of relative path
         # of the fmf metadata root towards the test metadata location
         self.test_dirs = {}
+        # fmf.Context instance, as used for test discovery
+        self.context = fmf.Context(**context) if context else fmf.Context()
 
         tree = fmf_tree.copy() if isinstance(fmf_tree, fmf.Tree) else fmf.Tree(fmf_tree)
-        ctx = fmf.Context(**context) if context else fmf.Context()
-        tree.adjust(context=ctx)
+        tree.adjust(context=self.context)
 
-        self.fmf_root = tree.root
+        # Path of the metadata root
+        self.root = Path(tree.root)
 
         # lookup the plan first
         plan = tree.find(plan_name)
@@ -146,23 +141,43 @@ class FMFTests:
                     self.tests[child.name] = child.data
                     # child.sources ie. ['/abs/path/to/some.fmf', '/abs/path/to/some/node.fmf']
                     self.test_dirs[child.name] = \
-                        Path(child.sources[-1]).parent.relative_to(self.fmf_root)
-
-    def as_fmftest(self, name):
-        return FMFTest(name, self.tests[name], self.test_dirs[name])
-
-    def as_fmftests(self):
-        for name, data in self.tests.items():
-            yield FMFTest(name, data, self.test_dirs[name])
+                        Path(child.sources[-1]).parent.relative_to(self.root)
 
     def match(self, regex):
         """
-        Return an iterable of FMFTest instances with test names matching the
-        specified regex via re.match(), just like how 'tmt' discovers tests.
+        Yield test names that match 'regex', simulating how tmt discovers tests.
         """
-        for name, data in self.tests.items():
-            if re.match(regex, name):
-                yield FMFTest(name, data, self.test_dirs[name])
+        yield from (name for name in self.tests if re.match(regex, name))
+
+
+def test_pkg_requires(data, key="require"):
+    """
+    Yield RPM package names specified by test 'data' (fmf metadata dict)
+    in the metadata 'key' (require or recommend), ignoring any non-RPM-package
+    requires/recommends.
+    """
+    for entry in listlike(data, key):
+        # skip type:library and type:path
+        if not isinstance(entry, str):
+            continue
+        # skip "fake RPMs" that begin with 'library('
+        if entry.startswith("library("):
+            continue
+        yield entry
+
+
+def all_pkg_requires(fmf_tests, key="require"):
+    """
+    Yield RPM package names from the plan and all tests discovered by
+    a class FMFTests instance 'fmf_tests', ignoring any non-RPM-package
+    requires/recommends.
+    """
+    # use a set to avoid duplicates
+    pkgs = set()
+    pkgs.update(fmf_tests.prepare_pkgs)
+    for data in fmf_tests.tests.values():
+        pkgs.update(test_pkg_requires(data, key))
+    yield from pkgs
 
 
 # Some extra notes for fmf.prune() arguments:
@@ -186,17 +201,17 @@ class FMFTests:
 # of tree metadata by the adjust expressions. Ie.
 #     {'distro': 'rhel-9.6.0', 'arch': 'x86_64'}
 
-Platform = collections.namedtuple("Platform", ["distro", "arch"])
-
-
-def combine_platforms(fmf_path, plan_name, platforms):
-    # TODO: document
-    fmf_tests = {}
-    tree = fmf.Tree(fmf_path)
-    for platform in platforms:
-        context = {"distro": platform.distro, "arch": platform.arch}
-        fmf_tests[platform] = FMFTests(tree, plan_name, context=context)
-    return fmf_tests
+#Platform = collections.namedtuple("Platform", ["distro", "arch"])
+#
+#
+#def combine_platforms(fmf_path, plan_name, platforms):
+#    # TODO: document
+#    fmf_tests = {}
+#    tree = fmf.Tree(fmf_path)
+#    for platform in platforms:
+#        context = {"distro": platform.distro, "arch": platform.arch}
+#        fmf_tests[platform] = FMFTests(tree, plan_name, context=context)
+#    return fmf_tests
 
 # TODO: in Orchestrator, when a Provisioner becomes free, have it pick a test
 #       from the appropriate tests[platform] per the Provisioner's platform

atex/orchestrator/__init__.py

@@ -1,59 +1,2 @@
-import importlib as _importlib
-import pkgutil as _pkgutil
-#import threading as _threading
-
-
-class Orchestrator:
-    """
-    A scheduler for parallel execution on multiple resources (machines/systems).
-
-    Given a list of Provisioner-derived class instances, it attempts to reserve
-    resources and uses them on-demand as they become available, calling run()
-    on each.
-
-    Note that run() and report() always run in a separate threads (are allowed
-    to block), and may access instance attributes, which are transparently
-    guarded by a thread-aware mutex.
-
-    """
-
-    def __init__(self):
-        pass
-        # TODO: configure via args, max workers, etc.
-
-#    def reserve(self, provisioner):
-#        # call provisioner.reserve(), return its return
-#        ...
-
-    def add_provisioner(self, provisioner):
-        # add to a self.* list of provisioners to be used for getting machines
-        ...
-
-    def run(self, provisioner):
-        # run tests, if destructive, call provisioner.release()
-        # returns anything
-        ...
-
-    def report(self):
-        # gets return from run
-        # writes it out to somewhere else
-        ...
-
-
-_submodules = [
-    info.name for info in _pkgutil.iter_modules(__spec__.submodule_search_locations)
-]
-
-__all__ = [*_submodules, Orchestrator.__name__]  # noqa: PLE0604
-
-
-def __dir__():
-    return __all__
-
-
-# lazily import submodules
-def __getattr__(attr):
-    if attr in _submodules:
-        return _importlib.import_module(f".{attr}", __name__)
-    else:
-        raise AttributeError(f"module '{__name__}' has no attribute '{attr}'")
+from .aggregator import CSVAggregator  # noqa: F401
+from .orchestrator import Orchestrator  # noqa: F401