atex 0.8__py3-none-any.whl → 0.10__py3-none-any.whl

atex/executor/reporter.py

@@ -1,70 +1,72 @@
 import os
 import json
-import ctypes
-import ctypes.util
-import contextlib
 from pathlib import Path
 
 from .. import util
 
 
-libc = ctypes.CDLL(ctypes.util.find_library("c"), use_errno=True)
-
-# int linkat(int olddirfd, const char *oldpath, int newdirfd, const char *newpath, int flags)
-libc.linkat.argtypes = (
-    ctypes.c_int,
-    ctypes.c_char_p,
-    ctypes.c_int,
-    ctypes.c_char_p,
-    ctypes.c_int,
-)
-libc.linkat.restype = ctypes.c_int
-
-# fcntl.h:#define AT_EMPTY_PATH                0x1000  /* Allow empty relative pathname */
-AT_EMPTY_PATH = 0x1000
-
-# fcntl.h:#define AT_FDCWD             -100    /* Special value used to indicate
-AT_FDCWD = -100
-
-
-def linkat(*args):
-    if (ret := libc.linkat(*args)) == -1:
-        errno = ctypes.get_errno()
-        raise OSError(errno, os.strerror(errno))
-    return ret
-
-
 class Reporter:
     """
     Collects reported results (in a format specified by RESULTS.md) for
     a specific test, storing them persistently.
     """
 
-    def __init__(self, json_file, files_dir):
-        """
-        'json_file' is a destination file (string or Path) for results.
+    # internal name, stored inside 'output_dir' and hardlinked to
+    # 'testout'-JSON-key-specified result entries; deleted on exit
+    TESTOUT = "testout.temp"
 
-        'files_dir' is a destination dir (string or Path) for uploaded files.
+    def __init__(self, output_dir, results_file, files_dir):
         """
-        self.json_file = json_file
-        self.files_dir = Path(files_dir)
-        self.json_fobj = None
+        'output_dir' is a destination dir (string or Path) for results reported
+        and files uploaded.
 
-    def __enter__(self):
-        if self.json_file.exists():
-            raise FileExistsError(f"{self.json_file} already exists")
-        self.json_fobj = open(self.json_file, "w")
+        'results_file' is a file name inside 'output_dir' the results will be
+        reported into.
+
+        'files_dir' is a dir name inside 'output_dir' any files will be
+        uploaded to.
+        """
+        output_dir = Path(output_dir)
+        self.testout_file = output_dir / self.TESTOUT
+        self.results_file = output_dir / results_file
+        self.files_dir = output_dir / files_dir
+        self.output_dir = output_dir
+        self.results_fobj = None
+        self.testout_fobj = None
+
+    def start(self):
+        if self.testout_file.exists():
+            raise FileExistsError(f"{self.testout_file} already exists")
+        self.testout_fobj = open(self.testout_file, "wb")
+
+        if self.results_file.exists():
+            raise FileExistsError(f"{self.results_file} already exists")
+        self.results_fobj = open(self.results_file, "w", newline="\n")
 
         if self.files_dir.exists():
             raise FileExistsError(f"{self.files_dir} already exists")
         self.files_dir.mkdir()
 
-        return self
+    def stop(self):
+        if self.results_fobj:
+            self.results_fobj.close()
+            self.results_fobj = None
+
+        if self.testout_fobj:
+            self.testout_fobj.close()
+            self.testout_fobj = None
+            Path(self.testout_file).unlink()
+
+    def __enter__(self):
+        try:
+            self.start()
+            return self
+        except Exception:
+            self.stop()
+            raise
 
     def __exit__(self, exc_type, exc_value, traceback):
-        if self.json_fobj:
-            self.json_fobj.close()
-            self.json_fobj = None
+        self.stop()
 
     def report(self, result_line):
         """
@@ -72,35 +74,28 @@ class Reporter:
 
         'result_line' is a dict in the format specified by RESULTS.md.
         """
-        json.dump(result_line, self.json_fobj, indent=None)
-        self.json_fobj.write("\n")
-        self.json_fobj.flush()
+        json.dump(result_line, self.results_fobj, indent=None)
+        self.results_fobj.write("\n")
+        self.results_fobj.flush()
 
-    @contextlib.contextmanager
-    def open_tmpfile(self, open_mode=os.O_WRONLY):
-        """
-        Open an anonymous (name-less) file for writing and yield its file
-        descriptor (int) as context, closing it when the context is exited.
-        """
-        flags = open_mode | os.O_TMPFILE
-        fd = os.open(self.files_dir, flags, 0o644)
-        try:
-            yield fd
-        finally:
-            os.close(fd)
+    def _dest_path(self, file_name, result_name=None):
+        result_name = util.normalize_path(result_name) if result_name else "."
+        # /path/to/files_dir / path/to/subtest / path/to/file.log
+        file_path = self.files_dir / result_name / util.normalize_path(file_name)
+        file_path.parent.mkdir(parents=True, exist_ok=True)
+        return file_path
 
-    def link_tmpfile_to(self, fd, file_name, result_name=None):
+    def open_file(self, file_name, result_name=None, mode="wb"):
         """
-        Store a file named 'file_name' in a directory relevant to 'result_name'
-        whose 'fd' (a file descriptor) was created by .open_tmpfile().
-
-        This function can be called multiple times with the same 'fd', and
-        does not close or otherwise alter the descriptor.
+        Open a file named 'file_name' in a directory relevant to 'result_name'.
+        Returns an opened file-like object that can be used in a context manager
+        just like with regular open().
 
-        If 'result_name' is not given, link files to the test (name) itself.
+        If 'result_name' (typically a subtest) is not given, open the file
+        for the test (name) itself.
         """
-        result_name = util.normalize_path(result_name) if result_name else "."
-        # /path/to/files_dir / path/to/subresult / path/to/file.log
-        file_path = self.files_dir / result_name / util.normalize_path(file_name)
-        file_path.parent.mkdir(parents=True, exist_ok=True)
-        linkat(fd, b"", AT_FDCWD, bytes(file_path), AT_EMPTY_PATH)
+        return open(self._dest_path(file_name, result_name), mode)
+
+    def link_testout(self, file_name, result_name=None):
+        # TODO: docstring
+        os.link(self.testout_file, self._dest_path(file_name, result_name))

atex/executor/scripts.py

@@ -1,6 +1,9 @@
+import os
 import collections
 from pathlib import Path
 
+import yaml
+
 from .. import util, fmf
 
 # name: fmf path to the test as string, ie. /some/test
@@ -49,7 +52,7 @@ def test_wrapper(*, test, tests_dir, test_exec):
     #       doing it here avoids unnecessary traffic (reading stdin) via ssh,
     #       even if it is fed from subprocess.DEVNULL on the runner
 
-    if util.in_debug_mode():
+    if os.environ.get("ATEX_DEBUG_TEST") == "1":
         out += "set -x\n"
 
     # use a subshell to limit the scope of the CWD change
@@ -102,7 +105,7 @@ def _install_packages(pkgs, extra_opts=None):
     """)  # noqa: E501
 
 
-def test_setup(*, test, wrapper_exec, test_exec, **kwargs):
+def test_setup(*, test, wrapper_exec, test_exec, test_yaml, **kwargs):
     """
     Generate a bash script that should prepare the remote end for test
     execution.
@@ -111,17 +114,17 @@ def test_setup(*, test, wrapper_exec, test_exec, **kwargs):
     scripts: a test script (contents of 'test' from FMF) and a wrapper script
     to run the test script.
 
+    'test' is a class Test instance.
+
     'wrapper_exec' is the remote path where the wrapper script should be put.
 
     'test_exec' is the remote path where the test script should be put.
 
-    'test' is a class Test instance.
-
     Any 'kwargs' are passed to test_wrapper().
     """
     out = "#!/bin/bash\n"
 
-    if util.in_debug_mode():
+    if os.environ.get("ATEX_DEBUG_TEST") == "1":
         out += "set -xe\n"
     else:
         out += "exec 1>/dev/null\n"
@@ -134,6 +137,11 @@ def test_setup(*, test, wrapper_exec, test_exec, **kwargs):
     if recommend := list(fmf.test_pkg_requires(test.data, "recommend")):
         out += _install_packages(recommend, ("--skip-broken",)) + "\n"
 
+    # write out test data
+    out += f"cat > '{test_yaml}' <<'ATEX_SETUP_EOF'\n"
+    out += yaml.dump(test.data).rstrip("\n")  # don't rely on trailing \n
+    out += "\nATEX_SETUP_EOF\n"
+
     # make the wrapper script
     out += f"cat > '{wrapper_exec}' <<'ATEX_SETUP_EOF'\n"
     out += test_wrapper(

atex/executor/testcontrol.py

@@ -95,7 +95,7 @@ class TestControl:
     processing test-issued commands, results and uploaded files.
     """
 
-    def __init__(self, *, control_fd, reporter, duration, testout_fd):
+    def __init__(self, *, reporter, duration, control_fd=None):
         """
         'control_fd' is a non-blocking file descriptor to be read.
 
@@ -103,16 +103,15 @@ class TestControl:
         and uploaded files will be written to.
 
         'duration' is a class Duration instance.
-
-        'testout_fd' is an optional file descriptor handle which the test uses
-        to write its output to - useful here for the 'result' control word and
-        its protocol, which allows "hardlinking" the fd to a real file name.
         """
-        self.control_fd = control_fd
-        self.stream = NonblockLineReader(control_fd)
         self.reporter = reporter
         self.duration = duration
-        self.testout_fd = testout_fd
+        if control_fd:
+            self.control_fd = control_fd
+            self.stream = NonblockLineReader(control_fd)
+        else:
+            self.control_fd = None
+            self.stream = None
         self.eof = False
         self.in_progress = None
         self.partial_results = collections.defaultdict(dict)
@@ -120,6 +119,20 @@ class TestControl:
         self.reconnect = None
         self.nameless_result_seen = False
 
+    def reassign(self, new_fd):
+        """
+        Assign a new control file descriptor to read test control from,
+        replacing a previous one. Useful on test reconnect.
+        """
+        err = "tried to assign new control fd while"
+        if self.in_progress:
+            raise BadControlError(f"{err} old one is reading non-control binary data")
+        elif self.stream and self.stream.bytes_read != 0:
+            raise BadControlError(f"{err} old one is in the middle of reading a control line")
+        self.eof = False
+        self.control_fd = new_fd
+        self.stream = NonblockLineReader(new_fd)
+
     def process(self):
         """
         Read from the control file descriptor and potentially perform any
@@ -143,7 +156,7 @@ class TestControl:
         except BufferFullError as e:
             raise BadControlError(str(e)) from None
 
-        util.debug(f"got control line: {line}")
+        util.extradebug(f"control line: {line} // eof: {self.stream.eof}")
 
         if self.stream.eof:
             self.eof = True
@@ -254,28 +267,28 @@ class TestControl:
             except ValueError as e:
                 raise BadReportJSONError(f"file entry {file_name} length: {str(e)}") from None
 
-            with self.reporter.open_tmpfile() as fd:
-                while file_length > 0:
-                    try:
-                        # try a more universal sendfile first, fall back to splice
+            try:
+                with self.reporter.open_file(file_name, name) as f:
+                    fd = f.fileno()
+                    while file_length > 0:
                         try:
-                            written = os.sendfile(fd, self.control_fd, None, file_length)
-                        except OSError as e:
-                            if e.errno == 22:  # EINVAL
-                                written = os.splice(self.control_fd, fd, file_length)
-                            else:
-                                raise
-                    except BlockingIOError:
+                            # try a more universal sendfile first, fall back to splice
+                            try:
+                                written = os.sendfile(fd, self.control_fd, None, file_length)
+                            except OSError as e:
+                                if e.errno == 22:  # EINVAL
+                                    written = os.splice(self.control_fd, fd, file_length)
+                                else:
+                                    raise
+                        except BlockingIOError:
+                            yield
+                            continue
+                        if written == 0:
+                            raise BadControlError("EOF when reading data")
+                        file_length -= written
                         yield
-                        continue
-                    if written == 0:
-                        raise BadControlError("EOF when reading data")
-                    file_length -= written
-                    yield
-                try:
-                    self.reporter.link_tmpfile_to(fd, file_name, name)
-                except FileExistsError:
-                    raise BadReportJSONError(f"file '{file_name}' already exists") from None
+            except FileExistsError:
+                raise BadReportJSONError(f"file '{file_name}' already exists") from None
 
         # either store partial result + return,
         # or load previous partial result and merge into it
@@ -304,7 +317,7 @@ class TestControl:
             if not testout:
                 raise BadReportJSONError("'testout' specified, but empty")
             try:
-                self.reporter.link_tmpfile_to(self.testout_fd, testout, name)
+                self.reporter.link_testout(testout, name)
             except FileExistsError:
                 raise BadReportJSONError(f"file '{testout}' already exists") from None
 

atex/fmf.py

@@ -1,4 +1,5 @@
 import re
+import collections
 from pathlib import Path
 
 # from system-wide sys.path
@@ -32,21 +33,44 @@ class FMFTests:
     """
     # TODO: usage example ^^^^
 
-    def __init__(self, fmf_tree, plan_name, context=None):
+    def __init__(
+        self, fmf_tree, plan_name=None, *,
+        names=None, filters=None, conditions=None, excludes=None,
+        context=None,
+    ):
         """
         'fmf_tree' is filesystem path somewhere inside fmf metadata tree,
         or a root fmf.Tree instance.
 
         'plan_name' is fmf identifier (like /some/thing) of a tmt plan
-        to use for discovering tests.
+        to use for discovering tests. If None, a dummy (empty) plan is used.
 
-        'context' is a dict like {'distro': 'rhel-9.6'} used for filtering
-        discovered tests.
+        'names', 'filters', 'conditions' and 'exclude' (all tuple/list)
+        are fmf tree filters (resolved by the fmf module), overriding any
+        existing tree filters in the plan's discover phase specifies, where:
+
+            'names' are test regexes like ["/some/test", "/another/test"]
+
+            'filters' are fmf-style filter expressions, as documented on
+            https://fmf.readthedocs.io/en/stable/modules.html#fmf.filter
+
+            'conditions' are python expressions whose namespace locals()
+            are set up to be a dictionary of the fmf tree. When any of the
+            expressions returns True, the tree is returned, ie.
+                ["environment['FOO'] == 'BAR'"]
+                ["'enabled' not in locals() or enabled"]
+            Note that KeyError is silently ignored and treated as False.
+
+            'excludes' are test regexes to exclude, format same as 'names'
+
+        'context' is a dict like {'distro': 'rhel-9.6'} used for additional
+        adjustment of the discovered fmf metadata.
         """
         # list of packages to install, as extracted from plan
         self.prepare_pkgs = []
         # list of scripts to run, as extracted from plan
         self.prepare_scripts = []
+        self.finish_scripts = []
         # dict of environment, as extracted from plan
         self.plan_env = {}
         # dict indexed by test name, value is dict with fmf-parsed metadata
@@ -54,21 +78,28 @@ class FMFTests:
         # dict indexed by test name, value is pathlib.Path of relative path
         # of the fmf metadata root towards the test metadata location
         self.test_dirs = {}
-        # fmf.Context instance, as used for test discovery
-        self.context = fmf.Context(**context) if context else fmf.Context()
 
+        # fmf.Context instance, as used for test discovery
+        context = fmf.Context(**context) if context else fmf.Context()
+        # allow the user to pass fmf.Tree directly, greatly speeding up the
+        # instantiation of multiple FMFTests instances
         tree = fmf_tree.copy() if isinstance(fmf_tree, fmf.Tree) else fmf.Tree(fmf_tree)
-        tree.adjust(context=self.context)
+        tree.adjust(context=context)
 
         # Path of the metadata root
         self.root = Path(tree.root)
 
         # lookup the plan first
-        plan = tree.find(plan_name)
-        if not plan:
-            raise ValueError(f"plan {plan_name} not found in {tree.root}")
-        if "test" in plan.data:
-            raise ValueError(f"plan {plan_name} appears to be a test")
+        if plan_name:
+            plan = tree.find(plan_name)
+            if not plan:
+                raise ValueError(f"plan {plan_name} not found in {tree.root}")
+            if "test" in plan.data:
+                raise ValueError(f"plan {plan_name} appears to be a test")
+        # fall back to a dummy plan
+        else:
+            class plan:  # noqa: N801
+                data = {}
 
         # gather and merge plan-defined environment variables
         #
@@ -88,13 +119,16 @@ class FMFTests:
         #     script:
         #       - some-command
         for entry in listlike(plan.data, "prepare"):
-            if "how" not in entry:
-                continue
-            if entry["how"] == "install":
+            if entry.get("how") == "install":
                 self.prepare_pkgs += listlike(entry, "package")
-            elif entry["how"] == "shell":
+            elif entry.get("how") == "shell":
                 self.prepare_scripts += listlike(entry, "script")
 
+        # gather all finish scripts, same as prepare scripts
+        for entry in listlike(plan.data, "finish"):
+            if entry.get("how") == "shell":
+                self.finish_scripts += listlike(entry, "script")
+
         # gather all tests selected by the plan
         #
         # discover:
@@ -105,49 +139,50 @@ class FMFTests:
         #       - some-test-regex
         #     exclude:
         #       - some-test-regex
-        if "discover" in plan.data:
-            discover = plan.data["discover"]
-            if not isinstance(discover, list):
-                discover = (discover,)
-
-            for entry in discover:
-                if entry.get("how") != "fmf":
-                    continue
-
-                filtering = {}
-                for meta_name in ("filter", "test", "exclude"):
-                    if value := listlike(entry, meta_name):
-                        filtering[meta_name] = value
-
-                children = tree.prune(
-                    names=filtering.get("test"),
-                    filters=filtering.get("filter"),
-                )
-                for child in children:
-                    # excludes not supported by .prune(), we have to do it here
-                    excludes = filtering.get("exclude")
-                    if excludes and any(re.match(x, child.name) for x in excludes):
-                        continue
-                    # only enabled tests
-                    if "enabled" in child.data and not child.data["enabled"]:
-                        continue
-                    # no manual tests and no stories
-                    if child.data.get("manual") or child.data.get("story"):
-                        continue
-                    # after adjusting above, any adjusts are useless, free some space
-                    if "adjust" in child.data:
-                        del child.data["adjust"]
-
-                    self.tests[child.name] = child.data
-                    # child.sources ie. ['/abs/path/to/some.fmf', '/abs/path/to/some/node.fmf']
-                    self.test_dirs[child.name] = \
-                        Path(child.sources[-1]).parent.relative_to(self.root)
-
-    def match(self, regex):
-        """
-        Yield test names that match 'regex', simulating how tmt discovers tests.
-        """
-        yield from (name for name in self.tests if re.match(regex, name))
+        plan_filters = collections.defaultdict(list)
+        for entry in listlike(plan.data, "discover"):
+            if entry.get("how") != "fmf":
+                continue
+            for meta_name in ("filter", "test", "exclude"):
+                if value := listlike(entry, meta_name):
+                    plan_filters[meta_name] += value
+
+        prune_kwargs = {}
+        if names:
+            prune_kwargs["names"] = names
+        elif "test" in plan_filters:
+            prune_kwargs["names"] = plan_filters["test"]
+        if filters:
+            prune_kwargs["filters"] = filters
+        elif "filter" in plan_filters:
+            prune_kwargs["filters"] = plan_filters["filter"]
+        if conditions:
+            prune_kwargs["conditions"] = conditions
+        if not excludes:
+            excludes = plan_filters.get("exclude")
+
+        # actually discover the tests
+        for child in tree.prune(**prune_kwargs):
+            # excludes not supported by .prune(), we have to do it here
+            if excludes and any(re.match(x, child.name) for x in excludes):
+                continue
+            # only tests
+            if "test" not in child.data:
+                continue
+            # only enabled tests
+            if "enabled" in child.data and not child.data["enabled"]:
+                continue
+            # no manual tests and no stories
+            if child.data.get("manual") or child.data.get("story"):
+                continue
+            # after adjusting above, any adjusts are useless, free some space
+            if "adjust" in child.data:
+                del child.data["adjust"]
+
+            self.tests[child.name] = child.data
+            # child.sources ie. ['/abs/path/to/some.fmf', '/abs/path/to/some/node.fmf']
+            self.test_dirs[child.name] = \
+                Path(child.sources[-1]).parent.relative_to(self.root)
 
 
 def test_pkg_requires(data, key="require"):
@@ -200,18 +235,3 @@ def all_pkg_requires(fmf_tests, key="require"):
 # context. Any other filters are applied afterwards to allow modification
 # of tree metadata by the adjust expressions. Ie.
 #     {'distro': 'rhel-9.6.0', 'arch': 'x86_64'}
-
-#Platform = collections.namedtuple("Platform", ["distro", "arch"])
-#
-#
-#def combine_platforms(fmf_path, plan_name, platforms):
-#    # TODO: document
-#    fmf_tests = {}
-#    tree = fmf.Tree(fmf_path)
-#    for platform in platforms:
-#        context = {"distro": platform.distro, "arch": platform.arch}
-#        fmf_tests[platform] = FMFTests(tree, plan_name, context=context)
-#    return fmf_tests
-
-# TODO: in Orchestrator, when a Provisioner becomes free, have it pick a test
-#       from the appropriate tests[platform] per the Provisioner's platform

atex/orchestrator/__init__.py

@@ -1,2 +1,76 @@
-from .aggregator import CSVAggregator  # noqa: F401
-from .orchestrator import Orchestrator  # noqa: F401
+import importlib as _importlib
+import pkgutil as _pkgutil
+import time as _time
+
+
+class OrchestratorError(Exception):
+    pass
+
+
+class Orchestrator:
+    """
+    A scheduler for parallel execution on multiple resources (machines/systems).
+
+    TODO: more description
+    """
+
+    def serve_once(self):
+        """
+        Run the orchestration logic, processing any outstanding requests
+        (for provisioning, new test execution, etc.) and returning once these
+        are taken care of.
+
+        Returns True to indicate that it should be called again by the user
+        (more work to be done), False once all testing is concluded.
+        """
+        raise NotImplementedError(f"'serve_once' not implemented for {self.__class__.__name__}")
+
+    def serve_forever(self):
+        """
+        Run the orchestration logic, blocking until all testing is concluded.
+        """
+        while self.serve_once():
+            _time.sleep(1)
+
+    def start(self):
+        """
+        Start the Orchestrator instance, opening any files / allocating
+        resources as necessary.
+        """
+        raise NotImplementedError(f"'start' not implemented for {self.__class__.__name__}")
+
+    def stop(self):
+        """
+        Stop the Orchestrator instance, freeing all allocated resources.
+        """
+        raise NotImplementedError(f"'stop' not implemented for {self.__class__.__name__}")
+
+    def __enter__(self):
+        try:
+            self.start()
+            return self
+        except Exception:
+            self.stop()
+            raise
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.stop()
+
+
+_submodules = [
+    info.name for info in _pkgutil.iter_modules(__spec__.submodule_search_locations)
+]
+
+__all__ = [*_submodules, Orchestrator.__name__]  # noqa: PLE0604
+
+
+def __dir__():
+    return __all__
+
+
+# lazily import submodules
+def __getattr__(attr):
+    if attr in _submodules:
+        return _importlib.import_module(f".{attr}", __name__)
+    else:
+        raise AttributeError(f"module '{__name__}' has no attribute '{attr}'")