atex 0.5__py3-none-any.whl → 0.8__py3-none-any.whl

atex/executor/testcontrol.py

@@ -0,0 +1,353 @@
+import os
+import collections
+import json
+
+from .. import util
+
+
+class BufferFullError(Exception):
+    pass
+
+
+class NonblockLineReader:
+    """
+    Kind of like io.BufferedReader but capable of reading from non-blocking
+    sources (both O_NONBLOCK sockets and os.set_blocking(False) descriptors),
+    re-assembling full lines from (potentially) multiple read() calls.
+
+    It also takes a file descriptor (not a file-like object) and takes extra
+    care to read one-byte-at-a-time to not read (and buffer) more data from the
+    source descriptor, allowing it to be used for in-kernel move, such as via
+    os.sendfile() or os.splice().
+    """
+
+    def __init__(self, src, maxlen=4096):
+        """
+        'src' is an opened file descriptor (integer).
+
+        'maxlen' is a maximum potential line length, incl. the newline
+        character - if reached, a BufferFullError is raised.
+        """
+        self.src = src
+        self.eof = False
+        self.buffer = bytearray(maxlen)
+        self.bytes_read = 0
+
+    def readline(self):
+        r"""
+        Read a line and return it, without the '\n' terminating character,
+        clearing the internal buffer upon return.
+
+        Returns None if nothing could be read (BlockingIOError) or if EOF
+        was reached.
+        """
+        while self.bytes_read < len(self.buffer):
+            try:
+                data = os.read(self.src, 1)
+            except BlockingIOError:
+                return None
+
+            # stream EOF
+            if len(data) == 0:
+                self.eof = True
+                return None
+
+            char = data[0]
+
+            if char == 0x0a:  # \n
+                line = self.buffer[0:self.bytes_read]
+                self.bytes_read = 0
+                return line
+            else:
+                self.buffer[self.bytes_read] = char
+                self.bytes_read += 1
+
+        raise BufferFullError(f"line buffer reached {len(self.buffer)} bytes")
+
+    def clear(self):
+        """
+        Clear the internal buffer, clearing any partially-read line data.
+        """
+        self.bytes_read = 0
+
+
+class BadControlError(Exception):
+    """
+    Raised by TestControl when abnormalities are detected in the control stream,
+    such as invalid syntax, unknown control word, or bad or unexpected data for
+    any given control word.
+    """
+    pass
+
+
+class BadReportJSONError(BadControlError):
+    """
+    Raised on a syntactical or semantical error caused by the test not following
+    the TEST_CONROL.md specification when passing JSON data to the 'result'
+    control word.
+    """
+    pass
+
+
+class TestControl:
+    """
+    An implementation of the protocol described by TEST_CONTROL.md,
+    processing test-issued commands, results and uploaded files.
+    """
+
+    def __init__(self, *, control_fd, reporter, duration, testout_fd):
+        """
+        'control_fd' is a non-blocking file descriptor to be read.
+
+        'reporter' is an instance of class Reporter all the results
+        and uploaded files will be written to.
+
+        'duration' is a class Duration instance.
+
+        'testout_fd' is an optional file descriptor handle which the test uses
+        to write its output to - useful here for the 'result' control word and
+        its protocol, which allows "hardlinking" the fd to a real file name.
+        """
+        self.control_fd = control_fd
+        self.stream = NonblockLineReader(control_fd)
+        self.reporter = reporter
+        self.duration = duration
+        self.testout_fd = testout_fd
+        self.eof = False
+        self.in_progress = None
+        self.partial_results = collections.defaultdict(dict)
+        self.exit_code = None
+        self.reconnect = None
+        self.nameless_result_seen = False
+
+    def process(self):
+        """
+        Read from the control file descriptor and potentially perform any
+        appropriate action based on commands read from the test.
+
+        Returns True if there is more data expected, False otherwise
+        (when the control file descriptor reached EOF).
+        """
+        # if a parser operation is in progress, continue calling it,
+        # avoid reading a control line
+        if self.in_progress:
+            try:
+                next(self.in_progress)
+                return
+            except StopIteration:
+                # parser is done, continue on to a control line
+                self.in_progress = None
+
+        try:
+            line = self.stream.readline()
+        except BufferFullError as e:
+            raise BadControlError(str(e)) from None
+
+        util.debug(f"got control line: {line}")
+
+        if self.stream.eof:
+            self.eof = True
+            return
+        # partial read or BlockingIOError, try next time
+        if line is None:
+            return
+        elif len(line) == 0:
+            raise BadControlError(r"empty control line (just '\n')")
+
+        line = line.decode()
+        word, _, arg = line.partition(" ")
+
+        if word == "result":
+            parser = self._parser_result(arg)
+        elif word == "duration":
+            parser = self._parser_duration(arg)
+        elif word == "exitcode":
+            parser = self._parser_exitcode(arg)
+        elif word == "reconnect":
+            parser = self._parser_reconnect(arg)
+        else:
+            raise BadControlError(f"unknown control word: {word}")
+
+        try:
+            next(parser)
+            # parser not done parsing, run it next time we're called
+            self.in_progress = parser
+        except StopIteration:
+            pass
+
+    @classmethod
+    def _merge(cls, dst, src):
+        """
+        Merge a 'src' dict into 'dst', using the rules described by
+        TEST_CONTROL.md for 'Partial results'.
+        """
+        for key, value in src.items():
+            # delete existing if new value is None (JSON null)
+            if value is None and key in dst:
+                del dst[key]
+                continue
+            # add new key
+            elif key not in dst:
+                dst[key] = value
+                continue
+
+            orig_value = dst[key]
+            # different type - replace
+            if type(value) is not type(orig_value):
+                dst[key] = value
+                continue
+
+            # nested dict, merge it recursively
+            if isinstance(value, dict):
+                cls._merge(orig_value, value)
+            # extensible list-like iterable, extend it
+            elif isinstance(value, (tuple, list)):
+                orig_value += value
+            # overridable types, doesn't make sense to extend them
+            elif isinstance(value, (str, int, float, bool, bytes, bytearray)):
+                dst[key] = value
+            # set-like, needs unioning
+            elif isinstance(value, set):
+                orig_value.update(value)
+            else:
+                raise BadReportJSONError(f"cannot merge type {type(value)}")
+
+    def _parser_result(self, arg):
+        try:
+            json_length = int(arg)
+        except ValueError as e:
+            raise BadControlError(f"reading json length: {str(e)}") from None
+
+        # read the full JSON
+        json_data = bytearray()
+        while json_length > 0:
+            try:
+                chunk = os.read(self.control_fd, json_length)
+            except BlockingIOError:
+                yield
+                continue
+            if chunk == b"":
+                raise BadControlError("EOF when reading data")
+            json_data += chunk
+            json_length -= len(chunk)
+            yield
+
+        # convert to native python dict
+        try:
+            result = json.loads(json_data)
+        except json.decoder.JSONDecodeError as e:
+            raise BadReportJSONError(f"JSON decode: {str(e)} caused by: {json_data}") from None
+
+        # note that this may be None (result for the test itself)
+        name = result.get("name")
+        if not name:
+            self.nameless_result_seen = True
+
+        # upload files
+        for entry in result.get("files", ()):
+            file_name = entry.get("name")
+            file_length = entry.get("length")
+            if not file_name or file_length is None:
+                raise BadReportJSONError(f"file entry missing 'name' or 'length': {entry}")
+            try:
+                file_length = int(file_length)
+            except ValueError as e:
+                raise BadReportJSONError(f"file entry {file_name} length: {str(e)}") from None
+
+            with self.reporter.open_tmpfile() as fd:
+                while file_length > 0:
+                    try:
+                        # try a more universal sendfile first, fall back to splice
+                        try:
+                            written = os.sendfile(fd, self.control_fd, None, file_length)
+                        except OSError as e:
+                            if e.errno == 22:  # EINVAL
+                                written = os.splice(self.control_fd, fd, file_length)
+                            else:
+                                raise
+                    except BlockingIOError:
+                        yield
+                        continue
+                    if written == 0:
+                        raise BadControlError("EOF when reading data")
+                    file_length -= written
+                    yield
+                try:
+                    self.reporter.link_tmpfile_to(fd, file_name, name)
+                except FileExistsError:
+                    raise BadReportJSONError(f"file '{file_name}' already exists") from None
+
+        # either store partial result + return,
+        # or load previous partial result and merge into it
+        partial = result.get("partial", False)
+        if partial:
+            # do not store the 'partial' key in the result
+            del result["partial"]
+            # note that nameless result will get None as dict key,
+            # which is perfectly fine
+            self._merge(self.partial_results[name], result)
+            # partial = do nothing
+            return
+
+        # if previously-stored partial result exist, merge the current one
+        # into it, but then use the merged result
+        # - avoid .get() or __getitem__() on defaultdict, it would create
+        #   a new key with an empty value if there was no partial result
+        if name in self.partial_results:
+            partial_result = self.partial_results[name]
+            del self.partial_results[name]
+            self._merge(partial_result, result)
+            result = partial_result
+
+        if "testout" in result:
+            testout = result.get("testout")
+            if not testout:
+                raise BadReportJSONError("'testout' specified, but empty")
+            try:
+                self.reporter.link_tmpfile_to(self.testout_fd, testout, name)
+            except FileExistsError:
+                raise BadReportJSONError(f"file '{testout}' already exists") from None
+
+        self.reporter.report(result)
+
+    def _parser_duration(self, arg):
+        if not arg:
+            raise BadControlError("duration argument empty")
+        # increment/decrement
+        if arg[0] == "+":
+            self.duration.increment(arg[1:])
+        elif arg[0] == "-":
+            self.duration.decrement(arg[1:])
+        # save/restore
+        elif arg == "save":
+            self.duration.save()
+        elif arg == "restore":
+            self.duration.restore()
+        else:
+            self.duration.set(arg)
+        # pretend to be a generator
+        if False:
+            yield
+
+    def _parser_exitcode(self, arg):
+        if not arg:
+            raise BadControlError("exitcode argument empty")
+        try:
+            code = int(arg)
+        except ValueError:
+            raise BadControlError(f"'{arg}' is not an integer exit code") from None
+        self.exit_code = code
+        # pretend to be a generator
+        if False:
+            yield
+
+    def _parser_reconnect(self, arg):
+        if not arg:
+            self.reconnect = "once"
+        elif arg == "always":
+            self.reconnect = "always"
+        else:
+            raise BadControlError(f"unknown reconnect arg: {arg}")
+        # pretend to be a generator
+        if False:
+            yield

atex/fmf.py

@@ -0,0 +1,217 @@
+import re
+from pathlib import Path
+
+# from system-wide sys.path
+import fmf
+
+
+def listlike(data, key):
+    """
+    Get a piece of fmf metadata as an iterable regardless of whether it was
+    defined as a dict or a list.
+
+    This is needed because many fmf metadata keys can be used either as
+        some_key: 123
+    or as lists via YAML syntax
+        some_key:
+          - 123
+          - 456
+    and, for simplicity, we want to always deal with lists (iterables).
+    """
+    if value := data.get(key):
+        return value if isinstance(value, list) else (value,)
+    else:
+        return ()
+
+
+class FMFTests:
+    """
+    FMF test metadata parsed from on-disk metadata using a specific plan name,
+    with all metadata dictionaries for all nodes being adjusted by that plan
+    and (optionally) a specified context.
+    """
+    # TODO: usage example ^^^^
+
+    def __init__(self, fmf_tree, plan_name, context=None):
+        """
+        'fmf_tree' is filesystem path somewhere inside fmf metadata tree,
+        or a root fmf.Tree instance.
+
+        'plan_name' is fmf identifier (like /some/thing) of a tmt plan
+        to use for discovering tests.
+
+        'context' is a dict like {'distro': 'rhel-9.6'} used for filtering
+        discovered tests.
+        """
+        # list of packages to install, as extracted from plan
+        self.prepare_pkgs = []
+        # list of scripts to run, as extracted from plan
+        self.prepare_scripts = []
+        # dict of environment, as extracted from plan
+        self.plan_env = {}
+        # dict indexed by test name, value is dict with fmf-parsed metadata
+        self.tests = {}
+        # dict indexed by test name, value is pathlib.Path of relative path
+        # of the fmf metadata root towards the test metadata location
+        self.test_dirs = {}
+        # fmf.Context instance, as used for test discovery
+        self.context = fmf.Context(**context) if context else fmf.Context()
+
+        tree = fmf_tree.copy() if isinstance(fmf_tree, fmf.Tree) else fmf.Tree(fmf_tree)
+        tree.adjust(context=self.context)
+
+        # Path of the metadata root
+        self.root = Path(tree.root)
+
+        # lookup the plan first
+        plan = tree.find(plan_name)
+        if not plan:
+            raise ValueError(f"plan {plan_name} not found in {tree.root}")
+        if "test" in plan.data:
+            raise ValueError(f"plan {plan_name} appears to be a test")
+
+        # gather and merge plan-defined environment variables
+        #
+        # environment:
+        #  - FOO: BAR
+        #    BAR: BAZ
+        for entry in listlike(plan.data, "environment"):
+            self.plan_env.update(entry)
+
+        # gather all prepare scripts / packages
+        #
+        # prepare:
+        #   - how: install
+        #     package:
+        #       - some-rpm-name
+        #   - how: shell
+        #     script:
+        #       - some-command
+        for entry in listlike(plan.data, "prepare"):
+            if "how" not in entry:
+                continue
+            if entry["how"] == "install":
+                self.prepare_pkgs += listlike(entry, "package")
+            elif entry["how"] == "shell":
+                self.prepare_scripts += listlike(entry, "script")
+
+        # gather all tests selected by the plan
+        #
+        # discover:
+        #   - how: fmf
+        #     filter:
+        #       - tag:some_tag
+        #     test:
+        #       - some-test-regex
+        #     exclude:
+        #       - some-test-regex
+        if "discover" in plan.data:
+            discover = plan.data["discover"]
+            if not isinstance(discover, list):
+                discover = (discover,)
+
+            for entry in discover:
+                if entry.get("how") != "fmf":
+                    continue
+
+                filtering = {}
+                for meta_name in ("filter", "test", "exclude"):
+                    if value := listlike(entry, meta_name):
+                        filtering[meta_name] = value
+
+                children = tree.prune(
+                    names=filtering.get("test"),
+                    filters=filtering.get("filter"),
+                )
+                for child in children:
+                    # excludes not supported by .prune(), we have to do it here
+                    excludes = filtering.get("exclude")
+                    if excludes and any(re.match(x, child.name) for x in excludes):
+                        continue
+                    # only enabled tests
+                    if "enabled" in child.data and not child.data["enabled"]:
+                        continue
+                    # no manual tests and no stories
+                    if child.data.get("manual") or child.data.get("story"):
+                        continue
+                    # after adjusting above, any adjusts are useless, free some space
+                    if "adjust" in child.data:
+                        del child.data["adjust"]
+
+                    self.tests[child.name] = child.data
+                    # child.sources ie. ['/abs/path/to/some.fmf', '/abs/path/to/some/node.fmf']
+                    self.test_dirs[child.name] = \
+                        Path(child.sources[-1]).parent.relative_to(self.root)
+
+    def match(self, regex):
+        """
+        Yield test names that match 'regex', simulating how tmt discovers tests.
+        """
+        yield from (name for name in self.tests if re.match(regex, name))
+
+
+def test_pkg_requires(data, key="require"):
+    """
+    Yield RPM package names specified by test 'data' (fmf metadata dict)
+    in the metadata 'key' (require or recommend), ignoring any non-RPM-package
+    requires/recommends.
+    """
+    for entry in listlike(data, key):
+        # skip type:library and type:path
+        if not isinstance(entry, str):
+            continue
+        # skip "fake RPMs" that begin with 'library('
+        if entry.startswith("library("):
+            continue
+        yield entry
+
+
+def all_pkg_requires(fmf_tests, key="require"):
+    """
+    Yield RPM package names from the plan and all tests discovered by
+    a class FMFTests instance 'fmf_tests', ignoring any non-RPM-package
+    requires/recommends.
+    """
+    # use a set to avoid duplicates
+    pkgs = set()
+    pkgs.update(fmf_tests.prepare_pkgs)
+    for data in fmf_tests.tests.values():
+        pkgs.update(test_pkg_requires(data, key))
+    yield from pkgs
+
+
+# Some extra notes for fmf.prune() arguments:
+#
+# Set 'names' to filter by a list of fmf node names, ie.
+#     ['/some/test', '/another/test']
+#
+# Set 'filters' to filter by a list of fmf-style filter expressions, see
+#     https://fmf.readthedocs.io/en/stable/modules.html#fmf.filter
+#
+# Set 'conditions' to filter by a list of python expressions whose namespace
+# locals() are set up to be a dictionary of the tree. When any of the
+# expressions returns True, the tree is returned, ie.
+#     ['environment["FOO"] == "BAR"']
+#     ['"enabled" not in locals() or enabled']
+# Note that KeyError is silently ignored and treated as False.
+#
+# Set 'context' to a dictionary to post-process the tree metadata with
+# adjust expressions (that may be present in a tree) using the specified
+# context. Any other filters are applied afterwards to allow modification
+# of tree metadata by the adjust expressions. Ie.
+#     {'distro': 'rhel-9.6.0', 'arch': 'x86_64'}
+
+#Platform = collections.namedtuple("Platform", ["distro", "arch"])
+#
+#
+#def combine_platforms(fmf_path, plan_name, platforms):
+#    # TODO: document
+#    fmf_tests = {}
+#    tree = fmf.Tree(fmf_path)
+#    for platform in platforms:
+#        context = {"distro": platform.distro, "arch": platform.arch}
+#        fmf_tests[platform] = FMFTests(tree, plan_name, context=context)
+#    return fmf_tests
+
+# TODO: in Orchestrator, when a Provisioner becomes free, have it pick a test
+#       from the appropriate tests[platform] per the Provisioner's platform

atex/orchestrator/__init__.py

@@ -0,0 +1,2 @@
+from .aggregator import CSVAggregator  # noqa: F401
+from .orchestrator import Orchestrator  # noqa: F401

atex/orchestrator/aggregator.py

@@ -0,0 +1,106 @@
+import csv
+import gzip
+import json
+import shutil
+import threading
+from pathlib import Path
+
+
+class CSVAggregator:
+    """
+    Collects reported results as a GZIP-ed CSV and files (logs) from multiple
+    test runs under a shared directory.
+    """
+
+    class _ExcelWithUnixNewline(csv.excel):
+        lineterminator = "\n"
+
+    def __init__(self, csv_file, storage_dir):
+        """
+        'csv_file' is a string/Path to a .csv.gz file with aggregated results.
+
+        'storage_dir' is a string/Path of the top-level parent for all
+        per-platform / per-test files uploaded by tests.
+        """
+        self.lock = threading.RLock()
+        self.storage_dir = Path(storage_dir)
+        self.csv_file = Path(csv_file)
+        self.csv_writer = None
+        self.results_gzip_handle = None
+
+    def open(self):
+        if self.csv_file.exists():
+            raise FileExistsError(f"{self.csv_file} already exists")
+        f = gzip.open(self.csv_file, "wt", newline="")
+        try:
+            self.csv_writer = csv.writer(f, dialect=self._ExcelWithUnixNewline)
+        except:
+            f.close()
+            raise
+        self.results_gzip_handle = f
+
+        if self.storage_dir.exists():
+            raise FileExistsError(f"{self.storage_dir} already exists")
+        self.storage_dir.mkdir()
+
+    def close(self):
+        self.results_gzip_handle.close()
+        self.results_gzip_handle = None
+        self.csv_writer = None
+
+    def __enter__(self):
+        self.open()
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.close()
+
+    def ingest(self, platform, test_name, json_file, files_dir):
+        """
+        Process 'json_file' (string/Path) for reported results and append them
+        to the overall aggregated CSV file, recursively copying over the dir
+        structure under 'files_dir' (string/Path) under the respective platform
+        and test name in the aggregated files storage dir.
+        """
+        # parse the JSON separately, before writing any CSV lines, to ensure
+        # that either all results from the test are ingested, or none at all
+        # (if one of the lines contains JSON errors)
+        csv_lines = []
+        with open(json_file) as json_fobj:
+            for raw_line in json_fobj:
+                result_line = json.loads(raw_line)
+
+                result_name = result_line.get("name")
+                if result_name:
+                    # sub-result; prefix test name
+                    result_name = f"{test_name}/{result_name}"
+                else:
+                    # result for test itself; use test name
+                    result_name = test_name
+
+                file_names = []
+                if "testout" in result_line:
+                    file_names.append(result_line["testout"])
+                if "files" in result_line:
+                    file_names += (f["name"] for f in result_line["files"])
+
+                csv_lines.append((
+                    platform,
+                    result_line["status"],
+                    result_name,
+                    result_line.get("note", ""),
+                    *file_names,
+                ))
+
+        with self.lock:
+            self.csv_writer.writerows(csv_lines)
+            self.results_gzip_handle.flush()
+
+        Path(json_file).unlink()
+
+        platform_dir = self.storage_dir / platform
+        platform_dir.mkdir(exist_ok=True)
+        test_dir = platform_dir / test_name.lstrip("/")
+        if test_dir.exists():
+            raise FileExistsError(f"{test_dir} already exists for {test_name}")
+        shutil.move(files_dir, test_dir, copy_function=shutil.copy)