atex 0.5__py3-none-any.whl → 0.8__py3-none-any.whl

atex/cli/minitmt.py

@@ -1,82 +0,0 @@
-import re
-import pprint
-
-#from .. import util
-from ..minitmt import fmf
-
-
-def _get_context(args):
-    context = {}
-    if args.context:
-        for c in args.context:
-            key, value = c.split('=', 1)
-            context[key] = value
-    return context or None
-
-
-def discover(args):
-    result = fmf.FMFData(args.root, args.plan, context=_get_context(args))
-    for test in result.tests:
-        print(test.name)
-
-
-def show(args):
-    result = fmf.FMFData(args.root, args.plan, context=_get_context(args))
-    for test in result.tests:
-        if re.match(args.test, test.name):
-            pprint.pprint(test.data)
-            break
-    else:
-        print(f"Not reachable via {args.plan} discovery: {args.test}")
-        raise SystemExit(1)
-
-
-def parse_args(parser):
-    parser.add_argument('--root', default='.', help="path to directory with fmf tests")
-    parser.add_argument('--context', '-c', help="tmt style key=value context", action='append')
-    cmds = parser.add_subparsers(
-        dest='_cmd', help="minitmt feature", metavar='<cmd>', required=True,
-    )
-
-    cmd = cmds.add_parser(
-        'discover', aliases=('di',),
-        help="list tests, post-processed by tmt plans",
-    )
-    cmd.add_argument('plan', help="tmt plan to use for discovery")
-
-    cmd = cmds.add_parser(
-        'show',
-        help="show fmf data of a test",
-    )
-    cmd.add_argument('plan', help="tmt plan to use for discovery")
-    cmd.add_argument('test', help="fmf style test regex")
-
-    cmd = cmds.add_parser(
-        'execute', aliases=('ex',),
-        help="run a plan (or test) on a remote system",
-    )
-    grp = cmd.add_mutually_exclusive_group()
-    grp.add_argument('--test', '-t', help="fmf style test regex")
-    grp.add_argument('--plan', '-p', help="tmt plan name (path) inside metadata root")
-    cmd.add_argument('--ssh-identity', '-i', help="path to a ssh keyfile for login")
-    cmd.add_argument('user_host', help="ssh style user@host of the remote")
-
-
-def main(args):
-    if args._cmd in ('discover', 'di'):
-        discover(args)
-    elif args._cmd == 'show':
-        show(args)
-    elif args._cmd in ('execute', 'ex'):
-        #execute(args)
-        print("not implemented yet")
-    else:
-        raise RuntimeError(f"unknown args: {args}")
-
-
-CLI_SPEC = {
-    'aliases': ('tmt',),
-    'help': "simple test executor using atex.minitmt",
-    'args': parse_args,
-    'main': main,
-}

atex/minitmt/__init__.py

@@ -1,115 +0,0 @@
-import os
-import random
-
-from pathlib import Path
-
-# TODO: TMT_PLAN_ENVIRONMENT_FILE
-
-# TODO: install rsync on the guest as part of setup
-
-# TODO: in Orchestrator, when a Provisioner becomes free, have it pick a test
-#       from the appropriate tests[platform] per the Provisioner's platform
-
-
-def _random_string(length):
-    return ''.join(
-        random.choices('abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789', k=length),
-    )
-
-
-class Preparator:
-    """
-    Set of utilities for preparing a newly acquired/reserved machine for
-    running tests, by installing global package requirements, copying all
-    tests over, executing tmt plan 'prepare' step, etc.
-    """
-    def __init__(self, ssh_conn):
-        self.conn = ssh_conn
-
-    def copy_tests(self):
-        pass
-
-    def run_prepare_scripts(self):
-        pass
-
-    def __enter__(self):
-        self.conn.connect()
-        return self
-
-    def __exit__(self, exc_type, exc_value, traceback):
-        self.conn.disconnect()
-
-
-# TODO: have Executor take a finished Preparator instance as input?
-#        - for extracting copied tests location
-#        - for extracting TMT_PLAN_ENVIRONMENT_FILE location
-#        - etc.
-
-
-class Executor:
-    """
-    Helper for running one test on a remote system and processing results
-    and uploaded files by that test.
-    """
-    def __init__(self, fmf_test, ssh_conn):
-        self.fmf_test = fmf_test
-        self.conn = ssh_conn
-        self.remote_socket = self.local_socket = None
-
-    def __enter__(self):
-        # generate a (hopefully) unique test control socket name
-        # and modify the SSHConn instance to use it
-        rand_name = f'atex-control-{_random_string(50)}.sock'
-        self.local_socket = Path(os.environ.get('TMPDIR', '/tmp')) / rand_name
-        self.remote_socket = f'/tmp/{rand_name}'
-        self.conn.options['RemoteForward'] = f'{self.remote_socket} {self.local_socket}'
-        self.conn.connect()
-        return self
-
-    def __exit__(self, exc_type, exc_value, traceback):
-        self.conn.ssh(f'rm -f {self.remote_socket}')
-        self.local_socket.unlink()
-        self.remote_socket = self.local_socket = None
-        self.conn.disconnect()
-
-    # execute all prepares (how:install and how:shell) via ssh
-    def prepare(self):
-        # TODO: check via __some_attr (named / prefixed after our class)
-        #       whether this reserved system has been prepared already ... ?
-        #        ^^^^ in Orchestrator
-        #
-        # TODO: copy root of fmf metadata to some /var/tmp/somedir to run tests from
-        #
-        # TODO: move prepare out, possibly to class-less function,
-        #       we don't want it running over an SSHConn that would set up socket forwarding
-        #       only to tear it back down, when executed from Orchestrator for setup only
-        #
-        # TODO: install rsync
-        pass
-
-    def run_script(self, script, duration=None, shell='/bin/bash', **kwargs):
-        self.conn.ssh(shell, input=script.encode())
-
-    # run one test via ssh and parse its results on-the-fly,
-    # write out logs
-    def run_test(self, fmf_test, reporter):
-        # TODO: pass environment from test fmf metadata
-        # TODO: watch for test duration, etc. metadata
-        # TODO: logging of stdout+stderr to hidden file, doing 'ln' from it to
-        #       test-named 'testout' files
-        #       - generate hidden name suffix via:
-        #         ''.join(random.choices('abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789', k=20))
-        output_logfile = \
-            reporter.files_dir(fmf_test.name) / f'.test_output_{self._random_string(50)}.log'
-        output_logfile = os.open(reporter.files_dir(fmf_test.name), os.O_WRONLY | os.O_CREAT)
-        try:
-            #self.conn.ssh(
-            pass
-        finally:
-            os.close(output_logfile)
-        # TODO: create temp dir on remote via 'mktemp -d', then call
-        #       self.conn.add_remote_forward(...) with socket path inside that tmpdir
-
-# TODO: run tests by passing stdout/stderr via pre-opened fd so we don't handle it in code
-
-# TODO: read unix socket as nonblocking, check test subprocess.Popen proc status every 0.1sec

atex/minitmt/fmf.py

@@ -1,168 +0,0 @@
-import re
-import collections
-from pathlib import Path
-
-# from system-wide sys.path
-import fmf
-
-# name: fmf path to the test as string, ie. /some/test
-# data: dict of the parsed fmf metadata (ie. {'tag': ... , 'environment': ...})
-# dir:  relative pathlib.Path of the test .fmf to repo root, ie. some/test
-#       (may be different from name for "virtual" tests that share the same dir)
-FMFTest = collections.namedtuple('FMFTest', ['name', 'data', 'dir'])
-
-
-class FMFData:
-    """
-    Helper class for reading and querying fmf metadata from the filesystem.
-    """
-    # TODO: usage example ^^^^
-
-    @staticmethod
-    def _listlike(data, key):
-        """
-        Get a piece of fmf metadata as an iterable regardless of whether it was
-        defined as a dict or a list.
-
-        This is needed because many fmf metadata keys can be used either as
-            some_key: 123
-        or as lists via YAML syntax
-            some_key:
-              - 123
-              - 456
-        and, for simplicity, we want to always deal with lists (iterables).
-        """
-        if value := data.get(key):
-            return value if isinstance(value, list) else (value,)
-        else:
-            return ()
-
-    def __init__(self, fmf_tree, plan_name, context=None):
-        """
-        'fmf_tree' is filesystem path somewhere inside fmf metadata tree,
-        or a root fmf.Tree instance.
-
-        'plan_name' is fmf identifier (like /some/thing) of a tmt plan
-        to use for discovering tests.
-
-        'context' is a dict like {'distro': 'rhel-9.6'} used for filtering
-        discovered tests.
-        """
-        self.prepare_pkgs = []
-        self.prepare_scripts = []
-        self.tests = []
-
-        tree = fmf_tree.copy() if isinstance(fmf_tree, fmf.Tree) else fmf.Tree(fmf_tree)
-        ctx = fmf.Context(**context) if context else fmf.Context()
-        tree.adjust(context=ctx)
-
-        self.fmf_root = tree.root
-
-        # lookup the plan first
-        plan = tree.find(plan_name)
-        if not plan:
-            raise ValueError(f"plan {plan_name} not found in {tree.root}")
-        if 'test' in plan.data:
-            raise ValueError(f"plan {plan_name} appears to be a test")
-
-        # gather all prepare scripts / packages
-        #
-        # prepare:
-        #   - how: install
-        #     package:
-        #       - some-rpm-name
-        #   - how: shell
-        #     script:
-        #       - some-command
-        for entry in self._listlike(plan.data, 'prepare'):
-            if 'how' not in entry:
-                continue
-            if entry['how'] == 'install':
-                self.prepare_pkgs += self._listlike(entry, 'package')
-            elif entry['how'] == 'shell':
-                self.prepare_scripts += self._listlike(entry, 'script')
-
-        # gather all tests selected by the plan
-        #
-        # discover:
-        #   - how: fmf
-        #     filter:
-        #       - tag:some_tag
-        #     test:
-        #       - some-test-regex
-        #     exclude:
-        #       - some-test-regex
-        if 'discover' in plan.data:
-            discover = plan.data['discover']
-            if not isinstance(discover, list):
-                discover = (discover,)
-
-            for entry in discover:
-                if entry.get('how') != 'fmf':
-                    continue
-
-                filtering = {}
-                for meta_name in ('filter', 'test', 'exclude'):
-                    if value := self._listlike(entry, meta_name):
-                        filtering[meta_name] = value
-
-                children = tree.prune(
-                    names=filtering.get('test'),
-                    filters=filtering.get('filter'),
-                )
-                for child in children:
-                    # excludes not supported by .prune(), we have to do it here
-                    excludes = filtering.get('exclude')
-                    if excludes and any(re.match(x, child.name) for x in excludes):
-                        continue
-                    # only enabled tests
-                    if 'enabled' in child.data and not child.data['enabled']:
-                        continue
-                    # no manual tests
-                    if child.data.get('manual'):
-                        continue
-                    # after adjusting above, any adjusts are useless, free some space
-                    if 'adjust' in child.data:
-                        del child.data['adjust']
-                    # ie. ['/abs/path/to/some.fmf', '/abs/path/to/some/node.fmf']
-                    source_dir = Path(child.sources[-1]).parent.relative_to(self.fmf_root)
-                    self.tests.append(
-                        FMFTest(name=child.name, data=child.data, dir=source_dir),
-                    )
-
-
-# Some extra notes for fmf.prune() arguments:
-#
-# Set 'names' to filter by a list of fmf node names, ie.
-#     ['/some/test', '/another/test']
-#
-# Set 'filters' to filter by a list of fmf-style filter expressions, see
-#     https://fmf.readthedocs.io/en/stable/modules.html#fmf.filter
-#
-# Set 'conditions' to filter by a list of python expressions whose namespace
-# locals() are set up to be a dictionary of the tree. When any of the
-# expressions returns True, the tree is returned, ie.
-#     ['environment["FOO"] == "BAR"']
-#     ['"enabled" not in locals() or enabled']
-# Note that KeyError is silently ignored and treated as False.
-#
-# Set 'context' to a dictionary to post-process the tree metadata with
-# adjust expressions (that may be present in a tree) using the specified
-# context. Any other filters are applied afterwards to allow modification
-# of tree metadata by the adjust expressions. Ie.
-#     {'distro': 'rhel-9.6.0', 'arch': 'x86_64'}
-
-Platform = collections.namedtuple('Platform', ['distro', 'arch'])
-
-
-def combine_platforms(fmf_path, plan_name, platforms):
-    # TODO: document
-    fmf_datas = {}
-    tree = fmf.Tree(fmf_path)
-    for platform in platforms:
-        context = {'distro': platform.distro, 'arch': platform.arch}
-        fmf_datas[platform] = FMFData(tree, plan_name, context=context)
-    return fmf_datas
-
-# TODO: in Orchestrator, when a Provisioner becomes free, have it pick a test
-#       from the appropriate tests[platform] per the Provisioner's platform

atex/minitmt/report.py

@@ -1,174 +0,0 @@
-import os
-import csv
-import gzip
-import ctypes
-import ctypes.util
-import contextlib
-from pathlib import Path
-
-from .. import util
-
-
-libc = ctypes.CDLL(ctypes.util.find_library('c'))
-
-# int linkat(int olddirfd, const char *oldpath, int newdirfd, const char *newpath, int flags)
-libc.linkat.argtypes = (
-    ctypes.c_int,
-    ctypes.c_char_p,
-    ctypes.c_int,
-    ctypes.c_char_p,
-    ctypes.c_int,
-)
-libc.linkat.restype = ctypes.c_int
-
-# fcntl.h:#define AT_EMPTY_PATH                0x1000  /* Allow empty relative pathname */
-AT_EMPTY_PATH = 0x1000
-
-# fcntl.h:#define AT_FDCWD             -100    /* Special value used to indicate
-AT_FDCWD = -100
-
-
-def linkat(*args):
-    if (ret := libc.linkat(*args)) == -1:
-        errno = ctypes.get_errno()
-        raise OSError(errno, os.strerror(errno))
-    return ret
-
-
-class CSVReporter(util.LockableClass):
-    """
-    Stores reported results as a GZIP-ed CSV and files (logs) under a related
-    directory.
-
-        with CSVReporter('file/storage/dir', 'results.csv.gz') as reporter:
-            sub = reporter.make_subreporter('rhel-9', 'x86_64')
-            sub({'name': '/some/test', 'status': 'pass'})
-            sub({'name': '/another/test', 'status': 'pass'})
-            ...
-            sub = reporter.make_subreporter('rhel-9', 'ppc64le')
-            ...
-            sock = accept_unix_connection()
-            reporter.store_file('/some/test', 'debug.log', sock, 1234)
-    """
-    class _ExcelWithUnixNewline(csv.excel):
-        lineterminator = '\n'
-
-    def __init__(self, storage_dir, results_file):
-        super().__init__()
-        self.storage_dir = Path(storage_dir)
-        if self.storage_dir.exists():
-            raise FileExistsError(f"{storage_dir} already exists")
-        self.results_file = Path(results_file)
-        if self.results_file.exists():
-            raise FileExistsError(f"{self.results_file} already exists")
-        self.storage_dir.mkdir()
-        self.csv_writer = None
-        self.results_gzip_handle = None
-
-    def __enter__(self):
-        f = gzip.open(self.results_file, 'wt', newline='')
-        try:
-            self.csv_writer = csv.writer(f, dialect=self._ExcelWithUnixNewline)
-        except:
-            f.close()
-            raise
-        self.results_gzip_handle = f
-        return self
-
-    def __exit__(self, exc_type, exc_value, traceback):
-        self.results_gzip_handle.close()
-        self.results_gzip_handle = None
-        self.csv_writer = None
-
-    def report(self, distro, arch, status, name, note, *files):
-        """
-        Persistently write out details of a test result.
-        """
-        with self.lock:
-            self.csv_writer.writerow((distro, arch, status, name, note, *files))
-
-    @staticmethod
-    def _normalize_path(path):
-        # the magic here is to treat any dangerous path as starting at /
-        # and resolve any weird constructs relative to /, and then simply
-        # strip off the leading / and use it as a relative path
-        path = path.lstrip('/')
-        path = os.path.normpath(f'/{path}')
-        return path[1:]
-
-    def make_subreporter(self, distro, arch):
-        """
-        Return a preconfigured reporter instance, suitable for use
-        by an Executor.
-        """
-        def reporter(result_line):
-            if 'files' in result_line:
-                files = (self._normalize_path(x['name']) for x in result_line['files'])
-            else:
-                files = ()
-            self.report(
-                distro, arch, result_line['status'], result_line['name'],
-                result_line.get('note', ''), *files,
-            )
-        return reporter
-
-    def _files_dir(self, result_name):
-        dir_path = self.storage_dir / result_name.lstrip('/')
-        dir_path.mkdir(parents=True, exist_ok=True)
-        return dir_path
-
-    def _files_file(self, result_name, file_name):
-        file_name = self._normalize_path(file_name)
-        return self._files_dir(result_name) / file_name
-
-    @contextlib.contextmanager
-    def open_tmpfile(self, open_mode=os.O_WRONLY):
-        flags = open_mode | os.O_TMPFILE
-        fd = os.open(self.storage_dir, flags, 0o644)
-        try:
-            yield fd
-        finally:
-            os.close(fd)
-#    def open_tmpfile(self, result_name, open_mode=os.O_WRONLY):
-#        """
-#        Open an anonymous (name-less) file for writing, in a directory relevant
-#        to 'result_name' and yield its file descriptor (int) as context, closing
-#        it when the context is exited.
-#        """
-#        flags = open_mode | os.O_TMPFILE
-#        fd = os.open(self._files_dir(result_name), flags, 0o644)
-#        try:
-#            yield fd
-#        finally:
-#            os.close(fd)
-
-    def link_tmpfile_to(self, result_name, file_name, fd):
-        """
-        Store a file named 'file_name' in a directory relevant to 'result_name'
-        whose 'fd' (a file descriptor) was created by open_tmpfile().
-
-        This function can be called multiple times with the same 'fd', and
-        does not close or otherwise alter the descriptor.
-        """
-        final_path = self._files_file(result_name, file_name)
-        linkat(fd, b'', AT_FDCWD, bytes(final_path), AT_EMPTY_PATH)
-
-    def store_file(self, result_name, file_name, in_fd, count):
-        """
-        Read 'count' bytes of binary data from an OS file descriptor 'in_fd'
-        and store them under 'result_name' as a file (or relative path)
-        named 'file_name', creating it.
-        """
-        final_path = self._files_file(result_name, file_name)
-        # be as efficient as possible, let the kernel handle big data
-        out_fd = None
-        try:
-            out_fd = os.open(final_path, os.O_WRONLY | os.O_CREAT)
-            while count > 0:
-                written = os.sendfile(out_fd, in_fd, None, count)
-                if written == 0:
-                    raise RuntimeError(f"got unexpected EOF when receiving {final_path}")
-                count -= written
-        finally:
-            if out_fd:
-                os.close(out_fd)

atex/minitmt/scripts.py

@@ -1,51 +0,0 @@
-#from .. import util
-
-#run_test = util.dedent(fr'''
-#    # create a temp dir for everything, send it to the controller
-#    tmpdir=$(mktemp -d /var/tmp/atex-XXXXXXXXX)
-#    echo "tmpdir=$tmpdir"
-#
-#    # remove transient files if interrupted
-#    trap "rm -rf \"$tmpdir\"" INT
-#
-#    # wait for result reporting unix socket to be created by sshd
-#    socket=$tmpdir/results.sock
-#    while [[ ! -e $socket ]]; do sleep 0.1; done
-#    echo "socket=$socket"
-#
-#    # tell the controller to start logging test output
-#    echo ---
-#
-#    # install test dependencies
-#    rpms=( {' '.join(requires)} )
-#    to_install=()
-#    for rpm in "${{rpms[@]}}"; do
-#        rpm -q --quiet "$rpm" || to_install+=("$rpm")
-#    done
-#    dnf -y --setopt=install_weak_deps=False install "${{to_install[@]}}"
-#
-#    # run the test
-#    ...
-#    rc=$?
-#
-#    # test finished, clean up
-#    rm -rf "$tmpdir"
-#
-#    exit $rc
-#''')
-
-# TODO: have another version of ^^^^ for re-execution of test after a reboot
-#       or disconnect that sets tmpdir= from us (reusing on-disk test CWD)
-#       rather than creating a new one
-#         - the second script needs to rm -f the unix socket before echoing
-#           something back to let us re-create it via a new ssh channel open
-#           because StreamLocalBindUnlink doesn't seem to work
-
-
-# TODO: call ssh with -oStreamLocalBindUnlink=yes to re-initialize
-#       the listening socket after guest reboot
-#
-#       -R /var/tmp/atex-BlaBla/results.sock:/var/tmp/controller.sock
-#
-#       (make sure to start listening on /var/tmp/controller.sock before
-#        calling ssh to run the test)

atex/minitmt/testme.py

@@ -1,3 +0,0 @@
-from ..util import debug
-
-debug("foo")

atex/orchestrator.py

@@ -1,38 +0,0 @@
-from . import util
-
-
-class Orchestrator:
-    """
-    A scheduler for parallel execution on multiple resources (machines/systems).
-
-    Given a list of Provisioner-derived class instances, it attempts to reserve
-    resources and uses them on-demand as they become available, calling run()
-    on each.
-
-    Note that run() and report() always run in a separate threads (are allowed
-    to block), and may access instance attributes, which are transparently
-    guarded by a thread-aware mutex.
-
-    """
-
-    def __init__(self):
-        pass
-        # TODO: configure via args, max workers, etc.
-
-#    def reserve(self, provisioner):
-#        # call provisioner.reserve(), return its return
-#        ...
-
-    def add_provisioner(self, provisioner):
-        # add to a self.* list of provisioners to be used for getting machines
-        ...
-
-    def run(self, provisioner):
-        # run tests, if destructive, call provisioner.release()
-        # returns anything
-        ...
-
-    def report(self):
-        # gets return from run
-        # writes it out to somewhere else
-        ...