atex 0.7__py3-none-any.whl → 0.8__py3-none-any.whl

atex/minitmt/executor.py

@@ -1,348 +0,0 @@
-import os
-import re
-import time
-import select
-import threading
-import contextlib
-import subprocess
-
-from .. import util
-from . import testcontrol, scripts, fmf
-
-
-class Duration:
-    """
-    A helper for parsing, keeping and manipulating test run time based on
-    FMF-defined 'duration' attribute.
-    """
-
-    def __init__(self, fmf_duration):
-        """
-        'fmf_duration' is the string specified as 'duration' in FMF metadata.
-        """
-        duration = self._fmf_to_seconds(fmf_duration)
-        self.end = time.monotonic() + duration
-        # keep track of only the first 'save' and the last 'restore',
-        # ignore any nested ones (as tracked by '_count')
-        self.saved = None
-        self.saved_count = 0
-
-    @staticmethod
-    def _fmf_to_seconds(string):
-        match = re.fullmatch(r"([0-9]+)([a-z]*)", string)
-        if not match:
-            raise RuntimeError(f"'duration' has invalid format: {string}")
-        length, unit = match.groups()
-        if unit == "m":
-            return int(length)*60
-        elif unit == "h":
-            return int(length)*60*60
-        elif unit == "d":
-            return int(length)*60*60*24
-        else:
-            return int(length)
-
-    def set(self, to):
-        self.end = time.monotonic() + self._fmf_to_seconds(to)
-
-    def increment(self, by):
-        self.end += self._fmf_to_seconds(by)
-
-    def decrement(self, by):
-        self.end -= self._fmf_to_seconds(by)
-
-    def save(self):
-        if self.saved_count == 0:
-            self.saved = self.end - time.monotonic()
-        self.saved_count += 1
-
-    def restore(self):
-        if self.saved_count > 1:
-            self.saved_count -= 1
-        elif self.saved_count == 1:
-            self.end = time.monotonic() + self.saved
-            self.saved_count = 0
-            self.saved = None
-
-    def out_of_time(self):
-        return time.monotonic() > self.end
-
-
-# SETUP global:
-# - create CSVAggregator instance on some destination dir
-
-# SETUP of new Provisioner instance:
-# - with SSHConn
-#   - rsync test repo to some tmpdir on the remote
-#   - install packages from plan
-#   - create TMT_PLAN_ENVIRONMENT_FILE and export it to plan setup scripts
-#   - run plan setup scripts
-
-# SETUP + CLEANUP for one test
-# - create Executor instance
-#   - pass TMT_PLAN_ENVIRONMENT_FILE to it
-#     - probably as general "environment variables" input,
-#       could be reused in the future for ie. TMT_PLAN_NAME or so
-#     - pass env from CLI -e switches
-#   - pass disconnected SSHConn to it
-#   - pass one FMFTest namedtuple to it (test to run)
-#   - pass test repo location on the remote host
-#   - pass CSVAggregator instance to it, so it can write results/logs
-# - with SSHConn
-#   - create wrapper script on the remote
-#   - run wrapper script, redirecting stderr to tmpfile via CSVAggregator
-#   - poll() / select() over test stdout, parse TEST_CONTROL protocol
-#   - on 0.1sec poll timeout
-#     - check if SSHConn master is alive, re-connect if not
-#     - check test duration against fmf-defined (control-adjusted) duration
-#   - ...
-# - make Executor return some complex status
-#   - whether testing was destructive (just leave SSHConn disconnected?)
-
-
-class TestAbortedError(Exception):
-    """
-    Raised when an infrastructure-related issue happened while running a test.
-    """
-    pass
-
-
-# TODO: automatic reporting of all partial results that were not finished
-class Executor:
-    """
-    Logic for running tests on a remote system and processing results
-    and uploaded files by those tests.
-    """
-
-    def __init__(self, remote, aggregator, remote_dir=None, env=None):
-        """
-        'remote' is a reserved class Remote instance, with an active connection.
-
-        'aggregator' is an instance of a ResultAggregator all the results
-        and uploaded files will be written to.
-
-        'remote_dir' is a path on the remote for storing tests and other
-        metadata. If unspecified, a tmpdir is created and used instead.
-
-        'env' is a dict of extra environment variables to pass to the
-        plan prepare scripts and tests.
-        """
-        self.lock = threading.RLock()
-        self.remote = remote
-        self.aggregator = aggregator
-        self.cancelled = False
-        self.remote_dir = remote_dir
-        self.plan_env = env.copy() if env else {}
-
-    def _get_remote_dir(self):
-        # TODO: do not mktemp here, do it in the parent, have remote_dir be mandatory,
-        #       renamed to 'tests_dir' (just the test repo), cleaned up by external logic
-        #       - handle custom metadata remote dir in run_test() and clean it up there
-        #         (for TMT_PLAN_ENVIRONMENT_FILE, wrapper, etc.)
-        if not self.remote_dir:
-            self.remote_dir = self.remote.cmd(
-                ("mktemp", "-d", "-p", "/var/tmp"),
-                func=util.subprocess_output,
-            )
-        return self.remote_dir
-
-    # TODO: do not do this in Executor
-    def upload_tests(self, tests_dir):
-        """
-        Upload a directory of all tests from a local 'tests_dir' path to
-        a temporary directory on the remote host.
-        """
-        remote_dir = self._get_remote_dir()
-        self.remote.rsync(
-            "-rv", "--delete", "--exclude=.git/",
-            f"{tests_dir}/",
-            f"remote:{remote_dir}/tests",
-        )
-
-    def setup_plan(self, fmf_tests):
-        """
-        Install packages and run scripts, presumably extracted from a TMT plan
-        provided as 'fmf_tests', an initialized FMFTests instance.
-
-        Also prepare additional environment for tests, ie. create and export
-        a path to TMT_PLAN_ENVIRONMENT_FILE.
-        """
-        # install packages from the plan
-        if fmf_tests.prepare_pkgs:
-            self.remote.cmd(
-                (
-                    "dnf", "-y", "--setopt=install_weak_deps=False",
-                    "install", *fmf_tests.prepare_pkgs,
-                ),
-                check=True,
-            )
-
-        # create TMT_PLAN_ENVIRONMENT_FILE
-        self.plan_env.update(fmf_tests.plan_env)
-        env_file = f"{self._get_remote_dir()}/TMT_PLAN_ENVIRONMENT_FILE"
-        self.remote.cmd(("truncate", "-s", "0", env_file), check=True)
-        self.plan_env["TMT_PLAN_ENVIRONMENT_FILE"] = env_file
-
-        # run prepare scripts
-        env_args = (f"{k}={v}" for k, v in self.plan_env.items())
-        for script in fmf_tests.prepare_scripts:
-            self.remote.cmd(
-                ("env", *env_args, "bash"),
-                input=script,
-                text=True,
-                check=True,
-            )
-
-    def run_test(self, fmf_test, env=None):
-        """
-        Run one test on the remote system.
-
-        'fmf_test' is a FMFTest namedtuple with the test to run.
-
-        'env' is a dict of extra environment variables to pass to the test.
-        """
-        env_vars = self.plan_env.copy()
-        for item in fmf.listlike(fmf_test.data, "environment"):
-            env_vars.update(item)
-        env_vars["ATEX_TEST_NAME"] = fmf_test.name
-        env_vars["TMT_TEST_NAME"] = fmf_test.name
-        if env:
-            env_vars.update(env)
-        env_args = (f"{k}={v}" for k, v in env_vars.items())
-
-        # run a setup script, preparing wrapper + test scripts
-        remote_dir = self._get_remote_dir()
-        setup_script = scripts.test_setup(
-            test=fmf_test,
-            tests_dir=f"{remote_dir}/tests",
-            wrapper_exec=f"{remote_dir}/wrapper.sh",
-            test_exec=f"{remote_dir}/test.sh",
-            debug=True,
-        )
-        self.remote.cmd(("bash",), input=setup_script, text=True, check=True)
-
-        with contextlib.ExitStack() as stack:
-            testout_fd = stack.enter_context(self.aggregator.open_tmpfile())
-
-            duration = Duration(fmf_test.data.get("duration", "5m"))
-
-            test_proc = None
-            control_fd = None
-            stack.callback(lambda: os.close(control_fd) if control_fd else None)
-
-            def abort(msg):
-                if test_proc:
-                    test_proc.kill()
-                    test_proc.wait()
-                self.remote.release()
-                raise TestAbortedError(msg) from None
-
-            try:
-                # TODO: probably enum
-                state = "starting_test"
-                while not duration.out_of_time():
-                    with self.lock:
-                        if self.cancelled:
-                            abort("cancel requested")
-                            return
-
-                    if state == "starting_test":
-                        control_fd, pipe_w = os.pipe()
-                        os.set_blocking(control_fd, False)
-                        control = testcontrol.TestControl(
-                            control_fd=control_fd,
-                            aggregator=self.aggregator,
-                            duration=duration,
-                            testout_fd=testout_fd,
-                        )
-                        # run the test in the background, letting it log output directly to
-                        # an opened file (we don't handle it, cmd client sends it to kernel)
-                        test_proc = self.remote.cmd(
-                            ("env", *env_args, f"{remote_dir}/wrapper.sh"),
-                            stdout=pipe_w,
-                            stderr=testout_fd,
-                            func=util.subprocess_Popen,
-                        )
-                        os.close(pipe_w)
-                        state = "reading_control"
-
-                    elif state == "reading_control":
-                        rlist, _, xlist = select.select((control_fd,), (), (control_fd,), 0.1)
-                        if xlist:
-                            abort(f"got exceptional condition on control_fd {control_fd}")
-                        elif rlist:
-                            control.process()
-                            if control.eof:
-                                os.close(control_fd)
-                                control_fd = None
-                                state = "waiting_for_exit"
-
-                    elif state == "waiting_for_exit":
-                        # control stream is EOF and it has nothing for us to read,
-                        # we're now just waiting for proc to cleanly terminate
-                        try:
-                            code = test_proc.wait(0.1)
-                            if code == 0:
-                                # wrapper exited cleanly, testing is done
-                                break
-                            else:
-                                # unexpected error happened (crash, disconnect, etc.)
-                                self.remote.disconnect()
-                                # if reconnect was requested, do so, otherwise abort
-                                if control.reconnect:
-                                    state = "reconnecting"
-                                    if control.reconnect != "always":
-                                        control.reconnect = None
-                                else:
-                                    abort(f"test wrapper unexpectedly exited with {code}")
-                            test_proc = None
-                        except subprocess.TimeoutExpired:
-                            pass
-
-                    elif state == "reconnecting":
-                        try:
-                            self.remote.connect(block=False)
-                            state = "reading_control"
-                        except BlockingIOError:
-                            pass
-
-                    else:
-                        raise AssertionError("reached unexpected state")
-
-                else:
-                    abort("test duration timeout reached")
-
-                # testing successful, do post-testing tasks
-
-                # test wrapper hasn't provided exitcode
-                if control.exit_code is None:
-                    abort("exitcode not reported, wrapper bug?")
-
-                # partial results that were never reported
-                if control.partial_results:
-                    control.result_seen = True  # partial result is also a result
-                    for result in control.partial_results.values():
-                        self.aggregator.report(result)
-
-                # test hasn't reported a single result, add an automatic one
-                # as specified in RESULTS.md
-                # {"status": "pass", "name": "/some/test", "testout": "output.txt"}
-                if not control.result_seen:
-                    self.aggregator.link_tmpfile_to(fmf_test.name, "output.txt", testout_fd)
-                    self.aggregator.report({
-                        "status": "pass" if control.exit_code == 0 else "fail",
-                        "name": fmf_test.name,
-                        "testout": "output.txt",
-                    })
-
-            except Exception:
-                # if the test hasn't reported a single result, but still
-                # managed to break something, provide at least the default log
-                # for manual investigation - otherwise test output disappears
-                if not control.result_seen:
-                    self.aggregator.link_tmpfile_to(fmf_test.name, "output.txt", testout_fd)
-                raise
-
-    def cancel(self):
-        with self.lock:
-            self.cancelled = True

atex/provision/nspawn/README

@@ -1,74 +0,0 @@
-The idea is to use systemd-nspawn containers on the host, binding
-/dev/kvm to each, thus avoiding the need for nested virt as our first layer
-of Contest tests will run in the containers (installing libvirtd, etc.)
-and the second layer (VMs created by tests) will use virtual machines,
-via a non-nested HVM.
-
-systemd-nspawn containers can have CPU core limits, memory limits, etc.
-done via cgroups, so we can provide some level of isolation/safety.
-
-
-systemd-nspawn can create its own veth via --network-veth=... and put it into
-a bridge automatically via --network-bridge=...
-
-We can then use NetworkManager + firewalld to pre-create a bridge with built-in
-DHCP and NAT to the outside, via something like
-
-  nmcli connection add type bridge ifname br0 con-name br0 ipv4.method shared ipv6.method ignore
-
-According to https://fedoramagazine.org/internet-connection-sharing-networkmanager/
-the ipv4.method=shared :
-
-    enables IP forwarding for the interface;
-    adds firewall rules and enables masquerading;
-    starts dnsmasq as a DHCP and DNS server.
-
-Specifically it should add MASQUERADE on packets *outgoing* from the bridge subnet,
-so shouldn't need any modification of the upstream eth0 device or any fw rules tied to it.
-
-There also seems to be ipv4.addresses 192.168.42.1/24 to modify the subnet?
-
-If that doesn't work, firewalld has an External zone that has <masquerade/>
-by default, so 
-
-  nmcli connection modify br0 connection.zone external
-
-should work.
-
-
---------
-
-TODO: We need some way to get DHCP leases for started containers (so we can connect
-      to the containerized sshd).
-
-      If there is no command for it via nmcli, it should be possible to just
-      extract it from wherever NetworkManager pointed dnsmasq to store its leases file.
-
-      We can then probably correlate --network-veth=... device from systemd-nspawn
-      (named after --machine=... name, prefixed with ve-* or vb-* if --network-bridge=* is used)
-      to the leased IP address.
-
-      ls -l /var/lib/NetworkManager/dnsmasq-*.leases
-
-      Or perhaps parse it out of 'ip neigh' to make sure the guest is *really* up.
-        - 'ip neigh' gives us MAC-to-IP, but device is always br0
-        - 'ip link show dev vb-contname' should give us the MAC for 'ip neigh'
-          - if container veth endpoint uses different mac, we can query bridge forward DB
-            via 'bridge fdb' to get all MACs that appeared on the veth
-
---------
-
-Containers can be installed via ie.
-
-dnf  --releasever=41 --installroot=/var/lib/machines/f41 --use-host-config \
-     --setopt=install_weak_deps=False \
-     install \
-     passwd dnf fedora-release vim-minimal util-linux systemd NetworkManager
-
-where --use-host-config re-uses host repositories.
-
-Maybe consider 'machinectl'-managed containers (start/terminate/kill/reboot/etc.)
-which are just repackaged systemd-nspawn@ services.
- - Especially since there is no concept of "throw away disk snapshot with container exit",
-   we always need some copy/clone of the --installroot for each instance of the container,
-   so using ie. 'machinectl clone ...' would provide a nice interface for it.

atex/provision/testingfarm/foo.py

@@ -1 +0,0 @@
-a = 123

atex-0.7.dist-info/RECORD

@@ -1,32 +0,0 @@
-atex/__init__.py,sha256=LdX67gprtHYeAkjLhFPKzpc7ECv2rHxUbHKDGbGXO1c,517
-atex/cli/__init__.py,sha256=erHv68SsybRbdgJ60013y9jVqY1ec-cb9T9ThPCJ_HY,2408
-atex/cli/minitmt.py,sha256=h1jHRGR0nqc7ANVTjmB5aeLron2A04vpzGGMQJLBliY,5793
-atex/cli/testingfarm.py,sha256=4oKWbkJwj0gs4_NnFP7w8fo2VvD5NpaPNG4HCeKm94I,6706
-atex/connection/__init__.py,sha256=O7rJDwf438lX2YgM8uHCC9gkmvaLYt3jRvtDq_FSzME,4107
-atex/connection/ssh.py,sha256=qVKeAY16pLGGS-d4TIF5A2DWWlNGR5qNlAmHo-jAIEc,13985
-atex/minitmt/__init__.py,sha256=wCda0QXwxlxkH4PpY0lGlNWT96_pUj7taNNP0Mo6TQk,522
-atex/minitmt/executor.py,sha256=zmFHo3M5hHDOzfZrXBPFSeqLmLK__q2KkT5A93ELEzY,13415
-atex/minitmt/fmf.py,sha256=HAjQkfrDLQLW2ZBvgQmleVpL_tfNSP9bUuqRt9BdjS4,7559
-atex/minitmt/scripts.py,sha256=WsLsQOmNxinpGUhOqqwN788zjcd5o79VC7I7UeGRwb8,5146
-atex/minitmt/testcontrol.py,sha256=l670ou1oPr5q3RA4QNQ0IgSh3uYvuvzcUW67NLNVgCA,12209
-atex/orchestrator/__init__.py,sha256=pMBBtev20KWAmQdth3RfuTwltXDm57CakDBpAsqyNYM,1592
-atex/orchestrator/aggregator.py,sha256=pTL6Ru70sin5WOc-dl3-bNSEvuOfWq7YLtduwiLX9q8,5463
-atex/provision/__init__.py,sha256=hJ7wbn6_LaCQVrIFilEcDA4z4LxglHYNEsxmizwr_OE,5573
-atex/provision/libvirt/VM_PROVISION,sha256=7pkZ-ozgTyK4qNGC-E-HUznr4IhbosWSASbB72Gknl8,2664
-atex/provision/libvirt/__init__.py,sha256=mAkGtciZsXdR9MVVrjm3OWNXZqTs_33-J1qAszFA0k4,768
-atex/provision/libvirt/setup-libvirt.sh,sha256=CXrEFdrj8CSHXQZCd2RWuRvTmw7QYFTVhZeLuhhXooI,1855
-atex/provision/nspawn/README,sha256=mIjOA6R5sM-1wGDm-7zBcEqax1NMSRzijwyeqVjGoqE,3127
-atex/provision/podman/README,sha256=kgP3vcTfWW9gcQzmXnyucjgWbqjNqm_ZM--pnqNTXRg,1345
-atex/provision/podman/host_container.sh,sha256=buCNz0BlsHY5I64sMSTGQHkvzEK0aeIhpGJXWCQVMXk,2283
-atex/provision/testingfarm/__init__.py,sha256=l4T4VOUQWKYEQXPe0OtfJmuW-aGSiSQ2mdOf6fSjpjU,605
-atex/provision/testingfarm/api.py,sha256=S-9m1D1CaG-1gEgh7oT1RkXPv2sfUw7Od7bW-Wv0uJQ,19262
-atex/provision/testingfarm/foo.py,sha256=WfGpn_saQb5x9Svx-gfViyXPVRF_u-gUjagUwLBClM0,8
-atex/util/__init__.py,sha256=PWU0STjLcQfe2pPaWKygRXQAudy8CNFdbnTGQ488mHs,1550
-atex/util/dedent.py,sha256=SEuJMtLzqz3dQ7g7qyZzEJ9VYynVlk52tQCJY-FveXo,603
-atex/util/log.py,sha256=oqJGXgHZkTgXVbD2-gzA2pqV3L8w0r_czYWIPfVz398,1776
-atex/util/subprocess.py,sha256=IQT9QHe2kMaaO_XPSry-DwObYstGsq6_QdwdbhYDjko,1826
-atex-0.7.dist-info/METADATA,sha256=VvPhdyVKdg_-kmR0y1WkClrYxw24yjoee3jGzkj6--g,5076
-atex-0.7.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-atex-0.7.dist-info/entry_points.txt,sha256=pLqJdcfeyQTgup2h6dWb6SvkHhtOl-W5Eg9zV8moK0o,39
-atex-0.7.dist-info/licenses/COPYING.txt,sha256=oEuj51jdmbXcCUy7pZ-KE0BNcJTR1okudRp5zQ0yWnU,670
-atex-0.7.dist-info/RECORD,,