atex 0.7__py3-none-any.whl → 0.9__py3-none-any.whl

atex/connection/ssh.py

@@ -18,6 +18,7 @@ import os
 import time
 import shlex
 import tempfile
+import threading
 import subprocess
 from pathlib import Path
 
@@ -38,7 +39,7 @@ DEFAULT_OPTIONS = {
 }
 
 
-class SSHError(Exception):
+class SSHError(ConnectionError):
     pass
 
 
@@ -68,7 +69,7 @@ def _shell_cmd(command, sudo=None):
     """
     Make a command line for running 'command' on the target system.
     """
-    quoted_args = (shlex.quote(arg) for arg in command)
+    quoted_args = (shlex.quote(str(arg)) for arg in command)
     if sudo:
         return " ".join((
             "exec", "sudo", "--no-update", "--non-interactive", "--user", sudo, "--", *quoted_args,
@@ -154,7 +155,6 @@ class StatelessSSHConn(Connection):
         If 'sudo' specifies a username, call sudo(8) on the remote shell
         to run under a different user on the remote host.
         """
-        super().__init__()
         self.options = DEFAULT_OPTIONS.copy()
         self.options.update(options)
         self.password = password
@@ -162,26 +162,24 @@ class StatelessSSHConn(Connection):
         self._tmpdir = None
         self._master_proc = None
 
-    def connect(self):
+    def connect(self, block=True):
         """
         Optional, .cmd() and .rsync() work without it, but it is provided here
         for compatibility with the Connection API.
         """
-        # TODO: just wait until .cmd(['true']) starts responding
+        # TODO: just wait until .cmd(['true']) starts responding ?
         pass
 
     def disconnect(self):
         pass
 
-#    def alive(self):
-#        return True
-
     # have options as kwarg to be compatible with other functions here
     def cmd(self, command, options=None, func=util.subprocess_run, **func_args):
         unified_options = self.options.copy()
         if options:
             unified_options.update(options)
-        unified_options["RemoteCommand"] = _shell_cmd(command, sudo=self.sudo)
+        if command:
+            unified_options["RemoteCommand"] = _shell_cmd(command, sudo=self.sudo)
         return func(
             _options_to_ssh(unified_options, password=self.password),
             skip_frames=1,
@@ -231,7 +229,7 @@ class ManagedSSHConn(Connection):
     to manage this complexity.
     """
 
-    # TODO: thread safety and locking via self.lock
+    # TODO: thread safety and locking via self.lock ?
 
     def __init__(self, options, *, password=None, sudo=None):
         """
@@ -243,7 +241,7 @@ class ManagedSSHConn(Connection):
         If 'sudo' specifies a username, call sudo(8) on the remote shell
         to run under a different user on the remote host.
         """
-        super().__init__()
+        self.lock = threading.RLock()
         self.options = DEFAULT_OPTIONS.copy()
         self.options.update(options)
         self.password = password
@@ -251,12 +249,6 @@ class ManagedSSHConn(Connection):
         self._tmpdir = None
         self._master_proc = None
 
-#    def __copy__(self):
-#        return type(self)(self.options, password=self.password)
-#
-#    def copy(self):
-#        return self.__copy__()
-
     def assert_master(self):
         proc = self._master_proc
         if not proc:
@@ -272,19 +264,13 @@ class ManagedSSHConn(Connection):
                 f"SSH ControlMaster on {self._tmpdir} exited with {code}{out}",
             )
 
-#    def alive(self):
-#        try:
-#            self.assert_master()
-#            return True
-#        except (NotConnectedError, DisconnectedError):
-#            return False
-
     def disconnect(self):
         proc = self._master_proc
         if not proc:
             return
+        util.debug(f"disconnecting: {self.options}")
         proc.kill()
-        # don"t zombie forever, return EPIPE on any attempts to write to us
+        # don't zombie forever, return EPIPE on any attempts to write to us
         proc.stdout.close()
         proc.wait()
         (self._tmpdir / "control.sock").unlink(missing_ok=True)
@@ -302,19 +288,26 @@ class ManagedSSHConn(Connection):
         sock = self._tmpdir / "control.sock"
 
         if not self._master_proc:
+            util.debug(f"connecting: {self.options}")
             options = self.options.copy()
             options["SessionType"] = "none"
             options["ControlMaster"] = "yes"
             options["ControlPath"] = sock
             self._master_proc = util.subprocess_Popen(
-                _options_to_ssh(options),
+                _options_to_ssh(options, password=self.password),
                 stdin=subprocess.DEVNULL,
                 stdout=subprocess.PIPE,
                 stderr=subprocess.STDOUT,
                 cwd=str(self._tmpdir),
+                start_new_session=True,  # resist Ctrl-C
             )
             os.set_blocking(self._master_proc.stdout.fileno(), False)
 
+        # NOTE: ideally, we would .read() before checking .poll() because
+        #       if the process writes a lot, it gets stuck in the pipe
+        #       (in kernel) and the process never ends; but output-appending
+        #       code would be obscure, and ssh(1) never outputs that much ..
+
         proc = self._master_proc
         if block:
             while proc.poll() is None:
@@ -325,7 +318,6 @@ class ManagedSSHConn(Connection):
                 code = proc.poll()
                 out = proc.stdout.read()
                 self._master_proc = None
-                # TODO: ConnectError should probably be generalized for Connection
                 raise ConnectError(
                     f"SSH ControlMaster failed to start on {self._tmpdir} with {code}:\n{out}",
                 )
@@ -334,54 +326,45 @@ class ManagedSSHConn(Connection):
             if code is not None:
                 out = proc.stdout.read()
                 self._master_proc = None
-                # TODO: ConnectError should probably be generalized for Connection
                 raise ConnectError(
                     f"SSH ControlMaster failed to start on {self._tmpdir} with {code}:\n{out}",
                 )
             elif not sock.exists():
                 raise BlockingIOError("SSH ControlMaster not yet ready")
 
-    def add_local_forward(self, *spec):
+    def forward(self, forward_type, *spec, cancel=False):
         """
         Add (one or more) ssh forwarding specifications as 'spec' to an
         already-connected instance. Each specification has to follow the
-        format of ssh client's LocalForward option (see ssh_config(5)).
-        """
-        self.assert_master()
-        options = self.options.copy()
-        options["LocalForward"] = spec
-        options["ControlPath"] = self._tmpdir / "control.sock"
-        util.subprocess_run(
-            _options_to_ssh(options, extra_cli_flags=("-O", "forward")),
-            skip_frames=1,
-            check=True,
-        )
+        format of LocalForward or RemoteForward (see ssh_config(5)).
+        Ie. "1234 1.2.3.4:22" or "0.0.0.0:1234 1.2.3.4:22".
 
-    def add_remote_forward(self, *spec):
-        """
-        Add (one or more) ssh forwarding specifications as 'spec' to an
-        already-connected instance. Each specification has to follow the
-        format of ssh client's RemoteForward option (see ssh_config(5)).
+        'forward_type' must be either LocalForward or RemoteForward.
+
+        If 'cancel' is True, cancel the forwarding instead of adding it.
         """
+        assert forward_type in ("LocalForward", "RemoteForward")
         self.assert_master()
-        options = self.options.copy()
-        options["RemoteForward"] = spec
+        options = DEFAULT_OPTIONS.copy()
+        options[forward_type] = spec
         options["ControlPath"] = self._tmpdir / "control.sock"
+        action = "forward" if not cancel else "cancel"
         util.subprocess_run(
-            _options_to_ssh(options, extra_cli_flags=("-O", "forward")),
+            _options_to_ssh(options, extra_cli_flags=("-O", action)),
             skip_frames=1,
             check=True,
         )
 
-    def cmd(self, command, options=None, func=util.subprocess_run, **func_args):
+    def cmd(self, command, *, options=None, func=util.subprocess_run, **func_args):
         self.assert_master()
         unified_options = self.options.copy()
         if options:
             unified_options.update(options)
-        unified_options["RemoteCommand"] = _shell_cmd(command, sudo=self.sudo)
+        if command:
+            unified_options["RemoteCommand"] = _shell_cmd(command, sudo=self.sudo)
         unified_options["ControlPath"] = self._tmpdir / "control.sock"
         return func(
-            _options_to_ssh(unified_options, password=self.password),
+            _options_to_ssh(unified_options),
             skip_frames=1,
             **func_args,
         )
@@ -396,7 +379,6 @@ class ManagedSSHConn(Connection):
             _rsync_host_cmd(
                 *args,
                 options=unified_options,
-                password=self.password,
                 sudo=self.sudo,
             ),
             skip_frames=1,

atex/executor/__init__.py

@@ -0,0 +1,2 @@
+from . import testcontrol  # noqa: F401
+from .executor import Executor  # noqa: F401

atex/executor/duration.py

@@ -0,0 +1,60 @@
+import re
+import time
+
+
+class Duration:
+    """
+    A helper for parsing, keeping and manipulating test run time based on
+    FMF-defined 'duration' attribute.
+    """
+
+    def __init__(self, fmf_duration):
+        """
+        'fmf_duration' is the string specified as 'duration' in FMF metadata.
+        """
+        duration = self._fmf_to_seconds(fmf_duration)
+        self.end = time.monotonic() + duration
+        # keep track of only the first 'save' and the last 'restore',
+        # ignore any nested ones (as tracked by '_count')
+        self.saved = None
+        self.saved_count = 0
+
+    @staticmethod
+    def _fmf_to_seconds(string):
+        match = re.fullmatch(r"([0-9]+)([a-z]*)", string)
+        if not match:
+            raise RuntimeError(f"'duration' has invalid format: {string}")
+        length, unit = match.groups()
+        if unit == "m":
+            return int(length)*60
+        elif unit == "h":
+            return int(length)*60*60
+        elif unit == "d":
+            return int(length)*60*60*24
+        else:
+            return int(length)
+
+    def set(self, to):
+        self.end = time.monotonic() + self._fmf_to_seconds(to)
+
+    def increment(self, by):
+        self.end += self._fmf_to_seconds(by)
+
+    def decrement(self, by):
+        self.end -= self._fmf_to_seconds(by)
+
+    def save(self):
+        if self.saved_count == 0:
+            self.saved = self.end - time.monotonic()
+        self.saved_count += 1
+
+    def restore(self):
+        if self.saved_count > 1:
+            self.saved_count -= 1
+        elif self.saved_count == 1:
+            self.end = time.monotonic() + self.saved
+            self.saved_count = 0
+            self.saved = None
+
+    def out_of_time(self):
+        return time.monotonic() > self.end

atex/executor/executor.py

@@ -0,0 +1,402 @@
+import os
+import enum
+import time
+import select
+import threading
+import contextlib
+import subprocess
+from pathlib import Path
+
+from .. import util, fmf
+from . import testcontrol, scripts
+from .duration import Duration
+from .reporter import Reporter
+
+
+class TestAbortedError(Exception):
+    """
+    Raised when an infrastructure-related issue happened while running a test.
+    """
+    pass
+
+
+class Executor:
+    """
+    Logic for running tests on a remote system and processing results
+    and uploaded files by those tests.
+
+        tests_repo = "path/to/cloned/tests"
+        tests_data = atex.fmf.FMFTests(tests_repo, "/plans/default")
+
+        with Executor(tests_data, conn) as e:
+            e.upload_tests()
+            e.plan_prepare()
+            Path("output_here").mkdir()
+            e.run_test("/some/test", "output_here")
+            e.run_test(...)
+            e.plan_finish()
+
+    One Executor instance may be used to run multiple tests sequentially.
+    In addition, multiple Executor instances can run in parallel on the same
+    host, provided each receives a unique class Connection instance to it.
+
+        conn.cmd(["mkdir", "-p", "/shared"])
+
+        with Executor(tests_data, conn, state_dir="/shared") as e:
+            e.upload_tests()
+            e.plan_prepare()
+
+        # in parallel (ie. threading or multiprocessing)
+        with Executor(tests_data, unique_conn, state_dir="/shared") as e:
+            e.run_test(...)
+    """
+
+    def __init__(self, fmf_tests, connection, *, env=None, state_dir=None):
+        """
+        'fmf_tests' is a class FMFTests instance with (discovered) tests.
+
+        'connection' is a class Connection instance, already fully connected.
+
+        'env' is a dict of extra environment variables to pass to the
+        plan prepare/finish scripts and to all tests.
+
+        'state_dir' is a string or Path specifying path on the remote system for
+        storing additional data, such as tests, execution wrappers, temporary
+        plan-exported variables, etc. If left as None, a tmpdir is used.
+        """
+        self.lock = threading.RLock()
+        self.fmf_tests = fmf_tests
+        self.conn = connection
+        self.env = env or {}
+        self.state_dir = state_dir
+        self.work_dir = None
+        self.tests_dir = None
+        self.plan_env_file = None
+        self.cancelled = False
+
+    def setup(self):
+        with self.lock:
+            state_dir = self.state_dir
+
+        # if user defined a state dir, have shared tests, but use per-instance
+        # work_dir for test wrappers, etc., identified by this instance's id(),
+        # which should be unique as long as this instance exists
+        if state_dir:
+            state_dir = Path(state_dir)
+            work_dir = state_dir / f"atex-{id(self)}"
+            self.conn.cmd(("mkdir", work_dir), check=True)
+            with self.lock:
+                self.tests_dir = state_dir / "tests"
+                self.plan_env_file = state_dir / "plan_env"
+                self.work_dir = work_dir
+
+        # else just create a tmpdir
+        else:
+            tmp_dir = self.conn.cmd(
+                # /var is not cleaned up by bootc, /var/tmp is
+                ("mktemp", "-d", "-p", "/var", "atex-XXXXXXXXXX"),
+                func=util.subprocess_output,
+            )
+            tmp_dir = Path(tmp_dir)
+            with self.lock:
+                self.tests_dir = tmp_dir / "tests"
+                self.plan_env_file = tmp_dir / "plan_env"
+                # use the tmpdir as work_dir, avoid extra mkdir over conn
+                self.work_dir = tmp_dir
+
+        # create / truncate the TMT_PLAN_ENVIRONMENT_FILE
+        self.conn.cmd(("truncate", "-s", "0", self.plan_env_file), check=True)
+
+    def cleanup(self):
+        with self.lock:
+            work_dir = self.work_dir
+
+        if work_dir:
+            self.conn.cmd(("rm", "-rf", work_dir), check=True)
+
+        with self.lock:
+            self.work_dir = None
+            self.tests_dir = None
+            self.plan_env_file = None
+
+    def __enter__(self):
+        try:
+            self.setup()
+            return self
+        except Exception:
+            self.cleanup()
+            raise
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.cleanup()
+
+    def cancel(self):
+        with self.lock:
+            self.cancelled = True
+
+    def upload_tests(self):
+        """
+        Upload a directory of all tests, the location of which was provided to
+        __init__() inside 'fmf_tests', to the remote host.
+        """
+        self.conn.rsync(
+            "-rv" if util.in_debug_mode() else "-rq",
+            "--delete", "--exclude=.git/",
+            f"{self.fmf_tests.root}/",
+            f"remote:{self.tests_dir}",
+        )
+
+    def _run_prepare_scripts(self, scripts):
+        # make envionment for 'prepare' scripts
+        env = {
+            **self.fmf_tests.plan_env,
+            **self.env,
+            "TMT_PLAN_ENVIRONMENT_FILE": self.plan_env_file,
+        }
+        env_args = (f"{k}={v}" for k, v in env.items())
+        # run the scripts
+        for script in scripts:
+            self.conn.cmd(
+                ("env", *env_args, "bash"),
+                input=script,
+                text=True,
+                check=True,
+                stdout=None if util.in_debug_mode() else subprocess.DEVNULL,
+                stderr=subprocess.STDOUT,
+            )
+
+    def plan_prepare(self):
+        """
+        Install packages and run scripts extracted from a TMT plan by a FMFTests
+        instance given during class initialization.
+
+        Also run additional scripts specified under the 'prepare' step inside
+        the fmf metadata of a plan.
+        """
+        # install packages from the plan
+        if self.fmf_tests.prepare_pkgs:
+            self.conn.cmd(
+                (
+                    "dnf", "-y", "--setopt=install_weak_deps=False",
+                    "install", *self.fmf_tests.prepare_pkgs,
+                ),
+                check=True,
+                stdout=None if util.in_debug_mode() else subprocess.DEVNULL,
+                stderr=subprocess.STDOUT,
+            )
+
+        # run 'prepare' scripts from the plan
+        if scripts := self.fmf_tests.prepare_scripts:
+            self._run_prepare_scripts(scripts)
+
+    def plan_finish(self):
+        """
+        Run any scripts specified under the 'finish' step inside
+        the fmf metadata of a plan.
+        """
+        if scripts := self.fmf_tests.finish_scripts:
+            self._run_prepare_scripts(scripts)
+
+    class State(enum.Enum):
+        STARTING_TEST = enum.auto()
+        READING_CONTROL = enum.auto()
+        WAITING_FOR_EXIT = enum.auto()
+        RECONNECTING = enum.auto()
+
+    def run_test(self, test_name, output_dir, *, env=None):
+        """
+        Run one test on the remote system.
+
+        'test_name' is a string with test name.
+
+        'output_dir' is a destination dir (string or Path) for results reported
+        and files uploaded by the test. Results are always stored in a line-JSON
+        format in a file named 'results', files are always uploaded to directory
+        named 'files', both inside 'output_dir'.
+        The path for 'output_dir' must already exist and be an empty directory
+        (ie. typically a tmpdir).
+
+        'env' is a dict of extra environment variables to pass to the test.
+
+        Returns an integer exit code of the test script.
+        """
+        output_dir = Path(output_dir)
+        test_data = self.fmf_tests.tests[test_name]
+
+        # run a setup script, preparing wrapper + test scripts
+        setup_script = scripts.test_setup(
+            test=scripts.Test(test_name, test_data, self.fmf_tests.test_dirs[test_name]),
+            tests_dir=self.tests_dir,
+            wrapper_exec=f"{self.work_dir}/wrapper.sh",
+            test_exec=f"{self.work_dir}/test.sh",
+            test_yaml=f"{self.work_dir}/metadata.yaml",
+        )
+        self.conn.cmd(("bash",), input=setup_script, text=True, check=True)
+
+        # start with fmf-plan-defined environment
+        env_vars = {
+            **self.fmf_tests.plan_env,
+            "TMT_PLAN_ENVIRONMENT_FILE": self.plan_env_file,
+            "TMT_TEST_NAME": test_name,
+            "TMT_TEST_METADATA": f"{self.work_dir}/metadata.yaml",
+        }
+        # append fmf-test-defined environment into it
+        for item in fmf.listlike(test_data, "environment"):
+            env_vars.update(item)
+        # append the Executor-wide environment passed to __init__()
+        env_vars.update(self.env)
+        # append variables given to this function call
+        if env:
+            env_vars.update(env)
+
+        with contextlib.ExitStack() as stack:
+            reporter = stack.enter_context(Reporter(output_dir, "results", "files"))
+            duration = Duration(test_data.get("duration", "5m"))
+            control = testcontrol.TestControl(reporter=reporter, duration=duration)
+
+            test_proc = None
+            control_fd = None
+            stack.callback(lambda: os.close(control_fd) if control_fd else None)
+
+            reconnects = 0
+
+            def abort(msg):
+                if test_proc:
+                    test_proc.kill()
+                    test_proc.wait()
+                raise TestAbortedError(msg) from None
+
+            exception = None
+
+            try:
+                state = self.State.STARTING_TEST
+                while not duration.out_of_time():
+                    with self.lock:
+                        if self.cancelled:
+                            abort("cancel requested")
+
+                    if state == self.State.STARTING_TEST:
+                        control_fd, pipe_w = os.pipe()
+                        os.set_blocking(control_fd, False)
+                        control.reassign(control_fd)
+                        # reconnect/reboot count (for compatibility)
+                        env_vars["TMT_REBOOT_COUNT"] = str(reconnects)
+                        env_vars["TMT_TEST_RESTART_COUNT"] = str(reconnects)
+                        # run the test in the background, letting it log output directly to
+                        # an opened file (we don't handle it, cmd client sends it to kernel)
+                        env_args = (f"{k}={v}" for k, v in env_vars.items())
+                        test_proc = self.conn.cmd(
+                            ("env", *env_args, f"{self.work_dir}/wrapper.sh"),
+                            stdout=pipe_w,
+                            stderr=reporter.testout_fobj.fileno(),
+                            func=util.subprocess_Popen,
+                        )
+                        os.close(pipe_w)
+                        state = self.State.READING_CONTROL
+
+                    elif state == self.State.READING_CONTROL:
+                        rlist, _, xlist = select.select((control_fd,), (), (control_fd,), 0.1)
+                        if xlist:
+                            abort(f"got exceptional condition on control_fd {control_fd}")
+                        elif rlist:
+                            control.process()
+                            if control.eof:
+                                os.close(control_fd)
+                                control_fd = None
+                                state = self.State.WAITING_FOR_EXIT
+
+                    elif state == self.State.WAITING_FOR_EXIT:
+                        # control stream is EOF and it has nothing for us to read,
+                        # we're now just waiting for proc to cleanly terminate
+                        try:
+                            code = test_proc.wait(0.1)
+                            if code == 0:
+                                # wrapper exited cleanly, testing is done
+                                break
+                            else:
+                                # unexpected error happened (crash, disconnect, etc.)
+                                self.conn.disconnect()
+                                # if reconnect was requested, do so, otherwise abort
+                                if control.reconnect:
+                                    state = self.State.RECONNECTING
+                                    if control.reconnect != "always":
+                                        control.reconnect = None
+                                else:
+                                    abort(
+                                        f"test wrapper unexpectedly exited with {code} and "
+                                        "reconnect was not sent via test control",
+                                    )
+                            test_proc = None
+                        except subprocess.TimeoutExpired:
+                            pass
+
+                    elif state == self.State.RECONNECTING:
+                        try:
+                            self.conn.connect(block=False)
+                            reconnects += 1
+                            state = self.State.STARTING_TEST
+                        except BlockingIOError:
+                            pass
+                        except ConnectionError:
+                            # can happen when ie. ssh is connecting over a LocalForward port,
+                            # causing 'read: Connection reset by peer' instead of timeout
+                            # - just retry again after a short delay
+                            time.sleep(0.5)
+
+                    else:
+                        raise AssertionError("reached unexpected state")
+
+                else:
+                    abort("test duration timeout reached")
+
+                # testing successful
+
+                # test wrapper hasn't provided exitcode
+                if control.exit_code is None:
+                    abort("exitcode not reported, wrapper bug?")
+
+                return control.exit_code
+
+            except Exception as e:
+                exception = e
+                raise
+
+            finally:
+                # partial results that were never reported
+                if control.partial_results:
+                    for result in control.partial_results.values():
+                        name = result.get("name")
+                        if not name:
+                            # partial result is also a result
+                            control.nameless_result_seen = True
+                        if testout := result.get("testout"):
+                            try:
+                                reporter.link_testout(testout, name)
+                            except FileExistsError:
+                                raise testcontrol.BadReportJSONError(
+                                    f"file '{testout}' already exists",
+                                ) from None
+                        reporter.report(result)
+
+                # if an unexpected infrastructure-related exception happened
+                if exception:
+                    try:
+                        reporter.link_testout("output.txt")
+                    except FileExistsError:
+                        pass
+                    reporter.report({
+                        "status": "infra",
+                        "note": repr(exception),
+                        "testout": "output.txt",
+                    })
+
+                # if the test hasn't reported a result for itself
+                elif not control.nameless_result_seen:
+                    try:
+                        reporter.link_testout("output.txt")
+                    except FileExistsError:
+                        pass
+                    reporter.report({
+                        "status": "pass" if control.exit_code == 0 else "fail",
+                        "testout": "output.txt",
+                    })