atex 0.5__py3-none-any.whl → 0.8__py3-none-any.whl

atex/orchestrator/orchestrator.py

@@ -0,0 +1,324 @@
+import time
+import tempfile
+import traceback
+import concurrent
+import collections
+from pathlib import Path
+
+from .. import util, executor
+
+
+class Orchestrator:
+    """
+    A scheduler for parallel execution on multiple resources (machines/systems).
+    """
+
+    SetupInfo = collections.namedtuple(
+        "SetupInfo",
+        (
+            # class Provisioner instance this machine is provided by
+            # (for logging purposes)
+            "provisioner",
+            # class Remote instance returned by the Provisioner
+            "remote",
+            # class Executor instance uploading tests / running setup or tests
+            "executor",
+        ),
+    )
+    RunningInfo = collections.namedtuple(
+        "RunningInfo",
+        (
+            # "inherit" from SetupInfo
+            *SetupInfo._fields,
+            # string with /test/name
+            "test_name",
+            # class tempfile.TemporaryDirectory instance with 'json_file' and 'files_dir'
+            "tmp_dir",
+        ),
+    )
+    FinishedInfo = collections.namedtuple(
+        "FinishedInfo",
+        (
+            # "inherit" from RunningInfo
+            *RunningInfo._fields,
+            # integer with exit code of the test
+            # (None if exception happened)
+            "exit_code",
+            # exception class instance if running the test failed
+            # (None if no exception happened (exit_code is defined))
+            "exception",
+        ),
+    )
+
+    def __init__(self, platform, fmf_tests, provisioners, aggregator, tmp_dir, *, max_reruns=2):
+        """
+        'platform' is a string with platform name.
+
+        'fmf_tests' is a class FMFTests instance of the tests to run.
+
+        'provisioners' is an iterable of class Provisioner instances.
+
+        'aggregator' is a class CSVAggregator instance.
+
+        'tmp_dir' is a string/Path to a temporary directory, to be used for
+        storing per-test results and uploaded files before being ingested
+        by the aggregator. Can be safely shared by Orchestrator instances.
+        """
+        self.platform = platform
+        self.fmf_tests = fmf_tests
+        self.provisioners = tuple(provisioners)
+        self.aggregator = aggregator
+        self.tmp_dir = tmp_dir
+        # tests still waiting to be run
+        self.to_run = set(fmf_tests.tests)
+        # running setup functions, as a list of SetupInfo items
+        self.running_setups = []
+        # running tests as a dict, indexed by test name, with RunningInfo values
+        self.running_tests = {}
+        # indexed by test name, value being integer of how many times
+        self.reruns = collections.defaultdict(lambda: max_reruns)
+        # thread queue for actively running tests
+        self.test_queue = util.ThreadQueue(daemon=False)
+        # thread queue for remotes being set up (uploading tests, etc.)
+        self.setup_queue = util.ThreadQueue(daemon=True)
+        # NOTE: running_setups and test_running are just for debugging and
+        #       cancellation, the execution flow itself uses ThreadQueues
+
+    @staticmethod
+    def _run_setup(sinfo):
+        sinfo.executor.setup()
+        sinfo.executor.upload_tests()
+        sinfo.executor.setup_plan()
+        # NOTE: we never run executor.cleanup() anywhere - instead, we assume
+        #       the remote (and its connection) was invalidated by the test,
+        #       so we just rely on remote.release() destroying the system
+        return sinfo
+
+    @classmethod
+    def _wrap_test(cls, rinfo, func, *args, **kwargs):
+        """
+        Wrap 'func' (test execution function) to preserve extra metadata
+        ('rinfo') and return it with the function return value.
+        """
+        try:
+            return cls.FinishedInfo(*rinfo, func(*args, **kwargs), None)
+        except Exception as e:
+            return cls.FinishedInfo(*rinfo, None, e)
+
+    def _run_new_test(self, sinfo):
+        """
+        'sinfo' is a SetupInfo instance.
+        """
+        next_test_name = self.next_test(self.to_run, self.fmf_tests)
+        assert next_test_name in self.to_run, "next_test() returned valid test name"
+
+        self.to_run.remove(next_test_name)
+
+        rinfo = self.RunningInfo(
+            *sinfo,
+            test_name=next_test_name,
+            tmp_dir=tempfile.TemporaryDirectory(
+                prefix=next_test_name.strip("/").replace("/","-") + "-",
+                dir=self.tmp_dir,
+                delete=False,
+            ),
+        )
+
+        tmp_dir_path = Path(rinfo.tmp_dir.name)
+        self.test_queue.start_thread(
+            target=self._wrap_test,
+            args=(
+                rinfo,
+                sinfo.executor.run_test,
+                next_test_name,
+                tmp_dir_path / "json_file",
+                tmp_dir_path / "files_dir",
+            ),
+        )
+
+        self.running_tests[next_test_name] = rinfo
+
+    def _process_finished_test(self, finfo):
+        """
+        'finfo' is a FinishedInfo instance.
+        """
+        test_id = f"'{finfo.test_name}' on '{finfo.remote}'"
+        tmp_dir_path = Path(finfo.tmp_dir.name)
+
+        # NOTE: document that we intentionally don't .cleanup() executioner below,
+        #       we rely on remote .release() destroying the OS, because we don't
+        #       want to risk .cleanup() blocking on dead ssh into the remote after
+        #       executing a destructive test
+
+        destructive = False
+
+        # if executor (or test) threw exception, schedule a re-run
+        if finfo.exception:
+            destructive = True
+            exc_str = "".join(traceback.format_exception(finfo.exception)).rstrip("\n")
+            util.info(f"unexpected exception happened while running {test_id}:\n{exc_str}")
+            finfo.remote.release()
+            if self.reruns[finfo.test_name] > 0:
+                self.reruns[finfo.test_name] -= 1
+                self.to_run.add(finfo.test_name)
+            else:
+                util.info(f"reruns for {test_id} exceeded, ignoring it")
+
+        # if the test exited as non-0, try a re-run
+        elif finfo.exit_code != 0:
+            destructive = True
+            finfo.remote.release()
+            if self.reruns[finfo.test_name] > 0:
+                util.info(
+                    f"{test_id} exited with non-zero: {finfo.exit_code}, re-running "
+                    f"({self.reruns[finfo.test_name]} reruns left)",
+                )
+                self.reruns[finfo.test_name] -= 1
+                self.to_run.add(finfo.test_name)
+            else:
+                util.info(
+                    f"{test_id} exited with non-zero: {finfo.exit_code}, "
+                    "all reruns exceeded, giving up",
+                )
+                # record the final result anyway
+                self.aggregator.ingest(
+                    self.platform,
+                    finfo.test_name,
+                    tmp_dir_path / "json_file",
+                    tmp_dir_path / "files_dir",
+                )
+                finfo.tmp_dir.cleanup()
+
+        # test finished successfully - ingest its results
+        else:
+            util.info(f"{test_id} finished successfully")
+            self.aggregator.ingest(
+                self.platform,
+                finfo.test_name,
+                tmp_dir_path / "json_file",
+                tmp_dir_path / "files_dir",
+            )
+            finfo.tmp_dir.cleanup()
+
+        # if the remote was not destroyed by traceback / failing test,
+        # check if the test always destroys it (even on success)
+        if not destructive:
+            test_data = self.fmf_tests.tests[finfo.test_name]
+            destructive = test_data.get("extra-atex", {}).get("destructive", False)
+
+        # if destroyed, release the remote
+        if destructive:
+            util.debug(f"{test_id} was destructive, releasing remote")
+            finfo.remote.release()
+
+        # if still not destroyed, run another test on it
+        # (without running plan setup, re-using already set up remote)
+        elif self.to_run:
+            sinfo = self.SetupInfo(
+                provisioner=finfo.provisioner,
+                remote=finfo.remote,
+                executor=finfo.executor,
+            )
+            util.debug(f"{test_id} was non-destructive, running next test")
+            self._run_new_test(sinfo)
+
+    def serve_once(self):
+        """
+        Run the orchestration logic, processing any outstanding requests
+        (for provisioning, new test execution, etc.) and returning once these
+        are taken care of.
+
+        Returns True to indicate that it should be called again by the user
+        (more work to be done), False once all testing is concluded.
+        """
+        util.debug(
+            f"to_run: {len(self.to_run)} tests / "
+            f"running: {len(self.running_tests)} tests, {len(self.running_setups)} setups",
+        )
+        # all done
+        if not self.to_run and not self.running_tests:
+            return False
+
+        # process all finished tests, potentially reusing remotes for executing
+        # further tests
+        while True:
+            try:
+                finfo = self.test_queue.get(block=False)
+            except util.ThreadQueue.Empty:
+                break
+            del self.running_tests[finfo.test_name]
+            self._process_finished_test(finfo)
+
+        # process any remotes with finished plan setup (uploaded tests,
+        # plan-defined pkgs / prepare scripts), start executing tests on them
+        while True:
+            try:
+                sinfo = self.setup_queue.get(block=False)
+            except util.ThreadQueue.Empty:
+                break
+            util.debug(f"setup finished for '{sinfo.remote}', running first test")
+            self.running_setups.remove(sinfo)
+            self._run_new_test(sinfo)
+
+        # try to get new remotes from Provisioners - if we get some, start
+        # running setup on them
+        for provisioner in self.provisioners:
+            while (remote := provisioner.get_remote(block=False)) is not None:
+                ex = executor.Executor(self.fmf_tests, remote)
+                sinfo = self.SetupInfo(
+                    provisioner=provisioner,
+                    remote=remote,
+                    executor=ex,
+                )
+                self.setup_queue.start_thread(
+                    target=self._run_setup,
+                    args=(sinfo,),
+                )
+                self.running_setups.append(sinfo)
+                util.debug(f"got remote '{remote}' from '{provisioner}', running setup")
+
+        return True
+
+    def serve_forever(self):
+        """
+        Run the orchestration logic, blocking until all testing is concluded.
+        """
+        while self.serve_once():
+            time.sleep(1)
+
+    def __enter__(self):
+        # start all provisioners
+        for prov in self.provisioners:
+            prov.start()
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        # cancel all running tests and wait for them to clean up (up to 0.1sec)
+        for rinfo in self.running_tests.values():
+            rinfo.executor.cancel()
+        self.test_queue.join()  # also ignore any exceptions raised
+
+        # stop all provisioners, also releasing all remotes
+        with concurrent.futures.ThreadPoolExecutor(max_workers=20) as ex:
+            for provisioner in self.provisioners:
+                for func in provisioner.stop_defer():
+                    ex.submit(func)
+
+    def next_test(self, tests, fmf_tests):  # noqa: ARG002, PLR6301
+        """
+        Return a test name (string) from a set of 'tests' (set of test name
+        strings) to be run next.
+
+        'fmf_tests' is a class FMFTests instance with additional test metadata.
+
+        This method is user-overridable, ie. by subclassing Orchestrator:
+
+            class CustomOrchestrator(Orchestrator):
+                @staticmethod
+                def next_test(tests):
+                    ...
+        """
+        # TODO: more advanced algorithm
+        #
+        # simple:
+        return next(iter(tests))

atex/provision/__init__.py

@@ -1,65 +1,33 @@
-import importlib
-import pkgutil
+import importlib as _importlib
+import pkgutil as _pkgutil
+import threading as _threading
 
-from .. import util
+from .. import connection as _connection
 
 
-class Provisioner(util.LockableClass):
+class Provisioner:
     """
-    A resource (machine/system) provider.
-
-    Any class derived from Provisioner serves as a mechanisms for requesting
-    a resource (machine/system), waiting for it to be reserved, providing ssh
-    details on how to connect to it, and releasing it when no longer useful.
-
-    The 4 main API points for this are reserve(), connection(), release() and
-    alive().
-    If necessary, these methods can share data via class instance attributes,
-    which are transparently guarded by a thread-aware mutex. For any complex
-    reads/writes, use 'self.lock' via a context manager.
-
-    Note that reserve() always runs in a separate thread (and thus may block),
-    and other functions (incl. release()) may be called at any time from
-    a different thread, even while reserve() is still running.
-    It is thus recommended for reserve() to store metadata in self.* as soon
-    as the metadata becomes available (some job ID, request UUID, Popen proc
-    object with PID, etc.) so that release() can free the resource at any time.
-
-    Once release()'d, the instance is never reused for reserve() again.
-    However connection(), release() and alive() may be called several times at
-    any time and need to handle it safely.
-    Ie. once released(), an instance must never return alive() == True.
-
-        # explicit method calls
-        res = Provisioner(...)
-        res.reserve()
-        conn = res.connection()
-        conn.connect()
-        conn.ssh('ls /')
-        conn.disconnect()
-        res.release()
-
-        # via a context manager
-        with Provisioner(...) as res:
-            with res.connection() as conn:
-                conn.ssh('ls /')
-
-    If a Provisioner class needs additional configuration, it should do so via
-    class (not instance) attributes, allowing it to be instantiated many times.
-
-        class ConfiguredProvisioner(Provisioner):
-            resource_hub = 'https://...'
-            login = 'joe'
-
-        # or dynamically
-        name = 'joe'
-        cls = type(
-            f'Provisioner_for_{name}',
-            (Provisioner,),
-            {'resource_hub': 'https://...', 'login': name},
-        )
-
-    These attributes can then be accessed from __init__ or any other function.
+    A remote resource (machine/system) provider.
+
+    The main interface is .get_remote() that returns a connected class Remote
+    instance for use by the user, to be .release()d when not needed anymore,
+    with Provisioner automatically getting a replacement for it, to be returned
+    via .get_remote() later.
+
+        p = Provisioner()
+        p.start()
+        remote = p.get_remote()
+        remote.cmd(["ls", "/"])
+        remote.release()
+        p.stop()
+
+        with Provisioner() as p:
+            remote = p.get_remote()
+            ...
+            remote.release()
+
+    Note that .stop() or .defer_stop() may be called from a different
+    thread, asynchronously to any other functions.
     """
 
     def __init__(self):
@@ -67,47 +35,90 @@ class Provisioner(util.LockableClass):
         Initialize the provisioner instance.
         If extending __init__, always call 'super().__init__()' at the top.
         """
-        super().__init__()
+        self.lock = _threading.RLock()
 
-    def reserve(self):
+    def get_remote(self, block=True):
         """
-        Send a reservation request for a resource and wait for it to be
-        reserved.
+        Get a connected class Remote instance.
+
+        If 'block' is True, wait for the remote to be available and connected,
+        otherwise return None if there is no Remote available yet.
         """
-        raise NotImplementedError(f"'reserve' not implemented for {self.__class__.__name__}")
+        raise NotImplementedError(f"'get_remote' not implemented for {self.__class__.__name__}")
 
-    def connection(self):
+    def start(self):
         """
-        Return an atex.ssh.SSHConn instance configured for connection to
-        the reserved resource, but not yet connected.
+        Start the Provisioner instance, start any provisioning-related
+        processes that lead to systems being reserved.
         """
-        raise NotImplementedError(f"'connection' not implemented for {self.__class__.__name__}")
+        raise NotImplementedError(f"'start' not implemented for {self.__class__.__name__}")
 
-    def release(self):
+    def stop(self):
         """
-        Release a reserved resource, or cancel a reservation-in-progress.
+        Stop the Provisioner instance, freeing all reserved resources,
+        calling .release() on all Remote instances that were created.
         """
-        raise NotImplementedError(f"'release' not implemented for {self.__class__.__name__}")
+        raise NotImplementedError(f"'stop' not implemented for {self.__class__.__name__}")
+
+    def stop_defer(self):
+        """
+        Enable an external caller to stop the Provisioner instance,
+        deferring resource deallocation to the caller.
+
+        Return an iterable of argument-free thread-safe callables that can be
+        called, possibly in parallel, to free up resources.
+        Ie. a list of 200 .release() functions, to be called in a thread pool
+        by the user, speeding up cleanup.
+        """
+        return self.stop
+
+    def __enter__(self):
+        self.start()
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.stop()
+
+
+class Remote(_connection.Connection):
+    """
+    Representation of a provisioned (reserved) remote system, providing
+    a Connection-like API in addition to system management helpers.
+
+    An instance of Remote is typically prepared by a Provisioner and lent out
+    for further use, to be .release()d by the user (if destroyed).
+    It is not meant for repeated reserve/release cycles, hence the lack
+    of .reserve().
+
+    Also note that Remote can be used via Context Manager, but does not
+    do automatic .release(), the manager only handles the built-in Connection.
+    The intention is for a Provisioner to run via its own Contest Manager and
+    release all Remotes upon exit.
+    If you need automatic release of one Remote, use a contextlib.ExitStack
+    with a callback, or a try/finally block.
+    """
 
-    def alive(self):
+    def release(self):
         """
-        Return True if the resource is still reserved, False otherwise.
+        Release (de-provision) the remote resource.
         """
-        raise NotImplementedError(f"'alive' not implemented for {self.__class__.__name__}")
-
-
-def find_provisioners():
-    provisioners = []
-    for info in pkgutil.iter_modules(__spec__.submodule_search_locations):
-        mod = importlib.import_module(f'.{info.name}', __name__)
-        # look for Provisioner-derived classes in the module
-        for attr in dir(mod):
-            if attr.startswith('_'):
-                continue
-            value = getattr(mod, attr)
-            try:
-                if issubclass(value, Provisioner):
-                    provisioners.append(attr)
-            except TypeError:
-                pass
-    return provisioners
+        raise NotImplementedError(f"'release' not implemented for {self.__class__.__name__}")
+
+
+_submodules = [
+    info.name for info in _pkgutil.iter_modules(__spec__.submodule_search_locations)
+]
+
+__all__ = [*_submodules, Provisioner.__name__, Remote.__name__]  # noqa: PLE0604
+
+
+def __dir__():
+    return __all__
+
+
+# lazily import submodules
+def __getattr__(attr):
+    if attr in _submodules:
+        return _importlib.import_module(f".{attr}", __name__)
+    else:
+        raise AttributeError(f"module '{__name__}' has no attribute '{attr}'")

atex/provision/libvirt/VM_PROVISION

@@ -49,3 +49,11 @@ FULLY CUSTOM INSTALLS:
 - basically virt-install creating a new domain (ignoring any pre-defined ones)
   - probably shouldn't be used by automation, only for one-VM-at-a-time on user request
     - (no free memory/disk checking, no libvirt locking, etc.)
+
+
+
+# ssh via ProxyJump allowing ssh keys specification
+ssh \
+    -o ProxyCommand='ssh -i /tmp/proxy_sshkey root@3.21.232.206 -W %h:%p' \
+    -i /tmp/destination_sshkey \
+    root@192.168.123.218

atex/provision/libvirt/__init__.py

@@ -1,8 +1,8 @@
-from .. import Provisioner as _Provisioner
+from .. import base
 from ... import util, ssh
 
 
-class LibvirtProvisioner(_Provisioner):
+class LibvirtProvisioner(base.Provisioner):
     number = 123
 
     def reserve(self):
@@ -12,9 +12,9 @@ class LibvirtProvisioner(_Provisioner):
     #       can be overriden by a getter function if you need to keep track
     #       how many times it was accessed
     def connection(self):
-        #return {'Hostname': '1.2.3.4', 'User': 'root', 'IdentityFile': ...}
+        #return {"Hostname": "1.2.3.4", "User": "root", "IdentityFile": ...}
         util.debug(f"returning ssh for {self.number}")
-        return ssh.SSHConn({'Hostname': '1.2.3.4', 'User': 'root'})
+        return ssh.SSHConn({"Hostname": "1.2.3.4", "User": "root"})
 
     def release(self):
         util.debug(f"releasing {self.number}")

atex/provision/podman/README

@@ -0,0 +1,59 @@
+
+making a podman image from the currently installed OS:
+
+1) dnf install into a separate installroot
+
+dnf
+    --installroot=$INSTALLROOT \
+    --setopt=install_weak_deps=False \
+    --setopt=tsflags=nodocs \
+    -y groupinstall minimal-environment
+
+as root (doesn't work well with unshare, maybe could work via bwrap (bubblewrap))
+
+maybe the unprivileged solution is pulling image from hub + installing @minimal-environment
+into it (perhaps via podman build)
+
+
+2) post process it
+
+echo -n > "$INSTALLROOT/etc/machine-id"
+echo container > "$INSTALLROOT/etc/hostname"
+
+rm -rf "$INSTALLROOT/etc/yum.repos.d"
+cp -f /etc/yum.repos.d/* "$INSTALLROOT/etc/yum.repos.d/."
+cp -f /etc/pki/rpm-gpg/* "$INSTALLROOT/etc/pki/rpm-gpg/."
+
+echo install_weak_deps=False >> "$INSTALLROOT/etc/dnf/dnf.conf"
+echo tsflags=nodocs >> "$INSTALLROOT/etc/dnf/dnf.conf"
+
+ln -sf \
+    /usr/lib/systemd/system/multi-user.target \
+    "$INSTALLROOT/etc/systemd/system/default.target"
+
+# disable auditd
+# disable other services
+# set root password
+
+dnf clean all --installroot="$INSTALLROOT"
+
+
+3) pack it
+
+tar --xattrs -C "$INSTALLROOT" -cvf tarball.tar .
+
+rm -rf "$INSTALLROOT"
+
+
+4) import it to podman
+
+podman import --change 'CMD ["/sbin/init"]' tarball.tar my-image-name
+
+
+5) run it
+
+podman {run,create} --systemd=always --cgroups=split ...
+
+
+
+------------------------------