atex 0.7__py3-none-any.whl → 0.9__py3-none-any.whl

atex/provision/testingfarm/testingfarm.py

@@ -0,0 +1,234 @@
+import time
+import tempfile
+import threading
+
+from ... import connection, util
+from .. import Provisioner, Remote
+
+from . import api
+
+
+class TestingFarmRemote(Remote, connection.ssh.ManagedSSHConn):
+    """
+    Built on the official Remote API, pulling in the Connection API
+    as implemented by ManagedSSHConn.
+    """
+
+    def __init__(self, request_id, ssh_options, *, release_hook):
+        """
+        'request_id' is a string with Testing Farm request UUID (for printouts).
+
+        'ssh_options' are a dict, passed to ManagedSSHConn __init__().
+
+        'release_hook' is a callable called on .release() in addition
+        to disconnecting the connection.
+        """
+        # NOTE: self.lock inherited from ManagedSSHConn
+        super().__init__(options=ssh_options)
+        self.request_id = request_id
+        self.release_hook = release_hook
+        self.release_called = False
+
+    def release(self):
+        with self.lock:
+            if self.release_called:
+                return
+            else:
+                self.release_called = True
+        self.release_hook(self)
+        self.disconnect()
+
+    # not /technically/ a valid repr(), but meh
+    def __repr__(self):
+        class_name = self.__class__.__name__
+        ssh_user = self.options.get("User", "unknown")
+        ssh_host = self.options.get("Hostname", "unknown")
+        ssh_port = self.options.get("Port", "unknown")
+        ssh_key = self.options.get("IdentityFile", "unknown")
+        return f"{class_name}({ssh_user}@{ssh_host}:{ssh_port}@{ssh_key}, {self.request_id})"
+
+
+class TestingFarmProvisioner(Provisioner):
+    # TODO: have max_systems as (min,default,max) tuple; have an algorithm that
+    #       starts at default and scales up/down as needed
+
+    def __init__(self, compose, arch="x86_64", *, max_systems=1, max_retries=10, **reserve_kwargs):
+        """
+        'compose' is a Testing Farm compose to prepare.
+
+        'arch' is an architecture associated with the compose.
+
+        'max_systems' is an int of how many systems to reserve (and keep
+        reserved) in an internal pool.
+
+        'max_retries' is a maximum number of provisioning (Testing Farm) errors
+        that will be reprovisioned before giving up.
+        """
+        self.lock = threading.RLock()
+        self.compose = compose
+        self.arch = arch
+        self.max_systems = max_systems
+        self.reserve_kwargs = reserve_kwargs
+        self.retries = max_retries
+
+        self._tmpdir = None
+        self.ssh_key = self.ssh_pubkey = None
+        self.queue = util.ThreadQueue(daemon=True)
+        self.tf_api = api.TestingFarmAPI()
+
+        # TF Reserve instances (not Remotes) actively being provisioned,
+        # in case we need to call their .release() on abort
+        self.reserving = []
+
+        # active TestingFarmRemote instances, ready to be handed over to the user,
+        # or already in use by the user
+        self.remotes = []
+
+    def _wait_for_reservation(self, tf_reserve, initial_delay):
+        # assuming this function will be called many times, attempt to
+        # distribute load on TF servers
+        # (we can sleep here as this code is running in a separate thread)
+        if initial_delay:
+            util.debug(f"delaying for {initial_delay}s to distribute load")
+            time.sleep(initial_delay)
+
+        # 'machine' is api.Reserve.ReservedMachine namedtuple
+        machine = tf_reserve.reserve()
+
+        # connect our Remote to the machine via its class Connection API
+        ssh_options = {
+            "Hostname": machine.host,
+            "User": machine.user,
+            "Port": machine.port,
+            "IdentityFile": machine.ssh_key,
+            "ConnectionAttempts": "1000",
+            "Compression": "yes",
+        }
+
+        def release_hook(remote):
+            # remove from the list of remotes inside this Provisioner
+            with self.lock:
+                try:
+                    self.remotes.remove(remote)
+                except ValueError:
+                    pass
+            # call TF API, cancel the request, etc.
+            tf_reserve.release()
+
+        remote = TestingFarmRemote(
+            tf_reserve.request.id,
+            ssh_options,
+            release_hook=release_hook,
+        )
+        remote.connect()
+
+        # since the system is fully ready, stop tracking its reservation
+        # and return the finished Remote instance
+        with self.lock:
+            self.remotes.append(remote)
+            self.reserving.remove(tf_reserve)
+
+        return remote
+
+    def _schedule_one_reservation(self, initial_delay=None):
+        # instantiate a class Reserve from the Testing Farm api module
+        # (which typically provides context manager, but we use its .reserve()
+        #  and .release() functions directly)
+        tf_reserve = api.Reserve(
+            compose=self.compose,
+            arch=self.arch,
+            ssh_key=self.ssh_key,
+            api=self.tf_api,
+            **self.reserve_kwargs,
+        )
+
+        # add it to self.reserving even before we schedule a provision,
+        # to avoid races on suddent abort
+        with self.lock:
+            self.reserving.append(tf_reserve)
+
+        # start a background wait
+        self.queue.start_thread(
+            target=self._wait_for_reservation,
+            target_args=(tf_reserve, initial_delay),
+        )
+
+    def start(self):
+        with self.lock:
+            self._tmpdir = tempfile.TemporaryDirectory()
+            self.ssh_key, self.ssh_pubkey = util.ssh_keygen(self._tmpdir.name)
+            # start up all initial reservations
+            for i in range(self.max_systems):
+                delay = (api.API_QUERY_DELAY / self.max_systems) * i
+                #self.queue.start_thread(target=self._schedule_one_reservation, args=(delay,))
+                self._schedule_one_reservation(delay)
+
+    def stop(self):
+        with self.lock:
+            # abort reservations in progress
+            while self.reserving:
+                # testingfarm api.Reserve instances
+                self.reserving.pop().release()
+            # cancel/release all Remotes ever created by us
+            while self.remotes:
+                # TestingFarmRemote instances
+                self.remotes.pop().release()
+            # explicitly remove the tmpdir rather than relying on destructor
+            self._tmpdir.cleanup()
+            self._tmpdir = None
+
+    def stop_defer(self):
+        callables = []
+        with self.lock:
+            callables += (f.release for f in self.reserving)
+            self.reserving = []
+            callables += (r.release for r in self.remotes)
+            self.remotes = []  # just in case
+            callables.append(self._tmpdir.cleanup)
+            self._tmpdir = None
+        return callables
+
+    def get_remote(self, block=True):
+        # fill .release()d remotes back up with reservations
+        with self.lock:
+            deficit = self.max_systems - len(self.remotes) - len(self.reserving)
+            for i in range(deficit):
+                delay = (api.API_QUERY_DELAY / deficit) * i
+                self._schedule_one_reservation(delay)
+
+        while True:
+            # otherwise wait on a queue of Remotes being provisioned
+            try:
+                return self.queue.get(block=block)  # thread-safe
+            except util.ThreadQueue.Empty:
+                # always non-blocking
+                return None
+            except (api.TestingFarmError, connection.ssh.SSHError) as e:
+                with self.lock:
+                    if self.retries > 0:
+                        util.warning(
+                            f"caught while reserving a TF system: {repr(e)}, "
+                            f"retrying ({self.retries} left)",
+                        )
+                        self.retries -= 1
+                        self._schedule_one_reservation()
+                        if block:
+                            continue
+                        else:
+                            return None
+                    else:
+                        util.warning(
+                            f"caught while reserving a TF system: {repr(e)}, "
+                            "exhausted all retries, giving up",
+                        )
+                        raise
+
+    # not /technically/ a valid repr(), but meh
+    def __repr__(self):
+        class_name = self.__class__.__name__
+        reserving = len(self.reserving)
+        remotes = len(self.remotes)
+        return (
+            f"{class_name}({self.compose} @ {self.arch}, {reserving} reserving, "
+            f"{remotes} remotes, {hex(id(self))})"
+        )

atex/util/__init__.py

@@ -1,7 +1,3 @@
-"""
-TODO some description about utilities
-"""
-
 import importlib as _importlib
 import pkgutil as _pkgutil
 import inspect as _inspect
@@ -39,8 +35,7 @@ def _import_submodules():
             if _inspect.ismodule(attr):
                 continue
             # do not override already processed objects (avoid duplicates)
-            if key in __all__:
-                raise AssertionError(f"tried to override already-imported '{key}'")
+            assert key not in __all__, f"tried to override already-imported '{key}'"
 
             globals()[key] = attr
             __all__.append(key)

atex/util/libvirt.py

@@ -0,0 +1,18 @@
+import importlib
+
+
+def import_libvirt():
+    try:
+        libvirt = importlib.import_module("libvirt")
+    except ModuleNotFoundError:
+        raise ModuleNotFoundError(
+            "No module named 'libvirt', you need to install it from the package"
+            " manager of your distro, ie. 'dnf install python3-libvirt' as it"
+            " requires distro-wide headers to compile. It won't work from PyPI."
+            " If using venv, create it with '--system-site-packages'.",
+        ) from None
+
+    # suppress console error printing, behave like a good python module
+    libvirt.registerErrorHandler(lambda _ctx, _err: None, None)
+
+    return libvirt

atex/util/log.py

@@ -5,6 +5,15 @@ from pathlib import Path
 _logger = logging.getLogger("atex")
 
 
+def in_debug_mode():
+    """
+    Return True if the root logger is using the DEBUG (or more verbose) level.
+    """
+    # TODO: use _logger.isEnabledFor() ?
+    root_level = logging.getLogger().level
+    return root_level > 0 and root_level <= logging.DEBUG
+
+
 def _format_msg(msg, *, skip_frames=0):
     stack = inspect.stack()
     if len(stack)-1 <= skip_frames:
@@ -23,11 +32,16 @@ def _format_msg(msg, *, skip_frames=0):
 
     # if the function has 'self' and it looks like a class instance,
     # prepend it to the function name
-    p_locals = parent.frame.f_locals
-    if "self" in p_locals:
-        self = p_locals["self"]
-        if hasattr(self, "__class__") and inspect.isclass(self.__class__):
-            function = f"{self.__class__.__name__}.{function}"
+    argvals = inspect.getargvalues(parent.frame)
+    if argvals.args:
+        if argvals.args[0] == "self":
+            self = argvals.locals["self"]
+            if hasattr(self, "__class__") and inspect.isclass(self.__class__):
+                function = f"{self.__class__.__name__}.{function}"
+        elif argvals.args[0] == "cls":
+            cls = argvals.locals["cls"]
+            if inspect.isclass(cls):
+                function = f"{cls.__name__}.{function}"
 
     # don't report module name of a function if it's the same as running module
     if parent.filename != module.filename:
@@ -42,12 +56,21 @@ def _format_msg(msg, *, skip_frames=0):
 
 
 def debug(msg, *, skip_frames=0):
-    _logger.debug(_format_msg(msg, skip_frames=skip_frames+1))
+    if in_debug_mode():
+        _logger.debug(_format_msg(msg, skip_frames=skip_frames+1))
+    else:
+        _logger.debug(msg)
 
 
 def info(msg, *, skip_frames=0):
-    _logger.info(_format_msg(msg, skip_frames=skip_frames+1))
+    if in_debug_mode():
+        _logger.info(_format_msg(msg, skip_frames=skip_frames+1))
+    else:
+        _logger.info(msg)
 
 
 def warning(msg, *, skip_frames=0):
-    _logger.warning(_format_msg(msg, skip_frames=skip_frames+1))
+    if in_debug_mode():
+        _logger.warning(_format_msg(msg, skip_frames=skip_frames+1))
+    else:
+        _logger.warning(msg)

atex/util/named_mapping.py

@@ -0,0 +1,158 @@
+"""
+Provides a namedtuple-inspired frozen mapping-backed data structure.
+
+    class MyMap(NamedMapping):
+        pass
+
+    m = MyMap(a=123, b=456)
+
+    m["a"]      # 123
+    m.a         # 123
+    m["a"] = 9  # KeyError (is read-only)
+    m.a = 9     # AttributeError (is read-only)
+
+Like namedtuple, you can specify required keys that always need to be given
+to the constructor:
+
+    class MyMap(NamedMapping, required=("key1", "key2")):
+        pass
+
+    m = MyMap(a=123, b=456, key1=999)  # KeyError (key2 not specified)
+
+Similarly, you can specify defaults (for required or non-required keys),
+as a dict, that are used if omitted from the constructor:
+
+    class MyMap(NamedMapping, defaults={"key": 678}):
+        pass
+
+    m = MyMap()  # will have m.key == 678
+
+A class instance can unpack via ** with the entirety of its mapping contents:
+
+    m = MyMap(key2=456)
+    both = {'key1': 123, **m}  # contains both keys
+
+You can also chain (append to) required / default values through inheritance:
+
+    class MyMap(NamedMapping, required=("key1",), defaults={"key2": 234}):
+        pass
+
+    class AnotherMap(MyMap, required=("key3",))
+        pass
+
+    m = AnotherMap()      # KeyError (key1 and key3 are required)
+
+    isinstance(m, MyMap)  # would be True
+
+When instantiating, it is also possible to copy just the required keys from
+another dict-like object (does not have to be a parent of the class):
+
+    class SmallMap(NamedMapping, required=("key1", "key2")):
+        pass
+
+    class BigMap(SmallMap, required=("key3", "key4")):
+        pass
+
+    b = BigMap(key1=123, key2=456, key3=789, key4=0)
+
+    s = SmallMap._from(b)             # will copy just key1 and key2
+    s = SmallMap._from(b, extra=555)  # can pass extra **kwargs to __init__
+    s = SmallMap(**b)                 # will copy all keys
+
+Note that this is a fairly basic implementation without __hash__, etc.
+"""
+
+import abc
+import collections
+
+
+class _NamedMappingMeta(abc.ABCMeta):
+    def __new__(
+        metacls, name, bases, namespace, *, required=None, default=None, **kwargs,  # noqa: N804
+    ):
+        new_required = []
+        for base in bases:
+            new_required.extend(getattr(base, "_required", ()))
+        if required:
+            new_required.extend(required)
+        namespace["_required"] = tuple(set(new_required))
+
+        new_default = {}
+        for base in bases:
+            new_default.update(getattr(base, "_default", {}))
+        if default:
+            new_default.update(default)
+        namespace["_default"] = new_default
+
+        return super().__new__(metacls, name, bases, namespace, **kwargs)
+
+
+class NamedMapping(collections.abc.Mapping, metaclass=_NamedMappingMeta):
+    __slots__ = ("_data",)
+
+    def __init__(self, **keys):
+        data = {}
+        if hasattr(self, "_default"):
+            data.update(self._default)
+        data.update(keys)
+        if hasattr(self, "_required"):
+            for key in self._required:
+                if key not in data:
+                    raise KeyError(f"'{self.__class__.__name__}' requires key '{key}'")
+        object.__setattr__(self, "_data", data)
+
+    @classmethod
+    def _from(cls, foreign, **keys):
+        """
+        (keys is like for __init__)
+        """
+        foreign_data = {}
+        if hasattr(cls, "_required"):
+            for key in cls._required:
+                if key in foreign:
+                    foreign_data[key] = foreign[key]
+        foreign_data.update(keys)
+        return cls(**foreign_data)
+
+    def __getattr__(self, item):
+        if item in ("_data", "_required", "_default"):
+            return super().__getattr__(item)
+        try:
+            return self._data[item]
+        except KeyError:
+            raise AttributeError(
+                f"'{self.__class__.__name__}' object has no attribute '{item}'",
+                name=item,
+            ) from None
+
+    def __setattr__(self, name, value):
+        raise AttributeError(
+            f"'{self}' is read-only, cannot set '{name}'",
+            name=name,
+            obj=value,
+        )
+
+    def __getitem__(self, key):
+        return self._data[key]
+
+    def __setitem__(self, key, value):
+        raise ValueError(f"'{self}' is read-only, cannot set '{key}'")
+
+    def __delitem__(self, key):
+        raise ValueError(f"'{self}' is read-only, cannot delete '{key}'")
+
+    def __contains__(self, key):
+        return key in self._data
+
+    def __iter__(self):
+        return iter(self._data)
+
+    def __len__(self):
+        return len(self._data)
+
+    def __repr__(self):
+        return (
+            f"{self.__class__.__name__}("
+            + ", ".join((f"{k}={repr(v)}" for k,v in self._data.items()))
+            + ")"
+        )

atex/util/path.py

@@ -0,0 +1,16 @@
+import os
+
+
+def normalize_path(path):
+    """
+    Transform a potentially dangerous path (leading slash, relative ../../../
+    leading beyond parent, etc.) to a safe one.
+
+    Always returns a relative path.
+    """
+    # the magic here is to treat any dangerous path as starting at /
+    # and resolve any weird constructs relative to /, and then simply
+    # strip off the leading / and use it as a relative path
+    path = path.lstrip("/")
+    path = os.path.normpath(f"/{path}")
+    return path[1:]

atex/util/ssh_keygen.py

@@ -0,0 +1,14 @@
+import subprocess
+from pathlib import Path
+
+from .subprocess import subprocess_run
+
+
+def ssh_keygen(dest_dir, key_type="rsa"):
+    dest_dir = Path(dest_dir)
+    subprocess_run(
+        ("ssh-keygen", "-t", key_type, "-N", "", "-f", dest_dir / f"key_{key_type}"),
+        stdout=subprocess.DEVNULL,
+        check=True,
+    )
+    return (dest_dir / "key_rsa", dest_dir / "key_rsa.pub")

atex/util/threads.py

@@ -0,0 +1,99 @@
+import queue
+import threading
+
+from .named_mapping import NamedMapping
+
+# TODO: documentation; this is like concurrent.futures, but with daemon=True support
+
+
+class ThreadQueue:
+    class ThreadReturn(NamedMapping, required=("thread", "returned", "exception")):
+        pass
+
+    Empty = queue.Empty
+
+    def __init__(self, daemon=False):
+        self.lock = threading.RLock()
+        self.queue = queue.SimpleQueue()
+        self.daemon = daemon
+        self.threads = set()
+
+    def _wrapper(self, func, func_args, func_kwargs, **user_kwargs):
+        current_thread = threading.current_thread()
+        try:
+            ret = func(*func_args, **func_kwargs)
+            result = self.ThreadReturn(
+                thread=current_thread,
+                returned=ret,
+                exception=None,
+                **user_kwargs,
+            )
+        except Exception as e:
+            result = self.ThreadReturn(
+                thread=current_thread,
+                returned=None,
+                exception=e,
+                **user_kwargs,
+            )
+        self.queue.put(result)
+
+    def start_thread(self, target, target_args=None, target_kwargs=None, **user_kwargs):
+        """
+        Start a new thread and call 'target' as a callable inside it, passing it
+        'target_args' as arguments and 'target_kwargs' as keyword arguments.
+
+        Any additional 'user_kwargs' specified are NOT passed to the callable,
+        but instead become part of the ThreadReturn namespace returned by the
+        .get_raw() method.
+        """
+        t = threading.Thread(
+            target=self._wrapper,
+            args=(target, target_args or (), target_kwargs or {}),
+            kwargs=user_kwargs,
+            daemon=self.daemon,
+        )
+        with self.lock:
+            self.threads.add(t)
+        t.start()
+
+    def get_raw(self, block=True, timeout=None):
+        """
+        Wait for and return the next available ThreadReturn instance on the
+        queue, as enqueued by a finished callable started by the .start_thread()
+        method.
+        """
+        with self.lock:
+            if block and timeout is None and not self.threads:
+                raise AssertionError("no threads are running, would block forever")
+        treturn = self.queue.get(block=block, timeout=timeout)
+        with self.lock:
+            self.threads.remove(treturn.thread)
+        return treturn
+
+    # get one return value from any thread's function, like .as_completed()
+    # or concurrent.futures.FIRST_COMPLETED
+    def get(self, block=True, timeout=None):
+        """
+        Wait for and return the next available return value of a callable
+        enqueued via the .start_thread() method.
+
+        If the callable raised an exception, the exception is re-raised here.
+        """
+        treturn = self.get_raw(block, timeout)
+        if treturn.exception is not None:
+            raise treturn.exception
+        else:
+            return treturn.returned
+
+    # wait for all threads to finish (ignoring queue contents)
+    def join(self):
+        """
+        Wait for all threads to finish, ignoring the state of the queue.
+        """
+        while True:
+            with self.lock:
+                try:
+                    thread = self.threads.pop()
+                except KeyError:
+                    break
+            thread.join()