atex 0.8__py3-none-any.whl → 0.10__py3-none-any.whl

atex/{provision → provisioner}/testingfarm/testingfarm.py

@@ -8,73 +8,64 @@ from .. import Provisioner, Remote
 from . import api
 
 
-class TestingFarmRemote(Remote, connection.ssh.ManagedSSHConn):
+class TestingFarmRemote(Remote, connection.ssh.ManagedSSHConnection):
     """
     Built on the official Remote API, pulling in the Connection API
-    as implemented by ManagedSSHConn.
+    as implemented by ManagedSSHConnection.
     """
 
-    def __init__(self, ssh_options, *, release_hook, provisioner):
+    def __init__(self, request_id, ssh_options, *, release_hook):
         """
-        'ssh_options' are a dict, passed to ManagedSSHConn __init__().
+        'request_id' is a string with Testing Farm request UUID (for printouts).
+
+        'ssh_options' are a dict, passed to ManagedSSHConnection __init__().
 
         'release_hook' is a callable called on .release() in addition
         to disconnecting the connection.
         """
-        # start with empty ssh options, we'll fill them in later
+        # NOTE: self.lock inherited from ManagedSSHConnection
         super().__init__(options=ssh_options)
+        self.request_id = request_id
         self.release_hook = release_hook
-        self.provisioner = provisioner
-        self.lock = threading.RLock()
         self.release_called = False
 
     def release(self):
         with self.lock:
-            if not self.release_called:
-                self.release_called = True
-            else:
+            if self.release_called:
                 return
-        self.release_hook(self)
+            else:
+                self.release_called = True
         self.disconnect()
+        self.release_hook(self)
 
     # not /technically/ a valid repr(), but meh
     def __repr__(self):
         class_name = self.__class__.__name__
-        compose = self.provisioner.compose
-        arch = self.provisioner.arch
-        return f"{class_name}({compose} @ {arch}, {hex(id(self))})"
-
-#    def alive(self):
-#        return self.valid
-
-    # TODO: def __str__(self):  as root@1.2.3.4 and arch, ranch, etc.
+        ssh_user = self.options.get("User", "unknown")
+        ssh_host = self.options.get("Hostname", "unknown")
+        ssh_port = self.options.get("Port", "unknown")
+        ssh_key = self.options.get("IdentityFile", "unknown")
+        return f"{class_name}({ssh_user}@{ssh_host}:{ssh_port}@{ssh_key}, {self.request_id})"
 
 
 class TestingFarmProvisioner(Provisioner):
-    # TODO: have max_systems as (min,default,max) tuple; have an algorithm that
-    #       starts at default and scales up/down as needed
+    absolute_max_remotes = 100
 
-    def __init__(self, compose, arch="x86_64", *, max_systems=1, timeout=60, max_retries=10):
+    def __init__(self, compose, arch="x86_64", *, max_retries=10, **reserve_kwargs):
         """
         'compose' is a Testing Farm compose to prepare.
 
         'arch' is an architecture associated with the compose.
 
-        'max_systems' is an int of how many systems to reserve (and keep
-        reserved) in an internal pool.
-
-        'timeout' is the maximum Testing Farm pipeline timeout (waiting for
-        a system + OS installation + reservation time).
-
         'max_retries' is a maximum number of provisioning (Testing Farm) errors
         that will be reprovisioned before giving up.
         """
-        super().__init__()
-        self.compose = compose  # TODO: translate "centos9" to "CentOS-Stream-9"
+        self.lock = threading.RLock()
+        self.compose = compose
         self.arch = arch
-        self.max_systems = max_systems
-        self.timeout = timeout
+        self.reserve_kwargs = reserve_kwargs
         self.retries = max_retries
+
         self._tmpdir = None
         self.ssh_key = self.ssh_pubkey = None
         self.queue = util.ThreadQueue(daemon=True)
@@ -104,7 +95,9 @@ class TestingFarmProvisioner(Provisioner):
             "Hostname": machine.host,
             "User": machine.user,
             "Port": machine.port,
-            "IdentityFile": machine.ssh_key,
+            "IdentityFile": machine.ssh_key.absolute(),
+            "ConnectionAttempts": "1000",
+            "Compression": "yes",
         }
 
         def release_hook(remote):
@@ -118,9 +111,9 @@ class TestingFarmProvisioner(Provisioner):
             tf_reserve.release()
 
         remote = TestingFarmRemote(
+            tf_reserve.request.id,
             ssh_options,
             release_hook=release_hook,
-            provisioner=self,
         )
         remote.connect()
 
@@ -139,9 +132,9 @@ class TestingFarmProvisioner(Provisioner):
         tf_reserve = api.Reserve(
             compose=self.compose,
             arch=self.arch,
-            timeout=self.timeout,
             ssh_key=self.ssh_key,
             api=self.tf_api,
+            **self.reserve_kwargs,
         )
 
         # add it to self.reserving even before we schedule a provision,
@@ -152,29 +145,24 @@ class TestingFarmProvisioner(Provisioner):
         # start a background wait
         self.queue.start_thread(
             target=self._wait_for_reservation,
-            args=(tf_reserve, initial_delay),
+            target_args=(tf_reserve, initial_delay),
         )
 
     def start(self):
         with self.lock:
             self._tmpdir = tempfile.TemporaryDirectory()
             self.ssh_key, self.ssh_pubkey = util.ssh_keygen(self._tmpdir.name)
-            # start up all initial reservations
-            for i in range(self.max_systems):
-                delay = (api.API_QUERY_DELAY / self.max_systems) * i
-                #self.queue.start_thread(target=self._schedule_one_reservation, args=(delay,))
-                self._schedule_one_reservation(delay)
 
     def stop(self):
         with self.lock:
             # abort reservations in progress
-            for tf_reserve in self.reserving:
-                tf_reserve.release()
-            self.reserving = []
+            while self.reserving:
+                # testingfarm api.Reserve instances
+                self.reserving.pop().release()
             # cancel/release all Remotes ever created by us
-            for remote in self.remotes:
-                remote.release()
-            self.remotes = []  # just in case
+            while self.remotes:
+                # TestingFarmRemote instances
+                self.remotes.pop().release()
             # explicitly remove the tmpdir rather than relying on destructor
             self._tmpdir.cleanup()
             self._tmpdir = None
@@ -190,14 +178,18 @@ class TestingFarmProvisioner(Provisioner):
             self._tmpdir = None
         return callables
 
-    def get_remote(self, block=True):
-        # fill .release()d remotes back up with reservations
+    def provision(self, count=1):
         with self.lock:
-            deficit = self.max_systems - len(self.remotes) - len(self.reserving)
-            for i in range(deficit):
-                delay = (api.API_QUERY_DELAY / deficit) * i
+            reservations = len(self.remotes) + len(self.reserving)
+            # clamp count to absolute_max_remotes
+            if count + reservations > self.absolute_max_remotes:
+                count = self.absolute_max_remotes - reservations
+            # spread out the request submissions
+            for i in range(count):
+                delay = (api.Request.api_query_limit / count) * i
                 self._schedule_one_reservation(delay)
 
+    def get_remote(self, block=True):
         while True:
             # otherwise wait on a queue of Remotes being provisioned
             try:

atex/util/libvirt.py

@@ -0,0 +1,18 @@
+import importlib
+
+
+def import_libvirt():
+    try:
+        libvirt = importlib.import_module("libvirt")
+    except ModuleNotFoundError:
+        raise ModuleNotFoundError(
+            "No module named 'libvirt', you need to install it from the package"
+            " manager of your distro, ie. 'dnf install python3-libvirt' as it"
+            " requires distro-wide headers to compile. It won't work from PyPI."
+            " If using venv, create it with '--system-site-packages'.",
+        ) from None
+
+    # suppress console error printing, behave like a good python module
+    libvirt.registerErrorHandler(lambda _ctx, _err: None, None)
+
+    return libvirt

atex/util/log.py

@@ -1,61 +1,71 @@
-import inspect
+import os
 import logging
-from pathlib import Path
+import inspect
 
 _logger = logging.getLogger("atex")
 
+# which functions to skip when determining the logger function caller;
+# typically, these are wrappers and we want to see their caller in the trace
+# instead of them
+#
+# ( file basename , qualname )
+# where qualname is '<module>' or 'funcname' or 'Classname.funcname'
+skip_levels = {
+    ("subprocess.py", "subprocess_run"),
+    ("subprocess.py", "subprocess_output"),
+    ("subprocess.py", "subprocess_Popen"),
+    ("subprocess.py", "subprocess_stream"),
+    ("subprocess.py", "subprocess_log"),
 
-def in_debug_mode():
-    """
-    Return True if the root logger is using the DEBUG (or more verbose) level.
-    """
-    root_level = logging.getLogger().level
-    return root_level > 0 and root_level <= logging.DEBUG
+    ("podman.py", "PodmanConnection.cmd"),
+    ("podman.py", "PodmanConnection.rsync"),
 
+    ("ssh.py", "StatelessSSHConnection.cmd"),
+    ("ssh.py", "StatelessSSHConnection.rsync"),
+    ("ssh.py", "ManagedSSHConnection.forward"),
+    ("ssh.py", "ManagedSSHConnection.cmd"),
+    ("ssh.py", "ManagedSSHConnection.rsync"),
+}
 
-def _format_msg(msg, *, skip_frames=0):
-    stack = inspect.stack()
-    if len(stack)-1 <= skip_frames:
-        raise SyntaxError("skip_frames exceeds call stack (frame count)")
-    stack = stack[skip_frames+1:]
 
-    # bottom of the stack, or runpy executed module
-    for frame_info in stack:
-        if frame_info.function == "<module>":
+def _log_msg(logger_func, *args, stacklevel=1, **kwargs):
+    # inspect.stack() is MUCH slower
+    caller = inspect.currentframe().f_back.f_back
+    extra_levels = 2  # skip this func and the debug/info/warning parent
+    while caller.f_back:
+        code = caller.f_code
+        # pathlib is much slower
+        basename = os.path.basename(code.co_filename)  # noqa: PTH119
+        qualname = code.co_qualname
+        if (basename, qualname) in skip_levels:
+            extra_levels += 1
+            caller = caller.f_back
+        else:
             break
-    module = frame_info
+    return logger_func(*args, stacklevel=stacklevel+extra_levels, **kwargs)
+
 
-    # last (topmost) function that isn't us
-    parent = stack[0]
-    function = parent.function
+def warning(*args, **kwargs):
+    return _log_msg(_logger.warning, *args, **kwargs)
 
-    # if the function has 'self' and it looks like a class instance,
-    # prepend it to the function name
-    p_locals = parent.frame.f_locals
-    if "self" in p_locals:
-        self = p_locals["self"]
-        if hasattr(self, "__class__") and inspect.isclass(self.__class__):
-            function = f"{self.__class__.__name__}.{function}"
 
-    # don't report module name of a function if it's the same as running module
-    if parent.filename != module.filename:
-        parent_modname = parent.frame.f_globals["__name__"]
-        # avoid everything having the package name prefixed
-        parent_modname = parent_modname.partition(".")[2] or parent_modname
-        return f"{parent_modname}.{function}:{parent.lineno}: {msg}"
-    elif parent.function != "<module>":
-        return f"{function}:{parent.lineno}: {msg}"
-    else:
-        return f"{Path(parent.filename).name}:{parent.lineno}: {msg}"
+def info(*args, **kwargs):
+    return _log_msg(_logger.info, *args, **kwargs)
 
 
-def debug(msg, *, skip_frames=0):
-    _logger.debug(_format_msg(msg, skip_frames=skip_frames+1))
+def debug(*args, **kwargs):
+    return _log_msg(_logger.debug, *args, **kwargs)
 
 
-def info(msg, *, skip_frames=0):
-    _logger.info(_format_msg(msg, skip_frames=skip_frames+1))
+# add a log level more verbose than logging.DEBUG, for verbose command
+# outputs, big JSON / XML printouts, and other outputs unsuitable for
+# large parallel runs; to be used in targeted debugging
+#
+# logging.DEBUG is 10, and programs tend to add TRACE as 5, so be somewhere
+# in between
+EXTRADEBUG = 8
+logging.addLevelName(EXTRADEBUG, "EXTRADEBUG")
 
 
-def warning(msg, *, skip_frames=0):
-    _logger.warning(_format_msg(msg, skip_frames=skip_frames+1))
+def extradebug(*args, **kwargs):
+    return _log_msg(_logger.log, EXTRADEBUG, *args, **kwargs)

atex/util/named_mapping.py

@@ -0,0 +1,158 @@
+"""
+Provides a namedtuple-inspired frozen mapping-backed data structure.
+
+    class MyMap(NamedMapping):
+        pass
+
+    m = MyMap(a=123, b=456)
+
+    m["a"]      # 123
+    m.a         # 123
+    m["a"] = 9  # KeyError (is read-only)
+    m.a = 9     # AttributeError (is read-only)
+
+Like namedtuple, you can specify required keys that always need to be given
+to the constructor:
+
+    class MyMap(NamedMapping, required=("key1", "key2")):
+        pass
+
+    m = MyMap(a=123, b=456, key1=999)  # KeyError (key2 not specified)
+
+Similarly, you can specify defaults (for required or non-required keys),
+as a dict, that are used if omitted from the constructor:
+
+    class MyMap(NamedMapping, defaults={"key": 678}):
+        pass
+
+    m = MyMap()  # will have m.key == 678
+
+A class instance can unpack via ** with the entirety of its mapping contents:
+
+    m = MyMap(key2=456)
+    both = {'key1': 123, **m}  # contains both keys
+
+You can also chain (append to) required / default values through inheritance:
+
+    class MyMap(NamedMapping, required=("key1",), defaults={"key2": 234}):
+        pass
+
+    class AnotherMap(MyMap, required=("key3",))
+        pass
+
+    m = AnotherMap()      # KeyError (key1 and key3 are required)
+
+    isinstance(m, MyMap)  # would be True
+
+When instantiating, it is also possible to copy just the required keys from
+another dict-like object (does not have to be a parent of the class):
+
+    class SmallMap(NamedMapping, required=("key1", "key2")):
+        pass
+
+    class BigMap(SmallMap, required=("key3", "key4")):
+        pass
+
+    b = BigMap(key1=123, key2=456, key3=789, key4=0)
+
+    s = SmallMap._from(b)             # will copy just key1 and key2
+    s = SmallMap._from(b, extra=555)  # can pass extra **kwargs to __init__
+    s = SmallMap(**b)                 # will copy all keys
+
+Note that this is a fairly basic implementation without __hash__, etc.
+"""
+
+import abc
+import collections
+
+
+class _NamedMappingMeta(abc.ABCMeta):
+    def __new__(
+        metacls, name, bases, namespace, *, required=None, default=None, **kwargs,  # noqa: N804
+    ):
+        new_required = []
+        for base in bases:
+            new_required.extend(getattr(base, "_required", ()))
+        if required:
+            new_required.extend(required)
+        namespace["_required"] = tuple(set(new_required))
+
+        new_default = {}
+        for base in bases:
+            new_default.update(getattr(base, "_default", {}))
+        if default:
+            new_default.update(default)
+        namespace["_default"] = new_default
+
+        return super().__new__(metacls, name, bases, namespace, **kwargs)
+
+
+class NamedMapping(collections.abc.Mapping, metaclass=_NamedMappingMeta):
+    __slots__ = ("_data",)
+
+    def __init__(self, **keys):
+        data = {}
+        if hasattr(self, "_default"):
+            data.update(self._default)
+        data.update(keys)
+        if hasattr(self, "_required"):
+            for key in self._required:
+                if key not in data:
+                    raise KeyError(f"'{self.__class__.__name__}' requires key '{key}'")
+        object.__setattr__(self, "_data", data)
+
+    @classmethod
+    def _from(cls, foreign, **keys):
+        """
+        (keys is like for __init__)
+        """
+        foreign_data = {}
+        if hasattr(cls, "_required"):
+            for key in cls._required:
+                if key in foreign:
+                    foreign_data[key] = foreign[key]
+        foreign_data.update(keys)
+        return cls(**foreign_data)
+
+    def __getattr__(self, item):
+        if item in ("_data", "_required", "_default"):
+            return super().__getattr__(item)
+        try:
+            return self._data[item]
+        except KeyError:
+            raise AttributeError(
+                f"'{self.__class__.__name__}' object has no attribute '{item}'",
+                name=item,
+            ) from None
+
+    def __setattr__(self, name, value):
+        raise AttributeError(
+            f"'{self}' is read-only, cannot set '{name}'",
+            name=name,
+            obj=value,
+        )
+
+    def __getitem__(self, key):
+        return self._data[key]
+
+    def __setitem__(self, key, value):
+        raise ValueError(f"'{self}' is read-only, cannot set '{key}'")
+
+    def __delitem__(self, key):
+        raise ValueError(f"'{self}' is read-only, cannot delete '{key}'")
+
+    def __contains__(self, key):
+        return key in self._data
+
+    def __iter__(self):
+        return iter(self._data)
+
+    def __len__(self):
+        return len(self._data)
+
+    def __repr__(self):
+        return (
+            f"{self.__class__.__name__}("
+            + ", ".join((f"{k}={repr(v)}" for k,v in self._data.items()))
+            + ")"
+        )

atex/util/subprocess.py

@@ -1,52 +1,86 @@
 import subprocess
 
-from .log import debug
+from .log import debug, extradebug
 
 
-def subprocess_run(cmd, *, skip_frames=0, **kwargs):
+def subprocess_run(cmd, **kwargs):
     """
     A simple wrapper for the real subprocess.run() that logs the command used.
     """
     # when logging, skip current stack frame - report the place we were called
     # from, not util.subprocess_run itself
-    debug(f"running: {cmd}", skip_frames=skip_frames+1)
+    debug(f"running: '{cmd}' with {kwargs=}")
     return subprocess.run(cmd, **kwargs)
 
 
-def subprocess_output(cmd, *, skip_frames=0, check=True, text=True, **kwargs):
+def subprocess_output(cmd, *, check=True, text=True, **kwargs):
     """
     A wrapper simulating subprocess.check_output() via a modern .run() API.
     """
-    debug(f"running: {cmd}", skip_frames=skip_frames+1)
+    debug(f"running: '{cmd}' with {check=}, {text=} and {kwargs=}")
     proc = subprocess.run(cmd, check=check, text=text, stdout=subprocess.PIPE, **kwargs)
     return proc.stdout.rstrip("\n") if text else proc.stdout
 
 
-def subprocess_Popen(cmd, *, skip_frames=0, **kwargs):  # noqa: N802
+def subprocess_Popen(cmd, **kwargs):  # noqa: N802
     """
     A simple wrapper for the real subprocess.Popen() that logs the command used.
     """
-    debug(f"running: {cmd}", skip_frames=skip_frames+1)
+    debug(f"running: '{cmd}' with {kwargs=}")
     return subprocess.Popen(cmd, **kwargs)
 
 
-def subprocess_stream(cmd, *, check=False, skip_frames=0, **kwargs):
+def subprocess_stream(cmd, *, stream="stdout", check=False, input=None, **kwargs):
     """
     Run 'cmd' via subprocess.Popen() and return an iterator over any lines
     the command outputs on stdout, in text mode.
 
+    The 'stream' is a subprocess.Popen attribute (either 'stdout' or 'stderr')
+    to read from.
+    To capture both stdout and stderr as yielded lines, use 'stream="stdout"'
+    and pass an additional 'stderr=subprocess.STDOUT'.
+
     With 'check' set to True, raise a CalledProcessError if the 'cmd' failed.
 
-    To capture both stdout and stderr as yielded lines, use subprocess.STDOUT.
+    Similarly, 'input' simulates the 'input' arg of subprocess.run().
+    Note that the input is written to stdin of the process *before* any outputs
+    are streamed, so it should be sufficiently small and/or not cause a deadlock
+    with the process waiting for outputs to be read before consuming more input.
+    Use 'stdin=subprocess.PIPE' and write to it manually if you need more.
     """
-    debug(f"running: {cmd}", skip_frames=skip_frames+1)
-    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, text=True, **kwargs)
+    all_kwargs = {
+        "text": True,
+        stream: subprocess.PIPE,
+    }
+    if input is not None:
+        all_kwargs["stdin"] = subprocess.PIPE
+    all_kwargs |= kwargs
+
+    debug(f"running: '{cmd}' with {all_kwargs=}")
+    proc = subprocess.Popen(cmd, **all_kwargs)
 
     def generate_lines():
-        for line in proc.stdout:
+        if input is not None:
+            proc.stdin.write(input)
+            proc.stdin.close()
+        line_stream = getattr(proc, stream)
+        for line in line_stream:
             yield line.rstrip("\n")
         code = proc.wait()
         if code > 0 and check:
             raise subprocess.CalledProcessError(cmd=cmd, returncode=code)
 
     return (proc, generate_lines())
+
+
+def subprocess_log(cmd, **kwargs):
+    """
+    A wrapper to stream every (text) line output from the process to the
+    logging module.
+
+    Uses subprocess_stream() to gather the lines.
+    """
+    debug(f"running: '{cmd}' with {kwargs=}")
+    _, lines = subprocess_stream(cmd, **kwargs)
+    for line in lines:
+        extradebug(line)