atex 0.8__py3-none-any.whl → 0.9__py3-none-any.whl

atex/provision/podman/podman.py

@@ -0,0 +1,274 @@
+import os
+import time
+import enum
+import threading
+import subprocess
+
+from ... import connection, util
+from .. import Provisioner, Remote
+
+
+class PodmanRemote(Remote, connection.podman.PodmanConn):
+    """
+    Built on the official Remote API, pulling in the Connection API
+    as implemented by ManagedSSHConn.
+    """
+
+    def __init__(self, image, container, *, release_hook):
+        """
+        'image' is an image tag (used for repr()).
+
+        'container' is a podman container id / name.
+
+        'release_hook' is a callable called on .release() in addition
+        to disconnecting the connection.
+        """
+        super().__init__(container=container)
+        self.lock = threading.RLock()
+        self.image = image
+        self.container = container
+        self.release_called = False
+        self.release_hook = release_hook
+
+    def release(self):
+        with self.lock:
+            if self.release_called:
+                return
+            else:
+                self.release_called = True
+        self.release_hook(self)
+        self.disconnect()
+        util.subprocess_run(
+            ("podman", "container", "rm", "-f", "-t", "0", self.container),
+            check=False,  # ignore if it fails
+            stdout=subprocess.DEVNULL,
+        )
+
+    # not /technically/ a valid repr(), but meh
+    def __repr__(self):
+        class_name = self.__class__.__name__
+
+        if "/" in self.image:
+            image = self.image.rsplit("/",1)[1]
+        elif len(self.image) > 20:
+            image = f"{self.image[:17]}..."
+        else:
+            image = self.image
+
+        name = f"{self.container[:17]}..." if len(self.container) > 20 else self.container
+
+        return f"{class_name}({image}, {name})"
+
+
+class PodmanProvisioner(Provisioner):
+    class State(enum.Enum):
+        WAITING_FOR_PULL = enum.auto()
+        CREATING_CONTAINER = enum.auto()
+        WAITING_FOR_CREATION = enum.auto()
+        SETTING_UP_CONTAINER = enum.auto()
+        WAITING_FOR_SETUP = enum.auto()
+
+    # NOTE: this uses a single Popen process to run podman commands,
+    #       to avoid double downloads/pulls, but also to avoid SQLite errors
+    #       when creating multiple containers in parallel
+
+    def __init__(self, image, run_options=None, *, pull=True, max_systems=1):
+        """
+        'image' is a string of image tag/id to create containers from.
+        It can be a local identifier or an URL.
+
+        'run_options' is an iterable with additional CLI options passed
+        to 'podman container run'.
+
+        'pull' specifies whether to attempt 'podman image pull' on the specified
+        image tag/id before any container creation.
+
+        'max_systems' is a maximum number of containers running at any one time.
+        """
+        self.lock = threading.RLock()
+        self.image = image
+        self.run_options = run_options or ()
+        self.pull = pull
+        self.max_systems = max_systems
+
+        self.image_id = None
+        self.container_id = None
+        self.worker = None
+        self.worker_output = bytearray()
+        self.state = None
+        # created PodmanRemote instances, ready to be handed over to the user,
+        # or already in use by the user
+        self.remotes = []
+
+    @staticmethod
+    def _spawn_proc(cmd):
+        proc = util.subprocess_Popen(
+            cmd,
+            stdin=subprocess.DEVNULL,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+        )
+        os.set_blocking(proc.stdout.fileno(), False)
+        return proc
+
+#    @staticmethod
+#    def _poll_proc(proc):
+#        # read from the process to un-block any kernel buffers
+#        try:
+#            out = proc.stdout.read()  # non-blocking
+#        except BlockingIOError:
+#            out = ""
+#        return (proc.poll(), out)
+
+    def _make_remote(self, container):
+        def release_hook(remote):
+            # remove from the list of remotes inside this Provisioner
+            with self.lock:
+                try:
+                    self.remotes.remove(remote)
+                except ValueError:
+                    pass
+
+        remote = PodmanRemote(
+            self.image,
+            container,
+            release_hook=release_hook,
+        )
+        self.remotes.append(remote)
+        return remote
+
+    def start(self):
+        if not self.image:
+            raise ValueError("image cannot be empty")
+
+        if not self.pull:
+            self.image_id = self.image
+            self.state = self.State.CREATING_CONTAINER
+        else:
+            self.worker = self._spawn_proc(
+                ("podman", "image", "pull", "--quiet", self.image),
+            )
+            self.state = self.State.WAITING_FOR_PULL
+
+    def stop(self):
+        with self.lock:
+            while self.remotes:
+                self.remotes.pop().release()
+            worker = self.worker
+            self.worker = None
+
+        if worker:
+            worker.kill()
+            # don"t zombie forever, return EPIPE on any attempts to write to us
+            worker.stdout.close()
+            worker.wait()
+
+    def stop_defer(self):
+        # avoid SQLite errors by removing containers sequentially
+        return self.stop
+
+    @staticmethod
+    def _nonblock_read(fobj):
+        """Return b'' if there was nothing to read, instead of None."""
+        data = fobj.read()
+        return b"" if data is None else data
+
+    def _get_remote_nonblock(self):
+        if self.state is None:
+            raise RuntimeError("the provisioner is in an invalid state")
+
+        # NOTE: these are not 'elif' statements explicitly to allow a next block
+        #       to follow a previous one (if the condition is met)
+        if self.state is self.State.WAITING_FOR_PULL:
+            self.worker_output += self._nonblock_read(self.worker.stdout)
+            rc = self.worker.poll()
+            if rc is None:
+                return None  # still running
+            elif rc != 0:
+                out = self.worker_output.decode().rstrip("\n")
+                self.worker_output.clear()
+                self.worker = None
+                self.state = None
+                raise RuntimeError(f"podman image pull failed with {rc}:\n{out}")
+            else:
+                self.image_id = self.worker_output.decode().rstrip("\n")
+                self.worker_output.clear()
+                self.worker = None
+                self.state = self.State.CREATING_CONTAINER
+
+        if self.state is self.State.CREATING_CONTAINER:
+            if len(self.remotes) < self.max_systems:
+                self.worker = self._spawn_proc(
+                    (
+                        "podman", "container", "run", "--quiet", "--detach", "--pull", "never",
+                        *self.run_options, self.image_id, "sleep", "inf",
+                    ),
+                )
+                self.state = self.State.WAITING_FOR_CREATION
+            else:
+                # too many remotes requested
+                return None
+
+        if self.state is self.State.WAITING_FOR_CREATION:
+            self.worker_output += self._nonblock_read(self.worker.stdout)
+            rc = self.worker.poll()
+            if rc is None:
+                return None  # still running
+            elif rc != 0:
+                out = self.worker_output.decode().rstrip("\n")
+                self.worker_output.clear()
+                self.worker = None
+                self.state = None
+                raise RuntimeError(f"podman run failed with {rc}:\n{out}")
+            else:
+                self.container_id = self.worker_output.decode().rstrip("\n")
+                self.worker_output.clear()
+                self.worker = None
+                self.state = self.State.SETTING_UP_CONTAINER
+
+        if self.state is self.State.SETTING_UP_CONTAINER:
+            cmd = ("dnf", "install", "-y", "-q", "--setopt=install_weak_deps=False", "rsync")
+            self.worker = self._spawn_proc(
+                ("podman", "container", "exec", self.container_id, *cmd),
+            )
+            self.state = self.State.WAITING_FOR_SETUP
+
+        if self.state is self.State.WAITING_FOR_SETUP:
+            self.worker_output += self._nonblock_read(self.worker.stdout)
+            rc = self.worker.poll()
+            if rc is None:
+                return None  # still running
+            elif rc != 0:
+                out = self.worker_output.decode().rstrip("\n")
+                self.worker_output.clear()
+                self.worker = None
+                self.state = None
+                raise RuntimeError(f"setting up failed with {rc}:\n{out}")
+            else:
+                # everything ready, give the Remote to the caller and reset
+                remote = self._make_remote(self.container_id)
+                self.worker_output.clear()
+                self.worker = None
+                self.state = self.State.CREATING_CONTAINER
+                return remote
+
+        raise AssertionError(f"reached end (invalid state {self.state}?)")
+
+    def get_remote(self, block=True):
+        if not block:
+            with self.lock:
+                return self._get_remote_nonblock()
+        else:
+            while True:
+                with self.lock:
+                    if remote := self._get_remote_nonblock():
+                        return remote
+                time.sleep(0.1)
+
+    # not /technically/ a valid repr(), but meh
+    def __repr__(self):
+        class_name = self.__class__.__name__
+        return (
+            f"{class_name}({self.image}, {len(self.remotes)}/{self.max_systems} remotes, "
+            f"{hex(id(self))})"
+        )

atex/provision/testingfarm/api.py

@@ -19,13 +19,11 @@ DEFAULT_API_URL = "https://api.testing-farm.io/v0.1"
 # how many seconds to sleep for during API polling
 API_QUERY_DELAY = 30
 
-RESERVE_TASK = {
-    "fmf": {
-        "url": "https://github.com/RHSecurityCompliance/atex",
-        "ref": "main",
-        "path": "tmt_tests",
-        "name": "/plans/reserve",
-    },
+DEFAULT_RESERVE_TEST = {
+    "url": "https://github.com/RHSecurityCompliance/atex-reserve",
+    "ref": "v0.9",
+    "path": ".",
+    "name": "/plans/reserve",
 }
 
 # final states of a request,
@@ -35,7 +33,19 @@ END_STATES = ("error", "complete", "canceled")
 # always have at most 10 outstanding HTTP requests to every given API host,
 # shared by all instances of all classes here, to avoid flooding the host
 # by multi-threaded users
-_http = urllib3.PoolManager(maxsize=10, block=True)
+_http = urllib3.PoolManager(
+    maxsize=10,
+    block=True,
+    retries=urllib3.Retry(
+        total=10,
+        # account for API restarts / short outages
+        backoff_factor=60,
+        backoff_max=600,
+        # retry on API server errors too, not just connection issues
+        status=10,
+        status_forcelist={403,404,408,429,500,502,503,504},
+    ),
+)
 
 
 class TestingFarmError(Exception):
@@ -370,7 +380,9 @@ class Reserve:
 
     def __init__(
         self, *, compose, arch="x86_64", pool=None, hardware=None, kickstart=None,
-        timeout=60, ssh_key=None, source_host=None, api=None,
+        timeout=60, ssh_key=None, source_host=None,
+        reserve_test=None, variables=None, secrets=None,
+        api=None,
     ):
         """
         'compose' (str) is the OS to install, chosen from the composes supported
@@ -403,18 +415,31 @@ class Reserve:
         facing address of the current system.
         Ignored on the 'redhat' ranch.
 
+        'reserve_test' is a dict with a fmf test specification to be run on the
+        target system to reserve it, ie.:
+            {
+                "url": "https://some-host/path/to/repo",
+                "ref": "main",
+                "name": "/plans/reserve",
+            }
+
+        'variables' and 'secrets' are dicts with environment variable key/values
+        exported for the reserve test - variables are visible via TF API,
+        secrets are not (but can still be extracted from pipeline log).
+
         'api' is a TestingFarmAPI instance - if unspecified, a sensible default
         will be used.
         """
-        util.info(f"Will reserve compose:{compose} on arch:{arch} for {timeout}min")
+        util.info(f"will reserve compose:{compose} on arch:{arch} for {timeout}min")
         spec = {
-            "test": RESERVE_TASK,
+            "test": {
+                "fmf": reserve_test or DEFAULT_RESERVE_TEST,
+            },
             "environments": [{
                 "arch": arch,
                 "os": {
                     "compose": compose,
                 },
-                "pool": pool,
                 "settings": {
                     "pipeline": {
                         "skip_guest_setup": True,
@@ -423,10 +448,8 @@ class Reserve:
                         "tags": {
                             "ArtemisUseSpot": "false",
                         },
-                        "security_group_rules_ingress": [],
                     },
                 },
-                "secrets": {},
             }],
             "settings": {
                 "pipeline": {
@@ -434,10 +457,16 @@ class Reserve:
                 },
             },
         }
+        spec_env = spec["environments"][0]
+        if pool:
+            spec_env["pool"] = pool
         if hardware:
-            spec["environments"][0]["hardware"] = hardware
+            spec_env["hardware"] = hardware
         if kickstart:
-            spec["environments"][0]["kickstart"] = kickstart
+            spec_env["kickstart"] = kickstart
+        if variables:
+            spec_env["variables"] = variables
+        spec_env["secrets"] = secrets.copy() if secrets else {}  # we need it for ssh pubkey
 
         self._spec = spec
         self._ssh_key = Path(ssh_key) if ssh_key else None
@@ -465,20 +494,25 @@ class Reserve:
                 raise RuntimeError("reservation already in progress")
 
         spec = self._spec.copy()
+        spec_env = spec["environments"][0]
 
-        try:
-            # add source_host firewall filter
+        # add source_host firewall filter on the public ranch
+        if self.api.whoami()["token"]["ranch"] == "public":
             source_host = self._source_host or f"{self._guess_host_ipv4()}/32"
-            ingress = \
-                spec["environments"][0]["settings"]["provisioning"]["security_group_rules_ingress"]
-            ingress.append({
+            ingress_rule = {
                 "type": "ingress",
                 "protocol": "-1",
                 "cidr": source_host,
                 "port_min": 0,
                 "port_max": 65535,
-            })
+            }
+            provisioning = spec_env["settings"]["provisioning"]
+            if "security_group_rules_ingress" in provisioning:
+                provisioning["security_group_rules_ingress"].append(ingress_rule)
+            else:
+                provisioning["security_group_rules_ingress"] = [ingress_rule]
 
+        try:
             # read user-provided ssh key, or generate one
             ssh_key = self._ssh_key
             if ssh_key:
@@ -491,8 +525,9 @@ class Reserve:
                     ssh_key, ssh_pubkey = util.ssh_keygen(self._tmpdir.name)
 
             pubkey_contents = ssh_pubkey.read_text().strip()
-            secrets = spec["environments"][0]["secrets"]
-            secrets["RESERVE_SSH_PUBKEY"] = pubkey_contents
+            # TODO: split ^^^ into 3 parts (key type, hash, comment), assert it,
+            #       and anonymize comment in case it contains a secret user/hostname
+            spec_env["secrets"]["RESERVE_SSH_PUBKEY"] = pubkey_contents
 
             with self.lock:
                 self.request = Request(api=self.api)
@@ -506,7 +541,11 @@ class Reserve:
                 # state by pipeline.log
                 util.debug(f"pipeline: {line}\033[0m")
                 # find hidden login details
-                m = re.search(r"\] Guest is ready: ArtemisGuest\([^,]+, (\w+)@([0-9\.]+), ", line)
+                m = re.search(
+                    # host address can be an IP address or a hostname
+                    r"\] Guest is ready: ArtemisGuest\([^,]+, (\w+)@([^,]+), arch=",
+                    line,
+                )
                 if m:
                     ssh_user, ssh_host = m.groups()
                     continue
@@ -564,7 +603,11 @@ class Reserve:
                 self._tmpdir = None
 
     def __enter__(self):
-        return self.reserve()
+        try:
+            return self.reserve()
+        except Exception:
+            self.release()
+            raise
 
     def __exit__(self, exc_type, exc_value, traceback):
         self.release()

atex/provision/testingfarm/testingfarm.py

@@ -14,47 +14,45 @@ class TestingFarmRemote(Remote, connection.ssh.ManagedSSHConn):
     as implemented by ManagedSSHConn.
     """
 
-    def __init__(self, ssh_options, *, release_hook, provisioner):
+    def __init__(self, request_id, ssh_options, *, release_hook):
         """
+        'request_id' is a string with Testing Farm request UUID (for printouts).
+
         'ssh_options' are a dict, passed to ManagedSSHConn __init__().
 
         'release_hook' is a callable called on .release() in addition
         to disconnecting the connection.
         """
-        # start with empty ssh options, we'll fill them in later
+        # NOTE: self.lock inherited from ManagedSSHConn
         super().__init__(options=ssh_options)
+        self.request_id = request_id
         self.release_hook = release_hook
-        self.provisioner = provisioner
-        self.lock = threading.RLock()
         self.release_called = False
 
     def release(self):
         with self.lock:
-            if not self.release_called:
-                self.release_called = True
-            else:
+            if self.release_called:
                 return
+            else:
+                self.release_called = True
         self.release_hook(self)
         self.disconnect()
 
     # not /technically/ a valid repr(), but meh
     def __repr__(self):
         class_name = self.__class__.__name__
-        compose = self.provisioner.compose
-        arch = self.provisioner.arch
-        return f"{class_name}({compose} @ {arch}, {hex(id(self))})"
-
-#    def alive(self):
-#        return self.valid
-
-    # TODO: def __str__(self):  as root@1.2.3.4 and arch, ranch, etc.
+        ssh_user = self.options.get("User", "unknown")
+        ssh_host = self.options.get("Hostname", "unknown")
+        ssh_port = self.options.get("Port", "unknown")
+        ssh_key = self.options.get("IdentityFile", "unknown")
+        return f"{class_name}({ssh_user}@{ssh_host}:{ssh_port}@{ssh_key}, {self.request_id})"
 
 
 class TestingFarmProvisioner(Provisioner):
     # TODO: have max_systems as (min,default,max) tuple; have an algorithm that
     #       starts at default and scales up/down as needed
 
-    def __init__(self, compose, arch="x86_64", *, max_systems=1, timeout=60, max_retries=10):
+    def __init__(self, compose, arch="x86_64", *, max_systems=1, max_retries=10, **reserve_kwargs):
         """
         'compose' is a Testing Farm compose to prepare.
 
@@ -63,18 +61,16 @@ class TestingFarmProvisioner(Provisioner):
         'max_systems' is an int of how many systems to reserve (and keep
         reserved) in an internal pool.
 
-        'timeout' is the maximum Testing Farm pipeline timeout (waiting for
-        a system + OS installation + reservation time).
-
         'max_retries' is a maximum number of provisioning (Testing Farm) errors
         that will be reprovisioned before giving up.
         """
-        super().__init__()
-        self.compose = compose  # TODO: translate "centos9" to "CentOS-Stream-9"
+        self.lock = threading.RLock()
+        self.compose = compose
         self.arch = arch
         self.max_systems = max_systems
-        self.timeout = timeout
+        self.reserve_kwargs = reserve_kwargs
         self.retries = max_retries
+
         self._tmpdir = None
         self.ssh_key = self.ssh_pubkey = None
         self.queue = util.ThreadQueue(daemon=True)
@@ -105,6 +101,8 @@ class TestingFarmProvisioner(Provisioner):
             "User": machine.user,
             "Port": machine.port,
             "IdentityFile": machine.ssh_key,
+            "ConnectionAttempts": "1000",
+            "Compression": "yes",
         }
 
         def release_hook(remote):
@@ -118,9 +116,9 @@ class TestingFarmProvisioner(Provisioner):
             tf_reserve.release()
 
         remote = TestingFarmRemote(
+            tf_reserve.request.id,
             ssh_options,
             release_hook=release_hook,
-            provisioner=self,
         )
         remote.connect()
 
@@ -139,9 +137,9 @@ class TestingFarmProvisioner(Provisioner):
         tf_reserve = api.Reserve(
             compose=self.compose,
             arch=self.arch,
-            timeout=self.timeout,
             ssh_key=self.ssh_key,
             api=self.tf_api,
+            **self.reserve_kwargs,
         )
 
         # add it to self.reserving even before we schedule a provision,
@@ -152,7 +150,7 @@ class TestingFarmProvisioner(Provisioner):
         # start a background wait
         self.queue.start_thread(
             target=self._wait_for_reservation,
-            args=(tf_reserve, initial_delay),
+            target_args=(tf_reserve, initial_delay),
         )
 
     def start(self):
@@ -168,13 +166,13 @@ class TestingFarmProvisioner(Provisioner):
     def stop(self):
         with self.lock:
             # abort reservations in progress
-            for tf_reserve in self.reserving:
-                tf_reserve.release()
-            self.reserving = []
+            while self.reserving:
+                # testingfarm api.Reserve instances
+                self.reserving.pop().release()
             # cancel/release all Remotes ever created by us
-            for remote in self.remotes:
-                remote.release()
-            self.remotes = []  # just in case
+            while self.remotes:
+                # TestingFarmRemote instances
+                self.remotes.pop().release()
             # explicitly remove the tmpdir rather than relying on destructor
             self._tmpdir.cleanup()
             self._tmpdir = None

atex/util/libvirt.py

@@ -0,0 +1,18 @@
+import importlib
+
+
+def import_libvirt():
+    try:
+        libvirt = importlib.import_module("libvirt")
+    except ModuleNotFoundError:
+        raise ModuleNotFoundError(
+            "No module named 'libvirt', you need to install it from the package"
+            " manager of your distro, ie. 'dnf install python3-libvirt' as it"
+            " requires distro-wide headers to compile. It won't work from PyPI."
+            " If using venv, create it with '--system-site-packages'.",
+        ) from None
+
+    # suppress console error printing, behave like a good python module
+    libvirt.registerErrorHandler(lambda _ctx, _err: None, None)
+
+    return libvirt

atex/util/log.py

@@ -9,6 +9,7 @@ def in_debug_mode():
     """
     Return True if the root logger is using the DEBUG (or more verbose) level.
     """
+    # TODO: use _logger.isEnabledFor() ?
     root_level = logging.getLogger().level
     return root_level > 0 and root_level <= logging.DEBUG
 
@@ -31,11 +32,16 @@ def _format_msg(msg, *, skip_frames=0):
 
     # if the function has 'self' and it looks like a class instance,
     # prepend it to the function name
-    p_locals = parent.frame.f_locals
-    if "self" in p_locals:
-        self = p_locals["self"]
-        if hasattr(self, "__class__") and inspect.isclass(self.__class__):
-            function = f"{self.__class__.__name__}.{function}"
+    argvals = inspect.getargvalues(parent.frame)
+    if argvals.args:
+        if argvals.args[0] == "self":
+            self = argvals.locals["self"]
+            if hasattr(self, "__class__") and inspect.isclass(self.__class__):
+                function = f"{self.__class__.__name__}.{function}"
+        elif argvals.args[0] == "cls":
+            cls = argvals.locals["cls"]
+            if inspect.isclass(cls):
+                function = f"{cls.__name__}.{function}"
 
     # don't report module name of a function if it's the same as running module
     if parent.filename != module.filename:
@@ -50,12 +56,21 @@ def _format_msg(msg, *, skip_frames=0):
 
 
 def debug(msg, *, skip_frames=0):
-    _logger.debug(_format_msg(msg, skip_frames=skip_frames+1))
+    if in_debug_mode():
+        _logger.debug(_format_msg(msg, skip_frames=skip_frames+1))
+    else:
+        _logger.debug(msg)
 
 
 def info(msg, *, skip_frames=0):
-    _logger.info(_format_msg(msg, skip_frames=skip_frames+1))
+    if in_debug_mode():
+        _logger.info(_format_msg(msg, skip_frames=skip_frames+1))
+    else:
+        _logger.info(msg)
 
 
 def warning(msg, *, skip_frames=0):
-    _logger.warning(_format_msg(msg, skip_frames=skip_frames+1))
+    if in_debug_mode():
+        _logger.warning(_format_msg(msg, skip_frames=skip_frames+1))
+    else:
+        _logger.warning(msg)