atex 0.9__py3-none-any.whl → 0.10__py3-none-any.whl

atex/{provision → provisioner}/libvirt/libvirt.py

@@ -4,6 +4,7 @@ import uuid
 import shlex
 import socket
 import random
+import textwrap
 import tempfile
 import threading
 import subprocess
@@ -38,14 +39,14 @@ def setup_event_loop():
     thread.start()
 
 
-class LibvirtCloningRemote(Remote, connection.ssh.ManagedSSHConn):
+class LibvirtCloningRemote(Remote, connection.ssh.ManagedSSHConnection):
     """
     TODO
     """
 
     def __init__(self, ssh_options, host, domain, source_image, *, release_hook):
         """
-        'ssh_options' are a dict, passed to ManagedSSHConn __init__().
+        'ssh_options' are a dict, passed to ManagedSSHConnection __init__().
 
         'host' is a str of libvirt host name (used for repr()).
 
@@ -57,7 +58,7 @@ class LibvirtCloningRemote(Remote, connection.ssh.ManagedSSHConn):
         'release_hook' is a callable called on .release() in addition
         to disconnecting the connection.
         """
-        # NOTE: self.lock inherited from ManagedSSHConn
+        # NOTE: self.lock inherited from ManagedSSHConnection
         super().__init__(options=ssh_options)
         self.host = host
         self.domain = domain
@@ -80,7 +81,7 @@ class LibvirtCloningRemote(Remote, connection.ssh.ManagedSSHConn):
         return f"{class_name}({self.host}, {self.domain}, {self.source_image})"
 
 
-# needs ManagedSSHConn due to .forward()
+# needs ManagedSSHConnection due to .forward()
 def reliable_ssh_local_fwd(conn, dest, retries=10):
     for _ in range(retries):
         # let the kernel give us a free port
@@ -128,7 +129,7 @@ class LibvirtCloningProvisioner(Provisioner):
         reserve_delay=3, reserve_time=3600, start_event_loop=True,
     ):
         """
-        'host' is a ManagedSSHConn class instance, connected to a libvirt host.
+        'host' is a ManagedSSHConnection class instance, connected to a libvirt host.
 
         'image' is a string with a libvirt storage volume name inside the
         given storage 'pool' that should be used as the source for cloning.
@@ -174,6 +175,7 @@ class LibvirtCloningProvisioner(Provisioner):
         self.signature = uuid.uuid4()
         self.reserve_end = None
         self.queue = util.ThreadQueue(daemon=True)
+        self.to_reserve = 0
 
         # use two libvirt connections - one to handle reservations and cloning,
         # and another for management and cleanup;
@@ -242,8 +244,8 @@ class LibvirtCloningProvisioner(Provisioner):
                 raise
 
         # parse XML definition of the domain
-        xmldesc = acquired.XMLDesc()
-        util.debug(f"domain {acquired.name()} XML:\n{xmldesc}")  # TODO: EXTRADEBUG log level
+        xmldesc = acquired.XMLDesc().rstrip("\n")
+        util.extradebug(f"domain {acquired.name()} XML:\n{textwrap.indent(xmldesc, '    ')}")
         xml_root = ET.fromstring(xmldesc)
         nvram_vol = nvram_path = None
 
@@ -325,7 +327,7 @@ class LibvirtCloningProvisioner(Provisioner):
         # set up ssh LocalForward to it
         port = reliable_ssh_local_fwd(self.host, f"{first_addr}:22")
 
-        # create a remote and connect it
+        # prepare release using variables from this scope
         def release_hook(remote):
             # un-forward the libvirt host ssh-forwarded port
             self.host.forward("LocalForward", f"127.0.0.1:{port} {first_addr}:22", cancel=True)
@@ -339,6 +341,7 @@ class LibvirtCloningProvisioner(Provisioner):
                     try:
                         domain = self.manage_conn.lookupByName(remote.domain)
                         locking.unlock(domain, self.signature)
+                        domain.destroy()
                     except libvirt.libvirtError as e:
                         if "Domain not found" not in str(e):
                             raise
@@ -348,11 +351,12 @@ class LibvirtCloningProvisioner(Provisioner):
                 except ValueError:
                     pass
 
+        # create a remote and connect it
         ssh_options = {
             "Hostname": "127.0.0.1",
             "User": self.domain_user,
             "Port": str(port),
-            "IdentityFile": self.domain_sshkey,
+            "IdentityFile": str(Path(self.domain_sshkey).absolute()),
             "ConnectionAttempts": "1000",
             "Compression": "yes",
         }
@@ -385,7 +389,7 @@ class LibvirtCloningProvisioner(Provisioner):
             ("virt-ssh-helper", "qemu:///system"),
             func=lambda *args, **_: args[0],
         )
-        # to make libvirt connect via our ManagedSSHConn, we need to give it
+        # to make libvirt connect via our ManagedSSHConnection, we need to give it
         # a specific ssh CLI, but libvirt URI command= takes only one argv[0]
         # and cannot pass arguments - we work around this by creating a temp
         # arg-less executable
@@ -410,8 +414,6 @@ class LibvirtCloningProvisioner(Provisioner):
             self.reserve_conn = self._open_libvirt_conn()
             self.manage_conn = self.reserve_conn  # for now
             self.reserve_end = int(time.time()) + self.reserve_time
-            # get an initial first remote
-            self.queue.start_thread(target=self._reserve_one)
 
     def stop(self):
         with self.lock:
@@ -443,11 +445,16 @@ class LibvirtCloningProvisioner(Provisioner):
             self.reserve_end = None
             # TODO: wait for threadqueue threads to join?
 
+    def provision(self, count=1):
+        with self.lock:
+            self.to_reserve += count
+
     def get_remote(self, block=True):
-        # if the reservation thread is not running, start one
         with self.lock:
-            if not self.queue.threads:
+            # if the reservation thread is not running, start one
+            if not self.queue.threads and self.to_reserve > 0:
                 self.queue.start_thread(target=self._reserve_one)
+                self.to_reserve -= 1
         try:
             return self.queue.get(block=block)
         except util.ThreadQueue.Empty:

atex/{provision → provisioner}/libvirt/locking.py

@@ -19,7 +19,9 @@ import time
 import random
 import xml.etree.ElementTree as ET
 
-import libvirt
+from ... import util
+
+libvirt = util.import_libvirt()
 
 
 def get_locks(domain, expired=False):

atex/provisioner/podman/__init__.py

@@ -0,0 +1,2 @@
+from .podman import PodmanProvisioner, PodmanRemote  # noqa: F401
+from .podman import pull_image, build_container_with_deps  # noqa: F401

atex/provisioner/podman/podman.py

@@ -0,0 +1,169 @@
+import tempfile
+import threading
+import subprocess
+
+from ... import connection, util
+from .. import Provisioner, Remote
+
+
+class PodmanRemote(Remote, connection.podman.PodmanConnection):
+    """
+    Built on the official Remote API, pulling in the Connection API
+    as implemented by ManagedSSHConnection.
+    """
+
+    def __init__(self, image, container, *, release_hook):
+        """
+        'image' is an image tag (used for repr()).
+
+        'container' is a podman container id / name.
+
+        'release_hook' is a callable called on .release() in addition
+        to disconnecting the connection.
+        """
+        super().__init__(container=container)
+        self.lock = threading.RLock()
+        self.image = image
+        self.container = container
+        self.release_called = False
+        self.release_hook = release_hook
+
+    def release(self):
+        with self.lock:
+            if self.release_called:
+                return
+            else:
+                self.release_called = True
+        self.release_hook(self)
+        self.disconnect()
+        util.subprocess_run(
+            ("podman", "container", "rm", "-f", "-t", "0", self.container),
+            check=False,  # ignore if it fails
+            stdout=subprocess.DEVNULL,
+        )
+
+    # not /technically/ a valid repr(), but meh
+    def __repr__(self):
+        class_name = self.__class__.__name__
+
+        if "/" in self.image:
+            image = self.image.rsplit("/",1)[1]
+        elif len(self.image) > 20:
+            image = f"{self.image[:17]}..."
+        else:
+            image = self.image
+
+        name = f"{self.container[:17]}..." if len(self.container) > 20 else self.container
+
+        return f"{class_name}({image}, {name})"
+
+
+class PodmanProvisioner(Provisioner):
+    def __init__(self, image, run_options=None):
+        """
+        'image' is a string of image tag/id to create containers from.
+        It can be a local identifier or an URL.
+
+        'run_options' is an iterable with additional CLI options passed
+        to 'podman container run'.
+        """
+        self.lock = threading.RLock()
+        self.image = image
+        self.run_options = run_options or ()
+
+        # created PodmanRemote instances, ready to be handed over to the user,
+        # or already in use by the user
+        self.remotes = []
+        self.to_create = 0
+
+    def start(self):
+        if not self.image:
+            raise ValueError("image cannot be empty")
+
+    def stop(self):
+        with self.lock:
+            while self.remotes:
+                self.remotes.pop().release()
+
+    def provision(self, count=1):
+        with self.lock:
+            self.to_create += count
+
+    def get_remote(self, block=True):
+        if self.to_create <= 0:
+            if block:
+                raise RuntimeError("no .provision() requested, would block forever")
+            else:
+                return None
+
+        proc = util.subprocess_run(
+            (
+                "podman", "container", "run", "--quiet", "--detach", "--pull", "never",
+                *self.run_options, self.image, "sleep", "inf",
+            ),
+            check=True,
+            text=True,
+            stdout=subprocess.PIPE,
+        )
+        container_id = proc.stdout.rstrip("\n")
+
+        def release_hook(remote):
+            # remove from the list of remotes inside this Provisioner
+            with self.lock:
+                try:
+                    self.remotes.remove(remote)
+                except ValueError:
+                    pass
+
+        remote = PodmanRemote(
+            self.image,
+            container_id,
+            release_hook=release_hook,
+        )
+
+        with self.lock:
+            self.remotes.append(remote)
+            self.to_create -= 1
+
+        return remote
+
+    # not /technically/ a valid repr(), but meh
+    def __repr__(self):
+        class_name = self.__class__.__name__
+        return (
+            f"{class_name}({self.image}, {len(self.remotes)} remotes, {hex(id(self))})"
+        )
+
+
+def pull_image(origin):
+    proc = util.subprocess_run(
+        ("podman", "image", "pull", "-q", origin),
+        check=True,
+        text=True,
+        stdout=subprocess.PIPE,
+    )
+    return proc.stdout.rstrip("\n")
+
+
+def build_container_with_deps(origin, tag=None, *, extra_pkgs=None):
+    tag_args = ("-t", tag) if tag else ()
+
+    pkgs = ["rsync"]
+    if extra_pkgs:
+        pkgs += extra_pkgs
+    pkgs_str = " ".join(pkgs)
+
+    with tempfile.NamedTemporaryFile("w+t", delete_on_close=False) as tmpf:
+        tmpf.write(util.dedent(fr"""
+            FROM {origin}
+            RUN dnf -y -q --setopt=install_weak_deps=False install {pkgs_str} >/dev/null
+            RUN dnf -y -q clean packages >/dev/null
+        """))
+        tmpf.close()
+        proc = util.subprocess_run(
+            ("podman", "image", "build", "-q", "-f", tmpf.name, *tag_args, "."),
+            check=True,
+            text=True,
+            stdout=subprocess.PIPE,
+        )
+        return proc.stdout.rstrip("\n")

atex/{provision → provisioner}/testingfarm/api.py

@@ -16,12 +16,9 @@ import urllib3
 
 DEFAULT_API_URL = "https://api.testing-farm.io/v0.1"
 
-# how many seconds to sleep for during API polling
-API_QUERY_DELAY = 30
-
 DEFAULT_RESERVE_TEST = {
     "url": "https://github.com/RHSecurityCompliance/atex-reserve",
-    "ref": "v0.9",
+    "ref": "v0.10",
     "path": ".",
     "name": "/plans/reserve",
 }
@@ -88,12 +85,13 @@ class TestingFarmAPI:
         self.api_url = url
         self.api_token = token or os.environ.get("TESTING_FARM_API_TOKEN")
 
-    def _query(self, method, path, *args, headers=None, **kwargs):
+    def _query(self, method, path, *args, headers=None, auth=True, **kwargs):
         url = f"{self.api_url}{path}"
-        if headers is not None:
-            headers["Authorization"] = f"Bearer {self.api_token}"
-        else:
-            headers = {"Authorization": f"Bearer {self.api_token}"}
+        if self.api_token and auth:
+            if headers is not None:
+                headers["Authorization"] = f"Bearer {self.api_token}"
+            else:
+                headers = {"Authorization": f"Bearer {self.api_token}"}
 
         reply = _http.request(method, url, *args, headers=headers, preload_content=False, **kwargs)
 
@@ -180,7 +178,7 @@ class TestingFarmAPI:
             fields["token_id"] = self.whoami()["token"]["id"]
             fields["user_id"] = self.whoami()["user"]["id"]
 
-        return self._query("GET", "/requests", fields=fields)
+        return self._query("GET", "/requests", fields=fields, auth=mine)
 
     def get_request(self, request_id):
         """
@@ -210,19 +208,22 @@ class Request:
     request.
     """
 
-    # TODO: maintain internal time.monotonic() clock and call .update() from
-    #       functions like .alive() if last update is > API_QUERY_DELAY
+    # actually query the TestingFarm API at most every X seconds,
+    # re-using cached state between updates
+    api_query_limit = 30
 
     def __init__(self, id=None, api=None, initial_data=None):
         """
         'id' is a Testing Farm request UUID
+
         'api' is a TestingFarmAPI instance - if unspecified, a sensible default
-        'initial_data' (dict) can be used to pre-fill an initial Request state
-        will be used.
+
+        'initial_data' (dict) can be used to pre-fill an initial Request state.
         """
         self.id = id
         self.api = api or TestingFarmAPI()
         self.data = initial_data or {}
+        self.next_query = 0
 
     def submit(self, spec):
         """
@@ -234,16 +235,12 @@ class Request:
         self.data = self.api.submit_request(spec)
         self.id = self.data["id"]
 
-    def update(self):
-        """
-        Query Testing Farm API to get a more up-to-date version of the request
-        metadata. Do not call too frequently.
-        This function is also used internally by others, you do not need to
-        always call it manually.
-        """
-        self.data = self.api.get_request(self.id)
-        # TODO: refresh internal time.monotonic() timer
-        return self.data
+    def _refresh(self):
+        if not self.id:
+            return
+        if time.monotonic() > self.next_query:
+            self.data = self.api.get_request(self.id)
+            self.next_query = time.monotonic() + self.api_query_limit
 
     def cancel(self):
         if not self.id:
@@ -254,35 +251,44 @@ class Request:
         return data
 
     def alive(self):
-        if "state" not in self.data:
-            self.update()
+        if not self.id:
+            return False
+        self._refresh()
         return self.data["state"] not in END_STATES
 
     def assert_alive(self):
         if not self.alive():
             state = self.data["state"]
-            raise GoneAwayError(f"request {self.data['id']} not alive anymore, entered: {state}")
+            raise GoneAwayError(f"request {self.id} not alive anymore, entered: {state}")
 
     def wait_for_state(self, state):
-        if "state" not in self.data:
-            self.update()
-        self.assert_alive()
-        while self.data["state"] != state:
-            time.sleep(API_QUERY_DELAY)
-            self.update()
-            self.assert_alive()
+        """
+        'state' is a str or a tuple of states to wait for.
+        """
+        watched = (state,) if isinstance(state, str) else state
+        while True:
+            self._refresh()
+            if self.data["state"] in watched:
+                break
+            # if the request ended in one of END_STATES and the above condition
+            # did not catch it, the wait will never end
+            if self.data["state"] in END_STATES:
+                raise GoneAwayError(f"request {self.id} ended with {self.data['state']}")
 
     def __repr__(self):
         return f"Request(id={self.id})"
 
     def __str__(self):
+        self._refresh()
         # python has no better dict-pretty-printing logic
         return json.dumps(self.data, sort_keys=True, indent=4)
 
     def __contains__(self, item):
+        self._refresh()
         return item in self.data
 
     def __getitem__(self, key):
+        self._refresh()
         return self.data[key]
 
 
@@ -292,6 +298,10 @@ class PipelineLogStreamer:
     to "stream" its contents over time (over many requests), never having to
     re-read old pipeline.log content.
     """
+
+    # how frequently to check for pipeline.log updates (seconds)
+    pipeline_query_limit = 30
+
     def __init__(self, request):
         self.request = request
 
@@ -324,8 +334,7 @@ class PipelineLogStreamer:
                 return log
 
             finally:
-                time.sleep(API_QUERY_DELAY)
-                self.request.update()
+                time.sleep(self.pipeline_query_limit)
 
     def __iter__(self):
         url = self._wait_for_entry()
@@ -356,8 +365,7 @@ class PipelineLogStreamer:
                     buffer = buffer[index+1:]
 
             finally:
-                time.sleep(API_QUERY_DELAY)
-                self.request.update()
+                time.sleep(self.pipeline_query_limit)
 
 
 class Reserve:
@@ -532,14 +540,17 @@ class Reserve:
             with self.lock:
                 self.request = Request(api=self.api)
                 self.request.submit(spec)
-            util.debug(f"submitted request:\n{textwrap.indent(str(self.request), '    ')}")
+            util.debug(f"submitted request {self.request.id}")
+            util.extradebug(
+                f"request {self.request.id}:\n{textwrap.indent(str(self.request), '    ')}",
+            )
 
             # wait for user/host to ssh to
             ssh_user = ssh_host = None
             for line in PipelineLogStreamer(self.request):
                 # the '\033[0m' is to reset colors sometimes left in a bad
                 # state by pipeline.log
-                util.debug(f"pipeline: {line}\033[0m")
+                util.extradebug(f"{line}\033[0m")
                 # find hidden login details
                 m = re.search(
                     # host address can be an IP address or a hostname
@@ -559,14 +570,12 @@ class Reserve:
             # (it will be failing to login for a while, until the reserve test
             #  installs our ssh pubkey into authorized_keys)
             ssh_attempt_cmd = (
-                "ssh", "-q", "-i", ssh_key, f"-oConnectionAttempts={API_QUERY_DELAY}",
+                "ssh", "-q", "-i", ssh_key.absolute(), "-oConnectionAttempts=60",
                "-oStrictHostKeyChecking=no", "-oUserKnownHostsFile=/dev/null",
                 f"{ssh_user}@{ssh_host}", "exit 123",
             )
             while True:
-                # wait for API_QUERY_DELAY between ssh retries, seems like GEFN sleep time
-                time.sleep(API_QUERY_DELAY)
-                self.request.update()
+                time.sleep(1)
                 self.request.assert_alive()
 
                 proc = util.subprocess_run(

atex/{provision → provisioner}/testingfarm/testingfarm.py

@@ -8,22 +8,22 @@ from .. import Provisioner, Remote
 from . import api
 
 
-class TestingFarmRemote(Remote, connection.ssh.ManagedSSHConn):
+class TestingFarmRemote(Remote, connection.ssh.ManagedSSHConnection):
     """
     Built on the official Remote API, pulling in the Connection API
-    as implemented by ManagedSSHConn.
+    as implemented by ManagedSSHConnection.
     """
 
     def __init__(self, request_id, ssh_options, *, release_hook):
         """
         'request_id' is a string with Testing Farm request UUID (for printouts).
 
-        'ssh_options' are a dict, passed to ManagedSSHConn __init__().
+        'ssh_options' are a dict, passed to ManagedSSHConnection __init__().
 
         'release_hook' is a callable called on .release() in addition
         to disconnecting the connection.
         """
-        # NOTE: self.lock inherited from ManagedSSHConn
+        # NOTE: self.lock inherited from ManagedSSHConnection
         super().__init__(options=ssh_options)
         self.request_id = request_id
         self.release_hook = release_hook
@@ -35,8 +35,8 @@ class TestingFarmRemote(Remote, connection.ssh.ManagedSSHConn):
                 return
             else:
                 self.release_called = True
-        self.release_hook(self)
         self.disconnect()
+        self.release_hook(self)
 
     # not /technically/ a valid repr(), but meh
     def __repr__(self):
@@ -49,25 +49,20 @@ class TestingFarmRemote(Remote, connection.ssh.ManagedSSHConn):
 
 
 class TestingFarmProvisioner(Provisioner):
-    # TODO: have max_systems as (min,default,max) tuple; have an algorithm that
-    #       starts at default and scales up/down as needed
+    absolute_max_remotes = 100
 
-    def __init__(self, compose, arch="x86_64", *, max_systems=1, max_retries=10, **reserve_kwargs):
+    def __init__(self, compose, arch="x86_64", *, max_retries=10, **reserve_kwargs):
         """
         'compose' is a Testing Farm compose to prepare.
 
         'arch' is an architecture associated with the compose.
 
-        'max_systems' is an int of how many systems to reserve (and keep
-        reserved) in an internal pool.
-
         'max_retries' is a maximum number of provisioning (Testing Farm) errors
         that will be reprovisioned before giving up.
         """
         self.lock = threading.RLock()
         self.compose = compose
         self.arch = arch
-        self.max_systems = max_systems
         self.reserve_kwargs = reserve_kwargs
         self.retries = max_retries
 
@@ -100,7 +95,7 @@ class TestingFarmProvisioner(Provisioner):
             "Hostname": machine.host,
             "User": machine.user,
             "Port": machine.port,
-            "IdentityFile": machine.ssh_key,
+            "IdentityFile": machine.ssh_key.absolute(),
             "ConnectionAttempts": "1000",
             "Compression": "yes",
         }
@@ -157,11 +152,6 @@ class TestingFarmProvisioner(Provisioner):
         with self.lock:
             self._tmpdir = tempfile.TemporaryDirectory()
             self.ssh_key, self.ssh_pubkey = util.ssh_keygen(self._tmpdir.name)
-            # start up all initial reservations
-            for i in range(self.max_systems):
-                delay = (api.API_QUERY_DELAY / self.max_systems) * i
-                #self.queue.start_thread(target=self._schedule_one_reservation, args=(delay,))
-                self._schedule_one_reservation(delay)
 
     def stop(self):
         with self.lock:
@@ -188,14 +178,18 @@ class TestingFarmProvisioner(Provisioner):
             self._tmpdir = None
         return callables
 
-    def get_remote(self, block=True):
-        # fill .release()d remotes back up with reservations
+    def provision(self, count=1):
         with self.lock:
-            deficit = self.max_systems - len(self.remotes) - len(self.reserving)
-            for i in range(deficit):
-                delay = (api.API_QUERY_DELAY / deficit) * i
+            reservations = len(self.remotes) + len(self.reserving)
+            # clamp count to absolute_max_remotes
+            if count + reservations > self.absolute_max_remotes:
+                count = self.absolute_max_remotes - reservations
+            # spread out the request submissions
+            for i in range(count):
+                delay = (api.Request.api_query_limit / count) * i
                 self._schedule_one_reservation(delay)
 
+    def get_remote(self, block=True):
         while True:
             # otherwise wait on a queue of Remotes being provisioned
             try: