atex 0.9__py3-none-any.whl → 0.11__py3-none-any.whl

atex/{provision → provisioner}/__init__.py

@@ -4,41 +4,74 @@ import pkgutil as _pkgutil
 from .. import connection as _connection
 
 
+class Remote(_connection.Connection):
+    """
+    Representation of a provisioned (reserved) remote system, providing
+    a Connection-like API in addition to system management helpers.
+
+    An instance of Remote is typically prepared by a Provisioner and returned
+    to the caller for use and an eventual .release().
+
+    Also note that Remote can be used via Context Manager, but does not
+    do automatic .release(), the manager only handles the built-in Connection.
+    The intention is for a Provisioner to run via its own Contest Manager and
+    release all Remotes upon exit.
+    If you need automatic release of one Remote, use a try/finally block.
+    """
+
+    def release(self):
+        """
+        Release (de-provision) the remote resource.
+        """
+        raise NotImplementedError(f"'release' not implemented for {self.__class__.__name__}")
+
+
 class Provisioner:
     """
     A remote resource (machine/system) provider.
 
-    The main interface is .get_remote() that returns a connected class Remote
-    instance for use by the user, to be .release()d when not needed anymore,
-    with Provisioner automatically getting a replacement for it, to be returned
-    via .get_remote() later.
+    The idea is to request machines (a.k.a. Remotes, or class Remote instances)
+    to be reserved via a non-blocking .provision() and for them to be retrieved
+    through blocking / non-blocking .get_remote() when they become available.
+
+    Each Remote has its own .release() for freeing (de-provisioning) it once
+    the user doesn't need it anymore. The Provisioner does this automatically
+    to all Remotes during .stop() or context manager exit.
 
         p = Provisioner()
         p.start()
+        p.provision(count=1)
         remote = p.get_remote()
         remote.cmd(["ls", "/"])
         remote.release()
         p.stop()
 
         with Provisioner() as p:
-            remote = p.get_remote()
+            p.provision(count=2)
+            remote1 = p.get_remote()
+            remote2 = p.get_remote()
             ...
-            remote.release()
-
-    TODO: mention how a Provisioner always needs to take care of release all Remotes
-          when .stop()ped or when context terminates; even the ones handed over to
-          the user
 
-    Note that .stop() or .defer_stop() may be called from a different
-    thread, asynchronously to any other functions.
+    Note that .provision() is a hint expressed by the caller, not a guarantee
+    that .get_remote() will ever return a Remote. Ie. the caller can call
+    .provision(count=math.inf) to receive as many remotes as the Provisioner
+    can possibly supply.
     """
 
+    def provision(self, count=1):
+        """
+        Request that 'count' machines be provisioned (reserved) for use,
+        to be returned at a later point by .get_remote().
+        """
+        raise NotImplementedError(f"'provision' not implemented for {self.__class__.__name__}")
+
     def get_remote(self, block=True):
         """
-        Get a connected class Remote instance.
+        Return a connected class Remote instance of a previously .provision()ed
+        remote system.
 
-        If 'block' is True, wait for the remote to be available and connected,
-        otherwise return None if there is no Remote available yet.
+        If 'block' is True, wait for the Remote to be available and connected,
+        otherwise return None if there is none available yet.
         """
         raise NotImplementedError(f"'get_remote' not implemented for {self.__class__.__name__}")
 
@@ -56,18 +89,6 @@ class Provisioner:
         """
         raise NotImplementedError(f"'stop' not implemented for {self.__class__.__name__}")
 
-    def stop_defer(self):
-        """
-        Enable an external caller to stop the Provisioner instance,
-        deferring resource deallocation to the caller.
-
-        Return an iterable of argument-free thread-safe callables that can be
-        called, possibly in parallel, to free up resources.
-        Ie. a list of 200 .release() functions, to be called in a thread pool
-        by the user, speeding up cleanup.
-        """
-        return (self.stop,)
-
     def __enter__(self):
         try:
             self.start()
@@ -80,31 +101,6 @@ class Provisioner:
         self.stop()
 
 
-class Remote(_connection.Connection):
-    """
-    Representation of a provisioned (reserved) remote system, providing
-    a Connection-like API in addition to system management helpers.
-
-    An instance of Remote is typically prepared by a Provisioner and lent out
-    for further use, to be .release()d by the user (if destroyed).
-    It is not meant for repeated reserve/release cycles, hence the lack
-    of .reserve().
-
-    Also note that Remote can be used via Context Manager, but does not
-    do automatic .release(), the manager only handles the built-in Connection.
-    The intention is for a Provisioner to run via its own Contest Manager and
-    release all Remotes upon exit.
-    If you need automatic release of one Remote, use a contextlib.ExitStack
-    with a callback, or a try/finally block.
-    """
-
-    def release(self):
-        """
-        Release (de-provision) the remote resource.
-        """
-        raise NotImplementedError(f"'release' not implemented for {self.__class__.__name__}")
-
-
 _submodules = [
     info.name for info in _pkgutil.iter_modules(__spec__.submodule_search_locations)
 ]

atex/{provision → provisioner}/libvirt/libvirt.py

@@ -4,6 +4,7 @@ import uuid
 import shlex
 import socket
 import random
+import textwrap
 import tempfile
 import threading
 import subprocess
@@ -38,14 +39,14 @@ def setup_event_loop():
     thread.start()
 
 
-class LibvirtCloningRemote(Remote, connection.ssh.ManagedSSHConn):
+class LibvirtCloningRemote(Remote, connection.ssh.ManagedSSHConnection):
     """
     TODO
     """
 
     def __init__(self, ssh_options, host, domain, source_image, *, release_hook):
         """
-        'ssh_options' are a dict, passed to ManagedSSHConn __init__().
+        'ssh_options' are a dict, passed to ManagedSSHConnection __init__().
 
         'host' is a str of libvirt host name (used for repr()).
 
@@ -57,7 +58,7 @@ class LibvirtCloningRemote(Remote, connection.ssh.ManagedSSHConn):
         'release_hook' is a callable called on .release() in addition
         to disconnecting the connection.
         """
-        # NOTE: self.lock inherited from ManagedSSHConn
+        # NOTE: self.lock inherited from ManagedSSHConnection
         super().__init__(options=ssh_options)
         self.host = host
         self.domain = domain
@@ -80,7 +81,7 @@ class LibvirtCloningRemote(Remote, connection.ssh.ManagedSSHConn):
         return f"{class_name}({self.host}, {self.domain}, {self.source_image})"
 
 
-# needs ManagedSSHConn due to .forward()
+# needs ManagedSSHConnection due to .forward()
 def reliable_ssh_local_fwd(conn, dest, retries=10):
     for _ in range(retries):
         # let the kernel give us a free port
@@ -128,7 +129,7 @@ class LibvirtCloningProvisioner(Provisioner):
         reserve_delay=3, reserve_time=3600, start_event_loop=True,
     ):
         """
-        'host' is a ManagedSSHConn class instance, connected to a libvirt host.
+        'host' is a ManagedSSHConnection class instance, connected to a libvirt host.
 
         'image' is a string with a libvirt storage volume name inside the
         given storage 'pool' that should be used as the source for cloning.
@@ -174,6 +175,7 @@ class LibvirtCloningProvisioner(Provisioner):
         self.signature = uuid.uuid4()
         self.reserve_end = None
         self.queue = util.ThreadQueue(daemon=True)
+        self.to_reserve = 0
 
         # use two libvirt connections - one to handle reservations and cloning,
         # and another for management and cleanup;
@@ -242,8 +244,8 @@ class LibvirtCloningProvisioner(Provisioner):
                 raise
 
         # parse XML definition of the domain
-        xmldesc = acquired.XMLDesc()
-        util.debug(f"domain {acquired.name()} XML:\n{xmldesc}")  # TODO: EXTRADEBUG log level
+        xmldesc = acquired.XMLDesc().rstrip("\n")
+        util.extradebug(f"domain {acquired.name()} XML:\n{textwrap.indent(xmldesc, '    ')}")
         xml_root = ET.fromstring(xmldesc)
         nvram_vol = nvram_path = None
 
@@ -258,7 +260,19 @@ class LibvirtCloningProvisioner(Provisioner):
             # by libvirt natively (because treating nvram as a storage pool
             # is a user hack)
             for p in conn.listAllStoragePools():
-                p.refresh()
+                # retry a few times to work around a libvirt race condition
+                for _ in range(10):
+                    try:
+                        p.refresh()
+                    except libvirt.libvirtError as e:
+                        if "domain is not running" in str(e):
+                            break
+                        elif "has asynchronous jobs running" in str(e):
+                            continue
+                        else:
+                            raise
+                    else:
+                        break
             try:
                 nvram_vol = conn.storageVolLookupByPath(nvram_path)
             except libvirt.libvirtError as e:
@@ -325,7 +339,7 @@ class LibvirtCloningProvisioner(Provisioner):
         # set up ssh LocalForward to it
         port = reliable_ssh_local_fwd(self.host, f"{first_addr}:22")
 
-        # create a remote and connect it
+        # prepare release using variables from this scope
         def release_hook(remote):
             # un-forward the libvirt host ssh-forwarded port
             self.host.forward("LocalForward", f"127.0.0.1:{port} {first_addr}:22", cancel=True)
@@ -339,6 +353,7 @@ class LibvirtCloningProvisioner(Provisioner):
                     try:
                         domain = self.manage_conn.lookupByName(remote.domain)
                         locking.unlock(domain, self.signature)
+                        domain.destroy()
                     except libvirt.libvirtError as e:
                         if "Domain not found" not in str(e):
                             raise
@@ -348,11 +363,12 @@ class LibvirtCloningProvisioner(Provisioner):
                 except ValueError:
                     pass
 
+        # create a remote and connect it
         ssh_options = {
             "Hostname": "127.0.0.1",
             "User": self.domain_user,
             "Port": str(port),
-            "IdentityFile": self.domain_sshkey,
+            "IdentityFile": str(Path(self.domain_sshkey).absolute()),
             "ConnectionAttempts": "1000",
             "Compression": "yes",
         }
@@ -385,7 +401,7 @@ class LibvirtCloningProvisioner(Provisioner):
             ("virt-ssh-helper", "qemu:///system"),
             func=lambda *args, **_: args[0],
         )
-        # to make libvirt connect via our ManagedSSHConn, we need to give it
+        # to make libvirt connect via our ManagedSSHConnection, we need to give it
         # a specific ssh CLI, but libvirt URI command= takes only one argv[0]
         # and cannot pass arguments - we work around this by creating a temp
         # arg-less executable
@@ -410,8 +426,6 @@ class LibvirtCloningProvisioner(Provisioner):
             self.reserve_conn = self._open_libvirt_conn()
             self.manage_conn = self.reserve_conn  # for now
             self.reserve_end = int(time.time()) + self.reserve_time
-            # get an initial first remote
-            self.queue.start_thread(target=self._reserve_one)
 
     def stop(self):
         with self.lock:
@@ -443,11 +457,16 @@ class LibvirtCloningProvisioner(Provisioner):
             self.reserve_end = None
             # TODO: wait for threadqueue threads to join?
 
+    def provision(self, count=1):
+        with self.lock:
+            self.to_reserve += count
+
     def get_remote(self, block=True):
-        # if the reservation thread is not running, start one
         with self.lock:
-            if not self.queue.threads:
+            # if the reservation thread is not running, start one
+            if not self.queue.threads and self.to_reserve > 0:
                 self.queue.start_thread(target=self._reserve_one)
+                self.to_reserve -= 1
         try:
             return self.queue.get(block=block)
         except util.ThreadQueue.Empty:

atex/{provision → provisioner}/libvirt/locking.py

@@ -19,7 +19,9 @@ import time
 import random
 import xml.etree.ElementTree as ET
 
-import libvirt
+from ... import util
+
+libvirt = util.import_libvirt()
 
 
 def get_locks(domain, expired=False):

atex/provisioner/podman/__init__.py

@@ -0,0 +1,2 @@
+from .podman import PodmanProvisioner, PodmanRemote  # noqa: F401
+from .podman import pull_image, build_container_with_deps  # noqa: F401

atex/provisioner/podman/podman.py

@@ -0,0 +1,169 @@
+import tempfile
+import threading
+import subprocess
+
+from ... import connection, util
+from .. import Provisioner, Remote
+
+
+class PodmanRemote(Remote, connection.podman.PodmanConnection):
+    """
+    Built on the official Remote API, pulling in the Connection API
+    as implemented by ManagedSSHConnection.
+    """
+
+    def __init__(self, image, container, *, release_hook):
+        """
+        'image' is an image tag (used for repr()).
+
+        'container' is a podman container id / name.
+
+        'release_hook' is a callable called on .release() in addition
+        to disconnecting the connection.
+        """
+        super().__init__(container=container)
+        self.lock = threading.RLock()
+        self.image = image
+        self.container = container
+        self.release_called = False
+        self.release_hook = release_hook
+
+    def release(self):
+        with self.lock:
+            if self.release_called:
+                return
+            else:
+                self.release_called = True
+        self.release_hook(self)
+        self.disconnect()
+        util.subprocess_run(
+            ("podman", "container", "rm", "-f", "-t", "0", self.container),
+            check=False,  # ignore if it fails
+            stdout=subprocess.DEVNULL,
+        )
+
+    # not /technically/ a valid repr(), but meh
+    def __repr__(self):
+        class_name = self.__class__.__name__
+
+        if "/" in self.image:
+            image = self.image.rsplit("/",1)[1]
+        elif len(self.image) > 20:
+            image = f"{self.image[:17]}..."
+        else:
+            image = self.image
+
+        name = f"{self.container[:17]}..." if len(self.container) > 20 else self.container
+
+        return f"{class_name}({image}, {name})"
+
+
+class PodmanProvisioner(Provisioner):
+    def __init__(self, image, run_options=None):
+        """
+        'image' is a string of image tag/id to create containers from.
+        It can be a local identifier or an URL.
+
+        'run_options' is an iterable with additional CLI options passed
+        to 'podman container run'.
+        """
+        self.lock = threading.RLock()
+        self.image = image
+        self.run_options = run_options or ()
+
+        # created PodmanRemote instances, ready to be handed over to the user,
+        # or already in use by the user
+        self.remotes = []
+        self.to_create = 0
+
+    def start(self):
+        if not self.image:
+            raise ValueError("image cannot be empty")
+
+    def stop(self):
+        with self.lock:
+            while self.remotes:
+                self.remotes.pop().release()
+
+    def provision(self, count=1):
+        with self.lock:
+            self.to_create += count
+
+    def get_remote(self, block=True):
+        if self.to_create <= 0:
+            if block:
+                raise RuntimeError("no .provision() requested, would block forever")
+            else:
+                return None
+
+        proc = util.subprocess_run(
+            (
+                "podman", "container", "run", "--quiet", "--detach", "--pull", "never",
+                *self.run_options, self.image, "sleep", "inf",
+            ),
+            check=True,
+            text=True,
+            stdout=subprocess.PIPE,
+        )
+        container_id = proc.stdout.rstrip("\n")
+
+        def release_hook(remote):
+            # remove from the list of remotes inside this Provisioner
+            with self.lock:
+                try:
+                    self.remotes.remove(remote)
+                except ValueError:
+                    pass
+
+        remote = PodmanRemote(
+            self.image,
+            container_id,
+            release_hook=release_hook,
+        )
+
+        with self.lock:
+            self.remotes.append(remote)
+            self.to_create -= 1
+
+        return remote
+
+    # not /technically/ a valid repr(), but meh
+    def __repr__(self):
+        class_name = self.__class__.__name__
+        return (
+            f"{class_name}({self.image}, {len(self.remotes)} remotes, {hex(id(self))})"
+        )
+
+
+def pull_image(origin):
+    proc = util.subprocess_run(
+        ("podman", "image", "pull", "-q", origin),
+        check=True,
+        text=True,
+        stdout=subprocess.PIPE,
+    )
+    return proc.stdout.rstrip("\n")
+
+
+def build_container_with_deps(origin, tag=None, *, extra_pkgs=None):
+    tag_args = ("-t", tag) if tag else ()
+
+    pkgs = ["rsync"]
+    if extra_pkgs:
+        pkgs += extra_pkgs
+    pkgs_str = " ".join(pkgs)
+
+    with tempfile.NamedTemporaryFile("w+t", delete_on_close=False) as tmpf:
+        tmpf.write(util.dedent(fr"""
+            FROM {origin}
+            RUN dnf -y -q --setopt=install_weak_deps=False install {pkgs_str} >/dev/null
+            RUN dnf -y -q clean packages >/dev/null
+        """))
+        tmpf.close()
+        proc = util.subprocess_run(
+            ("podman", "image", "build", "-q", "-f", tmpf.name, *tag_args, "."),
+            check=True,
+            text=True,
+            stdout=subprocess.PIPE,
+        )
+        return proc.stdout.rstrip("\n")