atex 0.13__py3-none-any.whl → 0.15__py3-none-any.whl

atex/provisioner/libvirt/libvirt.py

@@ -4,6 +4,7 @@ import uuid
 import shlex
 import socket
 import random
+import logging
 import textwrap
 import tempfile
 import threading
@@ -18,6 +19,8 @@ from . import locking
 
 libvirt = util.import_libvirt()
 
+logger = logging.getLogger("atex.provisioner.libvirt")
+
 # thread-safe bool
 libvirt_needs_setup = threading.Semaphore(1)
 
@@ -34,7 +37,7 @@ def setup_event_loop():
             time.sleep(0.5)
             libvirt.virEventRunDefaultImpl()
 
-    util.debug("starting libvirt event loop")
+    logger.debug("starting libvirt event loop")
     thread = threading.Thread(target=loop, name="libvirt_event_loop", daemon=True)
     thread.start()
 
@@ -46,17 +49,17 @@ class LibvirtCloningRemote(Remote, connection.ssh.ManagedSSHConnection):
 
     def __init__(self, ssh_options, host, domain, source_image, *, release_hook):
         """
-        'ssh_options' are a dict, passed to ManagedSSHConnection __init__().
+        - `ssh_options` are a dict, passed to ManagedSSHConnection `__init__()`.
 
-        'host' is a str of libvirt host name (used for repr()).
+        - `host` is a str of libvirt host name (used for `repr()`).
 
-        'domain' is a str of libvirt domain name (used for repr()).
+        - `domain` is a str of libvirt domain name (used for `repr()`).
 
-        'source_image' is a str of libvirt volume name that was cloned
-        for the domain to boot from (used for repr()).
+        - `source_image` is a str of libvirt volume name that was cloned
+          for the domain to boot from (used for `repr()`).
 
-        'release_hook' is a callable called on .release() in addition
-        to disconnecting the connection.
+        - `release_hook` is a callable called on `.release()` in addition
+          to disconnecting the connection.
         """
         # NOTE: self.lock inherited from ManagedSSHConnection
         super().__init__(options=ssh_options)
@@ -129,37 +132,42 @@ class LibvirtCloningProvisioner(Provisioner):
         reserve_delay=3, reserve_time=3600, start_event_loop=True,
     ):
         """
-        'host' is a ManagedSSHConnection class instance, connected to a libvirt host.
+        - `host` is a ManagedSSHConnection class instance, connected to
+          a libvirt host.
+
+        - `image` is a string with a libvirt storage volume name inside the
+          given storage `pool` that should be used as the source for cloning.
+
+        - `pool` is a libvirt storage pool used by all relevant domains on the
+          libvirt host **as well as** the would-be-cloned images.
 
-        'image' is a string with a libvirt storage volume name inside the
-        given storage 'pool' that should be used as the source for cloning.
+        - `domain_filter` is a regex string matching libvirt domain names to
+          attempt reservation on. Useful for including only ie. `auto-.*`
+          domains while leaving other domains on the same libvirt host
+          untouched.
 
-        'pool' is a libvirt storage pool used by all relevant domains on the
-        libvirt host **as well as** the would-be-cloned images.
+        - `domain_user` and `domain_sshkey` (strings) specify how to connect to
+          an OS booted from the pre-instaled `image`, as these credentials are
+          known only to the logic that created the `image` in the first place.
 
-        'domain_filter' is a regex string matching libvirt domain names to
-        attempt reservation on. Useful for including only ie. 'auto-.*' domains
-        while leaving other domains on the same libvirt host untouched.
+        - `reserve_delay` is an int of how many seconds to wait between trying
+          to lock libvirt domains, after every unsuccessful locking attempt.
 
-        'domain_user' and 'domain_sshkey' (strings) specify how to connect to
-        an OS booted from the pre-instaled 'image', as these credentials are
-        known only to the logic that created the 'image' in the first place.
+          Ie. with `delay=5` and 20 domains, the code will try to lock every
+          domain in `5*20=100` seconds before looping back to the first.
 
-        'reserve_delay' is an int of how many seconds to wait between trying to
-        lock libvirt domains, after every unsuccessful locking attempt.
-        Ie. with delay=5 and 20 domains, the code will try to lock every domain
-        in 5*20=100 seconds before looping back to the first.
+        - `reserve_time` is an int of maximum seconds to reserve a libvirt
+          domain for before other users can steal it for themselves. Note that
+          there is no automatic timeout release logic, it's just a hint for
+          others.
 
-        'reserve_time' is an int of maximum seconds to reserve a libvirt domain
-        for before other users can steal it for themselves. Note that there is
-        no automatic timeout release logic, it's just a hint for others.
+        - `start_event_loop` set to `True` starts a global default libvirt event
+          loop as part of `.start()` (or context manager enter) in a background
+          daemon thread.
 
-        'start_event_loop' set to True starts a global default libvirt event
-        loop as part of .start() (or context manager enter) in a background
-        daemon thread.
-        This is necessary to maintain connection keep-alives, but if you plan
-        on managing the loop yourself (have custom uses for the libvirt module),
-        setting False here avoids any meddling by this class.
+          This is necessary to maintain connection keep-alives, but if you plan
+          on managing the loop yourself (have custom uses for the libvirt
+          module), setting `False` here avoids any meddling by this class.
         """
         self.lock = threading.RLock()
         self.host = host
@@ -204,7 +212,7 @@ class LibvirtCloningProvisioner(Provisioner):
         xml_root = ET.fromstring(source_vol.XMLDesc())
         source_format = xml_root.find("target").find("format").get("type")
 
-        util.debug(
+        logger.info(
             f"found volume {source_vol.name()} (format:{source_format}) in pool {pool.name()}",
         )
 
@@ -214,7 +222,7 @@ class LibvirtCloningProvisioner(Provisioner):
         already_reserving = {conn.lookupByName(name) for name in already_reserving}
 
         # acquire (lock) a domain on the libvirt host
-        util.debug("attempting to acquire a domain")
+        logger.info("attempting to acquire a domain")
         acquired = None
         while not acquired:
             domains = []
@@ -229,7 +237,7 @@ class LibvirtCloningProvisioner(Provisioner):
             for domain in domains:
                 if locking.lock(domain, self.signature, self.reserve_end):
                     acquired = domain
-                    util.debug(f"acquired domain {acquired.name()}")
+                    logger.info(f"acquired domain {acquired.name()}")
                     break
                 time.sleep(self.reserve_delay)
 
@@ -245,7 +253,7 @@ class LibvirtCloningProvisioner(Provisioner):
 
         # parse XML definition of the domain
         xmldesc = acquired.XMLDesc().rstrip("\n")
-        util.extradebug(f"domain {acquired.name()} XML:\n{textwrap.indent(xmldesc, '    ')}")
+        logger.debug(f"domain {acquired.name()} XML:\n{textwrap.indent(xmldesc, '    ')}")
         xml_root = ET.fromstring(xmldesc)
         nvram_vol = nvram_path = None
 
@@ -279,7 +287,7 @@ class LibvirtCloningProvisioner(Provisioner):
                 if "Storage volume not found" not in str(e):
                     raise
         if nvram_vol:
-            util.debug(f"deleting nvram volume {nvram_vol.name()}")
+            logger.info(f"deleting nvram volume {nvram_vol.name()}")
             nvram_vol.delete()
 
         # try to find a disk that is a volume in the specified storage pool
@@ -298,7 +306,7 @@ class LibvirtCloningProvisioner(Provisioner):
             if xml_disk_source.get("pool") != pool.name():
                 continue
             disk_vol_name = xml_disk_source.get("volume")
-            util.debug(f"found a domain disk in XML: {disk_vol_name} for pool {pool.name()}")
+            logger.info(f"found a domain disk in XML: {disk_vol_name} for pool {pool.name()}")
             break
         else:
             raise RuntimeError("could not find any <disk> in <devices>")
@@ -322,7 +330,7 @@ class LibvirtCloningProvisioner(Provisioner):
         pool.createXMLFrom(new_volume, source_vol)
 
         # start the domain up
-        util.debug(f"starting up {acquired.name()}")
+        logger.info(f"starting up {acquired.name()}")
         acquired.create()  # like 'virsh start' NOT 'virsh create'
 
         # wait for an IP address leased by libvirt host
@@ -332,7 +340,7 @@ class LibvirtCloningProvisioner(Provisioner):
                 libvirt.VIR_DOMAIN_INTERFACE_ADDRESSES_SRC_LEASE,
             )
             time.sleep(1)
-        util.debug(f"found iface addrs: {addrs}")
+        logger.info(f"found iface addrs: {addrs}")
         first_iface = next(iter(addrs.values()))
         first_addr = next(iter(first_iface.values()))[0]["addr"]
 
@@ -414,7 +422,7 @@ class LibvirtCloningProvisioner(Provisioner):
             name = Path(f.name)
             name.chmod(0o0500)  # r-x------
             uri = f"qemu+ext:///system?command={urllib.parse.quote(str(name.absolute()))}"
-            util.debug(f"opening libvirt conn to {uri}")
+            logger.info(f"opening libvirt conn to {uri}")
             conn = libvirt.open(uri)
         conn.setKeepAlive(5, 3)
         return conn
@@ -429,7 +437,6 @@ class LibvirtCloningProvisioner(Provisioner):
 
     def stop(self):
         with self.lock:
-            #util.debug(f"SELF.RESERVING: {self.reserving} // SELF.REMOTES: {self.remotes}")
             # close reserving libvirt host connection
             # - this stops _reserve_one() from doing anything bad
             if self.reserve_conn:
@@ -445,8 +452,7 @@ class LibvirtCloningProvisioner(Provisioner):
                 try:
                     domain = self.manage_conn.lookupByName(self.reserving.pop())
                     locking.unlock(domain, self.signature)
-                except libvirt.libvirtError as e:
-                    util.debug(f"GOT ERROR: {str(e)}")
+                except libvirt.libvirtError:
                     pass
             # cancel/release all Remotes ever created by us
             while self.remotes:

atex/provisioner/libvirt/locking.py

@@ -9,8 +9,8 @@ Lock safety is ensured by libvirt retaining <metadata> content (tag) order,
 so each user can re-check (after locking) whether the race was won or lost,
 depending on whether the user's signature is on top of <metadata>.
 
-A timestamp (meaning now()+duration) is used as the lock tag value/text,
-and any expired timestamps (now() > timestamp) are ignored by the locking
+A timestamp (meaning `now()+duration`) is used as the lock tag value/text,
+and any expired timestamps (`now() > timestamp`) are ignored by the locking
 logic.
 """
 
@@ -26,11 +26,11 @@ libvirt = util.import_libvirt()
 
 def get_locks(domain, expired=False):
     """
-    Yield (signature,timestamp) tuples of atex-lock entries for a 'domain'.
+    Yield (signature,timestamp) tuples of atex-lock entries for a `domain`.
 
-    If 'expired' is True, yield also locks with an expired timestamp.
+    - If `expired` is `True`, yield also locks with an expired timestamp.
 
-    If a timestamp is missing, it is substituted with 0.
+    If a timestamp is missing, it is substituted with `0`.
     """
     xml_dump = ET.fromstring(domain.XMLDesc(libvirt.VIR_DOMAIN_XML_INACTIVE))
     metadata = xml_dump.find("metadata")
@@ -49,7 +49,7 @@ def get_locks(domain, expired=False):
 
 def unlock(domain, signature):
     """
-    Unlock a domain previously locked by lock().
+    Unlock a domain previously locked by `lock()`.
     """
     domain.setMetadata(
         libvirt.VIR_DOMAIN_METADATA_ELEMENT,
@@ -62,10 +62,10 @@ def unlock(domain, signature):
 
 def lock(domain, signature, timestamp):
     """
-    Attempt to lock a domain under 'signature' ownership,
-    writing out 'timestamp' as the lock tag content.
+    Attempt to lock a domain under `signature` ownership,
+    writing out `timestamp` as the lock tag content.
 
-    Returns True if the domain was successfully locked, False otherwise.
+    Returns `True` if the domain was successfully locked, `False` otherwise.
     """
     signature = str(signature)
     timestamp = int(timestamp)
@@ -102,17 +102,19 @@ def lock(domain, signature, timestamp):
 
 def lock_any(connection, signature, duration, filter_domains=lambda _: True):
     """
-    Given a libvirt 'connection', attempt to lock (reserve) any one
-    domain under 'signature' ownership for 'duration' seconds.
+    Given a libvirt `connection`, attempt to lock (reserve) any one
+    domain under `signature` ownership for `duration` seconds.
 
-    If 'filter_domain' is given as a callable, it is used to filter
-    domains considered for locking. It takes one argument (libvirt
-    domain object) and must return True (domain is eligible for locking)
-    or False (domain should be skipped).
-    For example: lambda dom: dom.name().startswith("foo-")
+    - If `filter_domain` is given as a callable, it is used to filter
+      domains considered for locking.
+
+      It takes one argument (libvirt domain object) and must return `True`
+      (domain is eligible for locking) or `False` (domain should be skipped).
+
+      For example: `lambda dom: dom.name().startswith("foo-")`
 
     Returns a libvirt domain object of a successfully locked domain,
-    or None if no domain could be locked.
+    or `None` if no domain could be locked.
     """
     domains = connection.listAllDomains(libvirt.VIR_CONNECT_LIST_DOMAINS_PERSISTENT)
     # try to avoid lock conflicts
@@ -129,11 +131,11 @@ def unlock_all(connection, signature=None, shutdown=False, filter_domains=lambda
     """
     Remove all locks for all domains.
 
-    If 'signature' is given, remove only locks matching the signature.
+    - If `signature` is given, remove only locks matching the signature.
 
-    If 'shutdown' is True, also forcibly shutdown (destroy) all domains.
+    - If `shutdown` is `True`, also forcibly shutdown (destroy) all domains.
 
-    If 'filter_domains' is given, it behaves like for lock_any().
+    - If `filter_domains` is given, it behaves like for `lock_any()`.
     """
     domains = connection.listAllDomains(libvirt.VIR_CONNECT_LIST_DOMAINS_PERSISTENT)
     for domain in filter(filter_domains, domains):
@@ -156,11 +158,11 @@ def cleanup_expired(connection, timestamp=None, filter_domains=lambda _: True):
     for the given signature, it is never removed, unless this function is used
     (by some maintenance service).
 
-    Note that unlock_all() cleans up all locks, incl. expired ones.
+    Note that `unlock_all()` cleans up all locks, incl. expired ones.
 
-    If 'timestamp' is given, it is used instead of the now() UTC timestamp.
+    - If `timestamp` is given, it is used instead of the `now()` UTC timestamp.
 
-    If 'filter_domains' is given, it behaves like for lock_any().
+    - If `filter_domains` is given, it behaves like for `lock_any()`.
     """
     now = int(timestamp) if timestamp is not None else int(time.time())
     domains = connection.listAllDomains(libvirt.VIR_CONNECT_LIST_DOMAINS_PERSISTENT)

atex/provisioner/podman/podman.py

@@ -14,12 +14,12 @@ class PodmanRemote(Remote, connection.podman.PodmanConnection):
 
     def __init__(self, image, container, *, release_hook):
         """
-        'image' is an image tag (used for repr()).
+        - `image` is an image tag (used for `repr()`).
 
-        'container' is a podman container id / name.
+        - `container` is a podman container ID / name.
 
-        'release_hook' is a callable called on .release() in addition
-        to disconnecting the connection.
+        - `release_hook` is a callable called on `.release()` in addition
+          to disconnecting the connection.
         """
         super().__init__(container=container)
         self.lock = threading.RLock()
@@ -61,11 +61,11 @@ class PodmanRemote(Remote, connection.podman.PodmanConnection):
 class PodmanProvisioner(Provisioner):
     def __init__(self, image, run_options=None):
         """
-        'image' is a string of image tag/id to create containers from.
-        It can be a local identifier or an URL.
+        - `image` is a string of image tag/ID to create containers from.
+          It can be a local identifier or an URL.
 
-        'run_options' is an iterable with additional CLI options passed
-        to 'podman container run'.
+        - `run_options` is an iterable with additional CLI options passed
+          to `podman container run`.
         """
         self.lock = threading.RLock()
         self.image = image