atex 0.4__py3-none-any.whl → 0.7__py3-none-any.whl

atex/orchestrator/aggregator.py

@@ -0,0 +1,163 @@
+"""
+Functions and utilities for persistently storing test results and files (logs).
+
+There is a global aggregator (ie. CSVAggregator) that handles all the results
+from all platforms (arches and distros), and several  per-platform aggregators
+that are used by test execution logic.
+
+    with CSVAggregator("results.csv.gz", "file/storage/dir") as global_aggr:
+        reporter = global_aggr.for_platform("rhel-9@x86_64")
+        reporter.report({"name": "/some/test", "status": "pass"})
+        with reporter.open_tmpfile() as fd:
+            os.write(fd, "some contents")
+            reporter.link_tmpfile_to("/some/test", "test.log", fd)
+"""
+
+import os
+import csv
+import gzip
+import ctypes
+import ctypes.util
+import threading
+import contextlib
+from pathlib import Path
+
+
+libc = ctypes.CDLL(ctypes.util.find_library("c"), use_errno=True)
+
+# int linkat(int olddirfd, const char *oldpath, int newdirfd, const char *newpath, int flags)
+libc.linkat.argtypes = (
+    ctypes.c_int,
+    ctypes.c_char_p,
+    ctypes.c_int,
+    ctypes.c_char_p,
+    ctypes.c_int,
+)
+libc.linkat.restype = ctypes.c_int
+
+# fcntl.h:#define AT_EMPTY_PATH                0x1000  /* Allow empty relative pathname */
+AT_EMPTY_PATH = 0x1000
+
+# fcntl.h:#define AT_FDCWD             -100    /* Special value used to indicate
+AT_FDCWD = -100
+
+
+def linkat(*args):
+    if (ret := libc.linkat(*args)) == -1:
+        errno = ctypes.get_errno()
+        raise OSError(errno, os.strerror(errno))
+    return ret
+
+
+def _normalize_path(path):
+    # the magic here is to treat any dangerous path as starting at /
+    # and resolve any weird constructs relative to /, and then simply
+    # strip off the leading / and use it as a relative path
+    path = path.lstrip("/")
+    path = os.path.normpath(f"/{path}")
+    return path[1:]
+
+
+class CSVAggregator:
+    """
+    Collects reported results as a GZIP-ed CSV and files (logs) under a related
+    directory.
+    """
+
+    class _ExcelWithUnixNewline(csv.excel):
+        lineterminator = "\n"
+
+    def __init__(self, results_file, storage_dir):
+        self.lock = threading.RLock()
+        self.storage_dir = Path(storage_dir)
+        self.results_file = Path(results_file)
+        self.csv_writer = None
+        self.results_gzip_handle = None
+
+    def __enter__(self):
+        if self.results_file.exists():
+            raise FileExistsError(f"{self.results_file} already exists")
+        f = gzip.open(self.results_file, "wt", newline="")
+        try:
+            self.csv_writer = csv.writer(f, dialect=self._ExcelWithUnixNewline)
+        except:
+            f.close()
+            raise
+        self.results_gzip_handle = f
+
+        if self.storage_dir.exists():
+            raise FileExistsError(f"{self.storage_dir} already exists")
+        self.storage_dir.mkdir()
+
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.results_gzip_handle.close()
+        self.results_gzip_handle = None
+        self.csv_writer = None
+
+    def report(self, platform, status, name, note, *files):
+        with self.lock:
+            self.csv_writer.writerow((platform, status, name, note, *files))
+
+    def for_platform(self, platform_string):
+        """
+        Return a ResultAggregator instance that writes results into this
+        CSVAgreggator instance.
+        """
+        def report(result_line):
+            file_names = []
+            if "testout" in result_line:
+                file_names.append(result_line["testout"])
+            if "files" in result_line:
+                file_names += (f["name"] for f in result_line["files"])
+            self.report(
+                platform_string, result_line["status"], result_line["name"],
+                result_line.get("note", ""), *file_names,
+            )
+        platform_dir = self.storage_dir / platform_string
+        platform_dir.mkdir(exist_ok=True)
+        return ResultAggregator(report, platform_dir)
+
+
+class ResultAggregator:
+    """
+    Collects reported results (in a format specified by RESULTS.md) for
+    a specific platform, storing them persistently.
+    """
+
+    def __init__(self, callback, storage_dir):
+        """
+        'callback' is a function to call to record a result, with the
+        result dict passed as an argument.
+
+        'storage_dir' is a directory for storing uploaded files.
+        """
+        self.report = callback
+        self.storage_dir = storage_dir
+
+    @contextlib.contextmanager
+    def open_tmpfile(self, open_mode=os.O_WRONLY):
+        """
+        Open an anonymous (name-less) file for writing and yield its file
+        descriptor (int) as context, closing it when the context is exited.
+        """
+        flags = open_mode | os.O_TMPFILE
+        fd = os.open(self.storage_dir, flags, 0o644)
+        try:
+            yield fd
+        finally:
+            os.close(fd)
+
+    def link_tmpfile_to(self, result_name, file_name, fd):
+        """
+        Store a file named 'file_name' in a directory relevant to 'result_name'
+        whose 'fd' (a file descriptor) was created by .open_tmpfile().
+
+        This function can be called multiple times with the same 'fd', and
+        does not close or otherwise alter the descriptor.
+        """
+        # /path/to/all/logs / some/test/name / path/to/file.log
+        file_path = self.storage_dir / result_name.lstrip("/") / _normalize_path(file_name)
+        file_path.parent.mkdir(parents=True, exist_ok=True)
+        linkat(fd, b"", AT_FDCWD, bytes(file_path), AT_EMPTY_PATH)

atex/provision/__init__.py

@@ -1,10 +1,11 @@
-import importlib
-import pkgutil
+import importlib as _importlib
+import pkgutil as _pkgutil
+import threading as _threading
 
-from .. import util
+from .. import connection as _connection
 
 
-class Provisioner(util.LockableClass):
+class Provisioner:
     """
     A resource (machine/system) provider.
 
@@ -67,47 +68,88 @@ class Provisioner(util.LockableClass):
         Initialize the provisioner instance.
         If extending __init__, always call 'super().__init__()' at the top.
         """
-        super().__init__()
-
-    def reserve(self):
-        """
-        Send a reservation request for a resource and wait for it to be
-        reserved.
-        """
-        raise NotImplementedError(f"'reserve' not implemented for {self.__class__.__name__}")
+        self.lock = _threading.RLock()
+
+#    def reserve(self):
+#        """
+#        Send a reservation request for a resource and wait for it to be
+#        reserved.
+#        """
+#        raise NotImplementedError(f"'reserve' not implemented for {self.__class__.__name__}")
+#
+#    def connection(self):
+#        """
+#        Return an atex.ssh.SSHConn instance configured for connection to
+#        the reserved resource, but not yet connected.
+#        """
+#        raise NotImplementedError(f"'connection' not implemented for {self.__class__.__name__}")
+#
+#    def release(self):
+#        """
+#        Release a reserved resource, or cancel a reservation-in-progress.
+#        """
+#        raise NotImplementedError(f"'release' not implemented for {self.__class__.__name__}")
+#
+#    def alive(self):
+#        """
+#        Return True if the resource is still reserved, False otherwise.
+#        """
+#        raise NotImplementedError(f"'alive' not implemented for {self.__class__.__name__}")
+
+
+class Remote(_connection.Connection):
+    """
+    Representation of a provisioned (reserved) remote system, providing
+    a Connection-like API in addition system management helpers.
+
+    An instance of Remote is typically prepared by a Provisioner and given
+    away for further use, to be .release()d by the user. It is not meant
+    for repeated reserve/release cycles, hence the lack of .reserve().
+
+    Also note that Remote can be used via Context Manager, but does not
+    do automatic .release(), the manager only handles the built-in Connection.
+    The intention is for a Provisioner to run via its own Contest Manager and
+    release all Remotes upon exit.
+    If you need automatic release of one Remote, use a contextlib.ExitStack
+    with a callback, or a try/finally block.
+    """
 
-    def connection(self):
-        """
-        Return an atex.ssh.SSHConn instance configured for connection to
-        the reserved resource, but not yet connected.
-        """
-        raise NotImplementedError(f"'connection' not implemented for {self.__class__.__name__}")
+    # TODO: pass platform as arg ?
+    #def __init__(self, platform, *args, **kwargs):
+    #    """
+    #    Initialize a new Remote instance based on a Connection instance.
+    #    If extending __init__, always call 'super().__init__(conn)' at the top.
+    #    """
+    #    self.lock = _threading.RLock()
+    #    self.platform = platform
 
     def release(self):
         """
-        Release a reserved resource, or cancel a reservation-in-progress.
+        Release (de-provision) the remote resource, freeing resources.
         """
         raise NotImplementedError(f"'release' not implemented for {self.__class__.__name__}")
 
     def alive(self):
         """
-        Return True if the resource is still reserved, False otherwise.
+        Return True if the remote resource is still valid and reserved.
         """
         raise NotImplementedError(f"'alive' not implemented for {self.__class__.__name__}")
 
 
-def find_provisioners():
-    provisioners = []
-    for info in pkgutil.iter_modules(__spec__.submodule_search_locations):
-        mod = importlib.import_module(f'.{info.name}', __name__)
-        # look for Provisioner-derived classes in the module
-        for attr in dir(mod):
-            if attr.startswith('_'):
-                continue
-            value = getattr(mod, attr)
-            try:
-                if issubclass(value, Provisioner):
-                    provisioners.append(attr)
-            except TypeError:
-                pass
-    return provisioners
+_submodules = [
+    info.name for info in _pkgutil.iter_modules(__spec__.submodule_search_locations)
+]
+
+__all__ = [*_submodules, Provisioner.__name__, Remote.__name__]  # noqa: PLE0604
+
+
+def __dir__():
+    return __all__
+
+
+# lazily import submodules
+def __getattr__(attr):
+    if attr in _submodules:
+        return _importlib.import_module(f".{attr}", __name__)
+    else:
+        raise AttributeError(f"module '{__name__}' has no attribute '{attr}'")

atex/provision/libvirt/VM_PROVISION

@@ -49,3 +49,11 @@ FULLY CUSTOM INSTALLS:
 - basically virt-install creating a new domain (ignoring any pre-defined ones)
   - probably shouldn't be used by automation, only for one-VM-at-a-time on user request
     - (no free memory/disk checking, no libvirt locking, etc.)
+
+
+
+# ssh via ProxyJump allowing ssh keys specification
+ssh \
+    -o ProxyCommand='ssh -i /tmp/proxy_sshkey root@3.21.232.206 -W %h:%p' \
+    -i /tmp/destination_sshkey \
+    root@192.168.123.218

atex/provision/libvirt/__init__.py

@@ -1,8 +1,8 @@
-from .. import Provisioner as _Provisioner
+from .. import base
 from ... import util, ssh
 
 
-class LibvirtProvisioner(_Provisioner):
+class LibvirtProvisioner(base.Provisioner):
     number = 123
 
     def reserve(self):
@@ -12,9 +12,9 @@ class LibvirtProvisioner(_Provisioner):
     #       can be overriden by a getter function if you need to keep track
     #       how many times it was accessed
     def connection(self):
-        #return {'Hostname': '1.2.3.4', 'User': 'root', 'IdentityFile': ...}
+        #return {"Hostname": "1.2.3.4", "User": "root", "IdentityFile": ...}
         util.debug(f"returning ssh for {self.number}")
-        return ssh.SSHConn({'Hostname': '1.2.3.4', 'User': 'root'})
+        return ssh.SSHConn({"Hostname": "1.2.3.4", "User": "root"})
 
     def release(self):
         util.debug(f"releasing {self.number}")

atex/provision/nspawn/README

@@ -0,0 +1,74 @@
+The idea is to use systemd-nspawn containers on the host, binding
+/dev/kvm to each, thus avoiding the need for nested virt as our first layer
+of Contest tests will run in the containers (installing libvirtd, etc.)
+and the second layer (VMs created by tests) will use virtual machines,
+via a non-nested HVM.
+
+systemd-nspawn containers can have CPU core limits, memory limits, etc.
+done via cgroups, so we can provide some level of isolation/safety.
+
+
+systemd-nspawn can create its own veth via --network-veth=... and put it into
+a bridge automatically via --network-bridge=...
+
+We can then use NetworkManager + firewalld to pre-create a bridge with built-in
+DHCP and NAT to the outside, via something like
+
+  nmcli connection add type bridge ifname br0 con-name br0 ipv4.method shared ipv6.method ignore
+
+According to https://fedoramagazine.org/internet-connection-sharing-networkmanager/
+the ipv4.method=shared :
+
+    enables IP forwarding for the interface;
+    adds firewall rules and enables masquerading;
+    starts dnsmasq as a DHCP and DNS server.
+
+Specifically it should add MASQUERADE on packets *outgoing* from the bridge subnet,
+so shouldn't need any modification of the upstream eth0 device or any fw rules tied to it.
+
+There also seems to be ipv4.addresses 192.168.42.1/24 to modify the subnet?
+
+If that doesn't work, firewalld has an External zone that has <masquerade/>
+by default, so 
+
+  nmcli connection modify br0 connection.zone external
+
+should work.
+
+
+--------
+
+TODO: We need some way to get DHCP leases for started containers (so we can connect
+      to the containerized sshd).
+
+      If there is no command for it via nmcli, it should be possible to just
+      extract it from wherever NetworkManager pointed dnsmasq to store its leases file.
+
+      We can then probably correlate --network-veth=... device from systemd-nspawn
+      (named after --machine=... name, prefixed with ve-* or vb-* if --network-bridge=* is used)
+      to the leased IP address.
+
+      ls -l /var/lib/NetworkManager/dnsmasq-*.leases
+
+      Or perhaps parse it out of 'ip neigh' to make sure the guest is *really* up.
+        - 'ip neigh' gives us MAC-to-IP, but device is always br0
+        - 'ip link show dev vb-contname' should give us the MAC for 'ip neigh'
+          - if container veth endpoint uses different mac, we can query bridge forward DB
+            via 'bridge fdb' to get all MACs that appeared on the veth
+
+--------
+
+Containers can be installed via ie.
+
+dnf  --releasever=41 --installroot=/var/lib/machines/f41 --use-host-config \
+     --setopt=install_weak_deps=False \
+     install \
+     passwd dnf fedora-release vim-minimal util-linux systemd NetworkManager
+
+where --use-host-config re-uses host repositories.
+
+Maybe consider 'machinectl'-managed containers (start/terminate/kill/reboot/etc.)
+which are just repackaged systemd-nspawn@ services.
+ - Especially since there is no concept of "throw away disk snapshot with container exit",
+   we always need some copy/clone of the --installroot for each instance of the container,
+   so using ie. 'machinectl clone ...' would provide a nice interface for it.

atex/provision/podman/README

@@ -0,0 +1,59 @@
+
+making a podman image from the currently installed OS:
+
+1) dnf install into a separate installroot
+
+dnf
+    --installroot=$INSTALLROOT \
+    --setopt=install_weak_deps=False \
+    --setopt=tsflags=nodocs \
+    -y groupinstall minimal-environment
+
+as root (doesn't work well with unshare, maybe could work via bwrap (bubblewrap))
+
+maybe the unprivileged solution is pulling image from hub + installing @minimal-environment
+into it (perhaps via podman build)
+
+
+2) post process it
+
+echo -n > "$INSTALLROOT/etc/machine-id"
+echo container > "$INSTALLROOT/etc/hostname"
+
+rm -rf "$INSTALLROOT/etc/yum.repos.d"
+cp -f /etc/yum.repos.d/* "$INSTALLROOT/etc/yum.repos.d/."
+cp -f /etc/pki/rpm-gpg/* "$INSTALLROOT/etc/pki/rpm-gpg/."
+
+echo install_weak_deps=False >> "$INSTALLROOT/etc/dnf/dnf.conf"
+echo tsflags=nodocs >> "$INSTALLROOT/etc/dnf/dnf.conf"
+
+ln -sf \
+    /usr/lib/systemd/system/multi-user.target \
+    "$INSTALLROOT/etc/systemd/system/default.target"
+
+# disable auditd
+# disable other services
+# set root password
+
+dnf clean all --installroot="$INSTALLROOT"
+
+
+3) pack it
+
+tar --xattrs -C "$INSTALLROOT" -cvf tarball.tar .
+
+rm -rf "$INSTALLROOT"
+
+
+4) import it to podman
+
+podman import --change 'CMD ["/sbin/init"]' tarball.tar my-image-name
+
+
+5) run it
+
+podman {run,create} --systemd=always --cgroups=split ...
+
+
+
+------------------------------

atex/provision/podman/host_container.sh

@@ -0,0 +1,74 @@
+#!/bin/bash
+
+if [[ $# -lt 1 ]]; then
+    echo "usage: $0 <podman-image-name>" >&2
+    exit 1
+fi
+image_name="$1"
+
+set -e -x
+
+tmpdir=$(mktemp -d -p /var/tmp)
+trap "rm -rf '$tmpdir'" EXIT
+
+installroot="$tmpdir/root"
+
+dnf \
+    --installroot="$installroot" \
+    --setopt=install_weak_deps=False \
+    --setopt=tsflags=nodocs \
+    -q -y groupinstall minimal-environment
+
+echo -n > "$installroot/etc/machine-id"
+#echo container > "$installroot/etc/hostname"
+
+cp -f /etc/yum.repos.d/* "$installroot/etc/yum.repos.d/."
+cp -f /etc/pki/rpm-gpg/* "$installroot/etc/pki/rpm-gpg/."
+
+echo install_weak_deps=False >> "$installroot/etc/dnf/dnf.conf"
+echo tsflags=nodocs >> "$installroot/etc/dnf/dnf.conf"
+
+ln -sf \
+    /usr/lib/systemd/system/multi-user.target \
+    "$installroot/etc/systemd/system/default.target"
+
+systemctl --root="$installroot" disable \
+    auditd.service crond.service rhsmcertd.service sshd.service
+
+#encrypted=$(openssl passwd -6 somepass)
+#usermod --root="$installroot" --password "$encrypted" root
+
+dnf clean packages --installroot="$installroot"
+
+tar --xattrs -C "$installroot" -cf "$tmpdir/packed.tar" .
+
+rm -rf "$installroot"
+
+podman import \
+    --change 'CMD ["/sbin/init"]' \
+    "$tmpdir/packed.tar" "$image_name"
+
+# start as
+# podmn {run,create} --systemd=always --cgroups=split --device /dev/kvm ...
+#
+# podman run -t -i \
+#   --systemd=always --cgroups=split \
+#   --device /dev/kvm \
+#   --network=bridge \
+#   --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_MODULE \
+#   --mount type=bind,src=/lib/modules,dst=/lib/modules,ro \
+#   --mount type=bind,src=/proc/sys/net,dst=/proc/sys/net,rw \
+#   my_container
+#
+# as unprivileged user:
+# podman run -t -i \
+#   --systemd=always --cgroups=split --network=bridge --privileged \
+#   my_container
+#
+# container setup:
+# dnf -y install libvirt-daemon qemu-kvm libvirt-client libvirt-daemon-driver-qemu virt-install libvirt-daemon-driver-storage libvirt-daemon-config-network
+# echo $'user = "root"\ngroup = "root"\nremember_owner = 0' >> /etc/libvirt/qemu.conf
+# systemctl start virtqemud.socket virtstoraged.socket virtnetworkd.socket
+# virsh net-start default
+# virt-install --install fedora40 --disk /var/lib/libvirt/images/foo.qcow2,size=20 --console pty --check disk_size=off --unattended --graphics none
+

atex/provision/testingfarm/__init__.py

@@ -0,0 +1,29 @@
+#from ... import connection
+
+from .. import Provisioner, Remote
+
+#from . import api
+
+
+class TestingFarmRemote(Remote):
+    def __init__(self, connection, request):
+        """
+        'connection' is a class Connection instance.
+
+        'request' is a testing farm Request class instance.
+        """
+        super().__init__(connection)
+        self.request = request
+        self.valid = True
+
+    def release(self):
+        self.disconnect()
+        self.request.cancel()
+        self.valid = False
+
+    def alive(self):
+        return self.valid
+
+
+class TestingFarmProvisioner(Provisioner):
+    pass