atex 0.8__py3-none-any.whl → 0.9__py3-none-any.whl

atex/cli/fmf.py

@@ -18,24 +18,42 @@ def _get_context(args):
     return context or None
 
 
+def make_fmftests(args):
+    return fmf.FMFTests(
+        args.root,
+        args.plan,
+        names=args.test or None,
+        filters=args.filter or None,
+        conditions=args.condition or None,
+        excludes=args.exclude or None,
+        context=_get_context(args),
+    )
+
+
+def requires(args):
+    result = make_fmftests(args)
+    all_pkgs = set()
+    all_pkgs.update(fmf.all_pkg_requires(result, key="require"))
+    all_pkgs.update(fmf.all_pkg_requires(result, key="recommend"))
+    for pkg in sorted(all_pkgs):
+        print(pkg)
+
+
 def discover(args):
-    result = fmf.FMFTests(args.root, args.plan, context=_get_context(args))
+    result = make_fmftests(args)
     for name in result.tests:
         print(name)
 
 
 def show(args):
-    result = fmf.FMFTests(args.root, args.plan, context=_get_context(args))
-    if tests := list(result.match(args.test)):
-        for test in tests:
-            print(f"\n--- {test.name} ---")
-            pprint.pprint(test.data)
-    else:
-        _fatal(f"Not reachable via {args.plan} discovery: {args.test}")
+    result = make_fmftests(args)
+    for name, data in result.tests.items():
+        print(f"\n--- {name} ---")
+        pprint.pprint(data)
 
 
 def prepare(args):
-    result = fmf.FMFTests(args.root, args.plan, context=_get_context(args))
+    result = make_fmftests(args)
     print("--- fmf root ---")
     print(str(result.root))
     print("--- prepare packages ---")
@@ -46,37 +64,69 @@ def prepare(args):
         print("--- prepare script ---")
         print(script)
         print("----------------------")
+    for script in result.finish_scripts:
+        print("--- finish script ---")
+        print(script)
+        print("----------------------")
 
 
-def parse_args(parser):
+def add_fmf_options(parser):
     parser.add_argument("--root", help="path to directory with fmf tests", default=".")
-    parser.add_argument("--context", "-c", help="tmt style key=value context", action="append")
+    parser.add_argument("--plan", help="plan name (defaults to dummy plan)")
+    parser.add_argument(
+        "--test", "-t", help="test name regex (replacing 'test' from plan)",
+        action="append",
+    )
+    parser.add_argument(
+        "--exclude", help="test name regex (replacing 'exclude' from plan)",
+        action="append",
+    )
+    parser.add_argument(
+        "--condition", help="fmf-style python condition",
+        action="append",
+    )
+    parser.add_argument(
+        "--filter", help="fmf-style expression filter (replacing 'filter' from plan)",
+        action="append",
+    )
+    parser.add_argument(
+        "--context", "-c", help="tmt style key=value context",
+        action="append",
+    )
+
+
+def parse_args(parser):
+    add_fmf_options(parser)
+
     cmds = parser.add_subparsers(
-        dest="_cmd", help="executor feature", metavar="<cmd>", required=True,
+        dest="_cmd", help="fmf feature", metavar="<cmd>", required=True,
+    )
+
+    cmds.add_parser(
+        "requires", aliases=("req",),
+        help="list requires/recommends of the plan and its tests",
     )
 
-    cmd = cmds.add_parser(
+    cmds.add_parser(
         "discover", aliases=("di",),
-        help="list tests, post-processed by tmt plans",
+        help="list tests, possibly post-processed by a tmt plan",
     )
-    cmd.add_argument("plan", help="tmt plan to use for discovery")
 
-    cmd = cmds.add_parser(
+    cmds.add_parser(
         "show",
-        help="show fmf data of a test",
+        help="show fmf metadata of test(s)",
     )
-    cmd.add_argument("plan", help="tmt plan to use for discovery")
-    cmd.add_argument("test", help="fmf style test regex")
 
-    cmd = cmds.add_parser(
+    cmds.add_parser(
         "prepare",
-        help="show prepare-related FMFTests details",
+        help="show prepare-related details from a plan",
     )
-    cmd.add_argument("plan", help="tmt plan to parse")
 
 
 def main(args):
-    if args._cmd in ("discover", "di"):
+    if args._cmd in ("requires", "req"):
+        requires(args)
+    elif args._cmd in ("discover", "di"):
         discover(args)
     elif args._cmd == "show":
         show(args)

atex/cli/libvirt.py

@@ -0,0 +1,127 @@
+import sys
+import re
+
+import libvirt
+
+from ..provision.libvirt import locking
+
+
+def _libvirt_open(url=None):
+    # pass no arguments if url is None
+    conn = libvirt.open(*((url,) if url else ()))
+    print(f"Connected to {conn.getHostname()} via {conn.getURI()}\n")
+    return conn
+
+
+def get_locks(args):
+    conn = _libvirt_open(args.connect)
+    domains = conn.listAllDomains(libvirt.VIR_CONNECT_LIST_DOMAINS_PERSISTENT)
+    for domain in sorted(domains, key=lambda d: d.name()):
+        print(f"{domain.name()}:")
+        for sig, stamp in locking.get_locks(domain, expired=args.expired):
+            print(f"    {sig} {stamp}")
+        print()
+
+
+def unlock(args):
+    conn = _libvirt_open(args.connect)
+    dom = conn.lookupByName(args.domain)
+    locking.unlock(dom, args.signature)
+
+
+def lock(args):
+    conn = _libvirt_open(args.connect)
+    dom = conn.lookupByName(args.domain)
+    if locking.lock(dom, args.signature, args.timestamp):
+        print("Succeeded.")
+        sys.exit(0)
+    else:
+        print("Failed (already locked).")
+        sys.exit(2)
+
+
+def unlock_all(args):
+    conn = _libvirt_open(args.connect)
+    if args.domains:
+        def domains(dom):
+            return bool(re.fullmatch(args.domains, dom.name()))
+    else:
+        def domains(_):
+            return True
+    locking.unlock_all(conn, args.signature, args.shutdown, domains)
+
+
+def cleanup_expired(args):
+    conn = _libvirt_open(args.connect)
+    if args.domains:
+        def domains(dom):
+            return bool(re.fullmatch(args.domains, dom.name()))
+    else:
+        def domains(_):
+            return True
+    locking.cleanup_expired(conn, args.timestamp, domains)
+
+
+def parse_args(parser):
+    parser.add_argument("--connect", "-c", help="Libvirt URL to connect to", metavar="URL")
+    cmds = parser.add_subparsers(
+        dest="_cmd", help="libvirt helper to run", metavar="<cmd>", required=True,
+    )
+
+    cmd = cmds.add_parser(
+        "get-locks",
+        help="List all locks (signatures)",
+    )
+    cmd.add_argument("--expired", help="List also expired locks", action="store_true")
+
+    cmd = cmds.add_parser(
+        "unlock",
+        help="Remove a lock signature from a domain",
+    )
+    cmd.add_argument("domain", help="Domain name")
+    cmd.add_argument("signature", help="Lock signature UUID")
+
+    cmd = cmds.add_parser(
+        "lock",
+        help="Lock a domain (exit 0:success, 2:fail)",
+    )
+    cmd.add_argument("domain", help="Domain name")
+    cmd.add_argument("signature", help="Lock signature UUID")
+    cmd.add_argument("timestamp", help="Expiration time for the lock")
+
+    cmd = cmds.add_parser(
+        "unlock-all",
+        help="Remove all lock signatures from all domains",
+    )
+    cmd.add_argument("--signature", help="Only remove this UUID")
+    cmd.add_argument("--shutdown", help="Also destroy the domains", action="store_true")
+    cmd.add_argument("--domains", help="Which domains names to impact", metavar="REGEX")
+
+    cmd = cmds.add_parser(
+        "cleanup-expired",
+        help="Remove expired lock signatures from all domains",
+    )
+    cmd.add_argument("--timestamp", help="Check against this instead of UTC now()")
+    cmd.add_argument("--domains", help="Which domains names to impact", metavar="REGEX")
+
+
+def main(args):
+    if args._cmd == "get-locks":
+        get_locks(args)
+    elif args._cmd == "unlock":
+        unlock(args)
+    elif args._cmd == "lock":
+        lock(args)
+    elif args._cmd == "unlock-all":
+        unlock_all(args)
+    elif args._cmd == "cleanup-expired":
+        cleanup_expired(args)
+    else:
+        raise RuntimeError(f"unknown args: {args}")
+
+
+CLI_SPEC = {
+    "help": "various utils for the Libvirt provisioner",
+    "args": parse_args,
+    "main": main,
+}

atex/cli/testingfarm.py

@@ -87,12 +87,19 @@ def reserve(args):
     else:
         hardware = None
 
+    if args.native_test:
+        test = tf.DEFAULT_RESERVE_TEST.copy()
+        test["name"] = "/plans/testing-farm-native"
+    else:
+        test = None
+
     api = _get_api(args)
     res = tf.Reserve(
         compose=args.compose,
         arch=args.arch,
         timeout=args.timeout,
         hardware=hardware,
+        reserve_test=test,
         api=api,
     )
     with res as m:
@@ -200,6 +207,11 @@ def parse_args(parser):
     cmd.add_argument("--timeout", "-t", help="pipeline timeout (in minutes)", type=int, default=60)
     cmd.add_argument("--ssh-key", help="path to a ssh private key file like 'id_rsa'")
     cmd.add_argument("--hvm", help="request a HVM virtualization capable HW", action="store_true")
+    cmd.add_argument(
+        "--native-test",
+        help="use the default testing farm reserve test",
+        action="store_true",
+    )
 
     cmd = cmds.add_parser(
         "watch-pipeline", aliases=("wp",),

atex/connection/__init__.py

@@ -1,6 +1,5 @@
 import importlib as _importlib
 import pkgutil as _pkgutil
-import threading as _threading
 
 from .. import util as _util
 
@@ -34,18 +33,21 @@ class Connection:
     ie. disconnect() might be called from a different thread while connect()
     or cmd() are still running.
     Similarly, multiple threads may run cmd() or rsync() independently.
-    """
 
-    def __init__(self):
-        """
-        Initialize the connection instance.
-        If extending __init__, always call 'super().__init__()' at the top.
-        """
-        self.lock = _threading.RLock()
+    TODO: document that any exceptions raised by a Connection should be children
+    of ConnectionError
+
+    If any connection-related error happens, a ConnectionError (or an exception
+    derived from it) must be raised.
+    """
 
     def __enter__(self):
-        self.connect()
-        return self
+        try:
+            self.connect()
+            return self
+        except Exception:
+            self.disconnect()
+            raise
 
     def __exit__(self, exc_type, exc_value, traceback):
         self.disconnect()
@@ -65,7 +67,7 @@ class Connection:
         """
         raise NotImplementedError(f"'disconnect' not implemented for {self.__class__.__name__}")
 
-    def cmd(self, command, func=_util.subprocess_run, **func_args):
+    def cmd(self, command, *, func=_util.subprocess_run, **func_args):
         """
         Execute a single command on the remote, using subprocess-like semantics.
 

atex/connection/podman.py

@@ -0,0 +1,63 @@
+"""
+Connection API implementation using the 'podman' CLI client.
+"""
+
+import subprocess
+
+from .. import util
+from . import Connection
+
+
+class PodmanConnError(ConnectionError):
+    pass
+
+
+class PodmanConn(Connection):
+    """
+    Implements the Connection API via 'podman container exec' on an
+    already-running container, it does not handle any image pulling,
+    container creation, starting or stopping.
+    """
+
+#    def __init__(self, container, *, user=None, workdir=None):
+#        """
+#        'container' is a string with either the full or partial podman
+#        container ID, or a container name, as recognized by podman CLI.
+#
+#        'user' is a string with a username or UID, possibly including a GID,
+#        passed to the podman CLI as --user.
+#
+#        'workdir' is a string specifying the CWD inside the container.
+#        """
+    def __init__(self, container):
+        self.container = container
+
+    def connect(self, block=True):
+        pass
+
+    def disconnect(self):
+        pass
+
+    # have options as kwarg to be compatible with other functions here
+    def cmd(self, command, *, func=util.subprocess_run, **func_args):
+        return func(
+            ("podman", "container", "exec", "-i", self.container, *command),
+            skip_frames=1,
+            **func_args,
+        )
+
+    def rsync(self, *args, func=util.subprocess_run, **func_args):
+        return func(
+            (
+                "rsync",
+                # use shell to strip off the destination argument rsync passes
+                #   cmd[0]=/bin/bash cmd[1]=-c cmd[2]=exec podman ... cmd[3]=destination
+                #   cmd[4]=rsync cmd[5]=--server cmd[6]=-vve.LsfxCIvu cmd[7]=. cmd[8]=.
+                "-e", f"/bin/bash -c 'exec podman container exec -i {self.container} \"$@\"'",
+                *args,
+            ),
+            skip_frames=1,
+            check=True,
+            stdin=subprocess.DEVNULL,
+            **func_args,
+        )

atex/connection/ssh.py

@@ -18,6 +18,7 @@ import os
 import time
 import shlex
 import tempfile
+import threading
 import subprocess
 from pathlib import Path
 
@@ -38,7 +39,7 @@ DEFAULT_OPTIONS = {
 }
 
 
-class SSHError(Exception):
+class SSHError(ConnectionError):
     pass
 
 
@@ -154,7 +155,6 @@ class StatelessSSHConn(Connection):
         If 'sudo' specifies a username, call sudo(8) on the remote shell
         to run under a different user on the remote host.
         """
-        super().__init__()
         self.options = DEFAULT_OPTIONS.copy()
         self.options.update(options)
         self.password = password
@@ -162,7 +162,7 @@ class StatelessSSHConn(Connection):
         self._tmpdir = None
         self._master_proc = None
 
-    def connect(self):
+    def connect(self, block=True):
         """
         Optional, .cmd() and .rsync() work without it, but it is provided here
         for compatibility with the Connection API.
@@ -178,7 +178,8 @@ class StatelessSSHConn(Connection):
         unified_options = self.options.copy()
         if options:
             unified_options.update(options)
-        unified_options["RemoteCommand"] = _shell_cmd(command, sudo=self.sudo)
+        if command:
+            unified_options["RemoteCommand"] = _shell_cmd(command, sudo=self.sudo)
         return func(
             _options_to_ssh(unified_options, password=self.password),
             skip_frames=1,
@@ -240,7 +241,7 @@ class ManagedSSHConn(Connection):
         If 'sudo' specifies a username, call sudo(8) on the remote shell
         to run under a different user on the remote host.
         """
-        super().__init__()
+        self.lock = threading.RLock()
         self.options = DEFAULT_OPTIONS.copy()
         self.options.update(options)
         self.password = password
@@ -267,8 +268,9 @@ class ManagedSSHConn(Connection):
         proc = self._master_proc
         if not proc:
             return
+        util.debug(f"disconnecting: {self.options}")
         proc.kill()
-        # don"t zombie forever, return EPIPE on any attempts to write to us
+        # don't zombie forever, return EPIPE on any attempts to write to us
         proc.stdout.close()
         proc.wait()
         (self._tmpdir / "control.sock").unlink(missing_ok=True)
@@ -286,19 +288,26 @@ class ManagedSSHConn(Connection):
         sock = self._tmpdir / "control.sock"
 
         if not self._master_proc:
+            util.debug(f"connecting: {self.options}")
             options = self.options.copy()
             options["SessionType"] = "none"
             options["ControlMaster"] = "yes"
             options["ControlPath"] = sock
             self._master_proc = util.subprocess_Popen(
-                _options_to_ssh(options),
+                _options_to_ssh(options, password=self.password),
                 stdin=subprocess.DEVNULL,
                 stdout=subprocess.PIPE,
                 stderr=subprocess.STDOUT,
                 cwd=str(self._tmpdir),
+                start_new_session=True,  # resist Ctrl-C
             )
             os.set_blocking(self._master_proc.stdout.fileno(), False)
 
+        # NOTE: ideally, we would .read() before checking .poll() because
+        #       if the process writes a lot, it gets stuck in the pipe
+        #       (in kernel) and the process never ends; but output-appending
+        #       code would be obscure, and ssh(1) never outputs that much ..
+
         proc = self._master_proc
         if block:
             while proc.poll() is None:
@@ -309,7 +318,6 @@ class ManagedSSHConn(Connection):
                 code = proc.poll()
                 out = proc.stdout.read()
                 self._master_proc = None
-                # TODO: ConnectError should probably be generalized for Connection
                 raise ConnectError(
                     f"SSH ControlMaster failed to start on {self._tmpdir} with {code}:\n{out}",
                 )
@@ -318,54 +326,45 @@ class ManagedSSHConn(Connection):
             if code is not None:
                 out = proc.stdout.read()
                 self._master_proc = None
-                # TODO: ConnectError should probably be generalized for Connection
                 raise ConnectError(
                     f"SSH ControlMaster failed to start on {self._tmpdir} with {code}:\n{out}",
                 )
             elif not sock.exists():
                 raise BlockingIOError("SSH ControlMaster not yet ready")
 
-    def add_local_forward(self, *spec):
+    def forward(self, forward_type, *spec, cancel=False):
         """
         Add (one or more) ssh forwarding specifications as 'spec' to an
         already-connected instance. Each specification has to follow the
-        format of ssh client's LocalForward option (see ssh_config(5)).
-        """
-        self.assert_master()
-        options = self.options.copy()
-        options["LocalForward"] = spec
-        options["ControlPath"] = self._tmpdir / "control.sock"
-        util.subprocess_run(
-            _options_to_ssh(options, extra_cli_flags=("-O", "forward")),
-            skip_frames=1,
-            check=True,
-        )
+        format of LocalForward or RemoteForward (see ssh_config(5)).
+        Ie. "1234 1.2.3.4:22" or "0.0.0.0:1234 1.2.3.4:22".
 
-    def add_remote_forward(self, *spec):
-        """
-        Add (one or more) ssh forwarding specifications as 'spec' to an
-        already-connected instance. Each specification has to follow the
-        format of ssh client's RemoteForward option (see ssh_config(5)).
+        'forward_type' must be either LocalForward or RemoteForward.
+
+        If 'cancel' is True, cancel the forwarding instead of adding it.
         """
+        assert forward_type in ("LocalForward", "RemoteForward")
         self.assert_master()
-        options = self.options.copy()
-        options["RemoteForward"] = spec
+        options = DEFAULT_OPTIONS.copy()
+        options[forward_type] = spec
         options["ControlPath"] = self._tmpdir / "control.sock"
+        action = "forward" if not cancel else "cancel"
         util.subprocess_run(
-            _options_to_ssh(options, extra_cli_flags=("-O", "forward")),
+            _options_to_ssh(options, extra_cli_flags=("-O", action)),
             skip_frames=1,
             check=True,
         )
 
-    def cmd(self, command, options=None, func=util.subprocess_run, **func_args):
+    def cmd(self, command, *, options=None, func=util.subprocess_run, **func_args):
         self.assert_master()
         unified_options = self.options.copy()
         if options:
             unified_options.update(options)
-        unified_options["RemoteCommand"] = _shell_cmd(command, sudo=self.sudo)
+        if command:
+            unified_options["RemoteCommand"] = _shell_cmd(command, sudo=self.sudo)
         unified_options["ControlPath"] = self._tmpdir / "control.sock"
         return func(
-            _options_to_ssh(unified_options, password=self.password),
+            _options_to_ssh(unified_options),
             skip_frames=1,
             **func_args,
         )
@@ -380,7 +379,6 @@ class ManagedSSHConn(Connection):
             _rsync_host_cmd(
                 *args,
                 options=unified_options,
-                password=self.password,
                 sudo=self.sudo,
             ),
             skip_frames=1,