atex 0.7__py3-none-any.whl → 0.9__py3-none-any.whl

atex/cli/fmf.py

@@ -0,0 +1,143 @@
+import sys
+import pprint
+
+from .. import fmf
+
+
+def _fatal(msg):
+    print(msg, file=sys.stderr)
+    sys.exit(1)
+
+
+def _get_context(args):
+    context = {}
+    if args.context:
+        for c in args.context:
+            key, value = c.split("=", 1)
+            context[key] = value
+    return context or None
+
+
+def make_fmftests(args):
+    return fmf.FMFTests(
+        args.root,
+        args.plan,
+        names=args.test or None,
+        filters=args.filter or None,
+        conditions=args.condition or None,
+        excludes=args.exclude or None,
+        context=_get_context(args),
+    )
+
+
+def requires(args):
+    result = make_fmftests(args)
+    all_pkgs = set()
+    all_pkgs.update(fmf.all_pkg_requires(result, key="require"))
+    all_pkgs.update(fmf.all_pkg_requires(result, key="recommend"))
+    for pkg in sorted(all_pkgs):
+        print(pkg)
+
+
+def discover(args):
+    result = make_fmftests(args)
+    for name in result.tests:
+        print(name)
+
+
+def show(args):
+    result = make_fmftests(args)
+    for name, data in result.tests.items():
+        print(f"\n--- {name} ---")
+        pprint.pprint(data)
+
+
+def prepare(args):
+    result = make_fmftests(args)
+    print("--- fmf root ---")
+    print(str(result.root))
+    print("--- prepare packages ---")
+    print("\n".join(result.prepare_pkgs))
+    print("--- plan environment ---")
+    print("\n".join("{k}={v}" for k,v in result.plan_env))
+    for script in result.prepare_scripts:
+        print("--- prepare script ---")
+        print(script)
+        print("----------------------")
+    for script in result.finish_scripts:
+        print("--- finish script ---")
+        print(script)
+        print("----------------------")
+
+
+def add_fmf_options(parser):
+    parser.add_argument("--root", help="path to directory with fmf tests", default=".")
+    parser.add_argument("--plan", help="plan name (defaults to dummy plan)")
+    parser.add_argument(
+        "--test", "-t", help="test name regex (replacing 'test' from plan)",
+        action="append",
+    )
+    parser.add_argument(
+        "--exclude", help="test name regex (replacing 'exclude' from plan)",
+        action="append",
+    )
+    parser.add_argument(
+        "--condition", help="fmf-style python condition",
+        action="append",
+    )
+    parser.add_argument(
+        "--filter", help="fmf-style expression filter (replacing 'filter' from plan)",
+        action="append",
+    )
+    parser.add_argument(
+        "--context", "-c", help="tmt style key=value context",
+        action="append",
+    )
+
+
+def parse_args(parser):
+    add_fmf_options(parser)
+
+    cmds = parser.add_subparsers(
+        dest="_cmd", help="fmf feature", metavar="<cmd>", required=True,
+    )
+
+    cmds.add_parser(
+        "requires", aliases=("req",),
+        help="list requires/recommends of the plan and its tests",
+    )
+
+    cmds.add_parser(
+        "discover", aliases=("di",),
+        help="list tests, possibly post-processed by a tmt plan",
+    )
+
+    cmds.add_parser(
+        "show",
+        help="show fmf metadata of test(s)",
+    )
+
+    cmds.add_parser(
+        "prepare",
+        help="show prepare-related details from a plan",
+    )
+
+
+def main(args):
+    if args._cmd in ("requires", "req"):
+        requires(args)
+    elif args._cmd in ("discover", "di"):
+        discover(args)
+    elif args._cmd == "show":
+        show(args)
+    elif args._cmd == "prepare":
+        prepare(args)
+    else:
+        raise RuntimeError(f"unknown args: {args}")
+
+
+CLI_SPEC = {
+    "help": "simple CLI interface to atex.fmf",
+    "args": parse_args,
+    "main": main,
+}

atex/cli/libvirt.py

@@ -0,0 +1,127 @@
+import sys
+import re
+
+import libvirt
+
+from ..provision.libvirt import locking
+
+
+def _libvirt_open(url=None):
+    # pass no arguments if url is None
+    conn = libvirt.open(*((url,) if url else ()))
+    print(f"Connected to {conn.getHostname()} via {conn.getURI()}\n")
+    return conn
+
+
+def get_locks(args):
+    conn = _libvirt_open(args.connect)
+    domains = conn.listAllDomains(libvirt.VIR_CONNECT_LIST_DOMAINS_PERSISTENT)
+    for domain in sorted(domains, key=lambda d: d.name()):
+        print(f"{domain.name()}:")
+        for sig, stamp in locking.get_locks(domain, expired=args.expired):
+            print(f"    {sig} {stamp}")
+        print()
+
+
+def unlock(args):
+    conn = _libvirt_open(args.connect)
+    dom = conn.lookupByName(args.domain)
+    locking.unlock(dom, args.signature)
+
+
+def lock(args):
+    conn = _libvirt_open(args.connect)
+    dom = conn.lookupByName(args.domain)
+    if locking.lock(dom, args.signature, args.timestamp):
+        print("Succeeded.")
+        sys.exit(0)
+    else:
+        print("Failed (already locked).")
+        sys.exit(2)
+
+
+def unlock_all(args):
+    conn = _libvirt_open(args.connect)
+    if args.domains:
+        def domains(dom):
+            return bool(re.fullmatch(args.domains, dom.name()))
+    else:
+        def domains(_):
+            return True
+    locking.unlock_all(conn, args.signature, args.shutdown, domains)
+
+
+def cleanup_expired(args):
+    conn = _libvirt_open(args.connect)
+    if args.domains:
+        def domains(dom):
+            return bool(re.fullmatch(args.domains, dom.name()))
+    else:
+        def domains(_):
+            return True
+    locking.cleanup_expired(conn, args.timestamp, domains)
+
+
+def parse_args(parser):
+    parser.add_argument("--connect", "-c", help="Libvirt URL to connect to", metavar="URL")
+    cmds = parser.add_subparsers(
+        dest="_cmd", help="libvirt helper to run", metavar="<cmd>", required=True,
+    )
+
+    cmd = cmds.add_parser(
+        "get-locks",
+        help="List all locks (signatures)",
+    )
+    cmd.add_argument("--expired", help="List also expired locks", action="store_true")
+
+    cmd = cmds.add_parser(
+        "unlock",
+        help="Remove a lock signature from a domain",
+    )
+    cmd.add_argument("domain", help="Domain name")
+    cmd.add_argument("signature", help="Lock signature UUID")
+
+    cmd = cmds.add_parser(
+        "lock",
+        help="Lock a domain (exit 0:success, 2:fail)",
+    )
+    cmd.add_argument("domain", help="Domain name")
+    cmd.add_argument("signature", help="Lock signature UUID")
+    cmd.add_argument("timestamp", help="Expiration time for the lock")
+
+    cmd = cmds.add_parser(
+        "unlock-all",
+        help="Remove all lock signatures from all domains",
+    )
+    cmd.add_argument("--signature", help="Only remove this UUID")
+    cmd.add_argument("--shutdown", help="Also destroy the domains", action="store_true")
+    cmd.add_argument("--domains", help="Which domains names to impact", metavar="REGEX")
+
+    cmd = cmds.add_parser(
+        "cleanup-expired",
+        help="Remove expired lock signatures from all domains",
+    )
+    cmd.add_argument("--timestamp", help="Check against this instead of UTC now()")
+    cmd.add_argument("--domains", help="Which domains names to impact", metavar="REGEX")
+
+
+def main(args):
+    if args._cmd == "get-locks":
+        get_locks(args)
+    elif args._cmd == "unlock":
+        unlock(args)
+    elif args._cmd == "lock":
+        lock(args)
+    elif args._cmd == "unlock-all":
+        unlock_all(args)
+    elif args._cmd == "cleanup-expired":
+        cleanup_expired(args)
+    else:
+        raise RuntimeError(f"unknown args: {args}")
+
+
+CLI_SPEC = {
+    "help": "various utils for the Libvirt provisioner",
+    "args": parse_args,
+    "main": main,
+}

atex/cli/testingfarm.py

@@ -1,4 +1,5 @@
 import sys
+import json
 import pprint
 
 from .. import util
@@ -49,6 +50,8 @@ def search_requests(args):
     reply = api.search_requests(
         state=args.state,
         mine=not args.all,
+        user_id=args.user_id,
+        token_id=args.token_id,
         ranch=args.ranch,
         created_before=args.before,
         created_after=args.after,
@@ -56,20 +59,24 @@ def search_requests(args):
     if not reply:
         return
 
-    for req in sorted(reply, key=lambda x: x["created"]):
-        req_id = req["id"]
-        created = req["created"].partition(".")[0]
+    if args.json:
+        for req in sorted(reply, key=lambda x: x["created"]):
+            print(json.dumps(req))
+    else:
+        for req in sorted(reply, key=lambda x: x["created"]):
+            req_id = req["id"]
+            created = req["created"].partition(".")[0]
 
-        envs = []
-        for env in req["environments_requested"]:
-            if "os" in env and env["os"] and "compose" in env["os"]:
-                compose = env["os"]["compose"]
-                arch = env["arch"]
-                if compose and arch:
-                    envs.append(f"{compose}@{arch}")
-        envs_str = ", ".join(envs)
+            envs = []
+            for env in req["environments_requested"]:
+                if "os" in env and env["os"] and "compose" in env["os"]:
+                    compose = env["os"]["compose"]
+                    arch = env["arch"]
+                    if compose and arch:
+                        envs.append(f"{compose}@{arch}")
+            envs_str = ", ".join(envs)
 
-        print(f"{created} {req_id} : {envs_str}")
+            print(f"{created} {req_id} : {envs_str}")
 
 
 def reserve(args):
@@ -80,12 +87,19 @@ def reserve(args):
     else:
         hardware = None
 
+    if args.native_test:
+        test = tf.DEFAULT_RESERVE_TEST.copy()
+        test["name"] = "/plans/testing-farm-native"
+    else:
+        test = None
+
     api = _get_api(args)
     res = tf.Reserve(
         compose=args.compose,
         arch=args.arch,
         timeout=args.timeout,
         hardware=hardware,
+        reserve_test=test,
         api=api,
     )
     with res as m:
@@ -177,9 +191,12 @@ def parse_args(parser):
     )
     cmd.add_argument("--state", help="request state (running, etc.)", required=True)
     cmd.add_argument("--all", help="all requests, not just owned by token", action="store_true")
-    cmd.add_argument("--ranch", help="Testing Farm ranch")
+    cmd.add_argument("--ranch", help="Testing Farm ranch (detected from token)")
+    cmd.add_argument("--user-id", help="'user_id' request field (detected from token)")
+    cmd.add_argument("--token-id", help="'token_id' request field (detected from token)")
     cmd.add_argument("--before", help="only requests created before ISO8601")
     cmd.add_argument("--after", help="only requests created after ISO8601")
+    cmd.add_argument("--json", help="full details, one request per line", action="store_true")
 
     cmd = cmds.add_parser(
         "reserve",
@@ -190,6 +207,11 @@ def parse_args(parser):
     cmd.add_argument("--timeout", "-t", help="pipeline timeout (in minutes)", type=int, default=60)
     cmd.add_argument("--ssh-key", help="path to a ssh private key file like 'id_rsa'")
     cmd.add_argument("--hvm", help="request a HVM virtualization capable HW", action="store_true")
+    cmd.add_argument(
+        "--native-test",
+        help="use the default testing farm reserve test",
+        action="store_true",
+    )
 
     cmd = cmds.add_parser(
         "watch-pipeline", aliases=("wp",),

atex/connection/__init__.py

@@ -1,6 +1,5 @@
 import importlib as _importlib
 import pkgutil as _pkgutil
-import threading as _threading
 
 from .. import util as _util
 
@@ -34,18 +33,21 @@ class Connection:
     ie. disconnect() might be called from a different thread while connect()
     or cmd() are still running.
     Similarly, multiple threads may run cmd() or rsync() independently.
-    """
 
-    def __init__(self):
-        """
-        Initialize the connection instance.
-        If extending __init__, always call 'super().__init__()' at the top.
-        """
-        self.lock = _threading.RLock()
+    TODO: document that any exceptions raised by a Connection should be children
+    of ConnectionError
+
+    If any connection-related error happens, a ConnectionError (or an exception
+    derived from it) must be raised.
+    """
 
     def __enter__(self):
-        self.connect()
-        return self
+        try:
+            self.connect()
+            return self
+        except Exception:
+            self.disconnect()
+            raise
 
     def __exit__(self, exc_type, exc_value, traceback):
         self.disconnect()
@@ -65,15 +67,7 @@ class Connection:
         """
         raise NotImplementedError(f"'disconnect' not implemented for {self.__class__.__name__}")
 
-    # TODO: is this needed? .. we probably want Remote.alive() instead
-    #def alive(self):
-    #    """
-    #    Return True if the connection was established and is active,
-    #    False otherwise.
-    #    """
-    #    raise NotImplementedError(f"'alive' not implemented for {self.__class__.__name__}")
-
-    def cmd(self, command, func=_util.subprocess_run, **func_args):
+    def cmd(self, command, *, func=_util.subprocess_run, **func_args):
         """
         Execute a single command on the remote, using subprocess-like semantics.
 

atex/connection/podman.py

@@ -0,0 +1,63 @@
+"""
+Connection API implementation using the 'podman' CLI client.
+"""
+
+import subprocess
+
+from .. import util
+from . import Connection
+
+
+class PodmanConnError(ConnectionError):
+    pass
+
+
+class PodmanConn(Connection):
+    """
+    Implements the Connection API via 'podman container exec' on an
+    already-running container, it does not handle any image pulling,
+    container creation, starting or stopping.
+    """
+
+#    def __init__(self, container, *, user=None, workdir=None):
+#        """
+#        'container' is a string with either the full or partial podman
+#        container ID, or a container name, as recognized by podman CLI.
+#
+#        'user' is a string with a username or UID, possibly including a GID,
+#        passed to the podman CLI as --user.
+#
+#        'workdir' is a string specifying the CWD inside the container.
+#        """
+    def __init__(self, container):
+        self.container = container
+
+    def connect(self, block=True):
+        pass
+
+    def disconnect(self):
+        pass
+
+    # have options as kwarg to be compatible with other functions here
+    def cmd(self, command, *, func=util.subprocess_run, **func_args):
+        return func(
+            ("podman", "container", "exec", "-i", self.container, *command),
+            skip_frames=1,
+            **func_args,
+        )
+
+    def rsync(self, *args, func=util.subprocess_run, **func_args):
+        return func(
+            (
+                "rsync",
+                # use shell to strip off the destination argument rsync passes
+                #   cmd[0]=/bin/bash cmd[1]=-c cmd[2]=exec podman ... cmd[3]=destination
+                #   cmd[4]=rsync cmd[5]=--server cmd[6]=-vve.LsfxCIvu cmd[7]=. cmd[8]=.
+                "-e", f"/bin/bash -c 'exec podman container exec -i {self.container} \"$@\"'",
+                *args,
+            ),
+            skip_frames=1,
+            check=True,
+            stdin=subprocess.DEVNULL,
+            **func_args,
+        )