atex 0.8__py3-none-any.whl → 0.10__py3-none-any.whl

atex/aggregator/__init__.py

@@ -0,0 +1,60 @@
+import importlib as _importlib
+import pkgutil as _pkgutil
+
+
+class Aggregator:
+    """
+    TODO: generic description, not JSON-specific
+    """
+
+    def ingest(self, platform, test_name, results_file, files_dir):
+        """
+        Process 'results_file' (string/Path) for reported results and append
+        them to the overall aggregated line-JSON file, recursively copying over
+        the dir structure under 'files_dir' (string/Path) under the respective
+        platform and test name in the aggregated storage dir.
+        """
+        raise NotImplementedError(f"'ingest' not implemented for {self.__class__.__name__}")
+
+    def start(self):
+        """
+        Start the Aggregator instance, opening any files / allocating resources
+        as necessary.
+        """
+        raise NotImplementedError(f"'start' not implemented for {self.__class__.__name__}")
+
+    def stop(self):
+        """
+        Stop the Aggregator instance, freeing all allocated resources.
+        """
+        raise NotImplementedError(f"'stop' not implemented for {self.__class__.__name__}")
+
+    def __enter__(self):
+        try:
+            self.start()
+            return self
+        except Exception:
+            self.close()
+            raise
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.stop()
+
+
+_submodules = [
+    info.name for info in _pkgutil.iter_modules(__spec__.submodule_search_locations)
+]
+
+__all__ = [*_submodules, Aggregator.__name__]  # noqa: PLE0604
+
+
+def __dir__():
+    return __all__
+
+
+# lazily import submodules
+def __getattr__(attr):
+    if attr in _submodules:
+        return _importlib.import_module(f".{attr}", __name__)
+    else:
+        raise AttributeError(f"module '{__name__}' has no attribute '{attr}'")

atex/aggregator/json.py

@@ -0,0 +1,96 @@
+import gzip
+import json
+import shutil
+import threading
+from pathlib import Path
+
+from . import Aggregator
+
+
+class JSONAggregator(Aggregator):
+    """
+    Collects reported results as a GZIP-ed line-JSON and files (logs) from
+    multiple test runs under a shared directory.
+
+    Note that the aggregated JSON file *does not* use the test-based JSON format
+    described by executor/RESULTS.md - both use JSON, but are very different.
+
+    This aggergated format uses a top-level array (on each line) with a fixed
+    field order:
+
+        platform, status, test name, subtest name, files, note
+
+    All these are strings except 'files', which is another (nested) array
+    of strings.
+
+    If a field is missing in the source result, it is translated to a null
+    value.
+    """
+
+    def __init__(self, json_file, storage_dir):
+        """
+        'json_file' is a string/Path to a .json.gz file with aggregated results.
+
+        'storage_dir' is a string/Path of the top-level parent for all
+        per-platform / per-test files uploaded by tests.
+        """
+        self.lock = threading.RLock()
+        self.storage_dir = Path(storage_dir)
+        self.json_file = Path(json_file)
+        self.json_gzip_fobj = None
+
+    def start(self):
+        if self.json_file.exists():
+            raise FileExistsError(f"{self.json_file} already exists")
+        self.json_gzip_fobj = gzip.open(self.json_file, "wt", newline="\n")
+
+        if self.storage_dir.exists():
+            raise FileExistsError(f"{self.storage_dir} already exists")
+        self.storage_dir.mkdir()
+
+    def stop(self):
+        if self.json_gzip_fobj:
+            self.json_gzip_fobj.close()
+            self.json_gzip_fobj = None
+
+    def ingest(self, platform, test_name, results_file, files_dir):
+        platform_dir = self.storage_dir / platform
+        test_dir = platform_dir / test_name.lstrip("/")
+        if test_dir.exists():
+            raise FileExistsError(f"{test_dir} already exists for {test_name}")
+
+        # parse the results separately, before writing any aggregated output,
+        # to ensure that either all results from the test are ingested, or none
+        # at all (ie. if one of the result lines contains JSON errors)
+        output_lines = []
+        with open(results_file) as results_fobj:
+            for raw_line in results_fobj:
+                result_line = json.loads(raw_line)
+
+                file_names = []
+                if "testout" in result_line:
+                    file_names.append(result_line["testout"])
+                if "files" in result_line:
+                    file_names += (f["name"] for f in result_line["files"])
+
+                output_line = (
+                    platform,
+                    result_line["status"],
+                    test_name,
+                    result_line.get("name"),  # subtest
+                    file_names,
+                    result_line.get("note"),
+                )
+                encoded = json.dumps(output_line, indent=None)
+                output_lines.append(encoded)
+
+        output_str = "\n".join(output_lines) + "\n"
+
+        with self.lock:
+            self.json_gzip_fobj.write(output_str)
+            self.json_gzip_fobj.flush()
+
+        Path(results_file).unlink()
+
+        platform_dir.mkdir(exist_ok=True)
+        shutil.move(files_dir, test_dir)

atex/cli/__init__.py

@@ -27,12 +27,18 @@ import pkgutil
 import argparse
 import logging
 
+from .. import util
+
 
 def setup_logging(level):
+    if level <= util.EXTRADEBUG:
+        fmt = "%(asctime)s %(name)s: %(filename)s:%(lineno)s: %(funcName)s(): %(message)s"
+    else:
+        fmt = "%(asctime)s %(name)s: %(message)s"
     logging.basicConfig(
         level=level,
         stream=sys.stderr,
-        format="%(asctime)s %(name)s: %(message)s",
+        format=fmt,
         datefmt="%Y-%m-%d %H:%M:%S",
     )
 
@@ -53,6 +59,10 @@ def main():
         "--debug", "-d", action="store_const", dest="loglevel", const=logging.DEBUG,
         help="enable extra debugging (logging.DEBUG)",
     )
+    log_grp.add_argument(
+        "--extra-debug", "-D", action="store_const", dest="loglevel", const=util.EXTRADEBUG,
+        help="enable extra debugging (atex.util.EXTRADEBUG)",
+    )
     log_grp.add_argument(
         "--quiet", "-q", action="store_const", dest="loglevel", const=logging.WARNING,
         help="be quiet during normal operation (logging.WARNING)",

atex/cli/fmf.py

@@ -18,24 +18,42 @@ def _get_context(args):
     return context or None
 
 
+def make_fmftests(args):
+    return fmf.FMFTests(
+        args.root,
+        args.plan,
+        names=args.test or None,
+        filters=args.filter or None,
+        conditions=args.condition or None,
+        excludes=args.exclude or None,
+        context=_get_context(args),
+    )
+
+
+def requires(args):
+    result = make_fmftests(args)
+    all_pkgs = set()
+    all_pkgs.update(fmf.all_pkg_requires(result, key="require"))
+    all_pkgs.update(fmf.all_pkg_requires(result, key="recommend"))
+    for pkg in sorted(all_pkgs):
+        print(pkg)
+
+
 def discover(args):
-    result = fmf.FMFTests(args.root, args.plan, context=_get_context(args))
+    result = make_fmftests(args)
     for name in result.tests:
         print(name)
 
 
 def show(args):
-    result = fmf.FMFTests(args.root, args.plan, context=_get_context(args))
-    if tests := list(result.match(args.test)):
-        for test in tests:
-            print(f"\n--- {test.name} ---")
-            pprint.pprint(test.data)
-    else:
-        _fatal(f"Not reachable via {args.plan} discovery: {args.test}")
+    result = make_fmftests(args)
+    for name, data in result.tests.items():
+        print(f"\n--- {name} ---")
+        pprint.pprint(data)
 
 
 def prepare(args):
-    result = fmf.FMFTests(args.root, args.plan, context=_get_context(args))
+    result = make_fmftests(args)
     print("--- fmf root ---")
     print(str(result.root))
     print("--- prepare packages ---")
@@ -46,37 +64,69 @@ def prepare(args):
         print("--- prepare script ---")
         print(script)
         print("----------------------")
+    for script in result.finish_scripts:
+        print("--- finish script ---")
+        print(script)
+        print("----------------------")
 
 
-def parse_args(parser):
+def add_fmf_options(parser):
     parser.add_argument("--root", help="path to directory with fmf tests", default=".")
-    parser.add_argument("--context", "-c", help="tmt style key=value context", action="append")
+    parser.add_argument("--plan", help="plan name (defaults to dummy plan)")
+    parser.add_argument(
+        "--test", "-t", help="test name regex (replacing 'test' from plan)",
+        action="append",
+    )
+    parser.add_argument(
+        "--exclude", help="test name regex (replacing 'exclude' from plan)",
+        action="append",
+    )
+    parser.add_argument(
+        "--condition", help="fmf-style python condition",
+        action="append",
+    )
+    parser.add_argument(
+        "--filter", help="fmf-style expression filter (replacing 'filter' from plan)",
+        action="append",
+    )
+    parser.add_argument(
+        "--context", "-c", help="tmt style key=value context",
+        action="append",
+    )
+
+
+def parse_args(parser):
+    add_fmf_options(parser)
+
     cmds = parser.add_subparsers(
-        dest="_cmd", help="executor feature", metavar="<cmd>", required=True,
+        dest="_cmd", help="fmf feature", metavar="<cmd>", required=True,
+    )
+
+    cmds.add_parser(
+        "requires", aliases=("req",),
+        help="list requires/recommends of the plan and its tests",
     )
 
-    cmd = cmds.add_parser(
+    cmds.add_parser(
         "discover", aliases=("di",),
-        help="list tests, post-processed by tmt plans",
+        help="list tests, possibly post-processed by a tmt plan",
     )
-    cmd.add_argument("plan", help="tmt plan to use for discovery")
 
-    cmd = cmds.add_parser(
+    cmds.add_parser(
         "show",
-        help="show fmf data of a test",
+        help="show fmf metadata of test(s)",
     )
-    cmd.add_argument("plan", help="tmt plan to use for discovery")
-    cmd.add_argument("test", help="fmf style test regex")
 
-    cmd = cmds.add_parser(
+    cmds.add_parser(
         "prepare",
-        help="show prepare-related FMFTests details",
+        help="show prepare-related details from a plan",
     )
-    cmd.add_argument("plan", help="tmt plan to parse")
 
 
 def main(args):
-    if args._cmd in ("discover", "di"):
+    if args._cmd in ("requires", "req"):
+        requires(args)
+    elif args._cmd in ("discover", "di"):
         discover(args)
     elif args._cmd == "show":
         show(args)

atex/cli/libvirt.py

@@ -0,0 +1,128 @@
+import sys
+import re
+
+from .. import util
+from ..provisioner.libvirt import locking
+
+libvirt = util.import_libvirt()
+
+
+def _libvirt_open(url=None):
+    # pass no arguments if url is None
+    conn = libvirt.open(*((url,) if url else ()))
+    print(f"Connected to {conn.getHostname()} via {conn.getURI()}\n")
+    return conn
+
+
+def get_locks(args):
+    conn = _libvirt_open(args.connect)
+    domains = conn.listAllDomains(libvirt.VIR_CONNECT_LIST_DOMAINS_PERSISTENT)
+    for domain in sorted(domains, key=lambda d: d.name()):
+        print(f"{domain.name()}:")
+        for sig, stamp in locking.get_locks(domain, expired=args.expired):
+            print(f"    {sig} {stamp}")
+        print()
+
+
+def unlock(args):
+    conn = _libvirt_open(args.connect)
+    dom = conn.lookupByName(args.domain)
+    locking.unlock(dom, args.signature)
+
+
+def lock(args):
+    conn = _libvirt_open(args.connect)
+    dom = conn.lookupByName(args.domain)
+    if locking.lock(dom, args.signature, args.timestamp):
+        print("Succeeded.")
+        sys.exit(0)
+    else:
+        print("Failed (already locked).")
+        sys.exit(2)
+
+
+def unlock_all(args):
+    conn = _libvirt_open(args.connect)
+    if args.domains:
+        def domains(dom):
+            return bool(re.fullmatch(args.domains, dom.name()))
+    else:
+        def domains(_):
+            return True
+    locking.unlock_all(conn, args.signature, args.shutdown, domains)
+
+
+def cleanup_expired(args):
+    conn = _libvirt_open(args.connect)
+    if args.domains:
+        def domains(dom):
+            return bool(re.fullmatch(args.domains, dom.name()))
+    else:
+        def domains(_):
+            return True
+    locking.cleanup_expired(conn, args.timestamp, domains)
+
+
+def parse_args(parser):
+    parser.add_argument("--connect", "-c", help="Libvirt URL to connect to", metavar="URL")
+    cmds = parser.add_subparsers(
+        dest="_cmd", help="libvirt helper to run", metavar="<cmd>", required=True,
+    )
+
+    cmd = cmds.add_parser(
+        "get-locks",
+        help="List all locks (signatures)",
+    )
+    cmd.add_argument("--expired", help="List also expired locks", action="store_true")
+
+    cmd = cmds.add_parser(
+        "unlock",
+        help="Remove a lock signature from a domain",
+    )
+    cmd.add_argument("domain", help="Domain name")
+    cmd.add_argument("signature", help="Lock signature UUID")
+
+    cmd = cmds.add_parser(
+        "lock",
+        help="Lock a domain (exit 0:success, 2:fail)",
+    )
+    cmd.add_argument("domain", help="Domain name")
+    cmd.add_argument("signature", help="Lock signature UUID")
+    cmd.add_argument("timestamp", help="Expiration time for the lock")
+
+    cmd = cmds.add_parser(
+        "unlock-all",
+        help="Remove all lock signatures from all domains",
+    )
+    cmd.add_argument("--signature", help="Only remove this UUID")
+    cmd.add_argument("--shutdown", help="Also destroy the domains", action="store_true")
+    cmd.add_argument("--domains", help="Which domains names to impact", metavar="REGEX")
+
+    cmd = cmds.add_parser(
+        "cleanup-expired",
+        help="Remove expired lock signatures from all domains",
+    )
+    cmd.add_argument("--timestamp", help="Check against this instead of UTC now()")
+    cmd.add_argument("--domains", help="Which domains names to impact", metavar="REGEX")
+
+
+def main(args):
+    if args._cmd == "get-locks":
+        get_locks(args)
+    elif args._cmd == "unlock":
+        unlock(args)
+    elif args._cmd == "lock":
+        lock(args)
+    elif args._cmd == "unlock-all":
+        unlock_all(args)
+    elif args._cmd == "cleanup-expired":
+        cleanup_expired(args)
+    else:
+        raise RuntimeError(f"unknown args: {args}")
+
+
+CLI_SPEC = {
+    "help": "various utils for the Libvirt provisioner",
+    "args": parse_args,
+    "main": main,
+}

atex/cli/testingfarm.py

@@ -1,9 +1,10 @@
 import sys
 import json
 import pprint
+import collections
 
 from .. import util
-from ..provision.testingfarm import api as tf
+from ..provisioner.testingfarm import api as tf
 
 
 def _get_api(args):
@@ -36,7 +37,6 @@ def composes(args):
 def get_request(args):
     api = _get_api(args)
     request = tf.Request(args.request_id, api=api)
-    request.update()
     print(str(request))
 
 
@@ -79,6 +79,44 @@ def search_requests(args):
             print(f"{created} {req_id} : {envs_str}")
 
 
+def stats(args):
+    api = _get_api(args)
+
+    def top_users_repos(requests):
+        tokens = collections.defaultdict(int)
+        repos = collections.defaultdict(int)
+        for req in requests:
+            tokens[req["token_id"]] += 1
+            if "fmf" in req["test"] and req["test"]["fmf"]:
+                repos[req["test"]["fmf"]["url"]] += 1
+            elif "tmt" in req["test"] and req["test"]["tmt"]:
+                repos[req["test"]["tmt"]["url"]] += 1
+
+        print("Top 10 token IDs:")
+        for token_id in sorted(tokens, key=lambda x: tokens[x], reverse=True)[:10]:
+            count = tokens[token_id]
+            print(f"{count:>5}  {token_id}")
+
+        print("Top 10 repo URLs:")
+        for repo_url in sorted(repos, key=lambda x: repos[x], reverse=True)[:10]:
+            count = repos[repo_url]
+            print(f"{count:>5}  {repo_url}")
+
+    def chain_without_none(*iterables):
+        for itr in iterables:
+            if itr is None:
+                continue
+            for item in itr:
+                if item is not None:
+                    yield item
+
+    queued_and_running = chain_without_none(
+        api.search_requests(state="queued", ranch=args.ranch, mine=False),
+        api.search_requests(state="running", ranch=args.ranch, mine=False),
+    )
+    top_users_repos(queued_and_running)
+
+
 def reserve(args):
     util.info(f"Reserving {args.compose} on {args.arch} for {args.timeout} minutes")
 
@@ -87,19 +125,25 @@ def reserve(args):
     else:
         hardware = None
 
+    if args.native_test:
+        test = tf.DEFAULT_RESERVE_TEST.copy()
+        test["name"] = "/plans/testing-farm-native"
+    else:
+        test = None
+
     api = _get_api(args)
     res = tf.Reserve(
         compose=args.compose,
         arch=args.arch,
         timeout=args.timeout,
         hardware=hardware,
+        reserve_test=test,
         api=api,
     )
     with res as m:
         util.info(f"Got machine: {m}")
         while True:
             try:
-                res.request.update()
                 res.request.assert_alive()
             except tf.GoneAwayError as e:
                 print(e)
@@ -191,6 +235,12 @@ def parse_args(parser):
     cmd.add_argument("--after", help="only requests created after ISO8601")
     cmd.add_argument("--json", help="full details, one request per line", action="store_true")
 
+    cmd = cmds.add_parser(
+        "stats",
+        help="print out TF usage statistics",
+    )
+    cmd.add_argument("ranch", help="Testing Farm ranch name")
+
     cmd = cmds.add_parser(
         "reserve",
         help="reserve a system and ssh into it",
@@ -200,6 +250,11 @@ def parse_args(parser):
     cmd.add_argument("--timeout", "-t", help="pipeline timeout (in minutes)", type=int, default=60)
     cmd.add_argument("--ssh-key", help="path to a ssh private key file like 'id_rsa'")
     cmd.add_argument("--hvm", help="request a HVM virtualization capable HW", action="store_true")
+    cmd.add_argument(
+        "--native-test",
+        help="use the default testing farm reserve test",
+        action="store_true",
+    )
 
     cmd = cmds.add_parser(
         "watch-pipeline", aliases=("wp",),
@@ -221,6 +276,8 @@ def main(args):
         cancel(args)
     elif args._cmd in ("search-requests", "sr"):
         search_requests(args)
+    elif args._cmd == "stats":
+        stats(args)
     elif args._cmd == "reserve":
         reserve(args)
     elif args._cmd in ("watch-pipeline", "wp"):

atex/connection/__init__.py

@@ -1,6 +1,5 @@
 import importlib as _importlib
 import pkgutil as _pkgutil
-import threading as _threading
 
 from .. import util as _util
 
@@ -34,18 +33,21 @@ class Connection:
     ie. disconnect() might be called from a different thread while connect()
     or cmd() are still running.
     Similarly, multiple threads may run cmd() or rsync() independently.
-    """
 
-    def __init__(self):
-        """
-        Initialize the connection instance.
-        If extending __init__, always call 'super().__init__()' at the top.
-        """
-        self.lock = _threading.RLock()
+    TODO: document that any exceptions raised by a Connection should be children
+    of ConnectionError
+
+    If any connection-related error happens, a ConnectionError (or an exception
+    derived from it) must be raised.
+    """
 
     def __enter__(self):
-        self.connect()
-        return self
+        try:
+            self.connect()
+            return self
+        except Exception:
+            self.disconnect()
+            raise
 
     def __exit__(self, exc_type, exc_value, traceback):
         self.disconnect()
@@ -65,7 +67,7 @@ class Connection:
         """
         raise NotImplementedError(f"'disconnect' not implemented for {self.__class__.__name__}")
 
-    def cmd(self, command, func=_util.subprocess_run, **func_args):
+    def cmd(self, command, *, func=_util.subprocess_run, **func_args):
         """
         Execute a single command on the remote, using subprocess-like semantics.
 

atex/connection/podman.py

@@ -0,0 +1,61 @@
+"""
+Connection API implementation using the 'podman' CLI client.
+"""
+
+import subprocess
+
+from .. import util
+from . import Connection
+
+
+class PodmanConnectionError(ConnectionError):
+    pass
+
+
+class PodmanConnection(Connection):
+    """
+    Implements the Connection API via 'podman container exec' on an
+    already-running container, it does not handle any image pulling,
+    container creation, starting or stopping.
+    """
+
+#    def __init__(self, container, *, user=None, workdir=None):
+#        """
+#        'container' is a string with either the full or partial podman
+#        container ID, or a container name, as recognized by podman CLI.
+#
+#        'user' is a string with a username or UID, possibly including a GID,
+#        passed to the podman CLI as --user.
+#
+#        'workdir' is a string specifying the CWD inside the container.
+#        """
+    def __init__(self, container):
+        self.container = container
+
+    def connect(self, block=True):
+        pass
+
+    def disconnect(self):
+        pass
+
+    # have options as kwarg to be compatible with other functions here
+    def cmd(self, command, *, func=util.subprocess_run, **func_args):
+        return func(
+            ("podman", "container", "exec", "-i", self.container, *command),
+            **func_args,
+        )
+
+    def rsync(self, *args, func=util.subprocess_run, **func_args):
+        return func(
+            (
+                "rsync",
+                # use shell to strip off the destination argument rsync passes
+                #   cmd[0]=/bin/bash cmd[1]=-c cmd[2]=exec podman ... cmd[3]=destination
+                #   cmd[4]=rsync cmd[5]=--server cmd[6]=-vve.LsfxCIvu cmd[7]=. cmd[8]=.
+                "-e", f"/bin/bash -c 'exec podman container exec -i {self.container} \"$@\"'",
+                *args,
+            ),
+            check=True,
+            stdin=subprocess.DEVNULL,
+            **func_args,
+        )