atex 0.9__py3-none-any.whl → 0.11__py3-none-any.whl

atex/aggregator/__init__.py

@@ -0,0 +1,62 @@
+import importlib as _importlib
+import pkgutil as _pkgutil
+
+
+class Aggregator:
+    """
+    TODO: generic description, not JSON-specific
+    """
+
+    def ingest(self, platform, test_name, test_results, test_files):
+        """
+        Process 'test_results' (string/Path) for as results reported by a test
+        ran by Executor, along with 'test_files' as files uploaded by that test,
+        aggregating them under 'platform' (string) as 'test_name' (string).
+
+        This is DESTRUCTIVE, the input results/files are consumed in the
+        process.
+        """
+        raise NotImplementedError(f"'ingest' not implemented for {self.__class__.__name__}")
+
+    def start(self):
+        """
+        Start the Aggregator instance, opening any files / allocating resources
+        as necessary.
+        """
+        raise NotImplementedError(f"'start' not implemented for {self.__class__.__name__}")
+
+    def stop(self):
+        """
+        Stop the Aggregator instance, freeing all allocated resources.
+        """
+        raise NotImplementedError(f"'stop' not implemented for {self.__class__.__name__}")
+
+    def __enter__(self):
+        try:
+            self.start()
+            return self
+        except Exception:
+            self.stop()
+            raise
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.stop()
+
+
+_submodules = [
+    info.name for info in _pkgutil.iter_modules(__spec__.submodule_search_locations)
+]
+
+__all__ = [*_submodules, Aggregator.__name__]  # noqa: PLE0604
+
+
+def __dir__():
+    return __all__
+
+
+# lazily import submodules
+def __getattr__(attr):
+    if attr in _submodules:
+        return _importlib.import_module(f".{attr}", __name__)
+    else:
+        raise AttributeError(f"module '{__name__}' has no attribute '{attr}'")

atex/aggregator/json.py

@@ -0,0 +1,279 @@
+import abc
+import gzip
+import lzma
+import json
+import shutil
+import threading
+from pathlib import Path
+
+from . import Aggregator
+
+
+def _verbatim_move(src, dst):
+    def copy_without_symlinks(src, dst):
+        return shutil.copy2(src, dst, follow_symlinks=False)
+    shutil.move(src, dst, copy_function=copy_without_symlinks)
+
+
+class JSONAggregator(Aggregator):
+    """
+    Collects reported results in a line-JSON output file and uploaded files
+    (logs) from multiple test runs under a shared directory.
+
+    Note that the aggregated JSON file *does not* use the test-based JSON format
+    described by executor/RESULTS.md - both use JSON, but are very different.
+
+    This aggergated format uses a top-level array (on each line) with a fixed
+    field order:
+
+        platform, status, test name, subtest name, files, note
+
+    All these are strings except 'files', which is another (nested) array
+    of strings.
+
+    If 'testout' is present in an input test result, it is prepended to
+    the list of 'files'.
+    If a field is missing in the source result, it is translated to a null
+    value.
+    """
+
+    def __init__(self, target, files):
+        """
+        'target' is a string/Path to a .json file for all ingested
+        results to be aggregated (written) to.
+
+        'files' is a string/Path of the top-level parent for all
+        per-platform / per-test files uploaded by tests.
+        """
+        self.lock = threading.RLock()
+        self.target = Path(target)
+        self.files = Path(files)
+        self.target_fobj = None
+
+    def start(self):
+        if self.target.exists():
+            raise FileExistsError(f"{self.target} already exists")
+        self.target_fobj = open(self.target, "w")
+
+        if self.files.exists():
+            raise FileExistsError(f"{self.files} already exists")
+        self.files.mkdir()
+
+    def stop(self):
+        if self.target_fobj:
+            self.target_fobj.close()
+            self.target_fobj = None
+
+    def _get_test_files_path(self, platform, test_name):
+        """
+        Return a directory path to where uploaded files should be stored
+        for a particular 'platform' and 'test_name'.
+        """
+        platform_files = self.files / platform
+        platform_files.mkdir(exist_ok=True)
+        test_files = platform_files / test_name.lstrip("/")
+        return test_files
+
+    @staticmethod
+    def _modify_file_list(test_files):
+        return test_files
+
+    @staticmethod
+    def _move_test_files(test_files, target_dir):
+        """
+        Move (or otherwise process) 'test_files' as directory of files uploaded
+        by the test, into the pre-computed 'target_dir' location (inside
+        a hierarchy of all files from all tests).
+        """
+        _verbatim_move(test_files, target_dir)
+
+    def _gen_test_results(self, input_fobj, platform, test_name):
+        """
+        Yield complete output JSON objects, one for each input result.
+        """
+        # 'testout' , 'files' and others are standard fields in the
+        # test control interface, see RESULTS.md for the Executor
+        for raw_line in input_fobj:
+            result_line = json.loads(raw_line)
+
+            file_names = []
+            # process the file specified by the 'testout' key
+            if "testout" in result_line:
+                file_names.append(result_line["testout"])
+            # process any additional files in the 'files' key
+            if "files" in result_line:
+                file_names += (f["name"] for f in result_line["files"])
+
+            file_names = self._modify_file_list(file_names)
+
+            output_line = (
+                platform,
+                result_line["status"],
+                test_name,
+                result_line.get("name"),  # subtest
+                file_names,
+                result_line.get("note"),
+            )
+            yield json.dumps(output_line, indent=None)
+
+    def ingest(self, platform, test_name, test_results, test_files):
+        target_test_files = self._get_test_files_path(platform, test_name)
+        if target_test_files.exists():
+            raise FileExistsError(f"{target_test_files} already exists for {test_name}")
+
+        # parse the results separately, before writing any aggregated output,
+        # to ensure that either ALL results from the test are ingested, or none
+        # at all (ie. if one of the result lines contains JSON errors)
+        with open(test_results) as test_results_fobj:
+            output_results = self._gen_test_results(test_results_fobj, platform, test_name)
+            output_json = "\n".join(output_results) + "\n"
+
+        with self.lock:
+            self.target_fobj.write(output_json)
+            self.target_fobj.flush()
+
+        # clean up the source test_results (Aggregator should 'mv', not 'cp')
+        Path(test_results).unlink()
+
+        # if the test_files dir is not empty
+        if any(test_files.iterdir()):
+            self._move_test_files(test_files, target_test_files)
+
+
+class CompressedJSONAggregator(JSONAggregator, abc.ABC):
+    compress_files = False
+    suffix = ""
+    exclude = ()
+
+    @abc.abstractmethod
+    def compressed_open(self, *args, **kwargs):
+        pass
+
+    def start(self):
+        if self.target.exists():
+            raise FileExistsError(f"{self.target_file} already exists")
+        self.target_fobj = self.compressed_open(self.target, "wt", newline="\n")
+
+        if self.files.exists():
+            raise FileExistsError(f"{self.storage_dir} already exists")
+        self.files.mkdir()
+
+    def _modify_file_list(self, test_files):
+        if self.compress_files and self.suffix:
+            return [
+                (name if name in self.exclude else f"{name}{self.suffix}")
+                for name in test_files
+            ]
+        else:
+            return super()._modify_file_list(test_files)
+
+    def _move_test_files(self, test_files, target_dir):
+        if not self.compress_files:
+            super()._move_test_files(test_files, target_dir)
+            return
+
+        for root, _, files in test_files.walk(top_down=False):
+            for file_name in files:
+                src_path = root / file_name
+                dst_path = target_dir / src_path.relative_to(test_files)
+
+                dst_path.parent.mkdir(parents=True, exist_ok=True)
+
+                # skip dirs, symlinks, device files, etc.
+                if not src_path.is_file(follow_symlinks=False) or file_name in self.exclude:
+                    _verbatim_move(src_path, dst_path)
+                    continue
+
+                if self.suffix:
+                    dst_path = dst_path.with_name(f"{dst_path.name}{self.suffix}")
+
+                with open(src_path, "rb") as plain_fobj:
+                    with self.compressed_open(dst_path, "wb") as compress_fobj:
+                        shutil.copyfileobj(plain_fobj, compress_fobj, 1048576)
+
+                src_path.unlink()
+
+            # we're walking bottom-up, so the local root should be empty now
+            root.rmdir()
+
+
+class GzipJSONAggregator(CompressedJSONAggregator):
+    """
+    Identical to JSONAggregator, but transparently Gzips either or both of
+    the output line-JSON file with results and the uploaded files.
+    """
+    def compressed_open(self, *args, **kwargs):
+        return gzip.open(*args, compresslevel=self.level, **kwargs)
+
+    def __init__(
+        self, target, files, *, compress_level=9,
+        compress_files=True, compress_files_suffix=".gz", compress_files_exclude=None,
+    ):
+        """
+        'target' is a string/Path to a .json.gz file for all ingested
+        results to be aggregated (written) to.
+
+        'files' is a string/Path of the top-level parent for all
+        per-platform / per-test files uploaded by tests.
+
+        'compress_level' specifies how much effort should be spent compressing,
+        (1 = fast, 9 = slow).
+
+        If 'compress_files' is True, compress also any files uploaded by tests.
+
+        The 'compress_files_suffix' is appended to any processed test-uploaded
+        files, and the respective 'files' results array is modified with the
+        new file names (as if the test uploaded compressed files already).
+        Set to "" (empty string) to use original file names and just compress
+        them transparently in-place.
+
+        'compress_files_exclude' is a tuple/list of strings (input 'files'
+        names) to skip when compressing. Their names also won't be modified.
+        """
+        super().__init__(target, files)
+        self.level = compress_level
+        self.compress_files = compress_files
+        self.suffix = compress_files_suffix
+        self.exclude = compress_files_exclude or ()
+
+
+class LZMAJSONAggregator(CompressedJSONAggregator):
+    """
+    Identical to JSONAggregator, but transparently compresses (via LZMA/XZ)
+    either or both of the output line-JSON file with results and the uploaded
+    files.
+    """
+    def compressed_open(self, *args, **kwargs):
+        return lzma.open(*args, preset=self.preset, **kwargs)
+
+    def __init__(
+        self, target, files, *, compress_preset=9,
+        compress_files=True, compress_files_suffix=".xz", compress_files_exclude=None,
+    ):
+        """
+        'target' is a string/Path to a .json.xz file for all ingested
+        results to be aggregated (written) to.
+
+        'files' is a string/Path of the top-level parent for all
+        per-platform / per-test files uploaded by tests.
+
+        'compress_preset' specifies how much effort should be spent compressing,
+        (1 = fast, 9 = slow). Optionally ORed with lzma.PRESET_EXTREME to spend
+        even more CPU time compressing.
+
+        If 'compress_files' is True, compress also any files uploaded by tests.
+
+        The 'compress_files_suffix' is appended to any processed test-uploaded
+        files, and the respective 'files' results array is modified with the
+        new file names (as if the test uploaded compressed files already).
+        Set to "" (empty string) to use original file names and just compress
+        them transparently in-place.
+
+        'compress_files_exclude' is a tuple/list of strings (input 'files'
+        names) to skip when compressing. Their names also won't be modified.
+        """
+        super().__init__(target, files)
+        self.preset = compress_preset
+        self.compress_files = compress_files
+        self.suffix = compress_files_suffix
+        self.exclude = compress_files_exclude or ()

atex/cli/__init__.py

@@ -27,12 +27,21 @@ import pkgutil
 import argparse
 import logging
 
+from .. import util
+
 
 def setup_logging(level):
+    if level <= util.EXTRADEBUG:
+        fmt = "%(asctime)s %(name)s: %(filename)s:%(lineno)s: %(funcName)s(): %(message)s"
+        # also print urllib3 headers
+        import http.client  # noqa: PLC0415
+        http.client.HTTPConnection.debuglevel = 5
+    else:
+        fmt = "%(asctime)s %(name)s: %(message)s"
     logging.basicConfig(
         level=level,
         stream=sys.stderr,
-        format="%(asctime)s %(name)s: %(message)s",
+        format=fmt,
         datefmt="%Y-%m-%d %H:%M:%S",
     )
 
@@ -53,6 +62,10 @@ def main():
         "--debug", "-d", action="store_const", dest="loglevel", const=logging.DEBUG,
         help="enable extra debugging (logging.DEBUG)",
     )
+    log_grp.add_argument(
+        "--extra-debug", "-D", action="store_const", dest="loglevel", const=util.EXTRADEBUG,
+        help="enable extra debugging (atex.util.EXTRADEBUG)",
+    )
     log_grp.add_argument(
         "--quiet", "-q", action="store_const", dest="loglevel", const=logging.WARNING,
         help="be quiet during normal operation (logging.WARNING)",

atex/cli/fmf.py

@@ -56,17 +56,17 @@ def prepare(args):
     result = make_fmftests(args)
     print("--- fmf root ---")
     print(str(result.root))
-    print("--- prepare packages ---")
+    print("\n--- prepare packages ---")
     print("\n".join(result.prepare_pkgs))
-    print("--- plan environment ---")
-    print("\n".join("{k}={v}" for k,v in result.plan_env))
+    print("\n--- plan environment ---")
+    print("\n".join(f"{k}={v}" for k,v in result.plan_env.items()))
     for script in result.prepare_scripts:
-        print("--- prepare script ---")
-        print(script)
+        print("\n--- prepare script ---")
+        print(script.rstrip("\n"))
         print("----------------------")
     for script in result.finish_scripts:
-        print("--- finish script ---")
-        print(script)
+        print("\n--- finish script ---")
+        print(script.rstrip("\n"))
         print("----------------------")
 
 

atex/cli/libvirt.py

@@ -1,9 +1,10 @@
 import sys
 import re
 
-import libvirt
+from .. import util
+from ..provisioner.libvirt import locking
 
-from ..provision.libvirt import locking
+libvirt = util.import_libvirt()
 
 
 def _libvirt_open(url=None):

atex/cli/testingfarm.py

@@ -1,9 +1,11 @@
 import sys
 import json
 import pprint
+import collections
+from datetime import datetime, timedelta, UTC
 
 from .. import util
-from ..provision.testingfarm import api as tf
+from ..provisioner.testingfarm import api as tf
 
 
 def _get_api(args):
@@ -36,7 +38,6 @@ def composes(args):
 def get_request(args):
     api = _get_api(args)
     request = tf.Request(args.request_id, api=api)
-    request.update()
     print(str(request))
 
 
@@ -79,6 +80,67 @@ def search_requests(args):
             print(f"{created} {req_id} : {envs_str}")
 
 
+def stats(args):
+    api = _get_api(args)
+
+    def top_users_repos(requests):
+        tokens = collections.defaultdict(int)
+        repos = collections.defaultdict(int)
+        for req in requests:
+            tokens[req["token_id"]] += 1
+            if "fmf" in req["test"] and req["test"]["fmf"]:
+                repos[req["test"]["fmf"]["url"]] += 1
+            elif "tmt" in req["test"] and req["test"]["tmt"]:
+                repos[req["test"]["tmt"]["url"]] += 1
+
+        top_tokens = sorted(tokens, key=lambda x: tokens[x], reverse=True)[:10]
+        top_repos = sorted(repos, key=lambda x: repos[x], reverse=True)[:10]
+        if not top_tokens or not top_repos:
+            return
+        digits = max(len(str(tokens[top_tokens[0]])), len(str(repos[top_repos[0]])))
+
+        print("Top 10 token IDs:")
+        for token_id in top_tokens:
+            count = tokens[token_id]
+            print(f"{count:>{digits}}  {token_id}")
+
+        print("Top 10 repo URLs:")
+        for repo_url in top_repos:
+            count = repos[repo_url]
+            print(f"{count:>{digits}}  {repo_url}")
+
+    def request_search_results():
+        for state in args.states.split(","):
+            result = api.search_requests(
+                state=state,
+                ranch=args.ranch,
+                mine=False,
+            )
+            if result:
+                yield from result
+
+    def multiday_request_search_results():
+        now = datetime.now(UTC)
+        for day in range(0,args.days):
+            before = now - timedelta(days=day)
+            after = now - timedelta(days=day+1)
+            for state in args.states.split(","):
+                result = api.search_requests(
+                    state=state,
+                    created_before=before.replace(microsecond=0).isoformat(),
+                    created_after=after.replace(microsecond=0).isoformat(),
+                    ranch=args.ranch,
+                    mine=False,
+                )
+                if result:
+                    yield from result
+
+    if args.days is not None:
+        top_users_repos(multiday_request_search_results())
+    else:
+        top_users_repos(request_search_results())
+
+
 def reserve(args):
     util.info(f"Reserving {args.compose} on {args.arch} for {args.timeout} minutes")
 
@@ -106,7 +168,6 @@ def reserve(args):
         util.info(f"Got machine: {m}")
         while True:
             try:
-                res.request.update()
                 res.request.assert_alive()
             except tf.GoneAwayError as e:
                 print(e)
@@ -198,6 +259,14 @@ def parse_args(parser):
     cmd.add_argument("--after", help="only requests created after ISO8601")
     cmd.add_argument("--json", help="full details, one request per line", action="store_true")
 
+    cmd = cmds.add_parser(
+        "stats",
+        help="print out TF usage statistics",
+    )
+    cmd.add_argument("--days", type=int, help="query last N days instead of all TF requests")
+    cmd.add_argument("ranch", help="Testing Farm ranch name")
+    cmd.add_argument("states", help="comma-separated TF request states")
+
     cmd = cmds.add_parser(
         "reserve",
         help="reserve a system and ssh into it",
@@ -233,6 +302,8 @@ def main(args):
         cancel(args)
     elif args._cmd in ("search-requests", "sr"):
         search_requests(args)
+    elif args._cmd == "stats":
+        stats(args)
     elif args._cmd == "reserve":
         reserve(args)
     elif args._cmd in ("watch-pipeline", "wp"):

atex/connection/podman.py

@@ -8,11 +8,11 @@ from .. import util
 from . import Connection
 
 
-class PodmanConnError(ConnectionError):
+class PodmanConnectionError(ConnectionError):
     pass
 
 
-class PodmanConn(Connection):
+class PodmanConnection(Connection):
     """
     Implements the Connection API via 'podman container exec' on an
     already-running container, it does not handle any image pulling,
@@ -42,7 +42,6 @@ class PodmanConn(Connection):
     def cmd(self, command, *, func=util.subprocess_run, **func_args):
         return func(
             ("podman", "container", "exec", "-i", self.container, *command),
-            skip_frames=1,
             **func_args,
         )
 
@@ -56,7 +55,6 @@ class PodmanConn(Connection):
                 "-e", f"/bin/bash -c 'exec podman container exec -i {self.container} \"$@\"'",
                 *args,
             ),
-            skip_frames=1,
             check=True,
             stdin=subprocess.DEVNULL,
             **func_args,

atex/connection/ssh.py

@@ -133,16 +133,16 @@ def _rsync_host_cmd(*args, options, password=None, sudo=None):
     )
 
 
-class StatelessSSHConn(Connection):
+class StatelessSSHConnection(Connection):
     """
     Implements the Connection API using a ssh(1) client using "standalone"
     (stateless) logic - connect() and disconnect() are no-op, .cmd() simply
     executes the ssh client and .rsync() executes 'rsync -e ssh'.
 
-    Compared to ManagedSSHConn, this may be slow for many .cmd() calls,
+    Compared to ManagedSSHConnection, this may be slow for many .cmd() calls,
     but every call is stateless, there is no persistent connection.
 
-    If you need only one .cmd(), this will be faster than ManagedSSHConn.
+    If you need only one .cmd(), this will be faster than ManagedSSHConnection.
     """
 
     def __init__(self, options, *, password=None, sudo=None):
@@ -182,7 +182,6 @@ class StatelessSSHConn(Connection):
             unified_options["RemoteCommand"] = _shell_cmd(command, sudo=self.sudo)
         return func(
             _options_to_ssh(unified_options, password=self.password),
-            skip_frames=1,
             **func_args,
         )
 
@@ -197,7 +196,6 @@ class StatelessSSHConn(Connection):
                 password=self.password,
                 sudo=self.sudo,
             ),
-            skip_frames=1,
             check=True,
             stdin=subprocess.DEVNULL,
             **func_args,
@@ -216,17 +214,15 @@ class StatelessSSHConn(Connection):
 # checks .assert_master() and manually signals the running clients
 # when it gets DisconnectedError from it.
 
-class ManagedSSHConn(Connection):
+class ManagedSSHConnection(Connection):
     """
     Implements the Connection API using one persistently-running ssh(1) client
     started in a 'ControlMaster' mode, with additional ssh clients using that
     session to execute remote commands. Similarly, .rsync() uses it too.
 
-    This is much faster than StatelessSSHConn when executing multiple commands,
-    but contains a complex internal state (what if ControlMaster disconnects?).
-
-    Hence why this implementation provides extra non-standard-Connection methods
-    to manage this complexity.
+    This is much faster than StatelessSSHConnection when executing multiple
+    commands, but contains a complex internal state (what if ControlMaster
+    disconnects?).
     """
 
     # TODO: thread safety and locking via self.lock ?
@@ -351,7 +347,6 @@ class ManagedSSHConn(Connection):
         action = "forward" if not cancel else "cancel"
         util.subprocess_run(
             _options_to_ssh(options, extra_cli_flags=("-O", action)),
-            skip_frames=1,
             check=True,
         )
 
@@ -365,7 +360,6 @@ class ManagedSSHConn(Connection):
         unified_options["ControlPath"] = self._tmpdir / "control.sock"
         return func(
             _options_to_ssh(unified_options),
-            skip_frames=1,
             **func_args,
         )
 
@@ -381,7 +375,6 @@ class ManagedSSHConn(Connection):
                 options=unified_options,
                 sudo=self.sudo,
             ),
-            skip_frames=1,
             check=True,
             stdin=subprocess.DEVNULL,
             **func_args,