atex 0.10__py3-none-any.whl → 0.11__py3-none-any.whl

atex/aggregator/__init__.py

@@ -7,12 +7,14 @@ class Aggregator:
     TODO: generic description, not JSON-specific
     """
 
-    def ingest(self, platform, test_name, results_file, files_dir):
+    def ingest(self, platform, test_name, test_results, test_files):
         """
-        Process 'results_file' (string/Path) for reported results and append
-        them to the overall aggregated line-JSON file, recursively copying over
-        the dir structure under 'files_dir' (string/Path) under the respective
-        platform and test name in the aggregated storage dir.
+        Process 'test_results' (string/Path) for as results reported by a test
+        ran by Executor, along with 'test_files' as files uploaded by that test,
+        aggregating them under 'platform' (string) as 'test_name' (string).
+
+        This is DESTRUCTIVE, the input results/files are consumed in the
+        process.
         """
         raise NotImplementedError(f"'ingest' not implemented for {self.__class__.__name__}")
 
@@ -34,7 +36,7 @@ class Aggregator:
             self.start()
             return self
         except Exception:
-            self.close()
+            self.stop()
             raise
 
     def __exit__(self, exc_type, exc_value, traceback):

atex/aggregator/json.py

@@ -1,4 +1,6 @@
+import abc
 import gzip
+import lzma
 import json
 import shutil
 import threading
@@ -7,10 +9,16 @@ from pathlib import Path
 from . import Aggregator
 
 
+def _verbatim_move(src, dst):
+    def copy_without_symlinks(src, dst):
+        return shutil.copy2(src, dst, follow_symlinks=False)
+    shutil.move(src, dst, copy_function=copy_without_symlinks)
+
+
 class JSONAggregator(Aggregator):
     """
-    Collects reported results as a GZIP-ed line-JSON and files (logs) from
-    multiple test runs under a shared directory.
+    Collects reported results in a line-JSON output file and uploaded files
+    (logs) from multiple test runs under a shared directory.
 
     Note that the aggregated JSON file *does not* use the test-based JSON format
     described by executor/RESULTS.md - both use JSON, but are very different.
@@ -23,74 +31,249 @@ class JSONAggregator(Aggregator):
     All these are strings except 'files', which is another (nested) array
     of strings.
 
+    If 'testout' is present in an input test result, it is prepended to
+    the list of 'files'.
     If a field is missing in the source result, it is translated to a null
     value.
     """
 
-    def __init__(self, json_file, storage_dir):
+    def __init__(self, target, files):
         """
-        'json_file' is a string/Path to a .json.gz file with aggregated results.
+        'target' is a string/Path to a .json file for all ingested
+        results to be aggregated (written) to.
 
-        'storage_dir' is a string/Path of the top-level parent for all
+        'files' is a string/Path of the top-level parent for all
         per-platform / per-test files uploaded by tests.
         """
         self.lock = threading.RLock()
-        self.storage_dir = Path(storage_dir)
-        self.json_file = Path(json_file)
-        self.json_gzip_fobj = None
+        self.target = Path(target)
+        self.files = Path(files)
+        self.target_fobj = None
 
     def start(self):
-        if self.json_file.exists():
-            raise FileExistsError(f"{self.json_file} already exists")
-        self.json_gzip_fobj = gzip.open(self.json_file, "wt", newline="\n")
+        if self.target.exists():
+            raise FileExistsError(f"{self.target} already exists")
+        self.target_fobj = open(self.target, "w")
 
-        if self.storage_dir.exists():
-            raise FileExistsError(f"{self.storage_dir} already exists")
-        self.storage_dir.mkdir()
+        if self.files.exists():
+            raise FileExistsError(f"{self.files} already exists")
+        self.files.mkdir()
 
     def stop(self):
-        if self.json_gzip_fobj:
-            self.json_gzip_fobj.close()
-            self.json_gzip_fobj = None
+        if self.target_fobj:
+            self.target_fobj.close()
+            self.target_fobj = None
+
+    def _get_test_files_path(self, platform, test_name):
+        """
+        Return a directory path to where uploaded files should be stored
+        for a particular 'platform' and 'test_name'.
+        """
+        platform_files = self.files / platform
+        platform_files.mkdir(exist_ok=True)
+        test_files = platform_files / test_name.lstrip("/")
+        return test_files
+
+    @staticmethod
+    def _modify_file_list(test_files):
+        return test_files
+
+    @staticmethod
+    def _move_test_files(test_files, target_dir):
+        """
+        Move (or otherwise process) 'test_files' as directory of files uploaded
+        by the test, into the pre-computed 'target_dir' location (inside
+        a hierarchy of all files from all tests).
+        """
+        _verbatim_move(test_files, target_dir)
+
+    def _gen_test_results(self, input_fobj, platform, test_name):
+        """
+        Yield complete output JSON objects, one for each input result.
+        """
+        # 'testout' , 'files' and others are standard fields in the
+        # test control interface, see RESULTS.md for the Executor
+        for raw_line in input_fobj:
+            result_line = json.loads(raw_line)
+
+            file_names = []
+            # process the file specified by the 'testout' key
+            if "testout" in result_line:
+                file_names.append(result_line["testout"])
+            # process any additional files in the 'files' key
+            if "files" in result_line:
+                file_names += (f["name"] for f in result_line["files"])
+
+            file_names = self._modify_file_list(file_names)
 
-    def ingest(self, platform, test_name, results_file, files_dir):
-        platform_dir = self.storage_dir / platform
-        test_dir = platform_dir / test_name.lstrip("/")
-        if test_dir.exists():
-            raise FileExistsError(f"{test_dir} already exists for {test_name}")
+            output_line = (
+                platform,
+                result_line["status"],
+                test_name,
+                result_line.get("name"),  # subtest
+                file_names,
+                result_line.get("note"),
+            )
+            yield json.dumps(output_line, indent=None)
+
+    def ingest(self, platform, test_name, test_results, test_files):
+        target_test_files = self._get_test_files_path(platform, test_name)
+        if target_test_files.exists():
+            raise FileExistsError(f"{target_test_files} already exists for {test_name}")
 
         # parse the results separately, before writing any aggregated output,
-        # to ensure that either all results from the test are ingested, or none
+        # to ensure that either ALL results from the test are ingested, or none
         # at all (ie. if one of the result lines contains JSON errors)
-        output_lines = []
-        with open(results_file) as results_fobj:
-            for raw_line in results_fobj:
-                result_line = json.loads(raw_line)
-
-                file_names = []
-                if "testout" in result_line:
-                    file_names.append(result_line["testout"])
-                if "files" in result_line:
-                    file_names += (f["name"] for f in result_line["files"])
-
-                output_line = (
-                    platform,
-                    result_line["status"],
-                    test_name,
-                    result_line.get("name"),  # subtest
-                    file_names,
-                    result_line.get("note"),
-                )
-                encoded = json.dumps(output_line, indent=None)
-                output_lines.append(encoded)
-
-        output_str = "\n".join(output_lines) + "\n"
+        with open(test_results) as test_results_fobj:
+            output_results = self._gen_test_results(test_results_fobj, platform, test_name)
+            output_json = "\n".join(output_results) + "\n"
 
         with self.lock:
-            self.json_gzip_fobj.write(output_str)
-            self.json_gzip_fobj.flush()
+            self.target_fobj.write(output_json)
+            self.target_fobj.flush()
+
+        # clean up the source test_results (Aggregator should 'mv', not 'cp')
+        Path(test_results).unlink()
+
+        # if the test_files dir is not empty
+        if any(test_files.iterdir()):
+            self._move_test_files(test_files, target_test_files)
+
+
+class CompressedJSONAggregator(JSONAggregator, abc.ABC):
+    compress_files = False
+    suffix = ""
+    exclude = ()
+
+    @abc.abstractmethod
+    def compressed_open(self, *args, **kwargs):
+        pass
+
+    def start(self):
+        if self.target.exists():
+            raise FileExistsError(f"{self.target_file} already exists")
+        self.target_fobj = self.compressed_open(self.target, "wt", newline="\n")
+
+        if self.files.exists():
+            raise FileExistsError(f"{self.storage_dir} already exists")
+        self.files.mkdir()
+
+    def _modify_file_list(self, test_files):
+        if self.compress_files and self.suffix:
+            return [
+                (name if name in self.exclude else f"{name}{self.suffix}")
+                for name in test_files
+            ]
+        else:
+            return super()._modify_file_list(test_files)
+
+    def _move_test_files(self, test_files, target_dir):
+        if not self.compress_files:
+            super()._move_test_files(test_files, target_dir)
+            return
+
+        for root, _, files in test_files.walk(top_down=False):
+            for file_name in files:
+                src_path = root / file_name
+                dst_path = target_dir / src_path.relative_to(test_files)
+
+                dst_path.parent.mkdir(parents=True, exist_ok=True)
 
-        Path(results_file).unlink()
+                # skip dirs, symlinks, device files, etc.
+                if not src_path.is_file(follow_symlinks=False) or file_name in self.exclude:
+                    _verbatim_move(src_path, dst_path)
+                    continue
 
-        platform_dir.mkdir(exist_ok=True)
-        shutil.move(files_dir, test_dir)
+                if self.suffix:
+                    dst_path = dst_path.with_name(f"{dst_path.name}{self.suffix}")
+
+                with open(src_path, "rb") as plain_fobj:
+                    with self.compressed_open(dst_path, "wb") as compress_fobj:
+                        shutil.copyfileobj(plain_fobj, compress_fobj, 1048576)
+
+                src_path.unlink()
+
+            # we're walking bottom-up, so the local root should be empty now
+            root.rmdir()
+
+
+class GzipJSONAggregator(CompressedJSONAggregator):
+    """
+    Identical to JSONAggregator, but transparently Gzips either or both of
+    the output line-JSON file with results and the uploaded files.
+    """
+    def compressed_open(self, *args, **kwargs):
+        return gzip.open(*args, compresslevel=self.level, **kwargs)
+
+    def __init__(
+        self, target, files, *, compress_level=9,
+        compress_files=True, compress_files_suffix=".gz", compress_files_exclude=None,
+    ):
+        """
+        'target' is a string/Path to a .json.gz file for all ingested
+        results to be aggregated (written) to.
+
+        'files' is a string/Path of the top-level parent for all
+        per-platform / per-test files uploaded by tests.
+
+        'compress_level' specifies how much effort should be spent compressing,
+        (1 = fast, 9 = slow).
+
+        If 'compress_files' is True, compress also any files uploaded by tests.
+
+        The 'compress_files_suffix' is appended to any processed test-uploaded
+        files, and the respective 'files' results array is modified with the
+        new file names (as if the test uploaded compressed files already).
+        Set to "" (empty string) to use original file names and just compress
+        them transparently in-place.
+
+        'compress_files_exclude' is a tuple/list of strings (input 'files'
+        names) to skip when compressing. Their names also won't be modified.
+        """
+        super().__init__(target, files)
+        self.level = compress_level
+        self.compress_files = compress_files
+        self.suffix = compress_files_suffix
+        self.exclude = compress_files_exclude or ()
+
+
+class LZMAJSONAggregator(CompressedJSONAggregator):
+    """
+    Identical to JSONAggregator, but transparently compresses (via LZMA/XZ)
+    either or both of the output line-JSON file with results and the uploaded
+    files.
+    """
+    def compressed_open(self, *args, **kwargs):
+        return lzma.open(*args, preset=self.preset, **kwargs)
+
+    def __init__(
+        self, target, files, *, compress_preset=9,
+        compress_files=True, compress_files_suffix=".xz", compress_files_exclude=None,
+    ):
+        """
+        'target' is a string/Path to a .json.xz file for all ingested
+        results to be aggregated (written) to.
+
+        'files' is a string/Path of the top-level parent for all
+        per-platform / per-test files uploaded by tests.
+
+        'compress_preset' specifies how much effort should be spent compressing,
+        (1 = fast, 9 = slow). Optionally ORed with lzma.PRESET_EXTREME to spend
+        even more CPU time compressing.
+
+        If 'compress_files' is True, compress also any files uploaded by tests.
+
+        The 'compress_files_suffix' is appended to any processed test-uploaded
+        files, and the respective 'files' results array is modified with the
+        new file names (as if the test uploaded compressed files already).
+        Set to "" (empty string) to use original file names and just compress
+        them transparently in-place.
+
+        'compress_files_exclude' is a tuple/list of strings (input 'files'
+        names) to skip when compressing. Their names also won't be modified.
+        """
+        super().__init__(target, files)
+        self.preset = compress_preset
+        self.compress_files = compress_files
+        self.suffix = compress_files_suffix
+        self.exclude = compress_files_exclude or ()

atex/cli/__init__.py

@@ -33,6 +33,9 @@ from .. import util
 def setup_logging(level):
     if level <= util.EXTRADEBUG:
         fmt = "%(asctime)s %(name)s: %(filename)s:%(lineno)s: %(funcName)s(): %(message)s"
+        # also print urllib3 headers
+        import http.client  # noqa: PLC0415
+        http.client.HTTPConnection.debuglevel = 5
     else:
         fmt = "%(asctime)s %(name)s: %(message)s"
     logging.basicConfig(

atex/cli/fmf.py

@@ -56,17 +56,17 @@ def prepare(args):
     result = make_fmftests(args)
     print("--- fmf root ---")
     print(str(result.root))
-    print("--- prepare packages ---")
+    print("\n--- prepare packages ---")
     print("\n".join(result.prepare_pkgs))
-    print("--- plan environment ---")
-    print("\n".join("{k}={v}" for k,v in result.plan_env))
+    print("\n--- plan environment ---")
+    print("\n".join(f"{k}={v}" for k,v in result.plan_env.items()))
     for script in result.prepare_scripts:
-        print("--- prepare script ---")
-        print(script)
+        print("\n--- prepare script ---")
+        print(script.rstrip("\n"))
         print("----------------------")
     for script in result.finish_scripts:
-        print("--- finish script ---")
-        print(script)
+        print("\n--- finish script ---")
+        print(script.rstrip("\n"))
         print("----------------------")
 
 

atex/cli/testingfarm.py

@@ -2,6 +2,7 @@ import sys
 import json
 import pprint
 import collections
+from datetime import datetime, timedelta, UTC
 
 from .. import util
 from ..provisioner.testingfarm import api as tf
@@ -92,29 +93,52 @@ def stats(args):
             elif "tmt" in req["test"] and req["test"]["tmt"]:
                 repos[req["test"]["tmt"]["url"]] += 1
 
+        top_tokens = sorted(tokens, key=lambda x: tokens[x], reverse=True)[:10]
+        top_repos = sorted(repos, key=lambda x: repos[x], reverse=True)[:10]
+        if not top_tokens or not top_repos:
+            return
+        digits = max(len(str(tokens[top_tokens[0]])), len(str(repos[top_repos[0]])))
+
         print("Top 10 token IDs:")
-        for token_id in sorted(tokens, key=lambda x: tokens[x], reverse=True)[:10]:
+        for token_id in top_tokens:
             count = tokens[token_id]
-            print(f"{count:>5}  {token_id}")
+            print(f"{count:>{digits}}  {token_id}")
 
         print("Top 10 repo URLs:")
-        for repo_url in sorted(repos, key=lambda x: repos[x], reverse=True)[:10]:
+        for repo_url in top_repos:
             count = repos[repo_url]
-            print(f"{count:>5}  {repo_url}")
-
-    def chain_without_none(*iterables):
-        for itr in iterables:
-            if itr is None:
-                continue
-            for item in itr:
-                if item is not None:
-                    yield item
-
-    queued_and_running = chain_without_none(
-        api.search_requests(state="queued", ranch=args.ranch, mine=False),
-        api.search_requests(state="running", ranch=args.ranch, mine=False),
-    )
-    top_users_repos(queued_and_running)
+            print(f"{count:>{digits}}  {repo_url}")
+
+    def request_search_results():
+        for state in args.states.split(","):
+            result = api.search_requests(
+                state=state,
+                ranch=args.ranch,
+                mine=False,
+            )
+            if result:
+                yield from result
+
+    def multiday_request_search_results():
+        now = datetime.now(UTC)
+        for day in range(0,args.days):
+            before = now - timedelta(days=day)
+            after = now - timedelta(days=day+1)
+            for state in args.states.split(","):
+                result = api.search_requests(
+                    state=state,
+                    created_before=before.replace(microsecond=0).isoformat(),
+                    created_after=after.replace(microsecond=0).isoformat(),
+                    ranch=args.ranch,
+                    mine=False,
+                )
+                if result:
+                    yield from result
+
+    if args.days is not None:
+        top_users_repos(multiday_request_search_results())
+    else:
+        top_users_repos(request_search_results())
 
 
 def reserve(args):
@@ -239,7 +263,9 @@ def parse_args(parser):
         "stats",
         help="print out TF usage statistics",
     )
+    cmd.add_argument("--days", type=int, help="query last N days instead of all TF requests")
     cmd.add_argument("ranch", help="Testing Farm ranch name")
+    cmd.add_argument("states", help="comma-separated TF request states")
 
     cmd = cmds.add_parser(
         "reserve",

atex/executor/executor.py

@@ -153,11 +153,11 @@ class Executor:
             **self.env,
             "TMT_PLAN_ENVIRONMENT_FILE": self.plan_env_file,
         }
-        env_args = (f"{k}={v}" for k, v in env.items())
+        env_args = tuple(f"{k}={v}" for k, v in env.items())
         # run the scripts
         for script in scripts:
             self.conn.cmd(
-                ("env", *env_args, "bash"),
+                ("env", "-C", self.tests_dir, *env_args, "bash"),
                 func=util.subprocess_log,
                 stderr=subprocess.STDOUT,
                 input=script,
@@ -387,7 +387,7 @@ class Executor:
                         pass
                     reporter.report({
                         "status": "infra",
-                        "note": repr(exception),
+                        "note": f"{type(exception).__name__}({exception})",
                         "testout": "output.txt",
                     })
 

atex/orchestrator/adhoc.py

@@ -1,6 +1,6 @@
 import tempfile
-import concurrent
 import collections
+import concurrent.futures
 from pathlib import Path
 
 from .. import util, executor
@@ -114,6 +114,8 @@ class AdHocOrchestrator(Orchestrator):
         self.setup_queue = util.ThreadQueue(daemon=True)
         # thread queue for remotes being released
         self.release_queue = util.ThreadQueue(daemon=True)
+        # thread queue for results being ingested
+        self.ingest_queue = util.ThreadQueue(daemon=False)
 
     def _run_new_test(self, info):
         """
@@ -125,7 +127,7 @@ class AdHocOrchestrator(Orchestrator):
         next_test_name = self.next_test(self.to_run, self.fmf_tests.tests, info)
         assert next_test_name in self.to_run, "next_test() returned valid test name"
 
-        util.info(f"starting '{next_test_name}' on {info.remote}")
+        util.info(f"{info.remote}: starting '{next_test_name}'")
 
         self.to_run.remove(next_test_name)
 
@@ -140,6 +142,7 @@ class AdHocOrchestrator(Orchestrator):
         )
 
         tmp_dir_path = Path(rinfo.tmp_dir.name)
+        tmp_dir_path.chmod(0o755)
         self.test_queue.start_thread(
             target=info.executor.run_test,
             target_args=(
@@ -176,6 +179,7 @@ class AdHocOrchestrator(Orchestrator):
 
         if not self.was_successful(finfo, test_data) and self.should_be_rerun(finfo, test_data):
             # re-run the test
+            util.info(f"{remote_with_test} failed, re-running")
             self.to_run.add(finfo.test_name)
         else:
             # ingest the result
@@ -183,15 +187,27 @@ class AdHocOrchestrator(Orchestrator):
             # a condition just in case Executor code itself threw an exception
             # and didn't even report the fallback 'infra' result
             if finfo.results is not None and finfo.files is not None:
-                self.aggregator.ingest(
-                    self.platform,
-                    finfo.test_name,
-                    finfo.results,
-                    finfo.files,
+                util.info(f"{remote_with_test} completed, ingesting result")
+
+                def ingest_and_cleanup(ingest, args, cleanup):
+                    ingest(*args)
+                    # also delete the tmpdir housing these
+                    cleanup()
+
+                self.ingest_queue.start_thread(
+                    ingest_and_cleanup,
+                    target_args=(
+                        # ingest func itself
+                        self.aggregator.ingest,
+                        # args for ingest
+                        (self.platform, finfo.test_name, finfo.results, finfo.files),
+                        # cleanup func itself
+                        finfo.tmp_dir.cleanup,
+                    ),
+                    test_name=finfo.test_name,
                 )
-                # also delete the tmpdir housing these
-                finfo.tmp_dir.cleanup()
-                # ingesting destroyed these
+
+                # ingesting destroys these
                 finfo = self.FinishedInfo._from(
                     finfo,
                     results=None,
@@ -207,6 +223,8 @@ class AdHocOrchestrator(Orchestrator):
                 finfo.remote.release,
                 remote=finfo.remote,
             )
+            # TODO: should this be conditioned by 'self.to_run:' ? to not uselessly fall
+            #       into setup spares and get immediately released after setup?
             finfo.provisioner.provision(1)
 
         # if still not destroyed, run another test on it
@@ -215,6 +233,14 @@ class AdHocOrchestrator(Orchestrator):
             util.debug(f"{remote_with_test} was non-destructive, running next test")
             self._run_new_test(finfo)
 
+        # no more tests to run, release the remote
+        else:
+            util.debug(f"{finfo.remote} no longer useful, releasing it")
+            self.release_queue.start_thread(
+                finfo.remote.release,
+                remote=finfo.remote,
+            )
+
     def serve_once(self):
         """
         Run the orchestration logic, processing any outstanding requests
@@ -225,7 +251,7 @@ class AdHocOrchestrator(Orchestrator):
         (more work to be done), False once all testing is concluded.
         """
         # all done
-        if not self.to_run and not self.running_tests and self.release_queue.qsize() == 0:
+        if not self.to_run and not self.running_tests:
             return False
 
         # process all finished tests, potentially reusing remotes for executing
@@ -263,7 +289,8 @@ class AdHocOrchestrator(Orchestrator):
             sinfo = treturn.sinfo
 
             if treturn.exception:
-                msg = f"{sinfo.remote}: setup failed with {repr(treturn.exception)}"
+                exc_str = f"{type(treturn.exception).__name__}({treturn.exception})"
+                msg = f"{sinfo.remote}: setup failed with {exc_str}"
                 self.release_queue.start_thread(
                     sinfo.remote.release,
                     remote=sinfo.remote,
@@ -286,6 +313,7 @@ class AdHocOrchestrator(Orchestrator):
                     treturn = self.setup_queue.get_raw(block=False)
                 except util.ThreadQueue.Empty:
                     break
+                util.debug(f"releasing extraneous set-up {treturn.sinfo.remote}")
                 self.release_queue.start_thread(
                     treturn.sinfo.remote.release,
                     remote=treturn.sinfo.remote,
@@ -311,15 +339,30 @@ class AdHocOrchestrator(Orchestrator):
         # gather returns from Remote.release() functions - check for exceptions
         # thrown, re-report them as warnings as they are not typically critical
         # for operation
-        try:
-            treturn = self.release_queue.get_raw(block=False)
-        except util.ThreadQueue.Empty:
-            pass
-        else:
-            if treturn.exception:
-                util.warning(f"{treturn.remote} release failed: {repr(treturn.exception)}")
+        while True:
+            try:
+                treturn = self.release_queue.get_raw(block=False)
+            except util.ThreadQueue.Empty:
+                break
+            else:
+                if treturn.exception:
+                    exc_str = f"{type(treturn.exception).__name__}({treturn.exception})"
+                    util.warning(f"{treturn.remote} release failed: {exc_str}")
+                else:
+                    util.debug(f"{treturn.remote} release completed")
+
+        # gather returns from Aggregator.ingest() calls - check for exceptions
+        while True:
+            try:
+                treturn = self.ingest_queue.get_raw(block=False)
+            except util.ThreadQueue.Empty:
+                break
             else:
-                util.debug(f"{treturn.remote}: completed .release()")
+                if treturn.exception:
+                    exc_str = f"{type(treturn.exception).__name__}({treturn.exception})"
+                    util.warning(f"'{treturn.test_name}' ingesting failed: {exc_str}")
+                else:
+                    util.debug(f"'{treturn.test_name}' ingesting completed")
 
         return True
 
@@ -342,16 +385,30 @@ class AdHocOrchestrator(Orchestrator):
         # cancel all running tests and wait for them to clean up (up to 0.1sec)
         for rinfo in self.running_tests.values():
             rinfo.executor.cancel()
-        self.test_queue.join()  # also ignore any exceptions raised
+        self.test_queue.join()    # also ignore any exceptions raised
+
+        # wait for all running ingestions to finish, print exceptions
+        # (we would rather stop provisioners further below than raise here)
+        while True:
+            try:
+                treturn = self.ingest_queue.get_raw(block=False)
+            except util.ThreadQueue.Empty:
+                break
+            else:
+                if treturn.exception:
+                    exc_str = f"{type(treturn.exception).__name__}({treturn.exception})"
+                    util.warning(f"'{treturn.test_name}' ingesting failed: {exc_str}")
+                else:
+                    util.debug(f"'{treturn.test_name}' ingesting completed")
+        self.ingest_queue.join()
 
         # stop all provisioners, also releasing all remotes
-        # TODO: don't parallelize here, remove .stop_defer() and parallelize in provisioners
+        # - parallelize up to 10 provisioners at a time
         if self.provisioners:
-            workers = min(len(self.provisioners), 20)
+            workers = min(len(self.provisioners), 10)
             with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as ex:
                 for provisioner in self.provisioners:
-                    for func in provisioner.stop_defer():
-                        ex.submit(func)
+                    ex.submit(provisioner.stop)
 
     @staticmethod
     def run_setup(sinfo):
@@ -426,7 +483,8 @@ class AdHocOrchestrator(Orchestrator):
 
         # executor (or test) threw exception
         if info.exception:
-            util.info(f"{remote_with_test} threw {repr(info.exception)} during test runtime")
+            exc_str = f"{type(info.exception).__name__}({info.exception})"
+            util.info(f"{remote_with_test} threw {exc_str} during test runtime")
             return False
 
         # the test exited as non-0
@@ -456,10 +514,10 @@ class AdHocOrchestrator(Orchestrator):
         #       of tests, counting reruns for each
         #        - allows the user to adjust counts per-test (ie. test_data metadata)
         #        - allows this template to be @staticmethod
-        if (reruns_left := self.reruns[info.test_name]) > 0:
-            util.info(f"{remote_with_test}: re-running ({reruns_left} reruns left)")
+        reruns_left = self.reruns[info.test_name]
+        util.info(f"{remote_with_test}: {reruns_left} reruns left")
+        if reruns_left > 0:
             self.reruns[info.test_name] -= 1
             return True
         else:
-            util.info(f"{remote_with_test}: reruns exceeded, giving up")
             return False

atex/orchestrator/contest.py

@@ -0,0 +1,94 @@
+from .. import util
+from .adhoc import AdHocOrchestrator
+
+
+# copy/pasted from the Contest repo, lib/virt.py
+def calculate_guest_tag(tags):
+    if "snapshottable" not in tags:
+        return None
+    name = "default"
+    if "with-gui" in tags:
+        name += "_gui"
+    if "uefi" in tags:
+        name += "_uefi"
+    if "fips" in tags:
+        name += "_fips"
+    return name
+
+
+class ContestOrchestrator(AdHocOrchestrator):
+    """
+    Orchestrator for the Contest test suite:
+        https://github.com/RHSecurityCompliance/contest
+
+    Includes SCAP content upload via rsync and other Contest-specific
+    optimizations (around VM snapshots and scheduling).
+    """
+    content_dir_on_remote = "/root/upstream-content"
+
+    def __init__(self, *args, content_dir, **kwargs):
+        self.content_dir = content_dir
+        super().__init__(*args, **kwargs)
+
+    def run_setup(self, sinfo):
+        super().run_setup(sinfo)
+        # upload pre-built content
+        sinfo.remote.rsync(
+            "-r", "--delete", "--exclude=.git/",
+            f"{self.content_dir}/",
+            f"remote:{self.content_dir_on_remote}",
+            func=util.subprocess_log,
+        )
+
+    @classmethod
+    def next_test(cls, to_run, all_tests, previous):
+        # fresh remote, prefer running destructive tests (which likely need
+        # clean OS) to get them out of the way and prevent them from running
+        # on a tainted OS later
+        if type(previous) is AdHocOrchestrator.SetupInfo:
+            for next_name in to_run:
+                next_tags = all_tests[next_name].get("tag", ())
+                util.debug(f"considering next_test for destructivity: {next_name}")
+                if "destructive" in next_tags:
+                    util.debug(f"chosen next_test: {next_name}")
+                    return next_name
+
+        # previous test was run and finished non-destructively,
+        # try to find a next test with the same Contest lib.virt guest tags
+        # as the previous one, allowing snapshot reuse by Contest
+        elif type(previous) is AdHocOrchestrator.FinishedInfo:
+            finished_tags = all_tests[previous.test_name].get("tag", ())
+            util.debug(f"previous finished test on {previous.remote}: {previous.test_name}")
+            # if Guest tag is None, don't bother searching
+            if finished_guest_tag := calculate_guest_tag(finished_tags):
+                for next_name in to_run:
+                    util.debug(f"considering next_test with tags {finished_tags}: {next_name}")
+                    next_tags = all_tests[next_name].get("tag", ())
+                    next_guest_tag = calculate_guest_tag(next_tags)
+                    if next_guest_tag and finished_guest_tag == next_guest_tag:
+                        util.debug(f"chosen next_test: {next_name}")
+                        return next_name
+
+        # fallback to the default next_test()
+        return super().next_test(to_run, all_tests, previous)
+
+    @classmethod
+    def destructive(cls, info, test_data):
+        # if Executor ended with an exception (ie. duration exceeded),
+        # consider the test destructive
+        if info.exception:
+            return True
+
+        # if the test returned non-0 exit code, it could have thrown
+        # a python exception of its own, or (if bash) aborted abruptly
+        # due to 'set -e', don't trust the remote, consider it destroyed
+        # (0 = pass, 2 = fail, anything else = bad)
+        if info.exit_code not in [0,2]:
+            return True
+
+        # if the test was destructive, assume the remote is destroyed
+        tags = test_data.get("tag", ())
+        if "destructive" in tags:
+            return True
+
+        return False

atex/provisioner/__init__.py

@@ -56,10 +56,6 @@ class Provisioner:
     that .get_remote() will ever return a Remote. Ie. the caller can call
     .provision(count=math.inf) to receive as many remotes as the Provisioner
     can possibly supply.
-
-    TODO: remove .defer_stop() (or stop_defer) and mention this below:
-    Note that .stop() or .defer_stop() may be called from a different
-    thread, asynchronously to any other functions.
     """
 
     def provision(self, count=1):
@@ -93,18 +89,6 @@ class Provisioner:
         """
         raise NotImplementedError(f"'stop' not implemented for {self.__class__.__name__}")
 
-    def stop_defer(self):
-        """
-        Enable an external caller to stop the Provisioner instance,
-        deferring resource deallocation to the caller.
-
-        Return an iterable of argument-free thread-safe callables that can be
-        called, possibly in parallel, to free up resources.
-        Ie. a list of 200 .release() functions, to be called in a thread pool
-        by the user, speeding up cleanup.
-        """
-        return (self.stop,)
-
     def __enter__(self):
         try:
             self.start()

atex/provisioner/libvirt/libvirt.py

@@ -260,7 +260,19 @@ class LibvirtCloningProvisioner(Provisioner):
             # by libvirt natively (because treating nvram as a storage pool
             # is a user hack)
             for p in conn.listAllStoragePools():
-                p.refresh()
+                # retry a few times to work around a libvirt race condition
+                for _ in range(10):
+                    try:
+                        p.refresh()
+                    except libvirt.libvirtError as e:
+                        if "domain is not running" in str(e):
+                            break
+                        elif "has asynchronous jobs running" in str(e):
+                            continue
+                        else:
+                            raise
+                    else:
+                        break
             try:
                 nvram_vol = conn.storageVolLookupByPath(nvram_path)
             except libvirt.libvirtError as e:

atex/provisioner/testingfarm/api.py

@@ -18,7 +18,7 @@ DEFAULT_API_URL = "https://api.testing-farm.io/v0.1"
 
 DEFAULT_RESERVE_TEST = {
     "url": "https://github.com/RHSecurityCompliance/atex-reserve",
-    "ref": "v0.10",
+    "ref": "0.11",
     "path": ".",
     "name": "/plans/reserve",
 }
@@ -34,10 +34,10 @@ _http = urllib3.PoolManager(
     maxsize=10,
     block=True,
     retries=urllib3.Retry(
-        total=10,
+        total=24,
         # account for API restarts / short outages
-        backoff_factor=60,
-        backoff_max=600,
+        backoff_factor=10,
+        backoff_max=3600,
         # retry on API server errors too, not just connection issues
         status=10,
         status_forcelist={403,404,408,429,500,502,503,504},
@@ -438,7 +438,6 @@ class Reserve:
         'api' is a TestingFarmAPI instance - if unspecified, a sensible default
         will be used.
         """
-        util.info(f"will reserve compose:{compose} on arch:{arch} for {timeout}min")
         spec = {
             "test": {
                 "fmf": reserve_test or DEFAULT_RESERVE_TEST,

atex/provisioner/testingfarm/testingfarm.py

@@ -1,6 +1,7 @@
 import time
 import tempfile
 import threading
+import concurrent.futures
 
 from ... import connection, util
 from .. import Provisioner, Remote
@@ -49,7 +50,12 @@ class TestingFarmRemote(Remote, connection.ssh.ManagedSSHConnection):
 
 
 class TestingFarmProvisioner(Provisioner):
+    # maximum number of TF requests the user can .provision(),
+    # as a last safety measure against Orchestrator(remotes=math.inf)
     absolute_max_remotes = 100
+    # number of parallel threads running HTTP DELETE calls to cancel
+    # TF requests on .stop() or Context Manager exit
+    stop_release_workers = 10
 
     def __init__(self, compose, arch="x86_64", *, max_retries=10, **reserve_kwargs):
         """
@@ -129,6 +135,7 @@ class TestingFarmProvisioner(Provisioner):
         # instantiate a class Reserve from the Testing Farm api module
         # (which typically provides context manager, but we use its .reserve()
         #  and .release() functions directly)
+        util.info(f"{repr(self)}: reserving new remote")
         tf_reserve = api.Reserve(
             compose=self.compose,
             arch=self.arch,
@@ -154,29 +161,25 @@ class TestingFarmProvisioner(Provisioner):
             self.ssh_key, self.ssh_pubkey = util.ssh_keygen(self._tmpdir.name)
 
     def stop(self):
-        with self.lock:
-            # abort reservations in progress
-            while self.reserving:
-                # testingfarm api.Reserve instances
-                self.reserving.pop().release()
-            # cancel/release all Remotes ever created by us
-            while self.remotes:
-                # TestingFarmRemote instances
-                self.remotes.pop().release()
-            # explicitly remove the tmpdir rather than relying on destructor
-            self._tmpdir.cleanup()
-            self._tmpdir = None
+        release_funcs = []
 
-    def stop_defer(self):
-        callables = []
         with self.lock:
-            callables += (f.release for f in self.reserving)
+            release_funcs += (f.release for f in self.reserving)
             self.reserving = []
-            callables += (r.release for r in self.remotes)
-            self.remotes = []  # just in case
-            callables.append(self._tmpdir.cleanup)
+            release_funcs += (r.release for r in self.remotes)
+            self.remotes = []  # just in case of a later .start()
+
+        # parallelize at most stop_release_workers TF API release (DELETE) calls
+        if release_funcs:
+            workers = min(len(release_funcs), self.stop_release_workers)
+            with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as ex:
+                for func in release_funcs:
+                    ex.submit(func)
+
+        with self.lock:
+            # explicitly remove the tmpdir rather than relying on destructor
+            self._tmpdir.cleanup()
             self._tmpdir = None
-        return callables
 
     def provision(self, count=1):
         with self.lock:
@@ -198,10 +201,11 @@ class TestingFarmProvisioner(Provisioner):
                 # always non-blocking
                 return None
             except (api.TestingFarmError, connection.ssh.SSHError) as e:
+                exc_str = f"{type(e).__name__}({e})"
                 with self.lock:
                     if self.retries > 0:
                         util.warning(
-                            f"caught while reserving a TF system: {repr(e)}, "
+                            f"caught while reserving a TF system: {exc_str}, "
                             f"retrying ({self.retries} left)",
                         )
                         self.retries -= 1
@@ -212,7 +216,7 @@ class TestingFarmProvisioner(Provisioner):
                             return None
                     else:
                         util.warning(
-                            f"caught while reserving a TF system: {repr(e)}, "
+                            f"caught while reserving a TF system: {exc_str}, "
                             "exhausted all retries, giving up",
                         )
                         raise

atex/util/subprocess.py

@@ -1,6 +1,6 @@
 import subprocess
 
-from .log import debug, extradebug
+from .log import extradebug
 
 
 def subprocess_run(cmd, **kwargs):
@@ -9,7 +9,7 @@ def subprocess_run(cmd, **kwargs):
     """
     # when logging, skip current stack frame - report the place we were called
     # from, not util.subprocess_run itself
-    debug(f"running: '{cmd}' with {kwargs=}")
+    extradebug(f"running: '{cmd}' with {kwargs=}")
     return subprocess.run(cmd, **kwargs)
 
 
@@ -17,7 +17,7 @@ def subprocess_output(cmd, *, check=True, text=True, **kwargs):
     """
     A wrapper simulating subprocess.check_output() via a modern .run() API.
     """
-    debug(f"running: '{cmd}' with {check=}, {text=} and {kwargs=}")
+    extradebug(f"running: '{cmd}' with {check=}, {text=} and {kwargs=}")
     proc = subprocess.run(cmd, check=check, text=text, stdout=subprocess.PIPE, **kwargs)
     return proc.stdout.rstrip("\n") if text else proc.stdout
 
@@ -26,7 +26,7 @@ def subprocess_Popen(cmd, **kwargs):  # noqa: N802
     """
     A simple wrapper for the real subprocess.Popen() that logs the command used.
     """
-    debug(f"running: '{cmd}' with {kwargs=}")
+    extradebug(f"running: '{cmd}' with {kwargs=}")
     return subprocess.Popen(cmd, **kwargs)
 
 
@@ -56,7 +56,7 @@ def subprocess_stream(cmd, *, stream="stdout", check=False, input=None, **kwargs
         all_kwargs["stdin"] = subprocess.PIPE
     all_kwargs |= kwargs
 
-    debug(f"running: '{cmd}' with {all_kwargs=}")
+    extradebug(f"running: '{cmd}' with {all_kwargs=}")
     proc = subprocess.Popen(cmd, **all_kwargs)
 
     def generate_lines():
@@ -80,7 +80,7 @@ def subprocess_log(cmd, **kwargs):
 
     Uses subprocess_stream() to gather the lines.
     """
-    debug(f"running: '{cmd}' with {kwargs=}")
+    extradebug(f"running: '{cmd}' with {kwargs=}")
     _, lines = subprocess_stream(cmd, **kwargs)
     for line in lines:
         extradebug(line)

{atex-0.10.dist-info → atex-0.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: atex
-Version: 0.10
+Version: 0.11
 Summary: Ad-hoc Test EXecutor
 Project-URL: Homepage, https://github.com/RHSecurityCompliance/atex
 License-Expression: GPL-3.0-or-later

{atex-0.10.dist-info → atex-0.11.dist-info}/RECORD

@@ -1,33 +1,34 @@
 atex/__init__.py,sha256=LdX67gprtHYeAkjLhFPKzpc7ECv2rHxUbHKDGbGXO1c,517
 atex/fmf.py,sha256=gkJXIaRO7_KvwJR-V6Tc1NVn4a9Hq7hoBLQLhxYIdbg,8834
-atex/aggregator/__init__.py,sha256=uNnYSyDGXjknxckI8MFfl-C8_gin8FwQchiq-UOyP6I,1744
-atex/aggregator/json.py,sha256=x1zim9O2olzBh185NYWo5N96fixB2oxCamoOZwmgR9w,3330
-atex/cli/__init__.py,sha256=X5XxkDEDXE4tJAjwt5ShRHCFTXDK-2zvxQ34opmueUc,2768
-atex/cli/fmf.py,sha256=HfbTgFbCwK4Nuyq6vtGutcq_4-4kj-tmoqzXUn3AYtY,3573
+atex/aggregator/__init__.py,sha256=8mN-glHdzR4icKAUGO4JPodsTrLMdJoeuZsO2CTbhyU,1773
+atex/aggregator/json.py,sha256=tpoUZoZM8EMYhZKwVr4LRtgEIDjRxC11BIKVXZKYPOs,10441
+atex/cli/__init__.py,sha256=Ew2z-gC0jvOmU_DqYgXVQla3p1rTnrz64I63q52aHv4,2899
+atex/cli/fmf.py,sha256=pvj_OIp6XT_nVUwziL7-v_HNbyAtuUmb7k_Ey_KkFJc,3616
 atex/cli/libvirt.py,sha256=6tt5ANb8XBBRXOQsYPTWILThKqf-gvt5AZh5Dctg2PA,3782
-atex/cli/testingfarm.py,sha256=HGlqrkhanUMo2CqKxmM3ACgptWtxm0gICyEGf7O6Qc0,9078
+atex/cli/testingfarm.py,sha256=ovgoogmIM2TglS7iQD3liMiEYYtcykS_HRRKbltpW2I,10131
 atex/connection/__init__.py,sha256=dj8ZBcEspom7Z_UjecfLGBRNvLZ3dyGR9q19i_B4xpY,3880
 atex/connection/podman.py,sha256=1T56gh1TgbcQWpTIJHL4NaxZOI6aMg7Xp7sn6PQQyBk,1911
 atex/connection/ssh.py,sha256=9A57b9YR_HI-kIu06Asic1y__JPVXEheDZxjbG2Qcsc,13460
 atex/executor/__init__.py,sha256=XCfhi7QDELjey7N1uzhMjc46Kp1Jsd5bOCf52I27SCE,85
 atex/executor/duration.py,sha256=x06sItKOZi6XA8KszQwZGpIb1Z_L-HWqIwZKo2SDo0s,1759
-atex/executor/executor.py,sha256=JLFR9cZjSlUdAlAlLct6WuzmYbtjGtSobxvsToQum6M,15738
+atex/executor/executor.py,sha256=toyLVQCDzfw381iEGrvOXoKPsd4SqxMZHwlDSTJGqKk,15792
 atex/executor/reporter.py,sha256=MceFmHFt0bTEClBZbRI1WnFbfMhR0e1noOzcu7gjKuQ,3403
 atex/executor/scripts.py,sha256=riJAQWsV-BFGkJwR2Dmf3R0ZRRZJs9w9iYnPpYaQNaE,5618
 atex/executor/testcontrol.py,sha256=mVrLwQUnDRfUq-5diz-80UvCWWxn1TkcBgmAKhKNb5E,12696
 atex/orchestrator/__init__.py,sha256=8Q1YknyibilXLjWRYkHm_Mr2HMm0SRw8Zv39KypeASM,2059
-atex/orchestrator/adhoc.py,sha256=GnvHLlCHeJ_nQ8doEjMuDzqmu4XZorI7ZzOtG_C08tU,18451
-atex/provisioner/__init__.py,sha256=2eepmEznq94tbam9VSWbsGFrZZpWeNSVlsTczGxjNuQ,4667
+atex/orchestrator/adhoc.py,sha256=QpYoPeyQzYFDBM1zgFJKMXH1RtdJixbH5whVX0OP-14,21003
+atex/orchestrator/contest.py,sha256=ADmRlsZPQx-MJ6fWHmBcJOIy3DSPnvwVheVL9Upwtg0,3703
+atex/provisioner/__init__.py,sha256=6hZxQlvTQ0yWWqCRCPqWMoYuim5wDMCcDIYHF-nIfMs,4013
 atex/provisioner/libvirt/VM_PROVISION,sha256=7pkZ-ozgTyK4qNGC-E-HUznr4IhbosWSASbB72Gknl8,2664
 atex/provisioner/libvirt/__init__.py,sha256=pKG5IpZSC2IHs5wL2ecQx_fd9AzAXEbZmDzA7RyZsfM,119
-atex/provisioner/libvirt/libvirt.py,sha256=rtxowv5DpgcWsGRXYF29n6S9x_cgXRVgqY41DiFu920,18431
+atex/provisioner/libvirt/libvirt.py,sha256=ZKctK2B51olvWvLxz2pZ2s6LtX_7EJ43LvlyJHnI1Ho,18955
 atex/provisioner/libvirt/locking.py,sha256=AXtDyidZNmUoMmrit26g9iTHDqInrzL_RSQEoc_EAXw,5669
 atex/provisioner/libvirt/setup-libvirt.sh,sha256=oCMy9SCnbC_QuAzO2sFwvB5ui1kMQ6uviHsgdXyoFXc,2428
 atex/provisioner/podman/__init__.py,sha256=dM0JzQXWX7edtWSc0KH0cMFXAjArFn2Vme4j_ZMsdYA,138
 atex/provisioner/podman/podman.py,sha256=ztRypoakSf-jF04iER58tEMUZ4Y6AuzIpNpFXp44bB4,4997
 atex/provisioner/testingfarm/__init__.py,sha256=kZncgLGdRCR4FMaRQr2GTwJ8vjlA-24ri8JO2ueZJuw,113
-atex/provisioner/testingfarm/api.py,sha256=UcMN61nBr3wqEd5KSR5Xhv1-TS7nSPFvk2byb6PdIs8,21811
-atex/provisioner/testingfarm/testingfarm.py,sha256=OI-a99xALaiYf-y5037WFVxY1g2H2y1xEKxHBdUQvfg,8271
+atex/provisioner/testingfarm/api.py,sha256=dlXe9brzHERawIx2UTv34u2tOSskdZtXD68-u1MnOHk,21726
+atex/provisioner/testingfarm/testingfarm.py,sha256=yvQzWat92B4UnJNZzCLI8mpAKf_QvHUKyKbjlk5123Q,8573
 atex/util/__init__.py,sha256=cWHFbtQ4mDlKe6lXyPDWRmWJOTcHDGfVuW_-GYa8hB0,1473
 atex/util/dedent.py,sha256=SEuJMtLzqz3dQ7g7qyZzEJ9VYynVlk52tQCJY-FveXo,603
 atex/util/libvirt.py,sha256=kDZmT6xLYEZkQNLZY98gJ2M48DDWXxHF8rQY9PnjB3U,660
@@ -35,10 +36,10 @@ atex/util/log.py,sha256=KVR7ep8n5wtghsvBFCtHiPsMAQBdAmK83E_Jec5t4cU,2230
 atex/util/named_mapping.py,sha256=UBMe9TetjV-DGPhjYjJ42YtC40FVPKAAEROXl9MA5fo,4700
 atex/util/path.py,sha256=x-kXqiWCVodfZWbEwtC5A8LFvutpDIPYv2m0boZSlXU,504
 atex/util/ssh_keygen.py,sha256=9yuSl2yBV7pG3Qfsf9tossVC00nbIUrAeLdbwTykpjk,384
-atex/util/subprocess.py,sha256=PQBxcQJPapP1ZLO4LqENyrxxCbNAxtJDNNlBV5DcD9k,2953
+atex/util/subprocess.py,sha256=_oQN8CNgGoH9GAR6nZlpujYe2HjXFBcCuIkLPw-IxJ4,2971
 atex/util/threads.py,sha256=c8hsEc-8SqJGodInorv_6JxpiHiSkGFGob4qbMmOD2M,3531
-atex-0.10.dist-info/METADATA,sha256=evOBYvVboY2T8eGrAKy64UpyeuFKSMLGz8pUz8Sstm8,3050
-atex-0.10.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-atex-0.10.dist-info/entry_points.txt,sha256=pLqJdcfeyQTgup2h6dWb6SvkHhtOl-W5Eg9zV8moK0o,39
-atex-0.10.dist-info/licenses/COPYING.txt,sha256=oEuj51jdmbXcCUy7pZ-KE0BNcJTR1okudRp5zQ0yWnU,670
-atex-0.10.dist-info/RECORD,,
+atex-0.11.dist-info/METADATA,sha256=3fRMLBrkoRIHwbY2GheyNkrx4mNVmL_95wlxGjZsORc,3050
+atex-0.11.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+atex-0.11.dist-info/entry_points.txt,sha256=pLqJdcfeyQTgup2h6dWb6SvkHhtOl-W5Eg9zV8moK0o,39
+atex-0.11.dist-info/licenses/COPYING.txt,sha256=oEuj51jdmbXcCUy7pZ-KE0BNcJTR1okudRp5zQ0yWnU,670
+atex-0.11.dist-info/RECORD,,

{atex-0.10.dist-info → atex-0.11.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.27.0
+Generator: hatchling 1.28.0
 Root-Is-Purelib: true
 Tag: py3-none-any