atex 0.10__py3-none-any.whl → 0.12__py3-none-any.whl

atex/aggregator/__init__.py

@@ -7,12 +7,14 @@ class Aggregator:
     TODO: generic description, not JSON-specific
     """
 
-    def ingest(self, platform, test_name, results_file, files_dir):
+    def ingest(self, platform, test_name, test_results, test_files):
         """
-        Process 'results_file' (string/Path) for reported results and append
-        them to the overall aggregated line-JSON file, recursively copying over
-        the dir structure under 'files_dir' (string/Path) under the respective
-        platform and test name in the aggregated storage dir.
+        Process 'test_results' (string/Path) for as results reported by a test
+        ran by Executor, along with 'test_files' as files uploaded by that test,
+        aggregating them under 'platform' (string) as 'test_name' (string).
+
+        This is DESTRUCTIVE, the input results/files are consumed in the
+        process.
         """
         raise NotImplementedError(f"'ingest' not implemented for {self.__class__.__name__}")
 
@@ -34,7 +36,7 @@ class Aggregator:
             self.start()
             return self
         except Exception:
-            self.close()
+            self.stop()
             raise
 
     def __exit__(self, exc_type, exc_value, traceback):

atex/aggregator/json.py

@@ -1,4 +1,6 @@
+import abc
 import gzip
+import lzma
 import json
 import shutil
 import threading
@@ -7,10 +9,16 @@ from pathlib import Path
 from . import Aggregator
 
 
+def _verbatim_move(src, dst):
+    def copy_without_symlinks(src, dst):
+        return shutil.copy2(src, dst, follow_symlinks=False)
+    shutil.move(src, dst, copy_function=copy_without_symlinks)
+
+
 class JSONAggregator(Aggregator):
     """
-    Collects reported results as a GZIP-ed line-JSON and files (logs) from
-    multiple test runs under a shared directory.
+    Collects reported results in a line-JSON output file and uploaded files
+    (logs) from multiple test runs under a shared directory.
 
     Note that the aggregated JSON file *does not* use the test-based JSON format
     described by executor/RESULTS.md - both use JSON, but are very different.
@@ -23,74 +31,249 @@ class JSONAggregator(Aggregator):
     All these are strings except 'files', which is another (nested) array
     of strings.
 
+    If 'testout' is present in an input test result, it is prepended to
+    the list of 'files'.
     If a field is missing in the source result, it is translated to a null
     value.
     """
 
-    def __init__(self, json_file, storage_dir):
+    def __init__(self, target, files):
         """
-        'json_file' is a string/Path to a .json.gz file with aggregated results.
+        'target' is a string/Path to a .json file for all ingested
+        results to be aggregated (written) to.
 
-        'storage_dir' is a string/Path of the top-level parent for all
+        'files' is a string/Path of the top-level parent for all
         per-platform / per-test files uploaded by tests.
         """
         self.lock = threading.RLock()
-        self.storage_dir = Path(storage_dir)
-        self.json_file = Path(json_file)
-        self.json_gzip_fobj = None
+        self.target = Path(target)
+        self.files = Path(files)
+        self.target_fobj = None
 
     def start(self):
-        if self.json_file.exists():
-            raise FileExistsError(f"{self.json_file} already exists")
-        self.json_gzip_fobj = gzip.open(self.json_file, "wt", newline="\n")
+        if self.target.exists():
+            raise FileExistsError(f"{self.target} already exists")
+        self.target_fobj = open(self.target, "w")
 
-        if self.storage_dir.exists():
-            raise FileExistsError(f"{self.storage_dir} already exists")
-        self.storage_dir.mkdir()
+        if self.files.exists():
+            raise FileExistsError(f"{self.files} already exists")
+        self.files.mkdir()
 
     def stop(self):
-        if self.json_gzip_fobj:
-            self.json_gzip_fobj.close()
-            self.json_gzip_fobj = None
+        if self.target_fobj:
+            self.target_fobj.close()
+            self.target_fobj = None
+
+    def _get_test_files_path(self, platform, test_name):
+        """
+        Return a directory path to where uploaded files should be stored
+        for a particular 'platform' and 'test_name'.
+        """
+        platform_files = self.files / platform
+        platform_files.mkdir(exist_ok=True)
+        test_files = platform_files / test_name.lstrip("/")
+        return test_files
+
+    @staticmethod
+    def _modify_file_list(test_files):
+        return test_files
+
+    @staticmethod
+    def _move_test_files(test_files, target_dir):
+        """
+        Move (or otherwise process) 'test_files' as directory of files uploaded
+        by the test, into the pre-computed 'target_dir' location (inside
+        a hierarchy of all files from all tests).
+        """
+        _verbatim_move(test_files, target_dir)
+
+    def _gen_test_results(self, input_fobj, platform, test_name):
+        """
+        Yield complete output JSON objects, one for each input result.
+        """
+        # 'testout' , 'files' and others are standard fields in the
+        # test control interface, see RESULTS.md for the Executor
+        for raw_line in input_fobj:
+            result_line = json.loads(raw_line)
+
+            file_names = []
+            # process the file specified by the 'testout' key
+            if "testout" in result_line:
+                file_names.append(result_line["testout"])
+            # process any additional files in the 'files' key
+            if "files" in result_line:
+                file_names += (f["name"] for f in result_line["files"])
+
+            file_names = self._modify_file_list(file_names)
 
-    def ingest(self, platform, test_name, results_file, files_dir):
-        platform_dir = self.storage_dir / platform
-        test_dir = platform_dir / test_name.lstrip("/")
-        if test_dir.exists():
-            raise FileExistsError(f"{test_dir} already exists for {test_name}")
+            output_line = (
+                platform,
+                result_line["status"],
+                test_name,
+                result_line.get("name"),  # subtest
+                file_names,
+                result_line.get("note"),
+            )
+            yield json.dumps(output_line, indent=None)
+
+    def ingest(self, platform, test_name, test_results, test_files):
+        target_test_files = self._get_test_files_path(platform, test_name)
+        if target_test_files.exists():
+            raise FileExistsError(f"{target_test_files} already exists for {test_name}")
 
         # parse the results separately, before writing any aggregated output,
-        # to ensure that either all results from the test are ingested, or none
+        # to ensure that either ALL results from the test are ingested, or none
         # at all (ie. if one of the result lines contains JSON errors)
-        output_lines = []
-        with open(results_file) as results_fobj:
-            for raw_line in results_fobj:
-                result_line = json.loads(raw_line)
-
-                file_names = []
-                if "testout" in result_line:
-                    file_names.append(result_line["testout"])
-                if "files" in result_line:
-                    file_names += (f["name"] for f in result_line["files"])
-
-                output_line = (
-                    platform,
-                    result_line["status"],
-                    test_name,
-                    result_line.get("name"),  # subtest
-                    file_names,
-                    result_line.get("note"),
-                )
-                encoded = json.dumps(output_line, indent=None)
-                output_lines.append(encoded)
-
-        output_str = "\n".join(output_lines) + "\n"
+        with open(test_results) as test_results_fobj:
+            output_results = self._gen_test_results(test_results_fobj, platform, test_name)
+            output_json = "\n".join(output_results) + "\n"
 
         with self.lock:
-            self.json_gzip_fobj.write(output_str)
-            self.json_gzip_fobj.flush()
+            self.target_fobj.write(output_json)
+            self.target_fobj.flush()
+
+        # clean up the source test_results (Aggregator should 'mv', not 'cp')
+        Path(test_results).unlink()
+
+        # if the test_files dir is not empty
+        if any(test_files.iterdir()):
+            self._move_test_files(test_files, target_test_files)
+
+
+class CompressedJSONAggregator(JSONAggregator, abc.ABC):
+    compress_files = False
+    suffix = ""
+    exclude = ()
+
+    @abc.abstractmethod
+    def compressed_open(self, *args, **kwargs):
+        pass
+
+    def start(self):
+        if self.target.exists():
+            raise FileExistsError(f"{self.target_file} already exists")
+        self.target_fobj = self.compressed_open(self.target, "wt", newline="\n")
+
+        if self.files.exists():
+            raise FileExistsError(f"{self.storage_dir} already exists")
+        self.files.mkdir()
+
+    def _modify_file_list(self, test_files):
+        if self.compress_files and self.suffix:
+            return [
+                (name if name in self.exclude else f"{name}{self.suffix}")
+                for name in test_files
+            ]
+        else:
+            return super()._modify_file_list(test_files)
+
+    def _move_test_files(self, test_files, target_dir):
+        if not self.compress_files:
+            super()._move_test_files(test_files, target_dir)
+            return
+
+        for root, _, files in test_files.walk(top_down=False):
+            for file_name in files:
+                src_path = root / file_name
+                dst_path = target_dir / src_path.relative_to(test_files)
+
+                dst_path.parent.mkdir(parents=True, exist_ok=True)
 
-        Path(results_file).unlink()
+                # skip dirs, symlinks, device files, etc.
+                if not src_path.is_file(follow_symlinks=False) or file_name in self.exclude:
+                    _verbatim_move(src_path, dst_path)
+                    continue
 
-        platform_dir.mkdir(exist_ok=True)
-        shutil.move(files_dir, test_dir)
+                if self.suffix:
+                    dst_path = dst_path.with_name(f"{dst_path.name}{self.suffix}")
+
+                with open(src_path, "rb") as plain_fobj:
+                    with self.compressed_open(dst_path, "wb") as compress_fobj:
+                        shutil.copyfileobj(plain_fobj, compress_fobj, 1048576)
+
+                src_path.unlink()
+
+            # we're walking bottom-up, so the local root should be empty now
+            root.rmdir()
+
+
+class GzipJSONAggregator(CompressedJSONAggregator):
+    """
+    Identical to JSONAggregator, but transparently Gzips either or both of
+    the output line-JSON file with results and the uploaded files.
+    """
+    def compressed_open(self, *args, **kwargs):
+        return gzip.open(*args, compresslevel=self.level, **kwargs)
+
+    def __init__(
+        self, target, files, *, compress_level=9,
+        compress_files=True, compress_files_suffix=".gz", compress_files_exclude=None,
+    ):
+        """
+        'target' is a string/Path to a .json.gz file for all ingested
+        results to be aggregated (written) to.
+
+        'files' is a string/Path of the top-level parent for all
+        per-platform / per-test files uploaded by tests.
+
+        'compress_level' specifies how much effort should be spent compressing,
+        (1 = fast, 9 = slow).
+
+        If 'compress_files' is True, compress also any files uploaded by tests.
+
+        The 'compress_files_suffix' is appended to any processed test-uploaded
+        files, and the respective 'files' results array is modified with the
+        new file names (as if the test uploaded compressed files already).
+        Set to "" (empty string) to use original file names and just compress
+        them transparently in-place.
+
+        'compress_files_exclude' is a tuple/list of strings (input 'files'
+        names) to skip when compressing. Their names also won't be modified.
+        """
+        super().__init__(target, files)
+        self.level = compress_level
+        self.compress_files = compress_files
+        self.suffix = compress_files_suffix
+        self.exclude = compress_files_exclude or ()
+
+
+class LZMAJSONAggregator(CompressedJSONAggregator):
+    """
+    Identical to JSONAggregator, but transparently compresses (via LZMA/XZ)
+    either or both of the output line-JSON file with results and the uploaded
+    files.
+    """
+    def compressed_open(self, *args, **kwargs):
+        return lzma.open(*args, preset=self.preset, **kwargs)
+
+    def __init__(
+        self, target, files, *, compress_preset=9,
+        compress_files=True, compress_files_suffix=".xz", compress_files_exclude=None,
+    ):
+        """
+        'target' is a string/Path to a .json.xz file for all ingested
+        results to be aggregated (written) to.
+
+        'files' is a string/Path of the top-level parent for all
+        per-platform / per-test files uploaded by tests.
+
+        'compress_preset' specifies how much effort should be spent compressing,
+        (1 = fast, 9 = slow). Optionally ORed with lzma.PRESET_EXTREME to spend
+        even more CPU time compressing.
+
+        If 'compress_files' is True, compress also any files uploaded by tests.
+
+        The 'compress_files_suffix' is appended to any processed test-uploaded
+        files, and the respective 'files' results array is modified with the
+        new file names (as if the test uploaded compressed files already).
+        Set to "" (empty string) to use original file names and just compress
+        them transparently in-place.
+
+        'compress_files_exclude' is a tuple/list of strings (input 'files'
+        names) to skip when compressing. Their names also won't be modified.
+        """
+        super().__init__(target, files)
+        self.preset = compress_preset
+        self.compress_files = compress_files
+        self.suffix = compress_files_suffix
+        self.exclude = compress_files_exclude or ()

atex/cli/__init__.py

@@ -33,6 +33,9 @@ from .. import util
 def setup_logging(level):
     if level <= util.EXTRADEBUG:
         fmt = "%(asctime)s %(name)s: %(filename)s:%(lineno)s: %(funcName)s(): %(message)s"
+        # also print urllib3 headers
+        import http.client  # noqa: PLC0415
+        http.client.HTTPConnection.debuglevel = 5
     else:
         fmt = "%(asctime)s %(name)s: %(message)s"
     logging.basicConfig(

atex/cli/fmf.py

@@ -56,17 +56,17 @@ def prepare(args):
     result = make_fmftests(args)
     print("--- fmf root ---")
     print(str(result.root))
-    print("--- prepare packages ---")
+    print("\n--- prepare packages ---")
     print("\n".join(result.prepare_pkgs))
-    print("--- plan environment ---")
-    print("\n".join("{k}={v}" for k,v in result.plan_env))
+    print("\n--- plan environment ---")
+    print("\n".join(f"{k}={v}" for k,v in result.plan_env.items()))
     for script in result.prepare_scripts:
-        print("--- prepare script ---")
-        print(script)
+        print("\n--- prepare script ---")
+        print(script.rstrip("\n"))
         print("----------------------")
     for script in result.finish_scripts:
-        print("--- finish script ---")
-        print(script)
+        print("\n--- finish script ---")
+        print(script.rstrip("\n"))
         print("----------------------")
 
 

atex/cli/testingfarm.py

@@ -31,7 +31,8 @@ def composes(args):
     comps = api.composes(ranch=args.ranch)
     comps_list = comps["composes"]
     for comp in comps_list:
-        print(comp["name"])
+        if comp["type"] == "compose":
+            print(comp["name"])
 
 
 def get_request(args):
@@ -47,36 +48,63 @@ def cancel(args):
 
 def search_requests(args):
     api = _get_api(args)
-    reply = api.search_requests(
-        state=args.state,
-        mine=not args.all,
-        user_id=args.user_id,
-        token_id=args.token_id,
-        ranch=args.ranch,
-        created_before=args.before,
-        created_after=args.after,
-    )
-    if not reply:
-        return
 
-    if args.json:
-        for req in sorted(reply, key=lambda x: x["created"]):
-            print(json.dumps(req))
+    func_kwargs = {
+        "mine": not args.all,
+        "user_id": args.user_id,
+        "token_id": args.token_id,
+        "ranch": args.ranch,
+        "created_before": args.before,
+        "created_after": args.after,
+    }
+
+    if args.page is not None:
+        reply = api.search_requests_paged(
+            state=args.state,
+            page=args.page,
+            **func_kwargs,
+        )
+        if not reply:
+            return
     else:
-        for req in sorted(reply, key=lambda x: x["created"]):
-            req_id = req["id"]
-            created = req["created"].partition(".")[0]
+        reply = api.search_requests(
+            state=args.state,
+            **func_kwargs,
+        )
+        if not reply:
+            return
+        reply = sorted(reply, key=lambda x: x["created"])
 
-            envs = []
-            for env in req["environments_requested"]:
-                if "os" in env and env["os"] and "compose" in env["os"]:
-                    compose = env["os"]["compose"]
-                    arch = env["arch"]
-                    if compose and arch:
-                        envs.append(f"{compose}@{arch}")
-            envs_str = ", ".join(envs)
+    if args.json:
+        for req in reply:
+            print(json.dumps(req))
+        return
 
-            print(f"{created} {req_id} : {envs_str}")
+    for req in reply:
+        req_id = req["id"]
+        created = req["created"].partition(".")[0]
+
+        if "fmf" in req["test"] and req["test"]["fmf"]:
+            test = req["test"]["fmf"]["url"]
+        elif "tmt" in req["test"] and req["test"]["tmt"]:
+            test = req["test"]["tmf"]["url"]
+        else:
+            test = ""
+
+        envs = []
+        for env in req["environments_requested"]:
+            if "os" in env and env["os"] and "compose" in env["os"]:
+                compose = env["os"]["compose"]
+                arch = env["arch"]
+                if compose and arch:
+                    envs.append(f"{compose}@{arch}")
+
+        print(f"{created} {req_id}", end="")
+        if test:
+            print(f" | test:{test}", end="")
+        if envs:
+            print(f" | envs:[{', '.join(envs)}]", end="")
+        print()
 
 
 def stats(args):
@@ -92,29 +120,46 @@ def stats(args):
             elif "tmt" in req["test"] and req["test"]["tmt"]:
                 repos[req["test"]["tmt"]["url"]] += 1
 
+        top_tokens = sorted(tokens, key=lambda x: tokens[x], reverse=True)[:10]
+        top_repos = sorted(repos, key=lambda x: repos[x], reverse=True)[:10]
+        if not top_tokens or not top_repos:
+            return
+        digits = max(len(str(tokens[top_tokens[0]])), len(str(repos[top_repos[0]])))
+
         print("Top 10 token IDs:")
-        for token_id in sorted(tokens, key=lambda x: tokens[x], reverse=True)[:10]:
+        for token_id in top_tokens:
             count = tokens[token_id]
-            print(f"{count:>5}  {token_id}")
+            print(f"{count:>{digits}}  {token_id}")
 
         print("Top 10 repo URLs:")
-        for repo_url in sorted(repos, key=lambda x: repos[x], reverse=True)[:10]:
+        for repo_url in top_repos:
             count = repos[repo_url]
-            print(f"{count:>5}  {repo_url}")
-
-    def chain_without_none(*iterables):
-        for itr in iterables:
-            if itr is None:
-                continue
-            for item in itr:
-                if item is not None:
-                    yield item
-
-    queued_and_running = chain_without_none(
-        api.search_requests(state="queued", ranch=args.ranch, mine=False),
-        api.search_requests(state="running", ranch=args.ranch, mine=False),
-    )
-    top_users_repos(queued_and_running)
+            print(f"{count:>{digits}}  {repo_url}")
+
+    def request_search_results():
+        if args.before is not None or args.after is not None:
+            for state in args.states.split(","):
+                reply = api.search_requests_paged(
+                    state=state,
+                    page=args.page,
+                    mine=False,
+                    ranch=args.ranch,
+                    created_before=args.before,
+                    created_after=args.after,
+                )
+                if reply:
+                    yield from reply
+        else:
+            for state in args.states.split(","):
+                reply = api.search_requests(
+                    state=state,
+                    mine=False,
+                    ranch=args.ranch,
+                )
+                if reply:
+                    yield from reply
+
+    top_users_repos(request_search_results())
 
 
 def reserve(args):
@@ -234,12 +279,17 @@ def parse_args(parser):
     cmd.add_argument("--before", help="only requests created before ISO8601")
     cmd.add_argument("--after", help="only requests created after ISO8601")
     cmd.add_argument("--json", help="full details, one request per line", action="store_true")
+    cmd.add_argument("--page", help="do paged search, page interval in secs", type=int)
 
     cmd = cmds.add_parser(
         "stats",
         help="print out TF usage statistics",
     )
+    cmd.add_argument("--before", help="only requests created before ISO8601")
+    cmd.add_argument("--after", help="only requests created after ISO8601")
+    cmd.add_argument("--page", help="do paged search, page interval in secs", type=int)
     cmd.add_argument("ranch", help="Testing Farm ranch name")
+    cmd.add_argument("states", help="comma-separated TF request states")
 
     cmd = cmds.add_parser(
         "reserve",

atex/executor/__init__.py

@@ -1,2 +1,23 @@
-from . import testcontrol  # noqa: F401
-from .executor import Executor  # noqa: F401
+class ExecutorError(Exception):
+    """
+    Raised by class Executor.
+    """
+    pass
+
+
+class TestSetupError(ExecutorError):
+    """
+    Raised when the preparation for test execution (ie. pkg install) fails.
+    """
+    pass
+
+
+class TestAbortedError(ExecutorError):
+    """
+    Raised when an infrastructure-related issue happened while running a test.
+    """
+    pass
+
+
+from . import testcontrol  # noqa: F401, E402
+from .executor import Executor  # noqa: F401, E402