atex 0.9__py3-none-any.whl → 0.10__py3-none-any.whl

atex/aggregator/__init__.py

@@ -0,0 +1,60 @@
+import importlib as _importlib
+import pkgutil as _pkgutil
+
+
+class Aggregator:
+    """
+    TODO: generic description, not JSON-specific
+    """
+
+    def ingest(self, platform, test_name, results_file, files_dir):
+        """
+        Process 'results_file' (string/Path) for reported results and append
+        them to the overall aggregated line-JSON file, recursively copying over
+        the dir structure under 'files_dir' (string/Path) under the respective
+        platform and test name in the aggregated storage dir.
+        """
+        raise NotImplementedError(f"'ingest' not implemented for {self.__class__.__name__}")
+
+    def start(self):
+        """
+        Start the Aggregator instance, opening any files / allocating resources
+        as necessary.
+        """
+        raise NotImplementedError(f"'start' not implemented for {self.__class__.__name__}")
+
+    def stop(self):
+        """
+        Stop the Aggregator instance, freeing all allocated resources.
+        """
+        raise NotImplementedError(f"'stop' not implemented for {self.__class__.__name__}")
+
+    def __enter__(self):
+        try:
+            self.start()
+            return self
+        except Exception:
+            self.close()
+            raise
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.stop()
+
+
+_submodules = [
+    info.name for info in _pkgutil.iter_modules(__spec__.submodule_search_locations)
+]
+
+__all__ = [*_submodules, Aggregator.__name__]  # noqa: PLE0604
+
+
+def __dir__():
+    return __all__
+
+
+# lazily import submodules
+def __getattr__(attr):
+    if attr in _submodules:
+        return _importlib.import_module(f".{attr}", __name__)
+    else:
+        raise AttributeError(f"module '{__name__}' has no attribute '{attr}'")

atex/{orchestrator/aggregator.py → aggregator/json.py}

@@ -4,8 +4,10 @@ import shutil
 import threading
 from pathlib import Path
 
+from . import Aggregator
 
-class JSONAggregator:
+
+class JSONAggregator(Aggregator):
     """
     Collects reported results as a GZIP-ed line-JSON and files (logs) from
     multiple test runs under a shared directory.
@@ -37,7 +39,7 @@ class JSONAggregator:
         self.json_file = Path(json_file)
         self.json_gzip_fobj = None
 
-    def open(self):
+    def start(self):
         if self.json_file.exists():
             raise FileExistsError(f"{self.json_file} already exists")
         self.json_gzip_fobj = gzip.open(self.json_file, "wt", newline="\n")
@@ -46,29 +48,12 @@ class JSONAggregator:
             raise FileExistsError(f"{self.storage_dir} already exists")
         self.storage_dir.mkdir()
 
-    def close(self):
+    def stop(self):
         if self.json_gzip_fobj:
             self.json_gzip_fobj.close()
             self.json_gzip_fobj = None
 
-    def __enter__(self):
-        try:
-            self.open()
-            return self
-        except Exception:
-            self.close()
-            raise
-
-    def __exit__(self, exc_type, exc_value, traceback):
-        self.close()
-
     def ingest(self, platform, test_name, results_file, files_dir):
-        """
-        Process 'results_file' (string/Path) for reported results and append
-        them to the overall aggregated line-JSON file, recursively copying over
-        the dir structure under 'files_dir' (string/Path) under the respective
-        platform and test name in the aggregated storage dir.
-        """
         platform_dir = self.storage_dir / platform
         test_dir = platform_dir / test_name.lstrip("/")
         if test_dir.exists():
@@ -92,7 +77,7 @@ class JSONAggregator:
                     platform,
                     result_line["status"],
                     test_name,
-                    result_line.get("name"),
+                    result_line.get("name"),  # subtest
                     file_names,
                     result_line.get("note"),
                 )

atex/cli/__init__.py

@@ -27,12 +27,18 @@ import pkgutil
 import argparse
 import logging
 
+from .. import util
+
 
 def setup_logging(level):
+    if level <= util.EXTRADEBUG:
+        fmt = "%(asctime)s %(name)s: %(filename)s:%(lineno)s: %(funcName)s(): %(message)s"
+    else:
+        fmt = "%(asctime)s %(name)s: %(message)s"
     logging.basicConfig(
         level=level,
         stream=sys.stderr,
-        format="%(asctime)s %(name)s: %(message)s",
+        format=fmt,
         datefmt="%Y-%m-%d %H:%M:%S",
     )
 
@@ -53,6 +59,10 @@ def main():
         "--debug", "-d", action="store_const", dest="loglevel", const=logging.DEBUG,
         help="enable extra debugging (logging.DEBUG)",
     )
+    log_grp.add_argument(
+        "--extra-debug", "-D", action="store_const", dest="loglevel", const=util.EXTRADEBUG,
+        help="enable extra debugging (atex.util.EXTRADEBUG)",
+    )
     log_grp.add_argument(
         "--quiet", "-q", action="store_const", dest="loglevel", const=logging.WARNING,
         help="be quiet during normal operation (logging.WARNING)",

atex/cli/libvirt.py

@@ -1,9 +1,10 @@
 import sys
 import re
 
-import libvirt
+from .. import util
+from ..provisioner.libvirt import locking
 
-from ..provision.libvirt import locking
+libvirt = util.import_libvirt()
 
 
 def _libvirt_open(url=None):

atex/cli/testingfarm.py

@@ -1,9 +1,10 @@
 import sys
 import json
 import pprint
+import collections
 
 from .. import util
-from ..provision.testingfarm import api as tf
+from ..provisioner.testingfarm import api as tf
 
 
 def _get_api(args):
@@ -36,7 +37,6 @@ def composes(args):
 def get_request(args):
     api = _get_api(args)
     request = tf.Request(args.request_id, api=api)
-    request.update()
     print(str(request))
 
 
@@ -79,6 +79,44 @@ def search_requests(args):
             print(f"{created} {req_id} : {envs_str}")
 
 
+def stats(args):
+    api = _get_api(args)
+
+    def top_users_repos(requests):
+        tokens = collections.defaultdict(int)
+        repos = collections.defaultdict(int)
+        for req in requests:
+            tokens[req["token_id"]] += 1
+            if "fmf" in req["test"] and req["test"]["fmf"]:
+                repos[req["test"]["fmf"]["url"]] += 1
+            elif "tmt" in req["test"] and req["test"]["tmt"]:
+                repos[req["test"]["tmt"]["url"]] += 1
+
+        print("Top 10 token IDs:")
+        for token_id in sorted(tokens, key=lambda x: tokens[x], reverse=True)[:10]:
+            count = tokens[token_id]
+            print(f"{count:>5}  {token_id}")
+
+        print("Top 10 repo URLs:")
+        for repo_url in sorted(repos, key=lambda x: repos[x], reverse=True)[:10]:
+            count = repos[repo_url]
+            print(f"{count:>5}  {repo_url}")
+
+    def chain_without_none(*iterables):
+        for itr in iterables:
+            if itr is None:
+                continue
+            for item in itr:
+                if item is not None:
+                    yield item
+
+    queued_and_running = chain_without_none(
+        api.search_requests(state="queued", ranch=args.ranch, mine=False),
+        api.search_requests(state="running", ranch=args.ranch, mine=False),
+    )
+    top_users_repos(queued_and_running)
+
+
 def reserve(args):
     util.info(f"Reserving {args.compose} on {args.arch} for {args.timeout} minutes")
 
@@ -106,7 +144,6 @@ def reserve(args):
         util.info(f"Got machine: {m}")
         while True:
             try:
-                res.request.update()
                 res.request.assert_alive()
             except tf.GoneAwayError as e:
                 print(e)
@@ -198,6 +235,12 @@ def parse_args(parser):
     cmd.add_argument("--after", help="only requests created after ISO8601")
     cmd.add_argument("--json", help="full details, one request per line", action="store_true")
 
+    cmd = cmds.add_parser(
+        "stats",
+        help="print out TF usage statistics",
+    )
+    cmd.add_argument("ranch", help="Testing Farm ranch name")
+
     cmd = cmds.add_parser(
         "reserve",
         help="reserve a system and ssh into it",
@@ -233,6 +276,8 @@ def main(args):
         cancel(args)
     elif args._cmd in ("search-requests", "sr"):
         search_requests(args)
+    elif args._cmd == "stats":
+        stats(args)
     elif args._cmd == "reserve":
         reserve(args)
     elif args._cmd in ("watch-pipeline", "wp"):

atex/connection/podman.py

@@ -8,11 +8,11 @@ from .. import util
 from . import Connection
 
 
-class PodmanConnError(ConnectionError):
+class PodmanConnectionError(ConnectionError):
     pass
 
 
-class PodmanConn(Connection):
+class PodmanConnection(Connection):
     """
     Implements the Connection API via 'podman container exec' on an
     already-running container, it does not handle any image pulling,
@@ -42,7 +42,6 @@ class PodmanConn(Connection):
     def cmd(self, command, *, func=util.subprocess_run, **func_args):
         return func(
             ("podman", "container", "exec", "-i", self.container, *command),
-            skip_frames=1,
             **func_args,
         )
 
@@ -56,7 +55,6 @@ class PodmanConn(Connection):
                 "-e", f"/bin/bash -c 'exec podman container exec -i {self.container} \"$@\"'",
                 *args,
             ),
-            skip_frames=1,
             check=True,
             stdin=subprocess.DEVNULL,
             **func_args,

atex/connection/ssh.py

@@ -133,16 +133,16 @@ def _rsync_host_cmd(*args, options, password=None, sudo=None):
     )
 
 
-class StatelessSSHConn(Connection):
+class StatelessSSHConnection(Connection):
     """
     Implements the Connection API using a ssh(1) client using "standalone"
     (stateless) logic - connect() and disconnect() are no-op, .cmd() simply
     executes the ssh client and .rsync() executes 'rsync -e ssh'.
 
-    Compared to ManagedSSHConn, this may be slow for many .cmd() calls,
+    Compared to ManagedSSHConnection, this may be slow for many .cmd() calls,
     but every call is stateless, there is no persistent connection.
 
-    If you need only one .cmd(), this will be faster than ManagedSSHConn.
+    If you need only one .cmd(), this will be faster than ManagedSSHConnection.
     """
 
     def __init__(self, options, *, password=None, sudo=None):
@@ -182,7 +182,6 @@ class StatelessSSHConn(Connection):
             unified_options["RemoteCommand"] = _shell_cmd(command, sudo=self.sudo)
         return func(
             _options_to_ssh(unified_options, password=self.password),
-            skip_frames=1,
             **func_args,
         )
 
@@ -197,7 +196,6 @@ class StatelessSSHConn(Connection):
                 password=self.password,
                 sudo=self.sudo,
             ),
-            skip_frames=1,
             check=True,
             stdin=subprocess.DEVNULL,
             **func_args,
@@ -216,17 +214,15 @@ class StatelessSSHConn(Connection):
 # checks .assert_master() and manually signals the running clients
 # when it gets DisconnectedError from it.
 
-class ManagedSSHConn(Connection):
+class ManagedSSHConnection(Connection):
     """
     Implements the Connection API using one persistently-running ssh(1) client
     started in a 'ControlMaster' mode, with additional ssh clients using that
     session to execute remote commands. Similarly, .rsync() uses it too.
 
-    This is much faster than StatelessSSHConn when executing multiple commands,
-    but contains a complex internal state (what if ControlMaster disconnects?).
-
-    Hence why this implementation provides extra non-standard-Connection methods
-    to manage this complexity.
+    This is much faster than StatelessSSHConnection when executing multiple
+    commands, but contains a complex internal state (what if ControlMaster
+    disconnects?).
     """
 
     # TODO: thread safety and locking via self.lock ?
@@ -351,7 +347,6 @@ class ManagedSSHConn(Connection):
         action = "forward" if not cancel else "cancel"
         util.subprocess_run(
             _options_to_ssh(options, extra_cli_flags=("-O", action)),
-            skip_frames=1,
             check=True,
         )
 
@@ -365,7 +360,6 @@ class ManagedSSHConn(Connection):
         unified_options["ControlPath"] = self._tmpdir / "control.sock"
         return func(
             _options_to_ssh(unified_options),
-            skip_frames=1,
             **func_args,
         )
 
@@ -381,7 +375,6 @@ class ManagedSSHConn(Connection):
                 options=unified_options,
                 sudo=self.sudo,
             ),
-            skip_frames=1,
             check=True,
             stdin=subprocess.DEVNULL,
             **func_args,

atex/executor/executor.py

@@ -26,9 +26,9 @@ class Executor:
     and uploaded files by those tests.
 
         tests_repo = "path/to/cloned/tests"
-        tests_data = atex.fmf.FMFTests(tests_repo, "/plans/default")
+        fmf_tests = atex.fmf.FMFTests(tests_repo, "/plans/default")
 
-        with Executor(tests_data, conn) as e:
+        with Executor(fmf_tests, conn) as e:
             e.upload_tests()
             e.plan_prepare()
             Path("output_here").mkdir()
@@ -42,12 +42,12 @@ class Executor:
 
         conn.cmd(["mkdir", "-p", "/shared"])
 
-        with Executor(tests_data, conn, state_dir="/shared") as e:
+        with Executor(fmf_tests, conn, state_dir="/shared") as e:
             e.upload_tests()
             e.plan_prepare()
 
         # in parallel (ie. threading or multiprocessing)
-        with Executor(tests_data, unique_conn, state_dir="/shared") as e:
+        with Executor(fmf_tests, unique_conn, state_dir="/shared") as e:
             e.run_test(...)
     """
 
@@ -74,7 +74,7 @@ class Executor:
         self.plan_env_file = None
         self.cancelled = False
 
-    def setup(self):
+    def start(self):
         with self.lock:
             state_dir = self.state_dir
 
@@ -107,7 +107,7 @@ class Executor:
         # create / truncate the TMT_PLAN_ENVIRONMENT_FILE
         self.conn.cmd(("truncate", "-s", "0", self.plan_env_file), check=True)
 
-    def cleanup(self):
+    def stop(self):
         with self.lock:
             work_dir = self.work_dir
 
@@ -121,14 +121,14 @@ class Executor:
 
     def __enter__(self):
         try:
-            self.setup()
+            self.start()
             return self
         except Exception:
-            self.cleanup()
+            self.stop()
             raise
 
     def __exit__(self, exc_type, exc_value, traceback):
-        self.cleanup()
+        self.stop()
 
     def cancel(self):
         with self.lock:
@@ -140,10 +140,10 @@ class Executor:
         __init__() inside 'fmf_tests', to the remote host.
         """
         self.conn.rsync(
-            "-rv" if util.in_debug_mode() else "-rq",
-            "--delete", "--exclude=.git/",
+            "-r", "--delete", "--exclude=.git/",
             f"{self.fmf_tests.root}/",
             f"remote:{self.tests_dir}",
+            func=util.subprocess_log,
         )
 
     def _run_prepare_scripts(self, scripts):
@@ -158,11 +158,10 @@ class Executor:
         for script in scripts:
             self.conn.cmd(
                 ("env", *env_args, "bash"),
+                func=util.subprocess_log,
+                stderr=subprocess.STDOUT,
                 input=script,
-                text=True,
                 check=True,
-                stdout=None if util.in_debug_mode() else subprocess.DEVNULL,
-                stderr=subprocess.STDOUT,
             )
 
     def plan_prepare(self):
@@ -180,9 +179,9 @@ class Executor:
                     "dnf", "-y", "--setopt=install_weak_deps=False",
                     "install", *self.fmf_tests.prepare_pkgs,
                 ),
-                check=True,
-                stdout=None if util.in_debug_mode() else subprocess.DEVNULL,
+                func=util.subprocess_log,
                 stderr=subprocess.STDOUT,
+                check=True,
             )
 
         # run 'prepare' scripts from the plan
@@ -336,7 +335,9 @@ class Executor:
                             reconnects += 1
                             state = self.State.STARTING_TEST
                         except BlockingIOError:
-                            pass
+                            # avoid 100% CPU spinning if the connection it too slow
+                            # to come up (ie. ssh ControlMaster socket file not created)
+                            time.sleep(0.5)
                         except ConnectionError:
                             # can happen when ie. ssh is connecting over a LocalForward port,
                             # causing 'read: Connection reset by peer' instead of timeout

atex/executor/scripts.py

@@ -1,7 +1,9 @@
+import os
 import collections
-import yaml
 from pathlib import Path
 
+import yaml
+
 from .. import util, fmf
 
 # name: fmf path to the test as string, ie. /some/test
@@ -50,7 +52,7 @@ def test_wrapper(*, test, tests_dir, test_exec):
     #       doing it here avoids unnecessary traffic (reading stdin) via ssh,
     #       even if it is fed from subprocess.DEVNULL on the runner
 
-    if util.in_debug_mode():
+    if os.environ.get("ATEX_DEBUG_TEST") == "1":
         out += "set -x\n"
 
     # use a subshell to limit the scope of the CWD change
@@ -122,7 +124,7 @@ def test_setup(*, test, wrapper_exec, test_exec, test_yaml, **kwargs):
     """
     out = "#!/bin/bash\n"
 
-    if util.in_debug_mode():
+    if os.environ.get("ATEX_DEBUG_TEST") == "1":
         out += "set -xe\n"
     else:
         out += "exec 1>/dev/null\n"

atex/executor/testcontrol.py

@@ -156,7 +156,7 @@ class TestControl:
         except BufferFullError as e:
             raise BadControlError(str(e)) from None
 
-        util.debug(f"got control line: {line} // eof: {self.stream.eof}")
+        util.extradebug(f"control line: {line} // eof: {self.stream.eof}")
 
         if self.stream.eof:
             self.eof = True

atex/orchestrator/__init__.py

@@ -1,3 +1,76 @@
-#from .aggregator import CSVAggregator, JSONAggregator  # noqa: F401
-from .aggregator import JSONAggregator  # noqa: F401
-from .orchestrator import Orchestrator, OrchestratorError, FailedSetupError  # noqa: F401
+import importlib as _importlib
+import pkgutil as _pkgutil
+import time as _time
+
+
+class OrchestratorError(Exception):
+    pass
+
+
+class Orchestrator:
+    """
+    A scheduler for parallel execution on multiple resources (machines/systems).
+
+    TODO: more description
+    """
+
+    def serve_once(self):
+        """
+        Run the orchestration logic, processing any outstanding requests
+        (for provisioning, new test execution, etc.) and returning once these
+        are taken care of.
+
+        Returns True to indicate that it should be called again by the user
+        (more work to be done), False once all testing is concluded.
+        """
+        raise NotImplementedError(f"'serve_once' not implemented for {self.__class__.__name__}")
+
+    def serve_forever(self):
+        """
+        Run the orchestration logic, blocking until all testing is concluded.
+        """
+        while self.serve_once():
+            _time.sleep(1)
+
+    def start(self):
+        """
+        Start the Orchestrator instance, opening any files / allocating
+        resources as necessary.
+        """
+        raise NotImplementedError(f"'start' not implemented for {self.__class__.__name__}")
+
+    def stop(self):
+        """
+        Stop the Orchestrator instance, freeing all allocated resources.
+        """
+        raise NotImplementedError(f"'stop' not implemented for {self.__class__.__name__}")
+
+    def __enter__(self):
+        try:
+            self.start()
+            return self
+        except Exception:
+            self.stop()
+            raise
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.stop()
+
+
+_submodules = [
+    info.name for info in _pkgutil.iter_modules(__spec__.submodule_search_locations)
+]
+
+__all__ = [*_submodules, Orchestrator.__name__]  # noqa: PLE0604
+
+
+def __dir__():
+    return __all__
+
+
+# lazily import submodules
+def __getattr__(attr):
+    if attr in _submodules:
+        return _importlib.import_module(f".{attr}", __name__)
+    else:
+        raise AttributeError(f"module '{__name__}' has no attribute '{attr}'")