atex 0.9__py3-none-any.whl → 0.11__py3-none-any.whl

atex/executor/executor.py

@@ -26,9 +26,9 @@ class Executor:
     and uploaded files by those tests.
 
         tests_repo = "path/to/cloned/tests"
-        tests_data = atex.fmf.FMFTests(tests_repo, "/plans/default")
+        fmf_tests = atex.fmf.FMFTests(tests_repo, "/plans/default")
 
-        with Executor(tests_data, conn) as e:
+        with Executor(fmf_tests, conn) as e:
             e.upload_tests()
             e.plan_prepare()
             Path("output_here").mkdir()
@@ -42,12 +42,12 @@ class Executor:
 
         conn.cmd(["mkdir", "-p", "/shared"])
 
-        with Executor(tests_data, conn, state_dir="/shared") as e:
+        with Executor(fmf_tests, conn, state_dir="/shared") as e:
             e.upload_tests()
             e.plan_prepare()
 
         # in parallel (ie. threading or multiprocessing)
-        with Executor(tests_data, unique_conn, state_dir="/shared") as e:
+        with Executor(fmf_tests, unique_conn, state_dir="/shared") as e:
             e.run_test(...)
     """
 
@@ -74,7 +74,7 @@ class Executor:
         self.plan_env_file = None
         self.cancelled = False
 
-    def setup(self):
+    def start(self):
         with self.lock:
             state_dir = self.state_dir
 
@@ -107,7 +107,7 @@ class Executor:
         # create / truncate the TMT_PLAN_ENVIRONMENT_FILE
         self.conn.cmd(("truncate", "-s", "0", self.plan_env_file), check=True)
 
-    def cleanup(self):
+    def stop(self):
         with self.lock:
             work_dir = self.work_dir
 
@@ -121,14 +121,14 @@ class Executor:
 
     def __enter__(self):
         try:
-            self.setup()
+            self.start()
             return self
         except Exception:
-            self.cleanup()
+            self.stop()
             raise
 
     def __exit__(self, exc_type, exc_value, traceback):
-        self.cleanup()
+        self.stop()
 
     def cancel(self):
         with self.lock:
@@ -140,10 +140,10 @@ class Executor:
         __init__() inside 'fmf_tests', to the remote host.
         """
         self.conn.rsync(
-            "-rv" if util.in_debug_mode() else "-rq",
-            "--delete", "--exclude=.git/",
+            "-r", "--delete", "--exclude=.git/",
             f"{self.fmf_tests.root}/",
             f"remote:{self.tests_dir}",
+            func=util.subprocess_log,
         )
 
     def _run_prepare_scripts(self, scripts):
@@ -153,16 +153,15 @@ class Executor:
             **self.env,
             "TMT_PLAN_ENVIRONMENT_FILE": self.plan_env_file,
         }
-        env_args = (f"{k}={v}" for k, v in env.items())
+        env_args = tuple(f"{k}={v}" for k, v in env.items())
         # run the scripts
         for script in scripts:
             self.conn.cmd(
-                ("env", *env_args, "bash"),
+                ("env", "-C", self.tests_dir, *env_args, "bash"),
+                func=util.subprocess_log,
+                stderr=subprocess.STDOUT,
                 input=script,
-                text=True,
                 check=True,
-                stdout=None if util.in_debug_mode() else subprocess.DEVNULL,
-                stderr=subprocess.STDOUT,
             )
 
     def plan_prepare(self):
@@ -180,9 +179,9 @@ class Executor:
                     "dnf", "-y", "--setopt=install_weak_deps=False",
                     "install", *self.fmf_tests.prepare_pkgs,
                 ),
-                check=True,
-                stdout=None if util.in_debug_mode() else subprocess.DEVNULL,
+                func=util.subprocess_log,
                 stderr=subprocess.STDOUT,
+                check=True,
             )
 
         # run 'prepare' scripts from the plan
@@ -336,7 +335,9 @@ class Executor:
                             reconnects += 1
                             state = self.State.STARTING_TEST
                         except BlockingIOError:
-                            pass
+                            # avoid 100% CPU spinning if the connection it too slow
+                            # to come up (ie. ssh ControlMaster socket file not created)
+                            time.sleep(0.5)
                         except ConnectionError:
                             # can happen when ie. ssh is connecting over a LocalForward port,
                             # causing 'read: Connection reset by peer' instead of timeout
@@ -386,7 +387,7 @@ class Executor:
                         pass
                     reporter.report({
                         "status": "infra",
-                        "note": repr(exception),
+                        "note": f"{type(exception).__name__}({exception})",
                         "testout": "output.txt",
                     })
 

atex/executor/scripts.py

@@ -1,7 +1,9 @@
+import os
 import collections
-import yaml
 from pathlib import Path
 
+import yaml
+
 from .. import util, fmf
 
 # name: fmf path to the test as string, ie. /some/test
@@ -50,7 +52,7 @@ def test_wrapper(*, test, tests_dir, test_exec):
     #       doing it here avoids unnecessary traffic (reading stdin) via ssh,
     #       even if it is fed from subprocess.DEVNULL on the runner
 
-    if util.in_debug_mode():
+    if os.environ.get("ATEX_DEBUG_TEST") == "1":
         out += "set -x\n"
 
     # use a subshell to limit the scope of the CWD change
@@ -122,7 +124,7 @@ def test_setup(*, test, wrapper_exec, test_exec, test_yaml, **kwargs):
     """
     out = "#!/bin/bash\n"
 
-    if util.in_debug_mode():
+    if os.environ.get("ATEX_DEBUG_TEST") == "1":
         out += "set -xe\n"
     else:
         out += "exec 1>/dev/null\n"

atex/executor/testcontrol.py

@@ -156,7 +156,7 @@ class TestControl:
         except BufferFullError as e:
             raise BadControlError(str(e)) from None
 
-        util.debug(f"got control line: {line} // eof: {self.stream.eof}")
+        util.extradebug(f"control line: {line} // eof: {self.stream.eof}")
 
         if self.stream.eof:
             self.eof = True

atex/orchestrator/__init__.py

@@ -1,3 +1,76 @@
-#from .aggregator import CSVAggregator, JSONAggregator  # noqa: F401
-from .aggregator import JSONAggregator  # noqa: F401
-from .orchestrator import Orchestrator, OrchestratorError, FailedSetupError  # noqa: F401
+import importlib as _importlib
+import pkgutil as _pkgutil
+import time as _time
+
+
+class OrchestratorError(Exception):
+    pass
+
+
+class Orchestrator:
+    """
+    A scheduler for parallel execution on multiple resources (machines/systems).
+
+    TODO: more description
+    """
+
+    def serve_once(self):
+        """
+        Run the orchestration logic, processing any outstanding requests
+        (for provisioning, new test execution, etc.) and returning once these
+        are taken care of.
+
+        Returns True to indicate that it should be called again by the user
+        (more work to be done), False once all testing is concluded.
+        """
+        raise NotImplementedError(f"'serve_once' not implemented for {self.__class__.__name__}")
+
+    def serve_forever(self):
+        """
+        Run the orchestration logic, blocking until all testing is concluded.
+        """
+        while self.serve_once():
+            _time.sleep(1)
+
+    def start(self):
+        """
+        Start the Orchestrator instance, opening any files / allocating
+        resources as necessary.
+        """
+        raise NotImplementedError(f"'start' not implemented for {self.__class__.__name__}")
+
+    def stop(self):
+        """
+        Stop the Orchestrator instance, freeing all allocated resources.
+        """
+        raise NotImplementedError(f"'stop' not implemented for {self.__class__.__name__}")
+
+    def __enter__(self):
+        try:
+            self.start()
+            return self
+        except Exception:
+            self.stop()
+            raise
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.stop()
+
+
+_submodules = [
+    info.name for info in _pkgutil.iter_modules(__spec__.submodule_search_locations)
+]
+
+__all__ = [*_submodules, Orchestrator.__name__]  # noqa: PLE0604
+
+
+def __dir__():
+    return __all__
+
+
+# lazily import submodules
+def __getattr__(attr):
+    if attr in _submodules:
+        return _importlib.import_module(f".{attr}", __name__)
+    else:
+        raise AttributeError(f"module '{__name__}' has no attribute '{attr}'")