atex 0.8__py3-none-any.whl → 0.10__py3-none-any.whl

atex/util/threads.py

@@ -1,48 +1,85 @@
-import collections
 import queue
 import threading
 
+from .named_mapping import NamedMapping
+
 # TODO: documentation; this is like concurrent.futures, but with daemon=True support
 
 
 class ThreadQueue:
-    ThreadReturn = collections.namedtuple("ThreadReturn", ("thread", "returned", "exception"))
+    class ThreadReturn(NamedMapping, required=("thread", "returned", "exception")):
+        pass
+
     Empty = queue.Empty
 
     def __init__(self, daemon=False):
+        self.lock = threading.RLock()
         self.queue = queue.SimpleQueue()
         self.daemon = daemon
         self.threads = set()
 
-    def _wrapper(self, func, *args, **kwargs):
+    def _wrapper(self, func, func_args, func_kwargs, **user_kwargs):
         current_thread = threading.current_thread()
         try:
-            ret = func(*args, **kwargs)
-            result = self.ThreadReturn(current_thread, ret, None)
+            ret = func(*func_args, **func_kwargs)
+            result = self.ThreadReturn(
+                thread=current_thread,
+                returned=ret,
+                exception=None,
+                **user_kwargs,
+            )
         except Exception as e:
-            result = self.ThreadReturn(current_thread, None, e)
+            result = self.ThreadReturn(
+                thread=current_thread,
+                returned=None,
+                exception=e,
+                **user_kwargs,
+            )
         self.queue.put(result)
 
-    def start_thread(self, target, name=None, args=None, kwargs=None):
-        args = args or ()
-        kwargs = kwargs or {}
+    def start_thread(self, target, target_args=None, target_kwargs=None, **user_kwargs):
+        """
+        Start a new thread and call 'target' as a callable inside it, passing it
+        'target_args' as arguments and 'target_kwargs' as keyword arguments.
+
+        Any additional 'user_kwargs' specified are NOT passed to the callable,
+        but instead become part of the ThreadReturn namespace returned by the
+        .get_raw() method.
+        """
         t = threading.Thread(
             target=self._wrapper,
-            name=name,
-            args=(target, *args),
-            kwargs=kwargs,
+            args=(target, target_args or (), target_kwargs or {}),
+            kwargs=user_kwargs,
             daemon=self.daemon,
         )
+        with self.lock:
+            self.threads.add(t)
         t.start()
-        self.threads.add(t)
+
+    def get_raw(self, block=True, timeout=None):
+        """
+        Wait for and return the next available ThreadReturn instance on the
+        queue, as enqueued by a finished callable started by the .start_thread()
+        method.
+        """
+        with self.lock:
+            if block and timeout is None and not self.threads:
+                raise AssertionError("no threads are running, would block forever")
+        treturn = self.queue.get(block=block, timeout=timeout)
+        with self.lock:
+            self.threads.remove(treturn.thread)
+        return treturn
 
     # get one return value from any thread's function, like .as_completed()
     # or concurrent.futures.FIRST_COMPLETED
     def get(self, block=True, timeout=None):
-        if block and timeout is None and not self.threads:
-            raise AssertionError("no threads are running, would block forever")
-        treturn = self.queue.get(block=block, timeout=timeout)
-        self.threads.remove(treturn.thread)
+        """
+        Wait for and return the next available return value of a callable
+        enqueued via the .start_thread() method.
+
+        If the callable raised an exception, the exception is re-raised here.
+        """
+        treturn = self.get_raw(block, timeout)
         if treturn.exception is not None:
             raise treturn.exception
         else:
@@ -50,6 +87,20 @@ class ThreadQueue:
 
     # wait for all threads to finish (ignoring queue contents)
     def join(self):
-        while self.threads:
-            t = self.threads.pop()
-            t.join()
+        """
+        Wait for all threads to finish, ignoring the state of the queue.
+        """
+        while True:
+            with self.lock:
+                try:
+                    thread = self.threads.pop()
+                except KeyError:
+                    break
+            thread.join()
+
+    def qsize(self):
+        """
+        Return the amount of elements .get() can retrieve before it raises
+        queue.Empty.
+        """
+        return self.queue.qsize()

atex-0.10.dist-info/METADATA

@@ -0,0 +1,86 @@
+Metadata-Version: 2.4
+Name: atex
+Version: 0.10
+Summary: Ad-hoc Test EXecutor
+Project-URL: Homepage, https://github.com/RHSecurityCompliance/atex
+License-Expression: GPL-3.0-or-later
+License-File: COPYING.txt
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.11
+Requires-Dist: fmf>=1.6
+Requires-Dist: pyyaml
+Requires-Dist: urllib3<3,>=2
+Description-Content-Type: text/markdown
+
+# ATEX = Ad-hoc Test EXecutor
+
+A collections of Python APIs to provision operating systems, collect
+and execute [FMF](https://github.com/teemtee/fmf/)-style tests, gather
+and organize their results and generate reports from those results.
+
+The name comes from a (fairly unique to FMF/TMT ecosystem) approach that
+allows provisioning a pool of systems and scheduling tests on them as one would
+on an ad-hoc pool of thread/process workers - once a worker becomes free,
+it receives a test to run.  
+This is in contrast to splitting a large list of N tests onto M workers
+like N/M, which yields significant time penalties due to tests having
+very varies runtimes.
+
+Above all, this project is meant to be a toolbox, not a silver-plate solution.
+Use its Python APIs to build a CLI tool for your specific use case.  
+The CLI tool provided here is just for demonstration / testing, not for serious
+use - we want to avoid huge modular CLIs for Every Possible Scenario. That's
+the job of the Python API. Any CLI should be simple by nature.
+
+---
+
+## License
+
+Unless specified otherwise, any content within this repository is distributed
+under the GNU GPLv3 license, see the [COPYING.txt](COPYING.txt) file for more.
+
+## Environment variables
+
+- `ATEX_DEBUG_TEST`
+  - Set to `1` to print out detailed runner-related trace within the test output
+    stream (as if it was printed out by the test).
+
+## Testing this project
+
+There are some limited sanity tests provided via `pytest`, although:
+
+- Some require additional variables (ie. Testing Farm) and will ERROR
+  without them.
+- Some take a long time (ie. Testing Farm) due to system provisioning
+  taking a long time, so install `pytest-xdist` and run with a large `-n`.
+
+Currently, the recommended approach is to split the execution:
+
+```
+# synchronously, because podman CLI has concurrency issues
+pytest tests/provision/test_podman.py
+
+# in parallel, because provisioning takes a long time
+export TESTING_FARM_API_TOKEN=...
+export TESTING_FARM_COMPOSE=...
+pytest -n 20 tests/provision/test_podman.py
+
+# fast enough for synchronous execution
+pytest tests/fmf
+```
+
+## Unsorted notes
+
+TODO: codestyle from contest
+
+```
+- this is not tmt, the goal is to make a python toolbox *for* making runcontest
+  style tools easily, not to replace those tools with tmt-style CLI syntax
+
+  - the whole point is to make usecase-targeted easy-to-use tools that don't
+    intimidate users with 1 KB long command line, and runcontest is a nice example
+
+  - TL;DR - use a modular pythonic approach, not a gluetool-style long CLI
+```

atex-0.10.dist-info/RECORD

@@ -0,0 +1,44 @@
+atex/__init__.py,sha256=LdX67gprtHYeAkjLhFPKzpc7ECv2rHxUbHKDGbGXO1c,517
+atex/fmf.py,sha256=gkJXIaRO7_KvwJR-V6Tc1NVn4a9Hq7hoBLQLhxYIdbg,8834
+atex/aggregator/__init__.py,sha256=uNnYSyDGXjknxckI8MFfl-C8_gin8FwQchiq-UOyP6I,1744
+atex/aggregator/json.py,sha256=x1zim9O2olzBh185NYWo5N96fixB2oxCamoOZwmgR9w,3330
+atex/cli/__init__.py,sha256=X5XxkDEDXE4tJAjwt5ShRHCFTXDK-2zvxQ34opmueUc,2768
+atex/cli/fmf.py,sha256=HfbTgFbCwK4Nuyq6vtGutcq_4-4kj-tmoqzXUn3AYtY,3573
+atex/cli/libvirt.py,sha256=6tt5ANb8XBBRXOQsYPTWILThKqf-gvt5AZh5Dctg2PA,3782
+atex/cli/testingfarm.py,sha256=HGlqrkhanUMo2CqKxmM3ACgptWtxm0gICyEGf7O6Qc0,9078
+atex/connection/__init__.py,sha256=dj8ZBcEspom7Z_UjecfLGBRNvLZ3dyGR9q19i_B4xpY,3880
+atex/connection/podman.py,sha256=1T56gh1TgbcQWpTIJHL4NaxZOI6aMg7Xp7sn6PQQyBk,1911
+atex/connection/ssh.py,sha256=9A57b9YR_HI-kIu06Asic1y__JPVXEheDZxjbG2Qcsc,13460
+atex/executor/__init__.py,sha256=XCfhi7QDELjey7N1uzhMjc46Kp1Jsd5bOCf52I27SCE,85
+atex/executor/duration.py,sha256=x06sItKOZi6XA8KszQwZGpIb1Z_L-HWqIwZKo2SDo0s,1759
+atex/executor/executor.py,sha256=JLFR9cZjSlUdAlAlLct6WuzmYbtjGtSobxvsToQum6M,15738
+atex/executor/reporter.py,sha256=MceFmHFt0bTEClBZbRI1WnFbfMhR0e1noOzcu7gjKuQ,3403
+atex/executor/scripts.py,sha256=riJAQWsV-BFGkJwR2Dmf3R0ZRRZJs9w9iYnPpYaQNaE,5618
+atex/executor/testcontrol.py,sha256=mVrLwQUnDRfUq-5diz-80UvCWWxn1TkcBgmAKhKNb5E,12696
+atex/orchestrator/__init__.py,sha256=8Q1YknyibilXLjWRYkHm_Mr2HMm0SRw8Zv39KypeASM,2059
+atex/orchestrator/adhoc.py,sha256=GnvHLlCHeJ_nQ8doEjMuDzqmu4XZorI7ZzOtG_C08tU,18451
+atex/provisioner/__init__.py,sha256=2eepmEznq94tbam9VSWbsGFrZZpWeNSVlsTczGxjNuQ,4667
+atex/provisioner/libvirt/VM_PROVISION,sha256=7pkZ-ozgTyK4qNGC-E-HUznr4IhbosWSASbB72Gknl8,2664
+atex/provisioner/libvirt/__init__.py,sha256=pKG5IpZSC2IHs5wL2ecQx_fd9AzAXEbZmDzA7RyZsfM,119
+atex/provisioner/libvirt/libvirt.py,sha256=rtxowv5DpgcWsGRXYF29n6S9x_cgXRVgqY41DiFu920,18431
+atex/provisioner/libvirt/locking.py,sha256=AXtDyidZNmUoMmrit26g9iTHDqInrzL_RSQEoc_EAXw,5669
+atex/provisioner/libvirt/setup-libvirt.sh,sha256=oCMy9SCnbC_QuAzO2sFwvB5ui1kMQ6uviHsgdXyoFXc,2428
+atex/provisioner/podman/__init__.py,sha256=dM0JzQXWX7edtWSc0KH0cMFXAjArFn2Vme4j_ZMsdYA,138
+atex/provisioner/podman/podman.py,sha256=ztRypoakSf-jF04iER58tEMUZ4Y6AuzIpNpFXp44bB4,4997
+atex/provisioner/testingfarm/__init__.py,sha256=kZncgLGdRCR4FMaRQr2GTwJ8vjlA-24ri8JO2ueZJuw,113
+atex/provisioner/testingfarm/api.py,sha256=UcMN61nBr3wqEd5KSR5Xhv1-TS7nSPFvk2byb6PdIs8,21811
+atex/provisioner/testingfarm/testingfarm.py,sha256=OI-a99xALaiYf-y5037WFVxY1g2H2y1xEKxHBdUQvfg,8271
+atex/util/__init__.py,sha256=cWHFbtQ4mDlKe6lXyPDWRmWJOTcHDGfVuW_-GYa8hB0,1473
+atex/util/dedent.py,sha256=SEuJMtLzqz3dQ7g7qyZzEJ9VYynVlk52tQCJY-FveXo,603
+atex/util/libvirt.py,sha256=kDZmT6xLYEZkQNLZY98gJ2M48DDWXxHF8rQY9PnjB3U,660
+atex/util/log.py,sha256=KVR7ep8n5wtghsvBFCtHiPsMAQBdAmK83E_Jec5t4cU,2230
+atex/util/named_mapping.py,sha256=UBMe9TetjV-DGPhjYjJ42YtC40FVPKAAEROXl9MA5fo,4700
+atex/util/path.py,sha256=x-kXqiWCVodfZWbEwtC5A8LFvutpDIPYv2m0boZSlXU,504
+atex/util/ssh_keygen.py,sha256=9yuSl2yBV7pG3Qfsf9tossVC00nbIUrAeLdbwTykpjk,384
+atex/util/subprocess.py,sha256=PQBxcQJPapP1ZLO4LqENyrxxCbNAxtJDNNlBV5DcD9k,2953
+atex/util/threads.py,sha256=c8hsEc-8SqJGodInorv_6JxpiHiSkGFGob4qbMmOD2M,3531
+atex-0.10.dist-info/METADATA,sha256=evOBYvVboY2T8eGrAKy64UpyeuFKSMLGz8pUz8Sstm8,3050
+atex-0.10.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+atex-0.10.dist-info/entry_points.txt,sha256=pLqJdcfeyQTgup2h6dWb6SvkHhtOl-W5Eg9zV8moK0o,39
+atex-0.10.dist-info/licenses/COPYING.txt,sha256=oEuj51jdmbXcCUy7pZ-KE0BNcJTR1okudRp5zQ0yWnU,670
+atex-0.10.dist-info/RECORD,,

atex/orchestrator/aggregator.py

@@ -1,106 +0,0 @@
-import csv
-import gzip
-import json
-import shutil
-import threading
-from pathlib import Path
-
-
-class CSVAggregator:
-    """
-    Collects reported results as a GZIP-ed CSV and files (logs) from multiple
-    test runs under a shared directory.
-    """
-
-    class _ExcelWithUnixNewline(csv.excel):
-        lineterminator = "\n"
-
-    def __init__(self, csv_file, storage_dir):
-        """
-        'csv_file' is a string/Path to a .csv.gz file with aggregated results.
-
-        'storage_dir' is a string/Path of the top-level parent for all
-        per-platform / per-test files uploaded by tests.
-        """
-        self.lock = threading.RLock()
-        self.storage_dir = Path(storage_dir)
-        self.csv_file = Path(csv_file)
-        self.csv_writer = None
-        self.results_gzip_handle = None
-
-    def open(self):
-        if self.csv_file.exists():
-            raise FileExistsError(f"{self.csv_file} already exists")
-        f = gzip.open(self.csv_file, "wt", newline="")
-        try:
-            self.csv_writer = csv.writer(f, dialect=self._ExcelWithUnixNewline)
-        except:
-            f.close()
-            raise
-        self.results_gzip_handle = f
-
-        if self.storage_dir.exists():
-            raise FileExistsError(f"{self.storage_dir} already exists")
-        self.storage_dir.mkdir()
-
-    def close(self):
-        self.results_gzip_handle.close()
-        self.results_gzip_handle = None
-        self.csv_writer = None
-
-    def __enter__(self):
-        self.open()
-        return self
-
-    def __exit__(self, exc_type, exc_value, traceback):
-        self.close()
-
-    def ingest(self, platform, test_name, json_file, files_dir):
-        """
-        Process 'json_file' (string/Path) for reported results and append them
-        to the overall aggregated CSV file, recursively copying over the dir
-        structure under 'files_dir' (string/Path) under the respective platform
-        and test name in the aggregated files storage dir.
-        """
-        # parse the JSON separately, before writing any CSV lines, to ensure
-        # that either all results from the test are ingested, or none at all
-        # (if one of the lines contains JSON errors)
-        csv_lines = []
-        with open(json_file) as json_fobj:
-            for raw_line in json_fobj:
-                result_line = json.loads(raw_line)
-
-                result_name = result_line.get("name")
-                if result_name:
-                    # sub-result; prefix test name
-                    result_name = f"{test_name}/{result_name}"
-                else:
-                    # result for test itself; use test name
-                    result_name = test_name
-
-                file_names = []
-                if "testout" in result_line:
-                    file_names.append(result_line["testout"])
-                if "files" in result_line:
-                    file_names += (f["name"] for f in result_line["files"])
-
-                csv_lines.append((
-                    platform,
-                    result_line["status"],
-                    result_name,
-                    result_line.get("note", ""),
-                    *file_names,
-                ))
-
-        with self.lock:
-            self.csv_writer.writerows(csv_lines)
-            self.results_gzip_handle.flush()
-
-        Path(json_file).unlink()
-
-        platform_dir = self.storage_dir / platform
-        platform_dir.mkdir(exist_ok=True)
-        test_dir = platform_dir / test_name.lstrip("/")
-        if test_dir.exists():
-            raise FileExistsError(f"{test_dir} already exists for {test_name}")
-        shutil.move(files_dir, test_dir, copy_function=shutil.copy)

atex/orchestrator/orchestrator.py

@@ -1,324 +0,0 @@
-import time
-import tempfile
-import traceback
-import concurrent
-import collections
-from pathlib import Path
-
-from .. import util, executor
-
-
-class Orchestrator:
-    """
-    A scheduler for parallel execution on multiple resources (machines/systems).
-    """
-
-    SetupInfo = collections.namedtuple(
-        "SetupInfo",
-        (
-            # class Provisioner instance this machine is provided by
-            # (for logging purposes)
-            "provisioner",
-            # class Remote instance returned by the Provisioner
-            "remote",
-            # class Executor instance uploading tests / running setup or tests
-            "executor",
-        ),
-    )
-    RunningInfo = collections.namedtuple(
-        "RunningInfo",
-        (
-            # "inherit" from SetupInfo
-            *SetupInfo._fields,
-            # string with /test/name
-            "test_name",
-            # class tempfile.TemporaryDirectory instance with 'json_file' and 'files_dir'
-            "tmp_dir",
-        ),
-    )
-    FinishedInfo = collections.namedtuple(
-        "FinishedInfo",
-        (
-            # "inherit" from RunningInfo
-            *RunningInfo._fields,
-            # integer with exit code of the test
-            # (None if exception happened)
-            "exit_code",
-            # exception class instance if running the test failed
-            # (None if no exception happened (exit_code is defined))
-            "exception",
-        ),
-    )
-
-    def __init__(self, platform, fmf_tests, provisioners, aggregator, tmp_dir, *, max_reruns=2):
-        """
-        'platform' is a string with platform name.
-
-        'fmf_tests' is a class FMFTests instance of the tests to run.
-
-        'provisioners' is an iterable of class Provisioner instances.
-
-        'aggregator' is a class CSVAggregator instance.
-
-        'tmp_dir' is a string/Path to a temporary directory, to be used for
-        storing per-test results and uploaded files before being ingested
-        by the aggregator. Can be safely shared by Orchestrator instances.
-        """
-        self.platform = platform
-        self.fmf_tests = fmf_tests
-        self.provisioners = tuple(provisioners)
-        self.aggregator = aggregator
-        self.tmp_dir = tmp_dir
-        # tests still waiting to be run
-        self.to_run = set(fmf_tests.tests)
-        # running setup functions, as a list of SetupInfo items
-        self.running_setups = []
-        # running tests as a dict, indexed by test name, with RunningInfo values
-        self.running_tests = {}
-        # indexed by test name, value being integer of how many times
-        self.reruns = collections.defaultdict(lambda: max_reruns)
-        # thread queue for actively running tests
-        self.test_queue = util.ThreadQueue(daemon=False)
-        # thread queue for remotes being set up (uploading tests, etc.)
-        self.setup_queue = util.ThreadQueue(daemon=True)
-        # NOTE: running_setups and test_running are just for debugging and
-        #       cancellation, the execution flow itself uses ThreadQueues
-
-    @staticmethod
-    def _run_setup(sinfo):
-        sinfo.executor.setup()
-        sinfo.executor.upload_tests()
-        sinfo.executor.setup_plan()
-        # NOTE: we never run executor.cleanup() anywhere - instead, we assume
-        #       the remote (and its connection) was invalidated by the test,
-        #       so we just rely on remote.release() destroying the system
-        return sinfo
-
-    @classmethod
-    def _wrap_test(cls, rinfo, func, *args, **kwargs):
-        """
-        Wrap 'func' (test execution function) to preserve extra metadata
-        ('rinfo') and return it with the function return value.
-        """
-        try:
-            return cls.FinishedInfo(*rinfo, func(*args, **kwargs), None)
-        except Exception as e:
-            return cls.FinishedInfo(*rinfo, None, e)
-
-    def _run_new_test(self, sinfo):
-        """
-        'sinfo' is a SetupInfo instance.
-        """
-        next_test_name = self.next_test(self.to_run, self.fmf_tests)
-        assert next_test_name in self.to_run, "next_test() returned valid test name"
-
-        self.to_run.remove(next_test_name)
-
-        rinfo = self.RunningInfo(
-            *sinfo,
-            test_name=next_test_name,
-            tmp_dir=tempfile.TemporaryDirectory(
-                prefix=next_test_name.strip("/").replace("/","-") + "-",
-                dir=self.tmp_dir,
-                delete=False,
-            ),
-        )
-
-        tmp_dir_path = Path(rinfo.tmp_dir.name)
-        self.test_queue.start_thread(
-            target=self._wrap_test,
-            args=(
-                rinfo,
-                sinfo.executor.run_test,
-                next_test_name,
-                tmp_dir_path / "json_file",
-                tmp_dir_path / "files_dir",
-            ),
-        )
-
-        self.running_tests[next_test_name] = rinfo
-
-    def _process_finished_test(self, finfo):
-        """
-        'finfo' is a FinishedInfo instance.
-        """
-        test_id = f"'{finfo.test_name}' on '{finfo.remote}'"
-        tmp_dir_path = Path(finfo.tmp_dir.name)
-
-        # NOTE: document that we intentionally don't .cleanup() executioner below,
-        #       we rely on remote .release() destroying the OS, because we don't
-        #       want to risk .cleanup() blocking on dead ssh into the remote after
-        #       executing a destructive test
-
-        destructive = False
-
-        # if executor (or test) threw exception, schedule a re-run
-        if finfo.exception:
-            destructive = True
-            exc_str = "".join(traceback.format_exception(finfo.exception)).rstrip("\n")
-            util.info(f"unexpected exception happened while running {test_id}:\n{exc_str}")
-            finfo.remote.release()
-            if self.reruns[finfo.test_name] > 0:
-                self.reruns[finfo.test_name] -= 1
-                self.to_run.add(finfo.test_name)
-            else:
-                util.info(f"reruns for {test_id} exceeded, ignoring it")
-
-        # if the test exited as non-0, try a re-run
-        elif finfo.exit_code != 0:
-            destructive = True
-            finfo.remote.release()
-            if self.reruns[finfo.test_name] > 0:
-                util.info(
-                    f"{test_id} exited with non-zero: {finfo.exit_code}, re-running "
-                    f"({self.reruns[finfo.test_name]} reruns left)",
-                )
-                self.reruns[finfo.test_name] -= 1
-                self.to_run.add(finfo.test_name)
-            else:
-                util.info(
-                    f"{test_id} exited with non-zero: {finfo.exit_code}, "
-                    "all reruns exceeded, giving up",
-                )
-                # record the final result anyway
-                self.aggregator.ingest(
-                    self.platform,
-                    finfo.test_name,
-                    tmp_dir_path / "json_file",
-                    tmp_dir_path / "files_dir",
-                )
-                finfo.tmp_dir.cleanup()
-
-        # test finished successfully - ingest its results
-        else:
-            util.info(f"{test_id} finished successfully")
-            self.aggregator.ingest(
-                self.platform,
-                finfo.test_name,
-                tmp_dir_path / "json_file",
-                tmp_dir_path / "files_dir",
-            )
-            finfo.tmp_dir.cleanup()
-
-        # if the remote was not destroyed by traceback / failing test,
-        # check if the test always destroys it (even on success)
-        if not destructive:
-            test_data = self.fmf_tests.tests[finfo.test_name]
-            destructive = test_data.get("extra-atex", {}).get("destructive", False)
-
-        # if destroyed, release the remote
-        if destructive:
-            util.debug(f"{test_id} was destructive, releasing remote")
-            finfo.remote.release()
-
-        # if still not destroyed, run another test on it
-        # (without running plan setup, re-using already set up remote)
-        elif self.to_run:
-            sinfo = self.SetupInfo(
-                provisioner=finfo.provisioner,
-                remote=finfo.remote,
-                executor=finfo.executor,
-            )
-            util.debug(f"{test_id} was non-destructive, running next test")
-            self._run_new_test(sinfo)
-
-    def serve_once(self):
-        """
-        Run the orchestration logic, processing any outstanding requests
-        (for provisioning, new test execution, etc.) and returning once these
-        are taken care of.
-
-        Returns True to indicate that it should be called again by the user
-        (more work to be done), False once all testing is concluded.
-        """
-        util.debug(
-            f"to_run: {len(self.to_run)} tests / "
-            f"running: {len(self.running_tests)} tests, {len(self.running_setups)} setups",
-        )
-        # all done
-        if not self.to_run and not self.running_tests:
-            return False
-
-        # process all finished tests, potentially reusing remotes for executing
-        # further tests
-        while True:
-            try:
-                finfo = self.test_queue.get(block=False)
-            except util.ThreadQueue.Empty:
-                break
-            del self.running_tests[finfo.test_name]
-            self._process_finished_test(finfo)
-
-        # process any remotes with finished plan setup (uploaded tests,
-        # plan-defined pkgs / prepare scripts), start executing tests on them
-        while True:
-            try:
-                sinfo = self.setup_queue.get(block=False)
-            except util.ThreadQueue.Empty:
-                break
-            util.debug(f"setup finished for '{sinfo.remote}', running first test")
-            self.running_setups.remove(sinfo)
-            self._run_new_test(sinfo)
-
-        # try to get new remotes from Provisioners - if we get some, start
-        # running setup on them
-        for provisioner in self.provisioners:
-            while (remote := provisioner.get_remote(block=False)) is not None:
-                ex = executor.Executor(self.fmf_tests, remote)
-                sinfo = self.SetupInfo(
-                    provisioner=provisioner,
-                    remote=remote,
-                    executor=ex,
-                )
-                self.setup_queue.start_thread(
-                    target=self._run_setup,
-                    args=(sinfo,),
-                )
-                self.running_setups.append(sinfo)
-                util.debug(f"got remote '{remote}' from '{provisioner}', running setup")
-
-        return True
-
-    def serve_forever(self):
-        """
-        Run the orchestration logic, blocking until all testing is concluded.
-        """
-        while self.serve_once():
-            time.sleep(1)
-
-    def __enter__(self):
-        # start all provisioners
-        for prov in self.provisioners:
-            prov.start()
-        return self
-
-    def __exit__(self, exc_type, exc_value, traceback):
-        # cancel all running tests and wait for them to clean up (up to 0.1sec)
-        for rinfo in self.running_tests.values():
-            rinfo.executor.cancel()
-        self.test_queue.join()  # also ignore any exceptions raised
-
-        # stop all provisioners, also releasing all remotes
-        with concurrent.futures.ThreadPoolExecutor(max_workers=20) as ex:
-            for provisioner in self.provisioners:
-                for func in provisioner.stop_defer():
-                    ex.submit(func)
-
-    def next_test(self, tests, fmf_tests):  # noqa: ARG002, PLR6301
-        """
-        Return a test name (string) from a set of 'tests' (set of test name
-        strings) to be run next.
-
-        'fmf_tests' is a class FMFTests instance with additional test metadata.
-
-        This method is user-overridable, ie. by subclassing Orchestrator:
-
-            class CustomOrchestrator(Orchestrator):
-                @staticmethod
-                def next_test(tests):
-                    ...
-        """
-        # TODO: more advanced algorithm
-        #
-        # simple:
-        return next(iter(tests))