atex 0.11__py3-none-any.whl → 0.13__py3-none-any.whl

atex/cli/__init__.py

@@ -93,4 +93,4 @@ def main():
     try:
         mains[args._module](args)
     except KeyboardInterrupt:
-        raise SystemExit() from None
+        raise SystemExit from None

atex/cli/testingfarm.py

@@ -2,7 +2,6 @@ import sys
 import json
 import pprint
 import collections
-from datetime import datetime, timedelta, UTC
 
 from .. import util
 from ..provisioner.testingfarm import api as tf
@@ -32,7 +31,8 @@ def composes(args):
     comps = api.composes(ranch=args.ranch)
     comps_list = comps["composes"]
     for comp in comps_list:
-        print(comp["name"])
+        if comp["type"] == "compose":
+            print(comp["name"])
 
 
 def get_request(args):
@@ -48,36 +48,63 @@ def cancel(args):
 
 def search_requests(args):
     api = _get_api(args)
-    reply = api.search_requests(
-        state=args.state,
-        mine=not args.all,
-        user_id=args.user_id,
-        token_id=args.token_id,
-        ranch=args.ranch,
-        created_before=args.before,
-        created_after=args.after,
-    )
-    if not reply:
-        return
 
-    if args.json:
-        for req in sorted(reply, key=lambda x: x["created"]):
-            print(json.dumps(req))
+    func_kwargs = {
+        "mine": not args.all,
+        "user_id": args.user_id,
+        "token_id": args.token_id,
+        "ranch": args.ranch,
+        "created_before": args.before,
+        "created_after": args.after,
+    }
+
+    if args.page is not None:
+        reply = api.search_requests_paged(
+            state=args.state,
+            page=args.page,
+            **func_kwargs,
+        )
+        if not reply:
+            return
     else:
-        for req in sorted(reply, key=lambda x: x["created"]):
-            req_id = req["id"]
-            created = req["created"].partition(".")[0]
+        reply = api.search_requests(
+            state=args.state,
+            **func_kwargs,
+        )
+        if not reply:
+            return
+        reply = sorted(reply, key=lambda x: x["created"])
 
-            envs = []
-            for env in req["environments_requested"]:
-                if "os" in env and env["os"] and "compose" in env["os"]:
-                    compose = env["os"]["compose"]
-                    arch = env["arch"]
-                    if compose and arch:
-                        envs.append(f"{compose}@{arch}")
-            envs_str = ", ".join(envs)
+    if args.json:
+        for req in reply:
+            print(json.dumps(req))
+        return
 
-            print(f"{created} {req_id} : {envs_str}")
+    for req in reply:
+        req_id = req["id"]
+        created = req["created"].partition(".")[0]
+
+        if "fmf" in req["test"] and req["test"]["fmf"]:
+            test = req["test"]["fmf"]["url"]
+        elif "tmt" in req["test"] and req["test"]["tmt"]:
+            test = req["test"]["tmf"]["url"]
+        else:
+            test = ""
+
+        envs = []
+        for env in req["environments_requested"]:
+            if "os" in env and env["os"] and "compose" in env["os"]:
+                compose = env["os"]["compose"]
+                arch = env["arch"]
+                if compose and arch:
+                    envs.append(f"{compose}@{arch}")
+
+        print(f"{created} {req_id}", end="")
+        if test:
+            print(f" | test:{test}", end="")
+        if envs:
+            print(f" | envs:[{', '.join(envs)}]", end="")
+        print()
 
 
 def stats(args):
@@ -110,35 +137,35 @@ def stats(args):
             print(f"{count:>{digits}}  {repo_url}")
 
     def request_search_results():
-        for state in args.states.split(","):
-            result = api.search_requests(
-                state=state,
-                ranch=args.ranch,
-                mine=False,
-            )
-            if result:
-                yield from result
-
-    def multiday_request_search_results():
-        now = datetime.now(UTC)
-        for day in range(0,args.days):
-            before = now - timedelta(days=day)
-            after = now - timedelta(days=day+1)
+        func_kwargs = {}
+        if args.before is not None:
+            func_kwargs["created_before"] = args.before
+        if args.after is not None:
+            func_kwargs["created_after"] = args.after
+
+        if args.page is not None:
             for state in args.states.split(","):
-                result = api.search_requests(
+                reply = api.search_requests_paged(
                     state=state,
-                    created_before=before.replace(microsecond=0).isoformat(),
-                    created_after=after.replace(microsecond=0).isoformat(),
+                    page=args.page,
+                    mine=False,
                     ranch=args.ranch,
+                    **func_kwargs,
+                )
+                if reply:
+                    yield from reply
+        else:
+            for state in args.states.split(","):
+                reply = api.search_requests(
+                    state=state,
                     mine=False,
+                    ranch=args.ranch,
+                    **func_kwargs,
                 )
-                if result:
-                    yield from result
+                if reply:
+                    yield from reply
 
-    if args.days is not None:
-        top_users_repos(multiday_request_search_results())
-    else:
-        top_users_repos(request_search_results())
+    top_users_repos(request_search_results())
 
 
 def reserve(args):
@@ -258,12 +285,15 @@ def parse_args(parser):
     cmd.add_argument("--before", help="only requests created before ISO8601")
     cmd.add_argument("--after", help="only requests created after ISO8601")
     cmd.add_argument("--json", help="full details, one request per line", action="store_true")
+    cmd.add_argument("--page", help="do paged search, page interval in secs", type=int)
 
     cmd = cmds.add_parser(
         "stats",
         help="print out TF usage statistics",
     )
-    cmd.add_argument("--days", type=int, help="query last N days instead of all TF requests")
+    cmd.add_argument("--before", help="only requests created before ISO8601")
+    cmd.add_argument("--after", help="only requests created after ISO8601")
+    cmd.add_argument("--page", help="do paged search, page interval in secs", type=int)
     cmd.add_argument("ranch", help="Testing Farm ranch name")
     cmd.add_argument("states", help="comma-separated TF request states")
 

atex/connection/ssh.py

@@ -36,6 +36,7 @@ DEFAULT_OPTIONS = {
     "TCPKeepAlive": "no",
     "EscapeChar": "none",
     "ExitOnForwardFailure": "yes",
+    "RequestTTY": "no",
 }
 
 

atex/executor/__init__.py

@@ -1,2 +1,23 @@
-from . import testcontrol  # noqa: F401
-from .executor import Executor  # noqa: F401
+class ExecutorError(Exception):
+    """
+    Raised by class Executor.
+    """
+    pass
+
+
+class TestSetupError(ExecutorError):
+    """
+    Raised when the preparation for test execution (ie. pkg install) fails.
+    """
+    pass
+
+
+class TestAbortedError(ExecutorError):
+    """
+    Raised when an infrastructure-related issue happened while running a test.
+    """
+    pass
+
+
+from . import testcontrol  # noqa: F401, E402
+from .executor import Executor  # noqa: F401, E402

atex/executor/executor.py

@@ -8,18 +8,11 @@ import subprocess
 from pathlib import Path
 
 from .. import util, fmf
-from . import testcontrol, scripts
+from . import TestSetupError, TestAbortedError, testcontrol, scripts
 from .duration import Duration
 from .reporter import Reporter
 
 
-class TestAbortedError(Exception):
-    """
-    Raised when an infrastructure-related issue happened while running a test.
-    """
-    pass
-
-
 class Executor:
     """
     Logic for running tests on a remote system and processing results
@@ -222,16 +215,6 @@ class Executor:
         output_dir = Path(output_dir)
         test_data = self.fmf_tests.tests[test_name]
 
-        # run a setup script, preparing wrapper + test scripts
-        setup_script = scripts.test_setup(
-            test=scripts.Test(test_name, test_data, self.fmf_tests.test_dirs[test_name]),
-            tests_dir=self.tests_dir,
-            wrapper_exec=f"{self.work_dir}/wrapper.sh",
-            test_exec=f"{self.work_dir}/test.sh",
-            test_yaml=f"{self.work_dir}/metadata.yaml",
-        )
-        self.conn.cmd(("bash",), input=setup_script, text=True, check=True)
-
         # start with fmf-plan-defined environment
         env_vars = {
             **self.fmf_tests.plan_env,
@@ -253,6 +236,28 @@ class Executor:
             duration = Duration(test_data.get("duration", "5m"))
             control = testcontrol.TestControl(reporter=reporter, duration=duration)
 
+            # run a setup script, preparing wrapper + test scripts
+            setup_script = scripts.test_setup(
+                test=scripts.Test(test_name, test_data, self.fmf_tests.test_dirs[test_name]),
+                tests_dir=self.tests_dir,
+                wrapper_exec=f"{self.work_dir}/wrapper.sh",
+                test_exec=f"{self.work_dir}/test.sh",
+                test_yaml=f"{self.work_dir}/metadata.yaml",
+            )
+            setup_proc = self.conn.cmd(
+                ("bash",),
+                input=setup_script,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,
+                text=True,
+            )
+            if setup_proc.returncode != 0:
+                reporter.report({
+                    "status": "infra",
+                    "note": f"TestSetupError({setup_proc.stdout})",
+                })
+                raise TestSetupError(setup_proc.stdout)
+
             test_proc = None
             control_fd = None
             stack.callback(lambda: os.close(control_fd) if control_fd else None)

atex/executor/reporter.py

@@ -85,16 +85,15 @@ class Reporter:
         file_path.parent.mkdir(parents=True, exist_ok=True)
         return file_path
 
-    def open_file(self, file_name, result_name=None, mode="wb"):
+    def open_fd(self, file_name, mode, result_name=None):
         """
         Open a file named 'file_name' in a directory relevant to 'result_name'.
-        Returns an opened file-like object that can be used in a context manager
-        just like with regular open().
+        Returns an opened file descriptor that can be closed with os.close().
 
         If 'result_name' (typically a subtest) is not given, open the file
         for the test (name) itself.
         """
-        return open(self._dest_path(file_name, result_name), mode)
+        return os.open(self._dest_path(file_name, result_name), mode)
 
     def link_testout(self, file_name, result_name=None):
         # TODO: docstring

atex/executor/scripts.py

@@ -86,7 +86,8 @@ def test_wrapper(*, test, tests_dir, test_exec):
     out += ")\n"
 
     # write test exitcode to test control stream
-    out += "echo exitcode $? >&$orig_stdout\n"
+    if os.environ.get("ATEX_DEBUG_NO_EXITCODE") != "1":
+        out += "echo exitcode $? >&$orig_stdout\n"
 
     # always exit the wrapper with 0 if test execution was normal
     out += "exit 0\n"
@@ -94,17 +95,6 @@ def test_wrapper(*, test, tests_dir, test_exec):
     return out
 
 
-def _install_packages(pkgs, extra_opts=None):
-    pkgs_str = " ".join(pkgs)
-    extra_opts = extra_opts or ()
-    dnf = ["dnf", "-y", "--setopt=install_weak_deps=False", "install", *extra_opts]
-    dnf_str = " ".join(dnf)
-    return util.dedent(fr"""
-        not_installed=$(rpm -q --qf '' {pkgs_str} | sed -nr 's/^package ([^ ]+) is not installed$/\1/p')
-        [[ $not_installed ]] && {dnf_str} $not_installed
-    """)  # noqa: E501
-
-
 def test_setup(*, test, wrapper_exec, test_exec, test_yaml, **kwargs):
     """
     Generate a bash script that should prepare the remote end for test
@@ -133,9 +123,19 @@ def test_setup(*, test, wrapper_exec, test_exec, test_yaml, **kwargs):
     # install test dependencies
     # - only strings (package names) in require/recommend are supported
     if require := list(fmf.test_pkg_requires(test.data, "require")):
-        out += _install_packages(require) + "\n"
+        pkgs_str = " ".join(require)
+        out += util.dedent(fr"""
+            not_installed=$(rpm -q --qf '' {pkgs_str} | sed -nr 's/^package ([^ ]+) is not installed$/\1/p')
+            [[ $not_installed ]] && dnf -y --setopt=install_weak_deps=False install $not_installed
+        """) + "\n"  # noqa: E501
     if recommend := list(fmf.test_pkg_requires(test.data, "recommend")):
-        out += _install_packages(recommend, ("--skip-broken",)) + "\n"
+        pkgs_str = " ".join(recommend)
+        out += util.dedent(fr"""
+            have_dnf5=$(command -v dnf5) || true
+            skip_bad="--skip-broken${{have_dnf5:+ --skip-unavailable}}"
+            not_installed=$(rpm -q --qf '' {pkgs_str} | sed -nr 's/^package ([^ ]+) is not installed$/\1/p')
+            [[ $not_installed ]] && dnf -y --setopt=install_weak_deps=False install $skip_bad $not_installed
+        """) + "\n"  # noqa: E501
 
     # write out test data
     out += f"cat > '{test_yaml}' <<'ATEX_SETUP_EOF'\n"

atex/executor/testcontrol.py

@@ -267,40 +267,45 @@ class TestControl:
             except ValueError as e:
                 raise BadReportJSONError(f"file entry {file_name} length: {str(e)}") from None
 
+            fd = self.reporter.open_fd(file_name, os.O_WRONLY | os.O_CREAT, name)
             try:
-                with self.reporter.open_file(file_name, name) as f:
-                    fd = f.fileno()
-                    while file_length > 0:
+                # Linux can't do splice(2) on O_APPEND fds, so we open it above
+                # as O_WRONLY and just seek to the end, simulating append
+                os.lseek(fd, 0, os.SEEK_END)
+
+                while file_length > 0:
+                    try:
+                        # try a more universal sendfile first, fall back to splice
                         try:
-                            # try a more universal sendfile first, fall back to splice
-                            try:
-                                written = os.sendfile(fd, self.control_fd, None, file_length)
-                            except OSError as e:
-                                if e.errno == 22:  # EINVAL
-                                    written = os.splice(self.control_fd, fd, file_length)
-                                else:
-                                    raise
-                        except BlockingIOError:
-                            yield
-                            continue
-                        if written == 0:
-                            raise BadControlError("EOF when reading data")
-                        file_length -= written
+                            written = os.sendfile(fd, self.control_fd, None, file_length)
+                        except OSError as e:
+                            if e.errno == 22:  # EINVAL
+                                written = os.splice(self.control_fd, fd, file_length)
+                            else:
+                                raise
+                    except BlockingIOError:
                         yield
-            except FileExistsError:
-                raise BadReportJSONError(f"file '{file_name}' already exists") from None
+                        continue
+                    if written == 0:
+                        raise BadControlError("EOF when reading data")
+                    file_length -= written
+                    yield
+            finally:
+                os.close(fd)
 
         # either store partial result + return,
         # or load previous partial result and merge into it
-        partial = result.get("partial", False)
-        if partial:
-            # do not store the 'partial' key in the result
+        partial = result.get("partial")
+        if partial is not None:
+            # do not store the 'partial' key in the result, even if False
             del result["partial"]
-            # note that nameless result will get None as dict key,
-            # which is perfectly fine
-            self._merge(self.partial_results[name], result)
-            # partial = do nothing
-            return
+            # if it exists and is True
+            if partial:
+                # note that nameless result will get None as dict key,
+                # which is perfectly fine
+                self._merge(self.partial_results[name], result)
+                # partial = do nothing
+                return
 
         # if previously-stored partial result exist, merge the current one
         # into it, but then use the merged result

atex/orchestrator/adhoc.py

@@ -1,5 +1,4 @@
 import tempfile
-import collections
 import concurrent.futures
 from pathlib import Path
 
@@ -60,7 +59,7 @@ class AdHocOrchestrator(Orchestrator):
 
     def __init__(
         self, platform, fmf_tests, provisioners, aggregator, tmp_dir, *,
-        max_remotes=1, max_spares=0, max_reruns=2, max_failed_setups=10, env=None,
+        max_remotes=1, max_spares=0, max_failed_setups=10, env=None,
     ):
         """
         'platform' is a string with platform name.
@@ -84,15 +83,15 @@ class AdHocOrchestrator(Orchestrator):
         speed up test reruns as Remote reservation happens asynchronously
         to test execution. Spares are reserved on top of 'max_remotes'.
 
-        'max_reruns' is an integer of how many times to re-try running a failed
-        test (which exited with non-0 or caused an Executor exception).
-
         'max_failed_setups' is an integer of how many times an Executor's
         plan setup (uploading tests, running prepare scripts, etc.) may fail
         before FailedSetupError is raised.
 
         'env' is a dict of extra environment variables to pass to Executor.
         """
+        if not fmf_tests.tests:
+            raise ValueError("'fmf_tests' has no tests (bad discover params?)")
+
         self.platform = platform
         self.fmf_tests = fmf_tests
         self.provisioners = tuple(provisioners)
@@ -101,11 +100,11 @@ class AdHocOrchestrator(Orchestrator):
         self.failed_setups_left = max_failed_setups
         self.max_remotes = max_remotes
         self.max_spares = max_spares
-        # indexed by test name, value being integer of how many times
-        self.reruns = collections.defaultdict(lambda: max_reruns)
         self.env = env
         # tests still waiting to be run
         self.to_run = set(fmf_tests.tests)
+        # number of Remotes being provisioned + set up (not running tests)
+        self.remotes_requested = 0
         # running tests as a dict, indexed by test name, with RunningInfo values
         self.running_tests = {}
         # thread queue for actively running tests
@@ -159,22 +158,6 @@ class AdHocOrchestrator(Orchestrator):
         'finfo' is a FinishedInfo instance.
         """
         test_data = self.fmf_tests.tests[finfo.test_name]
-
-        # TODO: somehow move logging from was_successful and should_be_rerun here,
-        #       probably print just some generic info from those functions that doesn't
-        #       imply any outcome, ie.
-        #           {remote_with_test} threw {exception}
-        #           {remote_with_test} exited with {code}
-        #           {remote_with_test} has {N} reruns left
-        #           {remote_with_test} has 0 reruns left
-        #       and then log the decision separately, here below, such as
-        #           {remote_with_test} failed, re-running
-        #           {remote_with_test} completed, ingesting result
-        #           {remote_with_test} was destructive, releasing remote
-        #           {remote_with_test} ...., running next test
-        #       That allows the user to override the functions, while keeping critical
-        #       flow reliably logged here.
-
         remote_with_test = f"{finfo.remote}: '{finfo.test_name}'"
 
         if not self.was_successful(finfo, test_data) and self.should_be_rerun(finfo, test_data):
@@ -215,31 +198,33 @@ class AdHocOrchestrator(Orchestrator):
                     tmp_dir=None,
                 )
 
-        # if destroyed, release the remote and request a replacement
-        # (Executor exception is always considered destructive)
-        if finfo.exception or self.destructive(finfo, test_data):
-            util.debug(f"{remote_with_test} was destructive, releasing remote")
+        # if there are still tests to be run and the last test was not
+        # destructive, just run a new test on it
+        if self.to_run and not (finfo.exception or self.destructive(finfo, test_data)):
+            util.debug(f"{remote_with_test} was non-destructive, running next test")
+            self._run_new_test(finfo)
+            return
+
+        # we are not running a new test right now, serve_once() might run it
+        # some time later, just decide what to do with the current remote
+
+        if self.remotes_requested >= len(self.to_run):
+            # we have enough remotes in the pipe to run every test,
+            # we don't need a new one - just release the current one
+            util.debug(f"{finfo.remote} no longer useful, releasing it")
             self.release_queue.start_thread(
                 finfo.remote.release,
                 remote=finfo.remote,
             )
-            # TODO: should this be conditioned by 'self.to_run:' ? to not uselessly fall
-            #       into setup spares and get immediately released after setup?
-            finfo.provisioner.provision(1)
-
-        # if still not destroyed, run another test on it
-        # (without running plan setup, re-using already set up remote)
-        elif self.to_run:
-            util.debug(f"{remote_with_test} was non-destructive, running next test")
-            self._run_new_test(finfo)
-
-        # no more tests to run, release the remote
         else:
-            util.debug(f"{finfo.remote} no longer useful, releasing it")
+            # we need more remotes and the last test was destructive,
+            # get a new one and let serve_once() run a test later
+            util.debug(f"{remote_with_test} was destructive, getting a new Remote")
             self.release_queue.start_thread(
                 finfo.remote.release,
                 remote=finfo.remote,
             )
+            finfo.provisioner.provision(1)
 
     def serve_once(self):
         """
@@ -286,6 +271,7 @@ class AdHocOrchestrator(Orchestrator):
             except util.ThreadQueue.Empty:
                 break
 
+            self.remotes_requested -= 1
             sinfo = treturn.sinfo
 
             if treturn.exception:
@@ -295,10 +281,11 @@ class AdHocOrchestrator(Orchestrator):
                     sinfo.remote.release,
                     remote=sinfo.remote,
                 )
-                if (reruns_left := self.failed_setups_left) > 0:
-                    util.warning(f"{msg}, re-trying ({reruns_left} setup retries left)")
+                if (retries_left := self.failed_setups_left) > 0:
+                    util.warning(f"{msg}, re-trying ({retries_left} setup retries left)")
                     self.failed_setups_left -= 1
                     sinfo.provisioner.provision(1)
+                    self.remotes_requested += 1
                 else:
                     util.warning(f"{msg}, setup retries exceeded, giving up")
                     raise FailedSetupError("setup retries limit exceeded, broken infra?")
@@ -318,8 +305,9 @@ class AdHocOrchestrator(Orchestrator):
                     treturn.sinfo.remote.release,
                     remote=treturn.sinfo.remote,
                 )
+                self.remotes_requested -= 1
 
-        # try to get new remotes from Provisioners - if we get some, start
+        # try to get new Remotes from Provisioners - if we get some, start
         # running setup on them
         for provisioner in self.provisioners:
             while (remote := provisioner.get_remote(block=False)) is not None:
@@ -371,9 +359,12 @@ class AdHocOrchestrator(Orchestrator):
         for prov in self.provisioners:
             prov.start()
 
+        # just the base remotes, no spares
+        self.remotes_requested = min(self.max_remotes, len(self.fmf_tests.tests))
+
         # start up initial reservations, balanced evenly across all available
         # provisioner instances
-        count = min(self.max_remotes, len(self.fmf_tests.tests)) + self.max_spares
+        count = self.remotes_requested + self.max_spares
         provisioners = self.provisioners[:count]
         for idx, prov in enumerate(provisioners):
             if count % len(provisioners) > idx:
@@ -480,24 +471,20 @@ class AdHocOrchestrator(Orchestrator):
         'test_data' is a dict of fully resolved fmf test metadata of that test.
         """
         remote_with_test = f"{info.remote}: '{info.test_name}'"
-
         # executor (or test) threw exception
         if info.exception:
             exc_str = f"{type(info.exception).__name__}({info.exception})"
             util.info(f"{remote_with_test} threw {exc_str} during test runtime")
             return False
-
         # the test exited as non-0
         if info.exit_code != 0:
             util.info(f"{remote_with_test} exited with non-zero: {info.exit_code}")
             return False
-
         # otherwise we good
         return True
 
-    # TODO: @staticmethod and remove ARG002
-    #@staticmethod
-    def should_be_rerun(self, info, test_data):  # noqa: ARG004, ARG002
+    @staticmethod
+    def should_be_rerun(info, test_data):  # noqa: ARG004
         """
         Return a boolean result whether a finished test failed in a way
         that another execution attempt might succeed, due to race conditions
@@ -507,17 +494,5 @@ class AdHocOrchestrator(Orchestrator):
 
         'test_data' is a dict of fully resolved fmf test metadata of that test.
         """
-        remote_with_test = f"{info.remote}: '{info.test_name}'"
-
-        # TODO: remove self.reruns and the whole X-reruns logic from AdHocOrchestrator,
-        #       leave it up to the user to wrap should_be_rerun() with an external dict
-        #       of tests, counting reruns for each
-        #        - allows the user to adjust counts per-test (ie. test_data metadata)
-        #        - allows this template to be @staticmethod
-        reruns_left = self.reruns[info.test_name]
-        util.info(f"{remote_with_test}: {reruns_left} reruns left")
-        if reruns_left > 0:
-            self.reruns[info.test_name] -= 1
-            return True
-        else:
-            return False
+        # never rerun by default
+        return False

atex/orchestrator/contest.py

@@ -1,3 +1,5 @@
+import collections
+
 from .. import util
 from .adhoc import AdHocOrchestrator
 
@@ -26,9 +28,18 @@ class ContestOrchestrator(AdHocOrchestrator):
     """
     content_dir_on_remote = "/root/upstream-content"
 
-    def __init__(self, *args, content_dir, **kwargs):
-        self.content_dir = content_dir
+    def __init__(self, *args, content_dir, max_reruns=1, **kwargs):
+        """
+        'content_dir' is a filesystem path to ComplianceAsCode/content local
+        directory, to be uploaded to the tested systems.
+
+        'max_reruns' is an integer of how many times to re-try running a failed
+        test (which exited with non-0 or caused an Executor exception).
+        """
         super().__init__(*args, **kwargs)
+        self.content_dir = content_dir
+        # indexed by test name, value being integer of how many times
+        self.reruns = collections.defaultdict(lambda: max_reruns)
 
     def run_setup(self, sinfo):
         super().run_setup(sinfo)
@@ -72,8 +83,8 @@ class ContestOrchestrator(AdHocOrchestrator):
         # fallback to the default next_test()
         return super().next_test(to_run, all_tests, previous)
 
-    @classmethod
-    def destructive(cls, info, test_data):
+    @staticmethod
+    def destructive(info, test_data):
         # if Executor ended with an exception (ie. duration exceeded),
         # consider the test destructive
         if info.exception:
@@ -92,3 +103,14 @@ class ContestOrchestrator(AdHocOrchestrator):
             return True
 
         return False
+
+    def should_be_rerun(self, info, test_data):  # noqa: ARG004, ARG002
+        remote_with_test = f"{info.remote}: '{info.test_name}'"
+
+        reruns_left = self.reruns[info.test_name]
+        util.info(f"{remote_with_test}: {reruns_left} reruns left")
+        if reruns_left > 0:
+            self.reruns[info.test_name] -= 1
+            return True
+        else:
+            return False

atex/provisioner/testingfarm/api.py

@@ -2,6 +2,7 @@ import os
 import re
 import time
 import tempfile
+import datetime
 import textwrap
 import threading
 import subprocess
@@ -14,11 +15,11 @@ from ... import util
 import json
 import urllib3
 
-DEFAULT_API_URL = "https://api.testing-farm.io/v0.1"
+DEFAULT_API_URL = "https://api.testing-farm.io"
 
 DEFAULT_RESERVE_TEST = {
     "url": "https://github.com/RHSecurityCompliance/atex-reserve",
-    "ref": "0.11",
+    "ref": "main",
     "path": ".",
     "name": "/plans/reserve",
 }
@@ -85,8 +86,8 @@ class TestingFarmAPI:
         self.api_url = url
         self.api_token = token or os.environ.get("TESTING_FARM_API_TOKEN")
 
-    def _query(self, method, path, *args, headers=None, auth=True, **kwargs):
-        url = f"{self.api_url}{path}"
+    def _query(self, method, path, *args, headers=None, auth=True, version="v0.1", **kwargs):
+        url = f"{self.api_url}/{version}{path}"
         if self.api_token and auth:
             if headers is not None:
                 headers["Authorization"] = f"Bearer {self.api_token}"
@@ -137,7 +138,7 @@ class TestingFarmAPI:
             if not self.api_token:
                 raise ValueError("composes() requires an auth token to identify ranch")
             ranch = self.whoami()["token"]["ranch"]
-        return self._query("GET", f"/composes/{ranch}")
+        return self._query("GET", f"/composes/{ranch}", version="v0.2")
 
     def search_requests(
         self, *, state, ranch=None,
@@ -180,6 +181,52 @@ class TestingFarmAPI:
 
         return self._query("GET", "/requests", fields=fields, auth=mine)
 
+    def search_requests_paged(self, *args, page=43200, **kwargs):
+        """
+        An unofficial wrapper for search_requests() that can search a large
+        interval incrementally (in "pages") and yield batches of results.
+
+        Needs 'created_after', with 'created_before' defaulting to now().
+
+        'page' specifies the time interval of one page, in seconds.
+
+        'args' and 'kwargs' are passed to search_requests().
+        """
+        assert "created_after" in kwargs, "at least 'created_after' is needed for paging"
+
+        def from_iso8601(date):
+            dt = datetime.datetime.fromisoformat(date)
+            # if no TZ is specified, treat it as UTC, not localtime
+            if dt.tzinfo is None:
+                dt = dt.replace(tzinfo=datetime.UTC)
+            # convert to UTC
+            else:
+                dt = dt.astimezone(datetime.UTC)
+            return dt
+
+        after = from_iso8601(kwargs["created_after"])
+        if kwargs.get("created_before"):
+            before = from_iso8601(kwargs["created_before"])
+        else:
+            before = datetime.datetime.now(datetime.UTC)
+
+        # scale down page size to fit between after/before
+        page = min(page, (before - after).total_seconds())
+
+        start = after
+        while start < before:
+            end = start + datetime.timedelta(seconds=page)
+            # clamp to real 'before'
+            end = min(end, before)
+            new_kwargs = kwargs | {
+                "created_after": start.isoformat(),
+                "created_before": end.isoformat(),
+            }
+            found = self.search_requests(*args, **new_kwargs)
+            if found is not None:
+                yield from found
+            start = end
+
     def get_request(self, request_id):
         """
         'request_id' is the UUID (string) of the request.
@@ -389,7 +436,7 @@ class Reserve:
     def __init__(
         self, *, compose, arch="x86_64", pool=None, hardware=None, kickstart=None,
         timeout=60, ssh_key=None, source_host=None,
-        reserve_test=None, variables=None, secrets=None,
+        reserve_test=None, variables=None, secrets=None, tags=None,
         api=None,
     ):
         """
@@ -435,6 +482,10 @@ class Reserve:
         exported for the reserve test - variables are visible via TF API,
         secrets are not (but can still be extracted from pipeline log).
 
+        'tags' is a dict of custom key/values to be submitted in TF Request as
+        environments->settings->provisioning->tags, useful for storing custom
+        metadata to be queried later.
+
         'api' is a TestingFarmAPI instance - if unspecified, a sensible default
         will be used.
         """
@@ -474,6 +525,8 @@ class Reserve:
         if variables:
             spec_env["variables"] = variables
         spec_env["secrets"] = secrets.copy() if secrets else {}  # we need it for ssh pubkey
+        if tags:
+            spec_env["settings"]["provisioning"]["tags"] |= tags
 
         self._spec = spec
         self._ssh_key = Path(ssh_key) if ssh_key else None
@@ -490,7 +543,7 @@ class Reserve:
         try:
             r = _http.request("GET", "https://ifconfig.me", headers=curl_agent)
             if r.status != 200:
-                raise ConnectionError()
+                raise ConnectionError
         except (ConnectionError, urllib3.exceptions.RequestError):
             r = _http.request("GET", "https://ifconfig.co", headers=curl_agent)
         return r.data.decode().strip()
@@ -570,7 +623,8 @@ class Reserve:
             #  installs our ssh pubkey into authorized_keys)
             ssh_attempt_cmd = (
                 "ssh", "-q", "-i", ssh_key.absolute(), "-oConnectionAttempts=60",
-               "-oStrictHostKeyChecking=no", "-oUserKnownHostsFile=/dev/null",
+                "-oStrictHostKeyChecking=no", "-oUserKnownHostsFile=/dev/null",
+                "-oBatchMode=yes",
                 f"{ssh_user}@{ssh_host}", "exit 123",
             )
             while True:

atex/util/log.py

@@ -30,7 +30,7 @@ skip_levels = {
 
 def _log_msg(logger_func, *args, stacklevel=1, **kwargs):
     # inspect.stack() is MUCH slower
-    caller = inspect.currentframe().f_back.f_back
+    caller = inspect.currentframe().f_back.f_back  # TODO: sys._getframe(2)
     extra_levels = 2  # skip this func and the debug/info/warning parent
     while caller.f_back:
         code = caller.f_code

{atex-0.11.dist-info → atex-0.13.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: atex
-Version: 0.11
+Version: 0.13
 Summary: Ad-hoc Test EXecutor
 Project-URL: Homepage, https://github.com/RHSecurityCompliance/atex
 License-Expression: GPL-3.0-or-later

{atex-0.11.dist-info → atex-0.13.dist-info}/RECORD

@@ -2,22 +2,22 @@ atex/__init__.py,sha256=LdX67gprtHYeAkjLhFPKzpc7ECv2rHxUbHKDGbGXO1c,517
 atex/fmf.py,sha256=gkJXIaRO7_KvwJR-V6Tc1NVn4a9Hq7hoBLQLhxYIdbg,8834
 atex/aggregator/__init__.py,sha256=8mN-glHdzR4icKAUGO4JPodsTrLMdJoeuZsO2CTbhyU,1773
 atex/aggregator/json.py,sha256=tpoUZoZM8EMYhZKwVr4LRtgEIDjRxC11BIKVXZKYPOs,10441
-atex/cli/__init__.py,sha256=Ew2z-gC0jvOmU_DqYgXVQla3p1rTnrz64I63q52aHv4,2899
+atex/cli/__init__.py,sha256=0YzZ4sIYvZ47yF9STA2znINIQfEoGAV1N0pduTA7NhI,2897
 atex/cli/fmf.py,sha256=pvj_OIp6XT_nVUwziL7-v_HNbyAtuUmb7k_Ey_KkFJc,3616
 atex/cli/libvirt.py,sha256=6tt5ANb8XBBRXOQsYPTWILThKqf-gvt5AZh5Dctg2PA,3782
-atex/cli/testingfarm.py,sha256=ovgoogmIM2TglS7iQD3liMiEYYtcykS_HRRKbltpW2I,10131
+atex/cli/testingfarm.py,sha256=b6IzoouYx-Qzuu7350QhyC-PW8576b7hyhiTDka_xak,10899
 atex/connection/__init__.py,sha256=dj8ZBcEspom7Z_UjecfLGBRNvLZ3dyGR9q19i_B4xpY,3880
 atex/connection/podman.py,sha256=1T56gh1TgbcQWpTIJHL4NaxZOI6aMg7Xp7sn6PQQyBk,1911
-atex/connection/ssh.py,sha256=9A57b9YR_HI-kIu06Asic1y__JPVXEheDZxjbG2Qcsc,13460
-atex/executor/__init__.py,sha256=XCfhi7QDELjey7N1uzhMjc46Kp1Jsd5bOCf52I27SCE,85
+atex/connection/ssh.py,sha256=6lama8q3tSQcWPRKN3B4lgFUGtuVA51tuKDiNh7IN0U,13484
+atex/executor/__init__.py,sha256=nmYJCbC36fRGGkjoniFJmsq-sqFw8YS2ndf4q_loVM0,471
 atex/executor/duration.py,sha256=x06sItKOZi6XA8KszQwZGpIb1Z_L-HWqIwZKo2SDo0s,1759
-atex/executor/executor.py,sha256=toyLVQCDzfw381iEGrvOXoKPsd4SqxMZHwlDSTJGqKk,15792
-atex/executor/reporter.py,sha256=MceFmHFt0bTEClBZbRI1WnFbfMhR0e1noOzcu7gjKuQ,3403
-atex/executor/scripts.py,sha256=riJAQWsV-BFGkJwR2Dmf3R0ZRRZJs9w9iYnPpYaQNaE,5618
-atex/executor/testcontrol.py,sha256=mVrLwQUnDRfUq-5diz-80UvCWWxn1TkcBgmAKhKNb5E,12696
+atex/executor/executor.py,sha256=WJXPWQo6VQhZgXORVVyvTDAdOQbbZz26E7FpwizbGIk,16126
+atex/executor/reporter.py,sha256=QbzBkaXuhI6lsTYrTlp7O5W9d6etR0KjDdH-J59cXWM,3357
+atex/executor/scripts.py,sha256=1u5ZEGJ7nIvkqbRK3uVusOkineVM8DXo4kAlH2MdQbg,5877
+atex/executor/testcontrol.py,sha256=iju_Cl32D8NHH1ePN1lykR1noP8-0eBDLQ5-V_9DqF0,12834
 atex/orchestrator/__init__.py,sha256=8Q1YknyibilXLjWRYkHm_Mr2HMm0SRw8Zv39KypeASM,2059
-atex/orchestrator/adhoc.py,sha256=QpYoPeyQzYFDBM1zgFJKMXH1RtdJixbH5whVX0OP-14,21003
-atex/orchestrator/contest.py,sha256=ADmRlsZPQx-MJ6fWHmBcJOIy3DSPnvwVheVL9Upwtg0,3703
+atex/orchestrator/adhoc.py,sha256=VUwHX71Vb6eRLzW3Z3KDZdck7p0PiwzAZrOuUKMkwtM,19667
+atex/orchestrator/contest.py,sha256=SuxT9uZtcs_DEsA3hHyKgrIWNrDeqCCWd3-hy3sHytY,4572
 atex/provisioner/__init__.py,sha256=6hZxQlvTQ0yWWqCRCPqWMoYuim5wDMCcDIYHF-nIfMs,4013
 atex/provisioner/libvirt/VM_PROVISION,sha256=7pkZ-ozgTyK4qNGC-E-HUznr4IhbosWSASbB72Gknl8,2664
 atex/provisioner/libvirt/__init__.py,sha256=pKG5IpZSC2IHs5wL2ecQx_fd9AzAXEbZmDzA7RyZsfM,119
@@ -27,19 +27,19 @@ atex/provisioner/libvirt/setup-libvirt.sh,sha256=oCMy9SCnbC_QuAzO2sFwvB5ui1kMQ6u
 atex/provisioner/podman/__init__.py,sha256=dM0JzQXWX7edtWSc0KH0cMFXAjArFn2Vme4j_ZMsdYA,138
 atex/provisioner/podman/podman.py,sha256=ztRypoakSf-jF04iER58tEMUZ4Y6AuzIpNpFXp44bB4,4997
 atex/provisioner/testingfarm/__init__.py,sha256=kZncgLGdRCR4FMaRQr2GTwJ8vjlA-24ri8JO2ueZJuw,113
-atex/provisioner/testingfarm/api.py,sha256=dlXe9brzHERawIx2UTv34u2tOSskdZtXD68-u1MnOHk,21726
+atex/provisioner/testingfarm/api.py,sha256=3vPpAmPisn99-NruZHUy_lS0I7A_IKHWGVGJcJBLBuI,23836
 atex/provisioner/testingfarm/testingfarm.py,sha256=yvQzWat92B4UnJNZzCLI8mpAKf_QvHUKyKbjlk5123Q,8573
 atex/util/__init__.py,sha256=cWHFbtQ4mDlKe6lXyPDWRmWJOTcHDGfVuW_-GYa8hB0,1473
 atex/util/dedent.py,sha256=SEuJMtLzqz3dQ7g7qyZzEJ9VYynVlk52tQCJY-FveXo,603
 atex/util/libvirt.py,sha256=kDZmT6xLYEZkQNLZY98gJ2M48DDWXxHF8rQY9PnjB3U,660
-atex/util/log.py,sha256=KVR7ep8n5wtghsvBFCtHiPsMAQBdAmK83E_Jec5t4cU,2230
+atex/util/log.py,sha256=GfdbLtpRkQoIkRU7AqWDWbJV7yZIpS4MsXhUomZqWjQ,2256
 atex/util/named_mapping.py,sha256=UBMe9TetjV-DGPhjYjJ42YtC40FVPKAAEROXl9MA5fo,4700
 atex/util/path.py,sha256=x-kXqiWCVodfZWbEwtC5A8LFvutpDIPYv2m0boZSlXU,504
 atex/util/ssh_keygen.py,sha256=9yuSl2yBV7pG3Qfsf9tossVC00nbIUrAeLdbwTykpjk,384
 atex/util/subprocess.py,sha256=_oQN8CNgGoH9GAR6nZlpujYe2HjXFBcCuIkLPw-IxJ4,2971
 atex/util/threads.py,sha256=c8hsEc-8SqJGodInorv_6JxpiHiSkGFGob4qbMmOD2M,3531
-atex-0.11.dist-info/METADATA,sha256=3fRMLBrkoRIHwbY2GheyNkrx4mNVmL_95wlxGjZsORc,3050
-atex-0.11.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-atex-0.11.dist-info/entry_points.txt,sha256=pLqJdcfeyQTgup2h6dWb6SvkHhtOl-W5Eg9zV8moK0o,39
-atex-0.11.dist-info/licenses/COPYING.txt,sha256=oEuj51jdmbXcCUy7pZ-KE0BNcJTR1okudRp5zQ0yWnU,670
-atex-0.11.dist-info/RECORD,,
+atex-0.13.dist-info/METADATA,sha256=Q3S7eSPkLtzLYvRgIatm_RYyPGXUa_K7dAqZ_GvV5rY,3050
+atex-0.13.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+atex-0.13.dist-info/entry_points.txt,sha256=pLqJdcfeyQTgup2h6dWb6SvkHhtOl-W5Eg9zV8moK0o,39
+atex-0.13.dist-info/licenses/COPYING.txt,sha256=oEuj51jdmbXcCUy7pZ-KE0BNcJTR1okudRp5zQ0yWnU,670
+atex-0.13.dist-info/RECORD,,