atex 0.7__py3-none-any.whl → 0.8__py3-none-any.whl

atex/util/log.py

@@ -5,6 +5,14 @@ from pathlib import Path
 _logger = logging.getLogger("atex")
 
 
+def in_debug_mode():
+    """
+    Return True if the root logger is using the DEBUG (or more verbose) level.
+    """
+    root_level = logging.getLogger().level
+    return root_level > 0 and root_level <= logging.DEBUG
+
+
 def _format_msg(msg, *, skip_frames=0):
     stack = inspect.stack()
     if len(stack)-1 <= skip_frames:

atex/util/path.py

@@ -0,0 +1,16 @@
+import os
+
+
+def normalize_path(path):
+    """
+    Transform a potentially dangerous path (leading slash, relative ../../../
+    leading beyond parent, etc.) to a safe one.
+
+    Always returns a relative path.
+    """
+    # the magic here is to treat any dangerous path as starting at /
+    # and resolve any weird constructs relative to /, and then simply
+    # strip off the leading / and use it as a relative path
+    path = path.lstrip("/")
+    path = os.path.normpath(f"/{path}")
+    return path[1:]

atex/util/ssh_keygen.py

@@ -0,0 +1,14 @@
+import subprocess
+from pathlib import Path
+
+from .subprocess import subprocess_run
+
+
+def ssh_keygen(dest_dir, key_type="rsa"):
+    dest_dir = Path(dest_dir)
+    subprocess_run(
+        ("ssh-keygen", "-t", key_type, "-N", "", "-f", dest_dir / f"key_{key_type}"),
+        stdout=subprocess.DEVNULL,
+        check=True,
+    )
+    return (dest_dir / "key_rsa", dest_dir / "key_rsa.pub")

atex/util/threads.py

@@ -0,0 +1,55 @@
+import collections
+import queue
+import threading
+
+# TODO: documentation; this is like concurrent.futures, but with daemon=True support
+
+
+class ThreadQueue:
+    ThreadReturn = collections.namedtuple("ThreadReturn", ("thread", "returned", "exception"))
+    Empty = queue.Empty
+
+    def __init__(self, daemon=False):
+        self.queue = queue.SimpleQueue()
+        self.daemon = daemon
+        self.threads = set()
+
+    def _wrapper(self, func, *args, **kwargs):
+        current_thread = threading.current_thread()
+        try:
+            ret = func(*args, **kwargs)
+            result = self.ThreadReturn(current_thread, ret, None)
+        except Exception as e:
+            result = self.ThreadReturn(current_thread, None, e)
+        self.queue.put(result)
+
+    def start_thread(self, target, name=None, args=None, kwargs=None):
+        args = args or ()
+        kwargs = kwargs or {}
+        t = threading.Thread(
+            target=self._wrapper,
+            name=name,
+            args=(target, *args),
+            kwargs=kwargs,
+            daemon=self.daemon,
+        )
+        t.start()
+        self.threads.add(t)
+
+    # get one return value from any thread's function, like .as_completed()
+    # or concurrent.futures.FIRST_COMPLETED
+    def get(self, block=True, timeout=None):
+        if block and timeout is None and not self.threads:
+            raise AssertionError("no threads are running, would block forever")
+        treturn = self.queue.get(block=block, timeout=timeout)
+        self.threads.remove(treturn.thread)
+        if treturn.exception is not None:
+            raise treturn.exception
+        else:
+            return treturn.returned
+
+    # wait for all threads to finish (ignoring queue contents)
+    def join(self):
+        while self.threads:
+            t = self.threads.pop()
+            t.join()

{atex-0.7.dist-info → atex-0.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: atex
-Version: 0.7
+Version: 0.8
 Summary: Ad-hoc Test EXecutor
 Project-URL: Homepage, https://github.com/RHSecurityCompliance/atex
 License-Expression: GPL-3.0-or-later
@@ -8,7 +8,7 @@ License-File: COPYING.txt
 Classifier: Operating System :: POSIX :: Linux
 Classifier: Programming Language :: Python :: 3
 Classifier: Topic :: Software Development :: Testing
-Requires-Python: >=3.9
+Requires-Python: >=3.11
 Requires-Dist: fmf>=1.6
 Requires-Dist: urllib3<3,>=2
 Description-Content-Type: text/markdown
@@ -45,8 +45,103 @@ BREAK. DO NOT USE IT (for now).
 Unless specified otherwise, any content within this repository is distributed
 under the GNU GPLv3 license, see the [COPYING.txt](COPYING.txt) file for more.
 
+## Parallelism and cleanup
+
+There are effectively 3 methods of running things in parallel in Python:
+
+- `threading.Thread` (and related `concurrent.futures` classes)
+- `multiprocessing.Process` (and related `concurrent.futures` classes)
+- `asyncio`
+
+and there is no clear winner (in terms of cleanup on `SIGTERM` or Ctrl-C):
+
+- `Thread` has signal handlers only in the main thread and is unable to
+  interrupt any running threads without super ugly workarounds like `sleep(1)`
+  in every thread, checking some "pls exit" variable
+- `Process` is too heavyweight and makes sharing native Python objects hard,
+  but it does handle signals in each process individually
+- `asyncio` handles interrupting perfectly (every `try`/`except`/`finally`
+  completes just fine, `KeyboardInterrupt` is raised in every async context),
+  but async python is still (3.14) too weird and unsupported
+  - `asyncio` effectively re-implements `subprocess` with a slightly different
+    API, same with `asyncio.Transport` and derivatives reimplementing `socket`
+  - 3rd party libraries like `requests` or `urllib3` don't support it, one needs
+    to resort to spawning these in separate threads anyway
+  - same with `os.*` functions and syscalls
+  - every thing exposed via API needs to have 2 copies - async and non-async,
+    making it unbearable
+  - other stdlib bugs, ie. "large" reads returning BlockingIOError sometimes
+
+The approach chosen by this project was to use `threading.Thread`, and
+implement thread safety for classes and their functions that need it.  
+For example:
+
+```python
+class MachineReserver:
+    def __init__(self):
+        self.lock = threading.RLock()
+        self.job = None
+        self.proc = None
+
+    def reserve(self, ...):
+        try:
+            ...
+            job = schedule_new_job_on_external_service()
+            with self.lock:
+                self.job = job
+            ...
+            while not reserved(self.job):
+                time.sleep(60)
+            ...
+            with self.lock:
+                self.proc = subprocess.Popen(["ssh", f"{user}@{host}", ...)
+            ...
+            return machine
+        except Exception:
+            self.abort()
+            raise
+
+    def abort(self):
+        with self.lock:
+            if self.job:
+                cancel_external_service(self.job)
+                self.job = None
+            if self.proc:
+                self.proc.kill()
+                self.proc = None
+```
+
+Here, it is expected for `.reserve()` to be called in a long-running thread that
+provisions a new machine on some external service, waits for it to be installed
+and reserved, connects an ssh session to it and returns it back.
+
+But equally, `.abort()` can be called from an external thread and clean up any
+non-pythonic resources (external jobs, processes, temporary files, etc.) at
+which point **we don't care what happens to .reserve()**, it will probably fail
+with some exception, but doesn't do any harm.
+
+Here is where `daemon=True` threads come in handy - we can simply call `.abort()`
+from a `KeyboardInterrupt` (or `SIGTERM`) handle in the main thread, and just
+exit, automatically killing any leftover threads that are uselessly sleeping.  
+(Realistically, we might want to spawn new threads to run many `.abort()`s in
+parallel, but the main thread can wait for those just fine.)
+
+It is not perfect, but it's probably the best Python can do.
+
+Note that races can still occur between a resource being reserved and written
+to `self.*` for `.abort()` to free, so resource de-allocation is not 100%
+guaranteed, but single-threaded interrupting has the same issue.  
+Do have fallbacks (ie. max reserve times on the external service).
+
+Also note that `.reserve()` and `.abort()` could be also called by a context
+manager as `__enter__` and `__exit__`, ie. by a non-threaded caller (running
+everything in the main thread).
+
+
 ## Unsorted notes
 
+TODO: codestyle from contest
+
 ```
 - this is not tmt, the goal is to make a python toolbox *for* making runcontest
   style tools easily, not to replace those tools with tmt-style CLI syntax

atex-0.8.dist-info/RECORD

@@ -0,0 +1,37 @@
+atex/__init__.py,sha256=LdX67gprtHYeAkjLhFPKzpc7ECv2rHxUbHKDGbGXO1c,517
+atex/fmf.py,sha256=ofbrJx2362qHAxERS-WulK4TMpbp0C4HQ-Js917Ll9w,7871
+atex/cli/__init__.py,sha256=erHv68SsybRbdgJ60013y9jVqY1ec-cb9T9ThPCJ_HY,2408
+atex/cli/fmf.py,sha256=5DbA-3rfbFZ41fJ5z7Tiz5FmuZhXNC7gRAQfIGX7pXc,2516
+atex/cli/testingfarm.py,sha256=wdN26TE9jZ0ozet-JBQQgIcRi0WIV3u_i-7_YYi_SUg,7248
+atex/connection/__init__.py,sha256=xFwGOvlFez1lIt1AD6WXgEEIbsF22pSpQFv41GEAGAI,3798
+atex/connection/ssh.py,sha256=vrrSfVdQoz5kWiZbiPuM8KGneMl2Tlb0VeJIHTFSSYs,13626
+atex/executor/__init__.py,sha256=XCfhi7QDELjey7N1uzhMjc46Kp1Jsd5bOCf52I27SCE,85
+atex/executor/duration.py,sha256=x06sItKOZi6XA8KszQwZGpIb1Z_L-HWqIwZKo2SDo0s,1759
+atex/executor/executor.py,sha256=QYYSlEfBZIm95NhM1gwd2ROeshSAavYu2DP_4TTHlQs,14770
+atex/executor/reporter.py,sha256=nW_Uls3R4Ev80a2ZNJl3nxAYrcYhXk5Cy9nAUMlYPrc,3326
+atex/executor/scripts.py,sha256=yE4Lbfu-TPkBcB5t15-t-tF79H8pBJWbWP6MKRSvKsw,5356
+atex/executor/testcontrol.py,sha256=-rfihfE6kryIGurFrHBPSS8ANaIJkzX-zfpOO8To-9o,12204
+atex/orchestrator/__init__.py,sha256=eF-6ix5rFEu85fBFzgSdTYau7bNTkIQndAU7QqeI-FA,105
+atex/orchestrator/aggregator.py,sha256=5-8nHVeW6kwImoEYOsQqsx6UBdbKc5xuj6qlg7dtOF8,3642
+atex/orchestrator/orchestrator.py,sha256=tQu_d8_9y3rOLHskb694NJKNvxplQWAZ2R452Sy3AXw,12056
+atex/provision/__init__.py,sha256=2d_hRVPxXF5BVbQ_Gn1OR-F2xuqRn8O0yyVbvSrtTIg,4043
+atex/provision/libvirt/VM_PROVISION,sha256=7pkZ-ozgTyK4qNGC-E-HUznr4IhbosWSASbB72Gknl8,2664
+atex/provision/libvirt/__init__.py,sha256=mAkGtciZsXdR9MVVrjm3OWNXZqTs_33-J1qAszFA0k4,768
+atex/provision/libvirt/setup-libvirt.sh,sha256=CXrEFdrj8CSHXQZCd2RWuRvTmw7QYFTVhZeLuhhXooI,1855
+atex/provision/podman/README,sha256=kgP3vcTfWW9gcQzmXnyucjgWbqjNqm_ZM--pnqNTXRg,1345
+atex/provision/podman/host_container.sh,sha256=buCNz0BlsHY5I64sMSTGQHkvzEK0aeIhpGJXWCQVMXk,2283
+atex/provision/testingfarm/__init__.py,sha256=kZncgLGdRCR4FMaRQr2GTwJ8vjlA-24ri8JO2ueZJuw,113
+atex/provision/testingfarm/api.py,sha256=jiEJhYxMTzRihayceHcnDnGKNZJisYWn2o_TAdCI2Xo,19943
+atex/provision/testingfarm/testingfarm.py,sha256=wp8W3bwOmQdO-UUOdqu_JLtOZTGaNg-wERFfLySwZmI,8587
+atex/util/__init__.py,sha256=cWHFbtQ4mDlKe6lXyPDWRmWJOTcHDGfVuW_-GYa8hB0,1473
+atex/util/dedent.py,sha256=SEuJMtLzqz3dQ7g7qyZzEJ9VYynVlk52tQCJY-FveXo,603
+atex/util/log.py,sha256=KZkuw4jl8YTUOHZ4wNBrfDeg16VpLa82-IZYFHfqwgk,1995
+atex/util/path.py,sha256=x-kXqiWCVodfZWbEwtC5A8LFvutpDIPYv2m0boZSlXU,504
+atex/util/ssh_keygen.py,sha256=9yuSl2yBV7pG3Qfsf9tossVC00nbIUrAeLdbwTykpjk,384
+atex/util/subprocess.py,sha256=IQT9QHe2kMaaO_XPSry-DwObYstGsq6_QdwdbhYDjko,1826
+atex/util/threads.py,sha256=bezDIEIMcQinmG7f5E2K6_mHJQOlwx7W3I9CKkCYAYA,1830
+atex-0.8.dist-info/METADATA,sha256=dvXW146ZvIfyzqPqGbKmhTNScLTZM7C5K0FLrGNGIJ0,8981
+atex-0.8.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+atex-0.8.dist-info/entry_points.txt,sha256=pLqJdcfeyQTgup2h6dWb6SvkHhtOl-W5Eg9zV8moK0o,39
+atex-0.8.dist-info/licenses/COPYING.txt,sha256=oEuj51jdmbXcCUy7pZ-KE0BNcJTR1okudRp5zQ0yWnU,670
+atex-0.8.dist-info/RECORD,,

atex/cli/minitmt.py

@@ -1,175 +0,0 @@
-import sys
-#import re
-import pprint
-#import subprocess
-from pathlib import Path
-
-from .. import connection, provision, minitmt
-from ..orchestrator import aggregator
-
-
-def _fatal(msg):
-    print(msg, file=sys.stderr)
-    sys.exit(1)
-
-
-def _get_context(args):
-    context = {}
-    if args.context:
-        for c in args.context:
-            key, value = c.split("=", 1)
-            context[key] = value
-    return context or None
-
-
-def discover(args):
-    result = minitmt.fmf.FMFTests(args.root, args.plan, context=_get_context(args))
-    for name in result.tests:
-        print(name)
-
-
-def show(args):
-    result = minitmt.fmf.FMFTests(args.root, args.plan, context=_get_context(args))
-    if tests := list(result.match(args.test)):
-        for test in tests:
-            print(f"\n--- {test.name} ---")
-            pprint.pprint(test.data)
-    else:
-        _fatal(f"Not reachable via {args.plan} discovery: {args.test}")
-
-
-def execute(args):
-    # remote system connection
-    ssh_keypath = Path(args.ssh_identity)
-    if not ssh_keypath.exists():
-        _fatal(f"SSH Identity {args.ssh_identity} does not exist")
-    ssh_options = {
-        "User": args.user,
-        "Hostname": args.host,
-        "IdentityFile": ssh_keypath,
-    }
-    env = dict(x.split("=",1) for x in args.env)
-
-    # dummy Remote that just wraps the connection
-    class DummyRemote(provision.Remote, connection.ssh.ManagedSSHConn):
-        @staticmethod
-        def release():
-            return
-
-        @staticmethod
-        def alive():
-            return True
-
-    # result aggregation
-    with aggregator.CSVAggregator(args.results_csv, args.results_dir) as csv_aggregator:
-        platform_aggregator = csv_aggregator.for_platform(args.platform)
-
-        # tests discovery and selection
-        result = minitmt.fmf.FMFTests(args.root, args.plan, context=_get_context(args))
-        if args.test:
-            tests = list(result.match(args.test))
-            if not tests:
-                _fatal(f"Not reachable via plan {args.plan} discovery: {args.test}")
-        else:
-            tests = list(result.as_fmftests())
-            if not tests:
-                _fatal(f"No tests found for plan {args.plan}")
-
-        # test run
-        with DummyRemote(ssh_options) as remote:
-            executor = minitmt.executor.Executor(remote, platform_aggregator, env=env)
-            executor.upload_tests(args.root)
-            executor.setup_plan(result)
-            for test in tests:
-                executor.run_test(test)
-
-
-def setup_script(args):
-    result = minitmt.fmf.FMFTests(args.root, args.plan, context=_get_context(args))
-    try:
-        test = result.as_fmftest(args.test)
-    except KeyError:
-        print(f"Not reachable via {args.plan} discovery: {args.test}")
-        raise SystemExit(1) from None
-    output = minitmt.scripts.test_setup(
-        test=test,
-        tests_dir=args.remote_root,
-        debug=args.script_debug,
-    )
-    print(output, end="")
-
-
-def parse_args(parser):
-    parser.add_argument("--root", help="path to directory with fmf tests", default=".")
-    parser.add_argument("--context", "-c", help="tmt style key=value context", action="append")
-    cmds = parser.add_subparsers(
-        dest="_cmd", help="minitmt feature", metavar="<cmd>", required=True,
-    )
-
-    cmd = cmds.add_parser(
-        "discover", aliases=("di",),
-        help="list tests, post-processed by tmt plans",
-    )
-    cmd.add_argument("plan", help="tmt plan to use for discovery")
-
-    cmd = cmds.add_parser(
-        "show",
-        help="show fmf data of a test",
-    )
-    cmd.add_argument("plan", help="tmt plan to use for discovery")
-    cmd.add_argument("test", help="fmf style test regex")
-
-    cmd = cmds.add_parser(
-        "execute", aliases=("ex",),
-        help="run a plan (or test) on a remote system",
-    )
-    #grp = cmd.add_mutually_exclusive_group()
-    #grp.add_argument("--test", "-t", help="fmf style test regex")
-    #grp.add_argument("--plan", "-p", help="tmt plan name (path) inside metadata root")
-    cmd.add_argument("--env", "-e", help="environment to pass to prepare/test", action="append")
-    cmd.add_argument("--test", "-t", help="fmf style test regex")
-    cmd.add_argument(
-        "--plan", "-p", help="tmt plan name (path) inside metadata root", required=True,
-    )
-    cmd.add_argument("--platform", help="platform name, ie. rhel9@x86_64", required=True)
-    cmd.add_argument("--user", help="ssh user to connect via", required=True)
-    cmd.add_argument("--host", help="ssh host to connect to", required=True)
-    cmd.add_argument(
-        "--ssh-identity", help="path to a ssh keyfile for login", required=True,
-    )
-    cmd.add_argument(
-        "--results-csv", help="path to would-be-created .csv.gz results", required=True,
-    )
-    cmd.add_argument(
-        "--results-dir", help="path to would-be-created dir for uploaded files", required=True,
-    )
-
-    cmd = cmds.add_parser(
-        "setup-script",
-        help="generate a script prepping tests for run",
-    )
-    cmd.add_argument("--remote-root", help="path to tests repo on the remote", required=True)
-    cmd.add_argument("--script-debug", help="do 'set -x' in the script", action="store_true")
-    cmd.add_argument("plan", help="tmt plan to use for discovery")
-    cmd.add_argument("test", help="full fmf test name (not regex)")
-
-
-def main(args):
-    if args._cmd in ("discover", "di"):
-        discover(args)
-    elif args._cmd == "show":
-        show(args)
-    elif args._cmd in ("execute", "ex"):
-        execute(args)
-    elif args._cmd == "setup-script":
-        setup_script(args)
-    else:
-        raise RuntimeError(f"unknown args: {args}")
-
-
-CLI_SPEC = {
-    "aliases": ("tmt",),
-    "help": "simple test executor using atex.minitmt",
-    "args": parse_args,
-    "main": main,
-}

atex/minitmt/__init__.py

@@ -1,23 +0,0 @@
-"""
-TODO Minitmt documentation - reference README, etc.
-"""
-
-import importlib as _importlib
-import pkgutil as _pkgutil
-
-__all__ = [
-    info.name for info in _pkgutil.iter_modules(__spec__.submodule_search_locations)
-]
-
-
-def __dir__():
-    return __all__
-
-
-# lazily import submodules
-def __getattr__(attr):
-    # importing a module known to exist
-    if attr in __all__:
-        return _importlib.import_module(f".{attr}", __name__)
-    else:
-        raise AttributeError(f"module '{__name__}' has no attribute '{attr}'")