atex 0.4__py3-none-any.whl → 0.7__py3-none-any.whl

atex/__init__.py

@@ -18,18 +18,8 @@ def __dir__():
 
 # lazily import submodules
 def __getattr__(attr):
-#    # from mod import *
-#    if attr == '__all__':
-#        print("importing all")
-#        for mod in __all__:
-#            _importlib.import_module(f'.{mod}', __name__)
-#        return __all__
-#    # accessing __all__, __getattr__, etc. directly
-#    elif attr in globals():
-#        print("importing globals")
-#        return globals()[attr]
     # importing a module known to exist
     if attr in __all__:
-        return _importlib.import_module(f'.{attr}', __name__)
+        return _importlib.import_module(f".{attr}", __name__)
     else:
-        raise AttributeError(f'module {__name__} has no attribute {attr}')
+        raise AttributeError(f"module '{__name__}' has no attribute '{attr}'")

atex/cli/__init__.py

@@ -32,16 +32,16 @@ def setup_logging(level):
     logging.basicConfig(
         level=level,
         stream=sys.stderr,
-        format='%(asctime)s %(name)s: %(message)s',
-        datefmt='%Y-%m-%d %H:%M:%S',
+        format="%(asctime)s %(name)s: %(message)s",
+        datefmt="%Y-%m-%d %H:%M:%S",
     )
 
 
 def collect_modules():
     for info in pkgutil.iter_modules(__spec__.submodule_search_locations):
-        mod = importlib.import_module(f'.{info.name}', __name__)
-        if not hasattr(mod, 'CLI_SPEC'):
-            raise ValueError(f"CLI submodule {info.name} does not define CLI_SPEC")
+        mod = importlib.import_module(f".{info.name}", __name__)
+        if not hasattr(mod, "CLI_SPEC"):
+            raise ValueError(f"CLI submodule '{info.name}' does not define CLI_SPEC")
         yield (info.name, mod.CLI_SPEC)
 
 
@@ -50,28 +50,28 @@ def main():
 
     log_grp = parser.add_mutually_exclusive_group()
     log_grp.add_argument(
-        '--debug', '-d', action='store_const', dest='loglevel', const=logging.DEBUG,
+        "--debug", "-d", action="store_const", dest="loglevel", const=logging.DEBUG,
         help="enable extra debugging (logging.DEBUG)",
     )
     log_grp.add_argument(
-        '--quiet', '-q', action='store_const', dest='loglevel', const=logging.WARNING,
+        "--quiet", "-q", action="store_const", dest="loglevel", const=logging.WARNING,
         help="be quiet during normal operation (logging.WARNING)",
     )
     parser.set_defaults(loglevel=logging.INFO)
 
     mains = {}
-    subparsers = parser.add_subparsers(dest='_module', metavar='<module>', required=True)
+    subparsers = parser.add_subparsers(dest="_module", metavar="<module>", required=True)
     for name, spec in collect_modules():
-        aliases = spec['aliases'] if 'aliases' in spec else ()
+        aliases = spec["aliases"] if "aliases" in spec else ()
         subp = subparsers.add_parser(
             name,
             aliases=aliases,
-            help=spec['help'],
+            help=spec["help"],
         )
-        spec['args'](subp)
-        mains[name] = spec['main']
+        spec["args"](subp)
+        mains[name] = spec["main"]
         for alias in aliases:
-            mains[alias] = spec['main']
+            mains[alias] = spec["main"]
 
     args = parser.parse_args()
 

atex/cli/minitmt.py

@@ -1,82 +1,175 @@
-import re
+import sys
+#import re
 import pprint
+#import subprocess
+from pathlib import Path
 
-#from .. import util
-from ..minitmt import fmf
+from .. import connection, provision, minitmt
+from ..orchestrator import aggregator
+
+
+def _fatal(msg):
+    print(msg, file=sys.stderr)
+    sys.exit(1)
 
 
 def _get_context(args):
     context = {}
     if args.context:
         for c in args.context:
-            key, value = c.split('=', 1)
+            key, value = c.split("=", 1)
             context[key] = value
     return context or None
 
 
 def discover(args):
-    result = fmf.FMFData(args.root, args.plan, context=_get_context(args))
-    for test in result.tests:
-        print(test.name)
+    result = minitmt.fmf.FMFTests(args.root, args.plan, context=_get_context(args))
+    for name in result.tests:
+        print(name)
 
 
 def show(args):
-    result = fmf.FMFData(args.root, args.plan, context=_get_context(args))
-    for test in result.tests:
-        if re.match(args.test, test.name):
+    result = minitmt.fmf.FMFTests(args.root, args.plan, context=_get_context(args))
+    if tests := list(result.match(args.test)):
+        for test in tests:
+            print(f"\n--- {test.name} ---")
             pprint.pprint(test.data)
-            break
     else:
+        _fatal(f"Not reachable via {args.plan} discovery: {args.test}")
+
+
+def execute(args):
+    # remote system connection
+    ssh_keypath = Path(args.ssh_identity)
+    if not ssh_keypath.exists():
+        _fatal(f"SSH Identity {args.ssh_identity} does not exist")
+    ssh_options = {
+        "User": args.user,
+        "Hostname": args.host,
+        "IdentityFile": ssh_keypath,
+    }
+    env = dict(x.split("=",1) for x in args.env)
+
+    # dummy Remote that just wraps the connection
+    class DummyRemote(provision.Remote, connection.ssh.ManagedSSHConn):
+        @staticmethod
+        def release():
+            return
+
+        @staticmethod
+        def alive():
+            return True
+
+    # result aggregation
+    with aggregator.CSVAggregator(args.results_csv, args.results_dir) as csv_aggregator:
+        platform_aggregator = csv_aggregator.for_platform(args.platform)
+
+        # tests discovery and selection
+        result = minitmt.fmf.FMFTests(args.root, args.plan, context=_get_context(args))
+        if args.test:
+            tests = list(result.match(args.test))
+            if not tests:
+                _fatal(f"Not reachable via plan {args.plan} discovery: {args.test}")
+        else:
+            tests = list(result.as_fmftests())
+            if not tests:
+                _fatal(f"No tests found for plan {args.plan}")
+
+        # test run
+        with DummyRemote(ssh_options) as remote:
+            executor = minitmt.executor.Executor(remote, platform_aggregator, env=env)
+            executor.upload_tests(args.root)
+            executor.setup_plan(result)
+            for test in tests:
+                executor.run_test(test)
+
+
+def setup_script(args):
+    result = minitmt.fmf.FMFTests(args.root, args.plan, context=_get_context(args))
+    try:
+        test = result.as_fmftest(args.test)
+    except KeyError:
         print(f"Not reachable via {args.plan} discovery: {args.test}")
-        raise SystemExit(1)
+        raise SystemExit(1) from None
+    output = minitmt.scripts.test_setup(
+        test=test,
+        tests_dir=args.remote_root,
+        debug=args.script_debug,
+    )
+    print(output, end="")
 
 
 def parse_args(parser):
-    parser.add_argument('--root', default='.', help="path to directory with fmf tests")
-    parser.add_argument('--context', '-c', help="tmt style key=value context", action='append')
+    parser.add_argument("--root", help="path to directory with fmf tests", default=".")
+    parser.add_argument("--context", "-c", help="tmt style key=value context", action="append")
     cmds = parser.add_subparsers(
-        dest='_cmd', help="minitmt feature", metavar='<cmd>', required=True,
+        dest="_cmd", help="minitmt feature", metavar="<cmd>", required=True,
     )
 
     cmd = cmds.add_parser(
-        'discover', aliases=('di',),
+        "discover", aliases=("di",),
         help="list tests, post-processed by tmt plans",
     )
-    cmd.add_argument('plan', help="tmt plan to use for discovery")
+    cmd.add_argument("plan", help="tmt plan to use for discovery")
 
     cmd = cmds.add_parser(
-        'show',
+        "show",
         help="show fmf data of a test",
     )
-    cmd.add_argument('plan', help="tmt plan to use for discovery")
-    cmd.add_argument('test', help="fmf style test regex")
+    cmd.add_argument("plan", help="tmt plan to use for discovery")
+    cmd.add_argument("test", help="fmf style test regex")
 
     cmd = cmds.add_parser(
-        'execute', aliases=('ex',),
+        "execute", aliases=("ex",),
         help="run a plan (or test) on a remote system",
     )
-    grp = cmd.add_mutually_exclusive_group()
-    grp.add_argument('--test', '-t', help="fmf style test regex")
-    grp.add_argument('--plan', '-p', help="tmt plan name (path) inside metadata root")
-    cmd.add_argument('--ssh-identity', '-i', help="path to a ssh keyfile for login")
-    cmd.add_argument('user_host', help="ssh style user@host of the remote")
+    #grp = cmd.add_mutually_exclusive_group()
+    #grp.add_argument("--test", "-t", help="fmf style test regex")
+    #grp.add_argument("--plan", "-p", help="tmt plan name (path) inside metadata root")
+    cmd.add_argument("--env", "-e", help="environment to pass to prepare/test", action="append")
+    cmd.add_argument("--test", "-t", help="fmf style test regex")
+    cmd.add_argument(
+        "--plan", "-p", help="tmt plan name (path) inside metadata root", required=True,
+    )
+    cmd.add_argument("--platform", help="platform name, ie. rhel9@x86_64", required=True)
+    cmd.add_argument("--user", help="ssh user to connect via", required=True)
+    cmd.add_argument("--host", help="ssh host to connect to", required=True)
+    cmd.add_argument(
+        "--ssh-identity", help="path to a ssh keyfile for login", required=True,
+    )
+    cmd.add_argument(
+        "--results-csv", help="path to would-be-created .csv.gz results", required=True,
+    )
+    cmd.add_argument(
+        "--results-dir", help="path to would-be-created dir for uploaded files", required=True,
+    )
+
+    cmd = cmds.add_parser(
+        "setup-script",
+        help="generate a script prepping tests for run",
+    )
+    cmd.add_argument("--remote-root", help="path to tests repo on the remote", required=True)
+    cmd.add_argument("--script-debug", help="do 'set -x' in the script", action="store_true")
+    cmd.add_argument("plan", help="tmt plan to use for discovery")
+    cmd.add_argument("test", help="full fmf test name (not regex)")
 
 
 def main(args):
-    if args._cmd in ('discover', 'di'):
+    if args._cmd in ("discover", "di"):
         discover(args)
-    elif args._cmd == 'show':
+    elif args._cmd == "show":
         show(args)
-    elif args._cmd in ('execute', 'ex'):
-        #execute(args)
-        print("not implemented yet")
+    elif args._cmd in ("execute", "ex"):
+        execute(args)
+    elif args._cmd == "setup-script":
+        setup_script(args)
     else:
         raise RuntimeError(f"unknown args: {args}")
 
 
 CLI_SPEC = {
-    'aliases': ('tmt',),
-    'help': "simple test executor using atex.minitmt",
-    'args': parse_args,
-    'main': main,
+    "aliases": ("tmt",),
+    "help": "simple test executor using atex.minitmt",
+    "args": parse_args,
+    "main": main,
 }

atex/cli/testingfarm.py

@@ -2,15 +2,15 @@ import sys
 import pprint
 
 from .. import util
-from .. import testingfarm as tf
+from ..provision.testingfarm import api as tf
 
 
 def _get_api(args):
     api_args = {}
     if args.url:
-        api_args['url'] = args.url
+        api_args["url"] = args.url
     if args.token:
-        api_args['token'] = args.token
+        api_args["token"] = args.token
     return tf.TestingFarmAPI(**api_args)
 
 
@@ -27,9 +27,9 @@ def whoami(args):
 def composes(args):
     api = _get_api(args)
     comps = api.composes(ranch=args.ranch)
-    comps_list = comps['composes']
+    comps_list = comps["composes"]
     for comp in comps_list:
-        print(comp['name'])
+        print(comp["name"])
 
 
 def get_request(args):
@@ -56,27 +56,27 @@ def search_requests(args):
     if not reply:
         return
 
-    for req in sorted(reply, key=lambda x: x['created']):
-        req_id = req['id']
-        created = req['created'].partition('.')[0]
+    for req in sorted(reply, key=lambda x: x["created"]):
+        req_id = req["id"]
+        created = req["created"].partition(".")[0]
 
         envs = []
-        for env in req['environments_requested']:
-            if 'os' in env and env['os'] and 'compose' in env['os']:
-                compose = env['os']['compose']
-                arch = env['arch']
+        for env in req["environments_requested"]:
+            if "os" in env and env["os"] and "compose" in env["os"]:
+                compose = env["os"]["compose"]
+                arch = env["arch"]
                 if compose and arch:
-                    envs.append(f'{compose}@{arch}')
-        envs_str = ', '.join(envs)
+                    envs.append(f"{compose}@{arch}")
+        envs_str = ", ".join(envs)
 
-        print(f'{created} {req_id} : {envs_str}')
+        print(f"{created} {req_id} : {envs_str}")
 
 
 def reserve(args):
     util.info(f"Reserving {args.compose} on {args.arch} for {args.timeout} minutes")
 
     if args.hvm:
-        hardware = {'virtualization': {'is-supported': True}}
+        hardware = {"virtualization": {"is-supported": True}}
     else:
         hardware = None
 
@@ -96,12 +96,12 @@ def reserve(args):
                 res.request.assert_alive()
             except tf.GoneAwayError as e:
                 print(e)
-                raise SystemExit(1)
+                raise SystemExit(1) from None
 
             proc = util.subprocess_run([
-                'ssh', '-q', '-i', m.ssh_key,
-                '-oStrictHostKeyChecking=no', '-oUserKnownHostsFile=/dev/null',
-                f'{m.user}@{m.host}',
+                "ssh", "-q", "-i", m.ssh_key,
+                "-oStrictHostKeyChecking=no", "-oUserKnownHostsFile=/dev/null",
+                f"{m.user}@{m.host}",
             ])
             if proc.returncode != 0:
                 print(
@@ -123,7 +123,7 @@ def watch_pipeline(args):
 
     util.info(f"Waiting for {args.request_id} to be 'running'")
     try:
-        request.wait_for_state('running')
+        request.wait_for_state("running")
     except tf.GoneAwayError:
         util.info(f"Request {args.request_id} already finished")
         return
@@ -132,96 +132,96 @@ def watch_pipeline(args):
     try:
         for line in tf.PipelineLogStreamer(request):
             sys.stdout.write(line)
-            sys.stdout.write('\n')
+            sys.stdout.write("\n")
     except tf.GoneAwayError:
         util.info(f"Request {args.request_id} finished, exiting")
 
 
 def parse_args(parser):
-    parser.add_argument('--url', help="Testing Farm API URL")
-    parser.add_argument('--token', help="Testing Farm API auth token")
+    parser.add_argument("--url", help="Testing Farm API URL")
+    parser.add_argument("--token", help="Testing Farm API auth token")
     cmds = parser.add_subparsers(
-        dest='_cmd', help="TF helper to run", metavar='<cmd>', required=True,
+        dest="_cmd", help="TF helper to run", metavar="<cmd>", required=True,
     )
 
     cmd = cmds.add_parser(
-        'whoami',
+        "whoami",
         help="print out details about active TF token",
     )
     cmd = cmds.add_parser(
-        'about',
+        "about",
         help="print out details about TF instance (url)",
     )
 
     cmd = cmds.add_parser(
-        'composes',
+        "composes",
         help="list all composes available on a given ranch",
     )
-    cmd.add_argument('ranch', nargs='?', help="Testing Farm ranch (autodetected if token)")
+    cmd.add_argument("ranch", nargs="?", help="Testing Farm ranch (autodetected if token)")
 
     cmd = cmds.add_parser(
-        'get-request', aliases=('gr',),
+        "get-request", aliases=("gr",),
         help="retrieve and print JSON of a Testing Farm request",
     )
-    cmd.add_argument('request_id', help="Testing Farm request UUID")
+    cmd.add_argument("request_id", help="Testing Farm request UUID")
 
     cmd = cmds.add_parser(
-        'cancel',
+        "cancel",
         help="cancel a Testing Farm request",
     )
-    cmd.add_argument('request_id', help="Testing Farm request UUID")
+    cmd.add_argument("request_id", help="Testing Farm request UUID")
 
     cmd = cmds.add_parser(
-        'search-requests', aliases=('sr',),
+        "search-requests", aliases=("sr",),
         help="return a list of requests matching the criteria",
     )
-    cmd.add_argument('--state', help="request state (running, etc.)", required=True)
-    cmd.add_argument('--all', help="all requests, not just owned by token", action='store_true')
-    cmd.add_argument('--ranch', help="Testing Farm ranch")
-    cmd.add_argument('--before', help="only requests created before ISO8601")
-    cmd.add_argument('--after', help="only requests created after ISO8601")
+    cmd.add_argument("--state", help="request state (running, etc.)", required=True)
+    cmd.add_argument("--all", help="all requests, not just owned by token", action="store_true")
+    cmd.add_argument("--ranch", help="Testing Farm ranch")
+    cmd.add_argument("--before", help="only requests created before ISO8601")
+    cmd.add_argument("--after", help="only requests created after ISO8601")
 
     cmd = cmds.add_parser(
-        'reserve',
+        "reserve",
         help="reserve a system and ssh into it",
     )
-    cmd.add_argument('--compose', '-c', help="OS compose to install", required=True)
-    cmd.add_argument('--arch', '-a', help="system HW architecture", default='x86_64')
-    cmd.add_argument('--timeout', '-t', help="pipeline timeout (in minutes)", type=int, default=60)
-    cmd.add_argument('--ssh-key', help="path to a ssh private key file like 'id_rsa'")
-    cmd.add_argument('--hvm', help="request a HVM virtualization capable HW", action='store_true')
+    cmd.add_argument("--compose", "-c", help="OS compose to install", required=True)
+    cmd.add_argument("--arch", "-a", help="system HW architecture", default="x86_64")
+    cmd.add_argument("--timeout", "-t", help="pipeline timeout (in minutes)", type=int, default=60)
+    cmd.add_argument("--ssh-key", help="path to a ssh private key file like 'id_rsa'")
+    cmd.add_argument("--hvm", help="request a HVM virtualization capable HW", action="store_true")
 
     cmd = cmds.add_parser(
-        'watch-pipeline', aliases=('wp',),
+        "watch-pipeline", aliases=("wp",),
         help="continuously output pipeline.log like 'tail -f'",
     )
-    cmd.add_argument('request_id', help="Testing Farm request UUID")
+    cmd.add_argument("request_id", help="Testing Farm request UUID")
 
 
 def main(args):
-    if args._cmd == 'whoami':
+    if args._cmd == "whoami":
         whoami(args)
-    elif args._cmd == 'about':
+    elif args._cmd == "about":
         about(args)
-    elif args._cmd == 'composes':
+    elif args._cmd == "composes":
         composes(args)
-    elif args._cmd in ('get-request', 'gr'):
+    elif args._cmd in ("get-request", "gr"):
         get_request(args)
-    elif args._cmd == 'cancel':
+    elif args._cmd == "cancel":
         cancel(args)
-    elif args._cmd in ('search-requests', 'sr'):
+    elif args._cmd in ("search-requests", "sr"):
         search_requests(args)
-    elif args._cmd == 'reserve':
+    elif args._cmd == "reserve":
         reserve(args)
-    elif args._cmd in ('watch-pipeline', 'wp'):
+    elif args._cmd in ("watch-pipeline", "wp"):
         watch_pipeline(args)
     else:
         raise RuntimeError(f"unknown args: {args}")
 
 
 CLI_SPEC = {
-    'aliases': ('tf',),
-    'help': "various utils for Testing Farm",
-    'args': parse_args,
-    'main': main,
+    "aliases": ("tf",),
+    "help": "various utils for Testing Farm",
+    "args": parse_args,
+    "main": main,
 }

atex/connection/__init__.py

@@ -0,0 +1,125 @@
+import importlib as _importlib
+import pkgutil as _pkgutil
+import threading as _threading
+
+from .. import util as _util
+
+
+class Connection:
+    """
+    A unified API for connecting to a remote system, running multiple commands,
+    rsyncing files to/from it and checking for connection state.
+
+        conn = Connection()
+        conn.connect()
+        proc = conn.cmd(["ls", "/"])
+        #proc = conn.cmd(["ls", "/"], func=subprocess.Popen)  # non-blocking
+        #output = conn.cmd(["ls", "/"], func=subprocess.check_output)  # stdout
+        conn.rsync("-v", "remote:/etc/passwd", "passwd")
+        conn.disconnect()
+
+        # or as try/except/finally
+        conn = Connection()
+        try:
+            conn.connect()
+            ...
+        finally:
+            conn.disconnect()
+
+        # or via Context Manager
+        with Connection() as conn:
+            ...
+
+    Note that internal connection handling must be implemented as thread-aware,
+    ie. disconnect() might be called from a different thread while connect()
+    or cmd() are still running.
+    Similarly, multiple threads may run cmd() or rsync() independently.
+    """
+
+    def __init__(self):
+        """
+        Initialize the connection instance.
+        If extending __init__, always call 'super().__init__()' at the top.
+        """
+        self.lock = _threading.RLock()
+
+    def __enter__(self):
+        self.connect()
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.disconnect()
+
+    def connect(self, block=True):
+        """
+        Establish a persistent connection to the remote.
+
+        If 'block' is True, wait for the connection to be up,
+        otherwise raise BlockingIOError if the connection is still down.
+        """
+        raise NotImplementedError(f"'connect' not implemented for {self.__class__.__name__}")
+
+    def disconnect(self):
+        """
+        Destroy the persistent connection to the remote.
+        """
+        raise NotImplementedError(f"'disconnect' not implemented for {self.__class__.__name__}")
+
+    # TODO: is this needed? .. we probably want Remote.alive() instead
+    #def alive(self):
+    #    """
+    #    Return True if the connection was established and is active,
+    #    False otherwise.
+    #    """
+    #    raise NotImplementedError(f"'alive' not implemented for {self.__class__.__name__}")
+
+    def cmd(self, command, func=_util.subprocess_run, **func_args):
+        """
+        Execute a single command on the remote, using subprocess-like semantics.
+
+        'command' is the command with arguments, as a tuple/list.
+
+        'func' is the subprocess function to use (.run(), .Popen, etc.).
+
+        'func_args' are further keyword arguments to pass to 'func'.
+        """
+        raise NotImplementedError(f"'cmd' not implemented for {self.__class__.__name__}")
+
+    def rsync(self, *args, func=_util.subprocess_run, **func_args):
+        """
+        Synchronize local/remote files/directories via 'rsync'.
+
+        Pass *args like rsync(1) CLI arguments, incl. option arguments, ie.
+            .rsync("-vr", "local_path/", "remote:remote_path")
+            .rsync("-z", "remote:remote_file" ".")
+
+        To indicate remote path, use any string followed by a colon, the remote
+        name does not matter as an internally-handled '-e' option dictates all
+        the connection details.
+
+        'func' is a subprocess function to use (.run(), .Popen, etc.).
+
+        'func_args' are further keyword arguments to pass to 'func'.
+
+        The remote must have rsync(1) already installed.
+        """
+        raise NotImplementedError(f"'rsync' not implemented for {self.__class__.__name__}")
+
+
+_submodules = [
+    info.name for info in _pkgutil.iter_modules(__spec__.submodule_search_locations)
+]
+
+__all__ = [*_submodules, Connection.__name__]  # noqa: PLE0604
+
+
+def __dir__():
+    return __all__
+
+
+# lazily import submodules
+def __getattr__(attr):
+    if attr in _submodules:
+        return _importlib.import_module(f".{attr}", __name__)
+    else:
+        raise AttributeError(f"module '{__name__}' has no attribute '{attr}'")