atex 0.1__py3-none-any.whl

atex/testingfarm.py

@@ -0,0 +1,523 @@
+import os
+import sys
+import re
+import time
+import tempfile
+import textwrap
+import subprocess
+import collections
+
+from pathlib import Path
+
+from . import util
+
+#from pprint import pprint as pp
+import json
+import urllib3
+
+DEFAULT_API_URL = 'https://api.testing-farm.io/v0.1'
+
+# how many seconds to sleep for during API polling
+API_QUERY_DELAY = 10
+
+RESERVE_TASK = {
+    'fmf': {
+#        'url': 'https://github.com/RHSecurityCompliance/atex',  # TODO
+#        'ref': 'main',
+#        'path': 'fmf_tests',
+#        'name': "/reserve",
+        'url': 'https://github.com/comps/tmt-experiments-public',
+        'ref': 'master',
+        'path': '.',
+        'test_name': '/reserve',
+    },
+}
+
+# final states of a request,
+# https://gitlab.com/testing-farm/nucleus/-/blob/main/api/src/tft/nucleus/api/core/schemes/test_request.py
+END_STATES = ('error', 'complete', 'canceled')
+
+# always have at most 3 outstanding HTTP requests to every given API host,
+# shared by all instances of all classes here, to avoid flooding the host
+# by multi-threaded users
+_http = urllib3.PoolManager(maxsize=3, block=True)
+
+
+class TestingFarmError(Exception):
+    pass
+
+
+class APIError(TestingFarmError):
+    pass
+
+
+class BadHTTPError(TestingFarmError):
+    pass
+
+
+# TODO: __init__ and __str__ so we pass just request ID, not a full message
+class GoneAwayError(TestingFarmError):
+    pass
+
+
+class TestingFarmAPI:
+    """
+    A python interface for the Testing Farm HTTP API, closely matching
+    functionality provided by it, returning python dictionaries that
+    correspond to JSON replies of the HTTP API functions.
+    """
+
+    def __init__(self, url=DEFAULT_API_URL, token=None):
+        """
+        'url' is Testing Farm API URL, a sensible default is used
+        if unspecified.
+
+        'token' is a secret API token generated by Testing Farm admins,
+        if empty, the TESTING_FARM_API_TOKEN env var is read instead.
+
+        Note that token-less operation is supported, with limited functionality.
+        """
+        self.api_url = url
+        self.api_token = token or os.environ.get('TESTING_FARM_API_TOKEN')
+
+    def _query(self, method, path, *args, headers=None, **kwargs):
+        url = f'{self.api_url}{path}'
+        if headers is not None:
+            headers['Authorization'] = f'Bearer {self.api_token}'
+        else:
+            headers = {'Authorization': f'Bearer {self.api_token}'}
+
+        reply = _http.request(method, url, *args, headers=headers, preload_content=False, **kwargs)
+
+        if reply.status != 200 and not reply.data:
+            raise APIError(f"got HTTP {reply.status} on {method} {url}")
+
+        if reply.headers.get('Content-Type') != 'application/json':
+            raise BadHTTPError(f"HTTP {reply.status} on {method} {url} is not application/json")
+
+        try:
+            decoded = reply.json()
+        except json.decoder.JSONDecodeError:
+            raise BadHTTPError(f"failed to decode JSON for {method} {url}: {reply.data}")
+
+        if reply.status != 200:
+            raise APIError(f"got HTTP {reply.status} on {method} {url}: {decoded}")
+
+        return decoded
+
+    def whoami(self):
+        if not self.api_token:
+            raise ValueError("whoami() requires an auth token")
+        if hasattr(self, '_whoami_cached'):
+            return self._whoami_cached
+        else:
+            self._whoami_cached = self._query('GET', '/whoami')
+            return self._whoami_cached
+
+    def about(self):
+        return self._query('GET', '/about')
+
+    def composes(self, ranch=None):
+        """
+        'ranch' is 'public' or 'redhat', autodetected if token was given.
+        """
+        if not ranch:
+            if not self.api_token:
+                raise ValueError("composes() requires an auth token to identify ranch")
+            ranch = self.whoami()['token']['ranch']
+        return self._query('GET', f'/composes/{ranch}')
+
+    def search_requests(
+        self, state, mine=True, ranch=None, created_before=None, created_after=None,
+    ):
+        """
+        'state' is one of 'running', 'queued', etc., and is required by the API.
+
+        If 'mine' is True and a token was given, return only requests for that
+        token (user), otherwise return *all* requests (use extra filters pls).
+
+        'ranch' is 'public' or 'redhat', or (probably?) all if left empty.
+
+        'created_*' take ISO 8601 formatted strings, as returned by the API
+        elsewhere, ie. 'YYYY-MM-DD' or 'YYYY-MM-DDTHH:MM:SS' (or with '.MS'),
+        without timezone.
+        """
+        fields = {'state': state}
+        if ranch:
+            fields['ranch'] = ranch
+        if created_before:
+            fields['created_before'] = created_before
+        if created_after:
+            fields['created_after'] = created_after
+
+        if mine:
+            if not self.api_token:
+                raise ValueError("search_requests(mine=True) requires an auth token")
+            fields['token_id'] = self.whoami()['token']['id']
+            fields['user_id'] = self.whoami()['user']['id']
+
+        return self._query('GET', '/requests', fields=fields)
+
+    def get_request(self, request_id):
+        """
+        'request_id' is the UUID (string) of the request.
+        """
+        return self._query('GET', f'/requests/{request_id}')
+
+    def submit_request(self, spec):
+        """
+        'spec' is a big dictionary with 'test', 'environment', 'settings', etc.
+        keys that specify what should be run and where.
+        """
+        if not self.api_token:
+            raise ValueError("submit_request() requires an auth token")
+        return self._query('POST', '/requests', json=spec)
+
+    def cancel_request(self, request_id):
+        """
+        'request_id' is the UUID (string) of the request.
+        """
+        return self._query('DELETE', f'/requests/{request_id}')
+
+
+class Request:
+    """
+    A higher-level API for submitting, querying, and cancelling a Testing Farm
+    request.
+    """
+
+    def __init__(self, id=None, api=None, initial_data=None):
+        """
+        'id' is a Testing Farm request UUID
+        'api' is a TestingFarmAPI instance - if unspecified, a sensible default
+        'initial_data' (dict) can be used to pre-fill an initial Request state
+        will be used.
+        """
+        self.id = id
+        self.api = api or TestingFarmAPI()
+        self.data = initial_data or {}
+
+    def submit(self, spec):
+        """
+        'spec' is a big dictionary with 'test', 'environment', 'settings', etc.
+        keys that specify what should be run and where.
+        """
+        if self.id:
+            raise ValueError("this Request instance already has 'id', refusing submit")
+        self.data = self.api.submit_request(spec)
+        self.id = self.data['id']
+
+    def update(self):
+        """
+        Query Testing Farm API to get a more up-to-date version of the request
+        metadata accessible via __str__().
+        """
+        self.data = self.api.get_request(self.id)
+        return self.data
+
+    def cancel(self):
+        if not self.id:
+            return
+        data = self.api.cancel_request(self.id)
+        self.id = None
+        self.data = {}
+        return data
+
+    def alive(self):
+        if 'state' not in self.data:
+            self.update()
+        return self.data['state'] not in END_STATES
+
+    def assert_alive(self):
+        if not self.alive():
+            raise GoneAwayError(f"request {self.data['id']} not alive anymore")
+
+    def wait_for_state(self, state):
+        if 'state' not in self.data:
+            self.update()
+        self.assert_alive()
+        while self.data['state'] != state:
+            time.sleep(API_QUERY_DELAY)
+            self.update()
+            self.assert_alive()
+
+    def __repr__(self):
+        return f'Request(id={self.id})'
+
+    def __str__(self):
+        # python has no better dict-pretty-printing logic
+        return json.dumps(self.data, sort_keys=True, indent=4)
+
+    def __contains__(self, item):
+        return item in self.data
+
+    def __getitem__(self, key):
+        return self.data[key]
+
+
+class PipelineLogStreamer:
+    """
+    Line buffer for querying Testing Farm pipeline.log using HTTP Range header
+    to "stream" its contents over time (over many requests), never having to
+    re-read old pipeline.log content.
+    """
+    def __init__(self, request):
+        self.request = request
+
+    def _wait_for_entry(self):
+        while True:
+            self.request.wait_for_state('running')
+
+            if 'run' in self.request and 'artifacts' in self.request['run']:
+                if artifacts := self.request['run']['artifacts']:
+                    return f'{artifacts}/pipeline.log'
+
+            time.sleep(API_QUERY_DELAY)
+            self.request.update()
+
+    def __iter__(self):
+        url = self._wait_for_entry()
+        buffer = ''
+        bytes_read = 0
+        while True:
+            self.request.assert_alive()
+
+            headers = {'Range': f'bytes={bytes_read}-'}
+            # load all returned data via .decode() rather than streaming it
+            # in chunks, because we don't want to leave the connection open
+            # (blocking others) while the user code runs between __next__ calls
+            reply = _http.request('GET', url, headers=headers)
+
+            # 416=Range Not Satisfiable, typically meaning "no new data to send"
+            if reply.status == 416:
+                time.sleep(API_QUERY_DELAY)
+                self.request.update()
+                continue
+            # 200=OK or 206=Partial Content
+            elif reply.status not in (200,206):
+                raise BadHTTPError(f"got {reply.status} when trying to GET {url}")
+
+            bytes_read += len(reply.data)
+            buffer += reply.data.decode(errors='ignore')
+
+            while (index := buffer.find('\n')) != -1:
+                yield buffer[:index]
+                buffer = buffer[index+1:]
+
+            time.sleep(API_QUERY_DELAY)
+            self.request.update()
+
+
+class Reserve:
+    r"""
+    An abstraction for (ab)using Testing Farm for OS reservations, by submitting
+    a dummy test that sets up user-provided SSH key access and enters a no-op
+    state, allowing ad-hoc access/use by the user.
+
+    When used in a context manager, it produces a ReservedMachine tuple with
+    connection details for an ssh client:
+
+        with Reserve(compose='CentOS-Stream-9', timeout=720) as m:
+            subprocess.run(['ssh', '-i', m.ssh_key, f'{m.user}@{m.host}', 'ls /'])
+    """
+
+    Reserved = collections.namedtuple('ReservedMachine', ['user', 'host', 'ssh_key', 'request'])
+
+    def __init__(
+        self, compose=None, arch='x86_64', pool=None, hardware=None, kickstart=None,
+        timeout=60, ssh_key=None, source_host=None, api=None,
+    ):
+        """
+        'compose' (str) is the OS to install, chosen from the composes supported
+        by the Testing Farm ranch of the authenticated user.
+
+        'arch' (str) is one of 'x86_64', 's390x', etc.
+
+        'pool' (str) is a name of a Testing Farm infrastructure pool.
+
+        'hardware' (dict) is a complex specification of hardware properties
+        the reserved system should have, see:
+        https://docs.testing-farm.io/Testing%20Farm/0.1/test-request.html#hardware
+
+        'kickstart' (dict) is a Beaker-style specification of Anaconda Kickstart
+        hacks, passed directly to Testing Farm POST /requests API.
+
+        'timeout' (int) is the maximum time IN MINUTES a Testing Farm request
+        is alive, which includes initial creation, waiting in queue, preparing
+        an OS, and the entire reservation period.
+        Make sure to set it high enough (not just the pure reservation time).
+
+        'ssh_key' (str) is a path to an OpenSSH private key file (with an
+        associated public key file in .pub), to be added to the reserved OS.
+        If unspecified, an attempt to read ~/.ssh/id_rsa will be made and if
+        that is also unsuccessful, a temporary keypair will be generated.
+
+        'source_host' (str) is an IPv4 network specified as ie. '1.2.3.4/32'
+        to be allowed incoming traffic to the reserved system (such as ssh).
+        If unspecified, an Internet service will be queried to get an outside-
+        facing address of the current system.
+        Ignored on the 'redhat' ranch.
+
+        'api' is a TestingFarmAPI instance - if unspecified, a sensible default
+        will be used.
+        """
+        spec = {
+            'test': RESERVE_TASK,
+            'environments': [{
+                'arch': arch,
+                'os': {
+                    'compose': compose,
+                },
+                'pool': pool,
+                'settings': {
+                    'pipeline': {
+                        'skip_guest_setup': True,
+                    },
+                    'provisioning': {
+                        'tags': {
+                            'ArtemisUseSpot': 'false',
+                        },
+                        'security_group_rules_ingress': [],
+                    },
+                },
+                'secrets': {},
+            }],
+            'settings': {
+                'pipeline': {
+                    'timeout': timeout,
+                },
+            },
+        }
+        if hardware:
+            spec['environments'][0]['hardware'] = hardware
+        if kickstart:
+            spec['environments'][0]['kickstart'] = kickstart
+
+        self._spec = spec
+        self._ssh_key = Path(ssh_key) if ssh_key else None
+        self._source_host = source_host
+        self.api = api or TestingFarmAPI()
+
+        self.request = None
+        self._tmpdir = None
+
+    @staticmethod
+    def _guess_host_ipv4():
+        curl_agent = {'User-Agent': 'curl/1.2.3'}
+        try:
+            r = _http.request('GET', 'https://ifconfig.me', headers=curl_agent)
+            if r.status != 200:
+                raise ConnectionError()
+        except (ConnectionError, urllib3.exceptions.RequestError):
+            r = _http.request('GET', 'https://ifconfig.co', headers=curl_agent)
+        return r.data.decode().strip()
+
+    @staticmethod
+    def _gen_ssh_keypair(tmpdir):
+        tmpdir = Path(tmpdir)
+        subprocess.run(
+            ['ssh-keygen', '-t', 'rsa', '-N', '', '-f', tmpdir / 'key_rsa'],
+            stdout=subprocess.DEVNULL,
+            check=True,
+        )
+        return (tmpdir / 'key_rsa', tmpdir / 'key_rsa.pub')
+
+    def __enter__(self):
+        spec = self._spec.copy()
+
+        try:
+            # add source_host firewall filter
+            source_host = self._source_host or f'{self._guess_host_ipv4()}/32'
+            ingress = \
+                spec['environments'][0]['settings']['provisioning']['security_group_rules_ingress']
+            ingress.append({
+                'type': 'ingress',
+                'protocol': '-1',
+                'cidr': source_host,
+                'port_min': 0,
+                'port_max': 65535,
+            })
+
+            # read user-provided ssh key, or generate one
+            ssh_key = self._ssh_key
+            if ssh_key:
+                if not ssh_key.exists():
+                    raise FileNotFoundError(f"{ssh_key} specified, but does not exist")
+                ssh_pubkey = Path(f'{ssh_key}.pub')
+            else:
+                self._tmpdir = tempfile.TemporaryDirectory()
+                ssh_key, ssh_pubkey = self._gen_ssh_keypair(self._tmpdir.name)
+
+            pubkey_contents = ssh_pubkey.read_text().strip()
+            secrets = spec['environments'][0]['secrets']
+            secrets['RESERVE_SSH_PUBKEY'] = pubkey_contents
+
+            self.request = Request(api=self.api)
+            self.request.submit(spec)
+            util.debug(f"submitted request:\n{textwrap.indent(str(self.request), '    ')}")
+
+            # wait for the request to become running
+            while self.request['state'] != 'running':
+                time.sleep(API_QUERY_DELAY)
+                self.request.update()
+                if self.request['state'] in END_STATES:
+                    raise GoneAwayError(f"request {self.request['id']} not alive anymore")
+
+            # wait for user/host to ssh to
+            ssh_user = ssh_host = None
+            for line in PipelineLogStreamer(self.request):
+                util.debug(f"pipeline: {line}")
+                # find hidden login details
+                m = re.search(r'\] Guest is ready: ArtemisGuest\([^,]+, (\w+)@([0-9\.]+), ', line)
+                if m:
+                    ssh_user, ssh_host = m.groups()
+                    continue
+                # but wait until much later despite having login, at least until
+                # the test starts running (and we get closer to it inserting our
+                # ~/.ssh/authorized_keys entry)
+                if ssh_user and re.search(r'\] starting tests execution', line):
+                    break
+
+            # wait for a successful connection over ssh
+            # (it will be failing to login for a while, until the reserve test
+            #  installs our ssh pubkey into authorized_keys)
+            ssh_attempt_cmd = [
+                'ssh', '-q', '-i', ssh_key, f'-oConnectionAttempts={API_QUERY_DELAY}',
+               '-oStrictHostKeyChecking=no', '-oUserKnownHostsFile=/dev/null',
+                f'{ssh_user}@{ssh_host}', 'exit 123',
+            ]
+            while True:
+                time.sleep(API_QUERY_DELAY)
+                self.request.update()
+                if self.request['state'] in END_STATES:
+                    raise GoneAwayError(f"request {self.request['id']} not alive anymore")
+
+                proc = util.subprocess_run(
+                    ssh_attempt_cmd,
+                    stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
+                )
+                if proc.returncode == 123:
+                    break
+
+            return self.Reserved(ssh_user, ssh_host, ssh_key, self.request)
+
+        except:
+            self.__exit__(*sys.exc_info())
+            raise
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        if self.request:
+            try:
+                self.request.cancel()
+            except APIError:
+                pass
+            finally:
+                self.request = None
+
+        if self._tmpdir:
+            self._tmpdir.cleanup()
+            self._tmpdir = None
+
+        # cancel request
+        # clear out stored self.request
+        pass

atex/util/__init__.py

@@ -0,0 +1,49 @@
+"""
+TODO some description about utilities
+"""
+
+import importlib as _importlib
+import pkgutil as _pkgutil
+import inspect as _inspect
+
+__all__ = []
+
+
+def __dir__():
+    return __all__
+
+
+# this is the equivalent of 'from .submod import *' for all submodules
+# (function to avoid polluting global namespace with extra variables)
+def _import_submodules():
+    for info in _pkgutil.iter_modules(__spec__.submodule_search_locations):
+        mod = _importlib.import_module(f'.{info.name}', __name__)
+
+        # if the module defines __all__, just use it
+        if hasattr(mod, '__all__'):
+            keys = mod.__all__
+        else:
+            # https://docs.python.org/3/reference/executionmodel.html#binding-of-names
+            keys = (x for x in dir(mod) if not x.startswith('_'))
+
+        for key in keys:
+            attr = getattr(mod, key)
+
+            # avoid objects that belong to other known modules
+            # (ie. imported function from another util module)
+            if hasattr(attr, '__module__'):
+                if attr.__module__ != mod.__name__:
+                    continue
+            # avoid some common pollution / imports
+            # (we don't want subpackages in here anyway)
+            if _inspect.ismodule(attr):
+                continue
+            # do not override already processed objects (avoid duplicates)
+            if key in __all__:
+                raise AssertionError(f"tried to override already-imported '{key}'")
+
+            globals()[key] = attr
+            __all__.append(key)
+
+
+_import_submodules()

atex/util/dedent.py

@@ -0,0 +1,25 @@
+r"""
+This allows raw blocks like
+
+    def func():
+        variable = dedent(fr'''
+            some
+            content
+        ''')
+
+without the leading or trailing newlines and any common leading whitespaces.
+You might think using '''\ would eliminate the first newline, but the string
+is 'raw', it doesn't have escapes.
+
+textwrap.dedent() does only the common leading whitespaces.
+"""
+
+import textwrap
+
+
+def dedent(text):
+    """
+    Like textwrap.dedent(), but also strip leading and trailing spaces/newlines
+    up to the content.
+    """
+    return textwrap.dedent(text.lstrip('\n').rstrip(' \n'))

atex/util/lockable_class.py

@@ -0,0 +1,38 @@
+import threading
+
+
+class LockableClass:
+    """
+    A class with (nearly) all attribute accesses protected by threading.RLock,
+    making them thread-safe at the cost of some speed.
+
+        class MyClass(LockableClass):
+            def writer(self):
+                self.attr = 222     # thread-safe instance access
+            def reader(self):
+                print(self.attr)    # thread-safe instance access
+            def complex(self):
+                with self.lock:     # thread-safe context
+                    self.attr += 1
+
+    Here, 'lock' is a reserved attribute name and must not be overriden
+    by a derived class.
+
+    If overriding '__init__', make sure to call 'super().__init__()' *before*
+    any attribute accesses in your '__init__'.
+    """
+    def __init__(self):
+        object.__setattr__(self, 'lock', threading.RLock())
+
+    def __getattribute__(self, name):
+        # optimize built-ins
+        if name.startswith('__') or name == 'lock':
+            return object.__getattribute__(self, name)
+        lock = object.__getattribute__(self, 'lock')
+        with lock:
+            return object.__getattribute__(self, name)
+
+    def __setattr__(self, name, value):
+        lock = object.__getattribute__(self, 'lock')
+        with lock:
+            object.__setattr__(self, name, value)

atex/util/log.py

@@ -0,0 +1,53 @@
+import inspect
+import logging
+from pathlib import Path
+
+_logger = logging.getLogger('atex')
+
+
+def _format_msg(msg, *, skip_frames=0):
+    stack = inspect.stack()
+    if len(stack)-1 <= skip_frames:
+        raise SyntaxError("skip_frames exceeds call stack (frame count)")
+    stack = stack[skip_frames+1:]
+
+    # bottom of the stack, or runpy executed module
+    for frame_info in stack:
+        if frame_info.function == '<module>':
+            break
+    module = frame_info
+
+    # last (topmost) function that isn't us
+    parent = stack[0]
+    function = parent.function
+
+    # if the function has 'self' and it looks like a class instance,
+    # prepend it to the function name
+    p_locals = parent.frame.f_locals
+    if 'self' in p_locals:
+        self = p_locals['self']
+        if hasattr(self, '__class__') and inspect.isclass(self.__class__):
+            function = f'{self.__class__.__name__}.{function}'
+
+    # don't report module name of a function if it's the same as running module
+    if parent.filename != module.filename:
+        parent_modname = parent.frame.f_globals['__name__']
+        # avoid everything having the package name prefixed
+        parent_modname = parent_modname.partition('.')[2] or parent_modname
+        return f'{parent_modname}.{function}:{parent.lineno}: {msg}'
+    elif parent.function != '<module>':
+        return f'{function}:{parent.lineno}: {msg}'
+    else:
+        return f'{Path(parent.filename).name}:{parent.lineno}: {msg}'
+
+
+def debug(msg, *, skip_frames=0):
+    _logger.debug(_format_msg(msg, skip_frames=skip_frames+1))
+
+
+def info(msg, *, skip_frames=0):
+    _logger.info(_format_msg(msg, skip_frames=skip_frames+1))
+
+
+def warning(msg, *, skip_frames=0):
+    _logger.warning(_format_msg(msg, skip_frames=skip_frames+1))