atex 0.5__py3-none-any.whl → 0.7__py3-none-any.whl

atex/{testingfarm.py → provision/testingfarm/api.py}

@@ -9,28 +9,28 @@ import collections
 
 from pathlib import Path
 
-from . import util
+from ... import util
 
 import json
 import urllib3
 
-DEFAULT_API_URL = 'https://api.testing-farm.io/v0.1'
+DEFAULT_API_URL = "https://api.testing-farm.io/v0.1"
 
 # how many seconds to sleep for during API polling
 API_QUERY_DELAY = 10
 
 RESERVE_TASK = {
-    'fmf': {
-        'url': 'https://github.com/RHSecurityCompliance/atex',
-        'ref': 'main',
-        'path': 'tmt_tests',
-        'name': "/plans/reserve",
+    "fmf": {
+        "url": "https://github.com/RHSecurityCompliance/atex",
+        "ref": "main",
+        "path": "tmt_tests",
+        "name": "/plans/reserve",
     },
 }
 
 # final states of a request,
 # https://gitlab.com/testing-farm/nucleus/-/blob/main/api/src/tft/nucleus/api/core/schemes/test_request.py
-END_STATES = ('error', 'complete', 'canceled')
+END_STATES = ("error", "complete", "canceled")
 
 # always have at most 3 outstanding HTTP requests to every given API host,
 # shared by all instances of all classes here, to avoid flooding the host
@@ -48,6 +48,8 @@ class APIError(TestingFarmError):
     pass
 
 
+# TODO docstrings for these:
+
 class BadHTTPError(TestingFarmError):
     pass
 
@@ -74,21 +76,21 @@ class TestingFarmAPI:
         Note that token-less operation is supported, with limited functionality.
         """
         self.api_url = url
-        self.api_token = token or os.environ.get('TESTING_FARM_API_TOKEN')
+        self.api_token = token or os.environ.get("TESTING_FARM_API_TOKEN")
 
     def _query(self, method, path, *args, headers=None, **kwargs):
-        url = f'{self.api_url}{path}'
+        url = f"{self.api_url}{path}"
         if headers is not None:
-            headers['Authorization'] = f'Bearer {self.api_token}'
+            headers["Authorization"] = f"Bearer {self.api_token}"
         else:
-            headers = {'Authorization': f'Bearer {self.api_token}'}
+            headers = {"Authorization": f"Bearer {self.api_token}"}
 
         reply = _http.request(method, url, *args, headers=headers, preload_content=False, **kwargs)
 
         if reply.status != 200 and not reply.data:
             raise APIError(f"got HTTP {reply.status} on {method} {url}", reply)
 
-        if reply.headers.get('Content-Type') != 'application/json':
+        if reply.headers.get("Content-Type") != "application/json":
             raise BadHTTPError(
                 f"HTTP {reply.status} on {method} {url} is not application/json",
                 reply,
@@ -97,7 +99,10 @@ class TestingFarmAPI:
         try:
             decoded = reply.json()
         except json.decoder.JSONDecodeError:
-            raise BadHTTPError(f"failed to decode JSON for {method} {url}: {reply.data}", reply)
+            raise BadHTTPError(
+                f"failed to decode JSON for {method} {url}: {reply.data}",
+                reply,
+            ) from None
 
         if reply.status != 200:
             raise APIError(f"got HTTP {reply.status} on {method} {url}: {decoded}", reply)
@@ -107,14 +112,14 @@ class TestingFarmAPI:
     def whoami(self):
         if not self.api_token:
             raise ValueError("whoami() requires an auth token")
-        if hasattr(self, '_whoami_cached'):
+        if hasattr(self, "_whoami_cached"):
             return self._whoami_cached
         else:
-            self._whoami_cached = self._query('GET', '/whoami')
+            self._whoami_cached = self._query("GET", "/whoami")
             return self._whoami_cached
 
     def about(self):
-        return self._query('GET', '/about')
+        return self._query("GET", "/about")
 
     def composes(self, ranch=None):
         """
@@ -123,8 +128,8 @@ class TestingFarmAPI:
         if not ranch:
             if not self.api_token:
                 raise ValueError("composes() requires an auth token to identify ranch")
-            ranch = self.whoami()['token']['ranch']
-        return self._query('GET', f'/composes/{ranch}')
+            ranch = self.whoami()["token"]["ranch"]
+        return self._query("GET", f"/composes/{ranch}")
 
     def search_requests(
         self, state, mine=True, ranch=None, created_before=None, created_after=None,
@@ -141,27 +146,27 @@ class TestingFarmAPI:
         elsewhere, ie. 'YYYY-MM-DD' or 'YYYY-MM-DDTHH:MM:SS' (or with '.MS'),
         without timezone.
         """
-        fields = {'state': state}
+        fields = {"state": state}
         if ranch:
-            fields['ranch'] = ranch
+            fields["ranch"] = ranch
         if created_before:
-            fields['created_before'] = created_before
+            fields["created_before"] = created_before
         if created_after:
-            fields['created_after'] = created_after
+            fields["created_after"] = created_after
 
         if mine:
             if not self.api_token:
                 raise ValueError("search_requests(mine=True) requires an auth token")
-            fields['token_id'] = self.whoami()['token']['id']
-            fields['user_id'] = self.whoami()['user']['id']
+            fields["token_id"] = self.whoami()["token"]["id"]
+            fields["user_id"] = self.whoami()["user"]["id"]
 
-        return self._query('GET', '/requests', fields=fields)
+        return self._query("GET", "/requests", fields=fields)
 
     def get_request(self, request_id):
         """
         'request_id' is the UUID (string) of the request.
         """
-        return self._query('GET', f'/requests/{request_id}')
+        return self._query("GET", f"/requests/{request_id}")
 
     def submit_request(self, spec):
         """
@@ -170,13 +175,13 @@ class TestingFarmAPI:
         """
         if not self.api_token:
             raise ValueError("submit_request() requires an auth token")
-        return self._query('POST', '/requests', json=spec)
+        return self._query("POST", "/requests", json=spec)
 
     def cancel_request(self, request_id):
         """
         'request_id' is the UUID (string) of the request.
         """
-        return self._query('DELETE', f'/requests/{request_id}')
+        return self._query("DELETE", f"/requests/{request_id}")
 
 
 class Request:
@@ -185,6 +190,9 @@ class Request:
     request.
     """
 
+    # TODO: maintain internal time.monotonic() clock and call .update() from
+    #       functions like .alive() if last update is > API_QUERY_DELAY
+
     def __init__(self, id=None, api=None, initial_data=None):
         """
         'id' is a Testing Farm request UUID
@@ -204,14 +212,17 @@ class Request:
         if self.id:
             raise ValueError("this Request instance already has 'id', refusing submit")
         self.data = self.api.submit_request(spec)
-        self.id = self.data['id']
+        self.id = self.data["id"]
 
     def update(self):
         """
         Query Testing Farm API to get a more up-to-date version of the request
-        metadata accessible via __str__().
+        metadata. Do not call too frequently.
+        This function is also used internally by others, you do not need to
+        always call it manually.
         """
         self.data = self.api.get_request(self.id)
+        # TODO: refresh internal time.monotonic() timer
         return self.data
 
     def cancel(self):
@@ -223,26 +234,26 @@ class Request:
         return data
 
     def alive(self):
-        if 'state' not in self.data:
+        if "state" not in self.data:
             self.update()
-        return self.data['state'] not in END_STATES
+        return self.data["state"] not in END_STATES
 
     def assert_alive(self):
         if not self.alive():
-            state = self.data['state']
+            state = self.data["state"]
             raise GoneAwayError(f"request {self.data['id']} not alive anymore, entered: {state}")
 
     def wait_for_state(self, state):
-        if 'state' not in self.data:
+        if "state" not in self.data:
             self.update()
         self.assert_alive()
-        while self.data['state'] != state:
+        while self.data["state"] != state:
             time.sleep(API_QUERY_DELAY)
             self.update()
             self.assert_alive()
 
     def __repr__(self):
-        return f'Request(id={self.id})'
+        return f"Request(id={self.id})"
 
     def __str__(self):
         # python has no better dict-pretty-printing logic
@@ -266,18 +277,18 @@ class PipelineLogStreamer:
 
     def _wait_for_entry(self):
         while True:
-            self.request.wait_for_state('running')
+            self.request.wait_for_state("running")
 
             try:
-                if 'run' not in self.request or 'artifacts' not in self.request['run']:
+                if "run" not in self.request or "artifacts" not in self.request["run"]:
                     continue
 
-                artifacts = self.request['run']['artifacts']
+                artifacts = self.request["run"]["artifacts"]
                 if not artifacts:
                     continue
 
-                log = f'{artifacts}/pipeline.log'
-                reply = _http.request('HEAD', log)
+                log = f"{artifacts}/pipeline.log"
+                reply = _http.request("HEAD", log)
                 # TF has a race condition of adding the .log entry without it being created
                 if reply.status == 404:
                     util.debug(f"got 404 for {log}, retrying")
@@ -285,6 +296,8 @@ class PipelineLogStreamer:
                 elif reply.status != 200:
                     raise APIError(f"got HTTP {reply.status} on HEAD {log}", reply)
 
+                util.info(f"artifacts: {artifacts}")
+
                 return log
 
             finally:
@@ -293,17 +306,17 @@ class PipelineLogStreamer:
 
     def __iter__(self):
         url = self._wait_for_entry()
-        buffer = ''
+        buffer = ""
         bytes_read = 0
         while True:
             self.request.assert_alive()
 
             try:
-                headers = {'Range': f'bytes={bytes_read}-'}
+                headers = {"Range": f"bytes={bytes_read}-"}
                 # load all returned data via .decode() rather than streaming it
                 # in chunks, because we don't want to leave the connection open
                 # (blocking others) while the user code runs between __next__ calls
-                reply = _http.request('GET', url, headers=headers)
+                reply = _http.request("GET", url, headers=headers)
 
                 # 416=Range Not Satisfiable, typically meaning "no new data to send"
                 if reply.status == 416:
@@ -313,9 +326,9 @@ class PipelineLogStreamer:
                     raise BadHTTPError(f"got {reply.status} when trying to GET {url}", reply)
 
                 bytes_read += len(reply.data)
-                buffer += reply.data.decode(errors='ignore')
+                buffer += reply.data.decode(errors="ignore")
 
-                while (index := buffer.find('\n')) != -1:
+                while (index := buffer.find("\n")) != -1:
                     yield buffer[:index]
                     buffer = buffer[index+1:]
 
@@ -333,14 +346,17 @@ class Reserve:
     When used in a context manager, it produces a ReservedMachine tuple with
     connection details for an ssh client:
 
-        with Reserve(compose='CentOS-Stream-9', timeout=720) as m:
-            subprocess.run(['ssh', '-i', m.ssh_key, f'{m.user}@{m.host}', 'ls /'])
+        with Reserve(compose="CentOS-Stream-9", timeout=720) as m:
+            subprocess.run(["ssh", "-i", m.ssh_key, f"{m.user}@{m.host}", "ls /"])
     """
 
-    Reserved = collections.namedtuple('ReservedMachine', ['user', 'host', 'ssh_key', 'request'])
+    Reserved = collections.namedtuple(
+        "ReservedMachine",
+        ("host", "port", "user", "ssh_key", "request"),
+    )
 
     def __init__(
-        self, compose=None, arch='x86_64', pool=None, hardware=None, kickstart=None,
+        self, *, compose, arch="x86_64", pool=None, hardware=None, kickstart=None,
         timeout=60, ssh_key=None, source_host=None, api=None,
     ):
         """
@@ -377,37 +393,38 @@ class Reserve:
         'api' is a TestingFarmAPI instance - if unspecified, a sensible default
         will be used.
         """
+        util.info(f"Will reserve compose:{compose} on arch:{arch} for {timeout}min")
         spec = {
-            'test': RESERVE_TASK,
-            'environments': [{
-                'arch': arch,
-                'os': {
-                    'compose': compose,
+            "test": RESERVE_TASK,
+            "environments": [{
+                "arch": arch,
+                "os": {
+                    "compose": compose,
                 },
-                'pool': pool,
-                'settings': {
-                    'pipeline': {
-                        'skip_guest_setup': True,
+                "pool": pool,
+                "settings": {
+                    "pipeline": {
+                        "skip_guest_setup": True,
                     },
-                    'provisioning': {
-                        'tags': {
-                            'ArtemisUseSpot': 'false',
+                    "provisioning": {
+                        "tags": {
+                            "ArtemisUseSpot": "false",
                         },
-                        'security_group_rules_ingress': [],
+                        "security_group_rules_ingress": [],
                     },
                 },
-                'secrets': {},
+                "secrets": {},
             }],
-            'settings': {
-                'pipeline': {
-                    'timeout': timeout,
+            "settings": {
+                "pipeline": {
+                    "timeout": timeout,
                 },
             },
         }
         if hardware:
-            spec['environments'][0]['hardware'] = hardware
+            spec["environments"][0]["hardware"] = hardware
         if kickstart:
-            spec['environments'][0]['kickstart'] = kickstart
+            spec["environments"][0]["kickstart"] = kickstart
 
         self._spec = spec
         self._ssh_key = Path(ssh_key) if ssh_key else None
@@ -419,39 +436,39 @@ class Reserve:
 
     @staticmethod
     def _guess_host_ipv4():
-        curl_agent = {'User-Agent': 'curl/1.2.3'}
+        curl_agent = {"User-Agent": "curl/1.2.3"}
         try:
-            r = _http.request('GET', 'https://ifconfig.me', headers=curl_agent)
+            r = _http.request("GET", "https://ifconfig.me", headers=curl_agent)
             if r.status != 200:
                 raise ConnectionError()
         except (ConnectionError, urllib3.exceptions.RequestError):
-            r = _http.request('GET', 'https://ifconfig.co', headers=curl_agent)
+            r = _http.request("GET", "https://ifconfig.co", headers=curl_agent)
         return r.data.decode().strip()
 
     @staticmethod
     def _gen_ssh_keypair(tmpdir):
         tmpdir = Path(tmpdir)
         subprocess.run(
-            ['ssh-keygen', '-t', 'rsa', '-N', '', '-f', tmpdir / 'key_rsa'],
+            ("ssh-keygen", "-t", "rsa", "-N", "", "-f", tmpdir / "key_rsa"),
             stdout=subprocess.DEVNULL,
             check=True,
         )
-        return (tmpdir / 'key_rsa', tmpdir / 'key_rsa.pub')
+        return (tmpdir / "key_rsa", tmpdir / "key_rsa.pub")
 
     def __enter__(self):
         spec = self._spec.copy()
 
         try:
             # add source_host firewall filter
-            source_host = self._source_host or f'{self._guess_host_ipv4()}/32'
+            source_host = self._source_host or f"{self._guess_host_ipv4()}/32"
             ingress = \
-                spec['environments'][0]['settings']['provisioning']['security_group_rules_ingress']
+                spec["environments"][0]["settings"]["provisioning"]["security_group_rules_ingress"]
             ingress.append({
-                'type': 'ingress',
-                'protocol': '-1',
-                'cidr': source_host,
-                'port_min': 0,
-                'port_max': 65535,
+                "type": "ingress",
+                "protocol": "-1",
+                "cidr": source_host,
+                "port_min": 0,
+                "port_max": 65535,
             })
 
             # read user-provided ssh key, or generate one
@@ -459,14 +476,14 @@ class Reserve:
             if ssh_key:
                 if not ssh_key.exists():
                     raise FileNotFoundError(f"{ssh_key} specified, but does not exist")
-                ssh_pubkey = Path(f'{ssh_key}.pub')
+                ssh_pubkey = Path(f"{ssh_key}.pub")
             else:
                 self._tmpdir = tempfile.TemporaryDirectory()
                 ssh_key, ssh_pubkey = self._gen_ssh_keypair(self._tmpdir.name)
 
             pubkey_contents = ssh_pubkey.read_text().strip()
-            secrets = spec['environments'][0]['secrets']
-            secrets['RESERVE_SSH_PUBKEY'] = pubkey_contents
+            secrets = spec["environments"][0]["secrets"]
+            secrets["RESERVE_SSH_PUBKEY"] = pubkey_contents
 
             self.request = Request(api=self.api)
             self.request.submit(spec)
@@ -477,24 +494,24 @@ class Reserve:
             for line in PipelineLogStreamer(self.request):
                 util.debug(f"pipeline: {line}")
                 # find hidden login details
-                m = re.search(r'\] Guest is ready: ArtemisGuest\([^,]+, (\w+)@([0-9\.]+), ', line)
+                m = re.search(r"\] Guest is ready: ArtemisGuest\([^,]+, (\w+)@([0-9\.]+), ", line)
                 if m:
                     ssh_user, ssh_host = m.groups()
                     continue
                 # but wait until much later despite having login, at least until
                 # the test starts running (and we get closer to it inserting our
                 # ~/.ssh/authorized_keys entry)
-                if ssh_user and re.search(r'\] starting tests execution', line):
+                if ssh_user and re.search(r"\] starting tests execution", line):
                     break
 
             # wait for a successful connection over ssh
             # (it will be failing to login for a while, until the reserve test
             #  installs our ssh pubkey into authorized_keys)
-            ssh_attempt_cmd = [
-                'ssh', '-q', '-i', ssh_key, f'-oConnectionAttempts={API_QUERY_DELAY}',
-               '-oStrictHostKeyChecking=no', '-oUserKnownHostsFile=/dev/null',
-                f'{ssh_user}@{ssh_host}', 'exit 123',
-            ]
+            ssh_attempt_cmd = (
+                "ssh", "-q", "-i", ssh_key, f"-oConnectionAttempts={API_QUERY_DELAY}",
+               "-oStrictHostKeyChecking=no", "-oUserKnownHostsFile=/dev/null",
+                f"{ssh_user}@{ssh_host}", "exit 123",
+            )
             while True:
                 # wait for API_QUERY_DELAY between ssh retries, seems like GEFN sleep time
                 time.sleep(API_QUERY_DELAY)
@@ -508,7 +525,13 @@ class Reserve:
                 if proc.returncode == 123:
                     break
 
-            return self.Reserved(ssh_user, ssh_host, ssh_key, self.request)
+            return self.Reserved(
+                host=ssh_host,
+                port=22,
+                user=ssh_user,
+                ssh_key=ssh_key,
+                request=self.request,
+            )
 
         except:
             self.__exit__(*sys.exc_info())

atex/provision/testingfarm/foo.py

@@ -0,0 +1 @@
+a = 123

atex/util/__init__.py

@@ -17,21 +17,21 @@ def __dir__():
 # (function to avoid polluting global namespace with extra variables)
 def _import_submodules():
     for info in _pkgutil.iter_modules(__spec__.submodule_search_locations):
-        mod = _importlib.import_module(f'.{info.name}', __name__)
+        mod = _importlib.import_module(f".{info.name}", __name__)
 
         # if the module defines __all__, just use it
-        if hasattr(mod, '__all__'):
+        if hasattr(mod, "__all__"):
             keys = mod.__all__
         else:
             # https://docs.python.org/3/reference/executionmodel.html#binding-of-names
-            keys = (x for x in dir(mod) if not x.startswith('_'))
+            keys = (x for x in dir(mod) if not x.startswith("_"))
 
         for key in keys:
             attr = getattr(mod, key)
 
             # avoid objects that belong to other known modules
             # (ie. imported function from another util module)
-            if hasattr(attr, '__module__'):
+            if hasattr(attr, "__module__"):
                 if attr.__module__ != mod.__name__:
                     continue
             # avoid some common pollution / imports

atex/util/dedent.py

@@ -22,4 +22,4 @@ def dedent(text):
     Like textwrap.dedent(), but also strip leading and trailing spaces/newlines
     up to the content.
     """
-    return textwrap.dedent(text.lstrip('\n').rstrip(' \n'))
+    return textwrap.dedent(text.lstrip("\n").rstrip(" \n"))

atex/util/log.py

@@ -2,7 +2,7 @@ import inspect
 import logging
 from pathlib import Path
 
-_logger = logging.getLogger('atex')
+_logger = logging.getLogger("atex")
 
 
 def _format_msg(msg, *, skip_frames=0):
@@ -13,7 +13,7 @@ def _format_msg(msg, *, skip_frames=0):
 
     # bottom of the stack, or runpy executed module
     for frame_info in stack:
-        if frame_info.function == '<module>':
+        if frame_info.function == "<module>":
             break
     module = frame_info
 
@@ -24,21 +24,21 @@ def _format_msg(msg, *, skip_frames=0):
     # if the function has 'self' and it looks like a class instance,
     # prepend it to the function name
     p_locals = parent.frame.f_locals
-    if 'self' in p_locals:
-        self = p_locals['self']
-        if hasattr(self, '__class__') and inspect.isclass(self.__class__):
-            function = f'{self.__class__.__name__}.{function}'
+    if "self" in p_locals:
+        self = p_locals["self"]
+        if hasattr(self, "__class__") and inspect.isclass(self.__class__):
+            function = f"{self.__class__.__name__}.{function}"
 
     # don't report module name of a function if it's the same as running module
     if parent.filename != module.filename:
-        parent_modname = parent.frame.f_globals['__name__']
+        parent_modname = parent.frame.f_globals["__name__"]
         # avoid everything having the package name prefixed
-        parent_modname = parent_modname.partition('.')[2] or parent_modname
-        return f'{parent_modname}.{function}:{parent.lineno}: {msg}'
-    elif parent.function != '<module>':
-        return f'{function}:{parent.lineno}: {msg}'
+        parent_modname = parent_modname.partition(".")[2] or parent_modname
+        return f"{parent_modname}.{function}:{parent.lineno}: {msg}"
+    elif parent.function != "<module>":
+        return f"{function}:{parent.lineno}: {msg}"
     else:
-        return f'{Path(parent.filename).name}:{parent.lineno}: {msg}'
+        return f"{Path(parent.filename).name}:{parent.lineno}: {msg}"
 
 
 def debug(msg, *, skip_frames=0):

atex/util/subprocess.py

@@ -3,29 +3,30 @@ import subprocess
 from .log import debug
 
 
-def _format_subprocess_cmd(cmd):
-    return cmd
-#    if isinstance(cmd, (list, tuple)):
-#        return ' '.join(str(x) for x in cmd)
-#    else:
-#        return cmd
-
-
 def subprocess_run(cmd, *, skip_frames=0, **kwargs):
     """
     A simple wrapper for the real subprocess.run() that logs the command used.
     """
     # when logging, skip current stack frame - report the place we were called
     # from, not util.subprocess_run itself
-    debug(f'running: {_format_subprocess_cmd(cmd)}', skip_frames=skip_frames+1)
+    debug(f"running: {cmd}", skip_frames=skip_frames+1)
     return subprocess.run(cmd, **kwargs)
 
 
+def subprocess_output(cmd, *, skip_frames=0, check=True, text=True, **kwargs):
+    """
+    A wrapper simulating subprocess.check_output() via a modern .run() API.
+    """
+    debug(f"running: {cmd}", skip_frames=skip_frames+1)
+    proc = subprocess.run(cmd, check=check, text=text, stdout=subprocess.PIPE, **kwargs)
+    return proc.stdout.rstrip("\n") if text else proc.stdout
+
+
 def subprocess_Popen(cmd, *, skip_frames=0, **kwargs):  # noqa: N802
     """
     A simple wrapper for the real subprocess.Popen() that logs the command used.
     """
-    debug(f'running: {_format_subprocess_cmd(cmd)}', skip_frames=skip_frames+1)
+    debug(f"running: {cmd}", skip_frames=skip_frames+1)
     return subprocess.Popen(cmd, **kwargs)
 
 
@@ -38,12 +39,12 @@ def subprocess_stream(cmd, *, check=False, skip_frames=0, **kwargs):
 
     To capture both stdout and stderr as yielded lines, use subprocess.STDOUT.
     """
-    debug(f'running: {_format_subprocess_cmd(cmd)}', skip_frames=skip_frames+1)
-    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, universal_newlines=True, **kwargs)
+    debug(f"running: {cmd}", skip_frames=skip_frames+1)
+    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, text=True, **kwargs)
 
     def generate_lines():
         for line in proc.stdout:
-            yield line.rstrip('\n')
+            yield line.rstrip("\n")
         code = proc.wait()
         if code > 0 and check:
             raise subprocess.CalledProcessError(cmd=cmd, returncode=code)

{atex-0.5.dist-info → atex-0.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: atex
-Version: 0.5
+Version: 0.7
 Summary: Ad-hoc Test EXecutor
 Project-URL: Homepage, https://github.com/RHSecurityCompliance/atex
 License-Expression: GPL-3.0-or-later