atex 0.5__py3-none-any.whl → 0.8__py3-none-any.whl

atex/{testingfarm.py → provision/testingfarm/api.py}

@@ -1,41 +1,41 @@
 import os
-import sys
 import re
 import time
 import tempfile
 import textwrap
+import threading
 import subprocess
 import collections
 
 from pathlib import Path
 
-from . import util
+from ... import util
 
 import json
 import urllib3
 
-DEFAULT_API_URL = 'https://api.testing-farm.io/v0.1'
+DEFAULT_API_URL = "https://api.testing-farm.io/v0.1"
 
 # how many seconds to sleep for during API polling
-API_QUERY_DELAY = 10
+API_QUERY_DELAY = 30
 
 RESERVE_TASK = {
-    'fmf': {
-        'url': 'https://github.com/RHSecurityCompliance/atex',
-        'ref': 'main',
-        'path': 'tmt_tests',
-        'name': "/plans/reserve",
+    "fmf": {
+        "url": "https://github.com/RHSecurityCompliance/atex",
+        "ref": "main",
+        "path": "tmt_tests",
+        "name": "/plans/reserve",
     },
 }
 
 # final states of a request,
 # https://gitlab.com/testing-farm/nucleus/-/blob/main/api/src/tft/nucleus/api/core/schemes/test_request.py
-END_STATES = ('error', 'complete', 'canceled')
+END_STATES = ("error", "complete", "canceled")
 
-# always have at most 3 outstanding HTTP requests to every given API host,
+# always have at most 10 outstanding HTTP requests to every given API host,
 # shared by all instances of all classes here, to avoid flooding the host
 # by multi-threaded users
-_http = urllib3.PoolManager(maxsize=3, block=True)
+_http = urllib3.PoolManager(maxsize=10, block=True)
 
 
 class TestingFarmError(Exception):
@@ -48,6 +48,8 @@ class APIError(TestingFarmError):
     pass
 
 
+# TODO docstrings for these:
+
 class BadHTTPError(TestingFarmError):
     pass
 
@@ -74,21 +76,21 @@ class TestingFarmAPI:
         Note that token-less operation is supported, with limited functionality.
         """
         self.api_url = url
-        self.api_token = token or os.environ.get('TESTING_FARM_API_TOKEN')
+        self.api_token = token or os.environ.get("TESTING_FARM_API_TOKEN")
 
     def _query(self, method, path, *args, headers=None, **kwargs):
-        url = f'{self.api_url}{path}'
+        url = f"{self.api_url}{path}"
         if headers is not None:
-            headers['Authorization'] = f'Bearer {self.api_token}'
+            headers["Authorization"] = f"Bearer {self.api_token}"
         else:
-            headers = {'Authorization': f'Bearer {self.api_token}'}
+            headers = {"Authorization": f"Bearer {self.api_token}"}
 
         reply = _http.request(method, url, *args, headers=headers, preload_content=False, **kwargs)
 
         if reply.status != 200 and not reply.data:
             raise APIError(f"got HTTP {reply.status} on {method} {url}", reply)
 
-        if reply.headers.get('Content-Type') != 'application/json':
+        if reply.headers.get("Content-Type") != "application/json":
             raise BadHTTPError(
                 f"HTTP {reply.status} on {method} {url} is not application/json",
                 reply,
@@ -97,7 +99,10 @@ class TestingFarmAPI:
         try:
             decoded = reply.json()
         except json.decoder.JSONDecodeError:
-            raise BadHTTPError(f"failed to decode JSON for {method} {url}: {reply.data}", reply)
+            raise BadHTTPError(
+                f"failed to decode JSON for {method} {url}: {reply.data}",
+                reply,
+            ) from None
 
         if reply.status != 200:
             raise APIError(f"got HTTP {reply.status} on {method} {url}: {decoded}", reply)
@@ -107,14 +112,14 @@ class TestingFarmAPI:
     def whoami(self):
         if not self.api_token:
             raise ValueError("whoami() requires an auth token")
-        if hasattr(self, '_whoami_cached'):
+        if hasattr(self, "_whoami_cached"):
             return self._whoami_cached
         else:
-            self._whoami_cached = self._query('GET', '/whoami')
+            self._whoami_cached = self._query("GET", "/whoami")
             return self._whoami_cached
 
     def about(self):
-        return self._query('GET', '/about')
+        return self._query("GET", "/about")
 
     def composes(self, ranch=None):
         """
@@ -123,45 +128,55 @@ class TestingFarmAPI:
         if not ranch:
             if not self.api_token:
                 raise ValueError("composes() requires an auth token to identify ranch")
-            ranch = self.whoami()['token']['ranch']
-        return self._query('GET', f'/composes/{ranch}')
+            ranch = self.whoami()["token"]["ranch"]
+        return self._query("GET", f"/composes/{ranch}")
 
     def search_requests(
-        self, state, mine=True, ranch=None, created_before=None, created_after=None,
+        self, *, state, ranch=None,
+        mine=True, user_id=None, token_id=None,
+        created_before=None, created_after=None,
     ):
         """
         'state' is one of 'running', 'queued', etc., and is required by the API.
 
+        'ranch' is 'public' or 'redhat', or (probably?) all if left empty.
+
         If 'mine' is True and a token was given, return only requests for that
         token (user), otherwise return *all* requests (use extra filters pls).
 
-        'ranch' is 'public' or 'redhat', or (probably?) all if left empty.
+        'user_id' and 'token_id' are search API parameters - if not given and
+        'mine' is True, these are extracted from a user-provided token.
 
         'created_*' take ISO 8601 formatted strings, as returned by the API
         elsewhere, ie. 'YYYY-MM-DD' or 'YYYY-MM-DDTHH:MM:SS' (or with '.MS'),
         without timezone.
         """
-        fields = {'state': state}
+        fields = {"state": state}
         if ranch:
-            fields['ranch'] = ranch
+            fields["ranch"] = ranch
         if created_before:
-            fields['created_before'] = created_before
+            fields["created_before"] = created_before
         if created_after:
-            fields['created_after'] = created_after
-
-        if mine:
+            fields["created_after"] = created_after
+
+        if user_id or token_id:
+            if user_id:
+                fields["user_id"] = user_id
+            if token_id:
+                fields["token_id"] = token_id
+        elif mine:
             if not self.api_token:
                 raise ValueError("search_requests(mine=True) requires an auth token")
-            fields['token_id'] = self.whoami()['token']['id']
-            fields['user_id'] = self.whoami()['user']['id']
+            fields["token_id"] = self.whoami()["token"]["id"]
+            fields["user_id"] = self.whoami()["user"]["id"]
 
-        return self._query('GET', '/requests', fields=fields)
+        return self._query("GET", "/requests", fields=fields)
 
     def get_request(self, request_id):
         """
         'request_id' is the UUID (string) of the request.
         """
-        return self._query('GET', f'/requests/{request_id}')
+        return self._query("GET", f"/requests/{request_id}")
 
     def submit_request(self, spec):
         """
@@ -170,13 +185,13 @@ class TestingFarmAPI:
         """
         if not self.api_token:
             raise ValueError("submit_request() requires an auth token")
-        return self._query('POST', '/requests', json=spec)
+        return self._query("POST", "/requests", json=spec)
 
     def cancel_request(self, request_id):
         """
         'request_id' is the UUID (string) of the request.
         """
-        return self._query('DELETE', f'/requests/{request_id}')
+        return self._query("DELETE", f"/requests/{request_id}")
 
 
 class Request:
@@ -185,6 +200,9 @@ class Request:
     request.
     """
 
+    # TODO: maintain internal time.monotonic() clock and call .update() from
+    #       functions like .alive() if last update is > API_QUERY_DELAY
+
     def __init__(self, id=None, api=None, initial_data=None):
         """
         'id' is a Testing Farm request UUID
@@ -204,14 +222,17 @@ class Request:
         if self.id:
             raise ValueError("this Request instance already has 'id', refusing submit")
         self.data = self.api.submit_request(spec)
-        self.id = self.data['id']
+        self.id = self.data["id"]
 
     def update(self):
         """
         Query Testing Farm API to get a more up-to-date version of the request
-        metadata accessible via __str__().
+        metadata. Do not call too frequently.
+        This function is also used internally by others, you do not need to
+        always call it manually.
         """
         self.data = self.api.get_request(self.id)
+        # TODO: refresh internal time.monotonic() timer
         return self.data
 
     def cancel(self):
@@ -223,26 +244,26 @@ class Request:
         return data
 
     def alive(self):
-        if 'state' not in self.data:
+        if "state" not in self.data:
             self.update()
-        return self.data['state'] not in END_STATES
+        return self.data["state"] not in END_STATES
 
     def assert_alive(self):
         if not self.alive():
-            state = self.data['state']
+            state = self.data["state"]
             raise GoneAwayError(f"request {self.data['id']} not alive anymore, entered: {state}")
 
     def wait_for_state(self, state):
-        if 'state' not in self.data:
+        if "state" not in self.data:
             self.update()
         self.assert_alive()
-        while self.data['state'] != state:
+        while self.data["state"] != state:
             time.sleep(API_QUERY_DELAY)
             self.update()
             self.assert_alive()
 
     def __repr__(self):
-        return f'Request(id={self.id})'
+        return f"Request(id={self.id})"
 
     def __str__(self):
         # python has no better dict-pretty-printing logic
@@ -266,25 +287,30 @@ class PipelineLogStreamer:
 
     def _wait_for_entry(self):
         while True:
-            self.request.wait_for_state('running')
+            self.request.wait_for_state("running")
 
             try:
-                if 'run' not in self.request or 'artifacts' not in self.request['run']:
+                if "run" not in self.request or "artifacts" not in self.request["run"]:
                     continue
 
-                artifacts = self.request['run']['artifacts']
+                artifacts = self.request["run"]["artifacts"]
                 if not artifacts:
                     continue
 
-                log = f'{artifacts}/pipeline.log'
-                reply = _http.request('HEAD', log)
-                # TF has a race condition of adding the .log entry without it being created
-                if reply.status == 404:
-                    util.debug(f"got 404 for {log}, retrying")
+                log = f"{artifacts}/pipeline.log"
+                reply = _http.request("HEAD", log)
+                # 404: TF has a race condition of adding the .log entry without
+                #      it being created
+                # 403: happens on internal OSCI artifacts server, probably
+                #      due to similar reasons (folder exists without log)
+                if reply.status in (404,403):
+                    util.debug(f"got {reply.status} for {log}, retrying")
                     continue
                 elif reply.status != 200:
                     raise APIError(f"got HTTP {reply.status} on HEAD {log}", reply)
 
+                util.info(f"artifacts: {artifacts}")
+
                 return log
 
             finally:
@@ -293,17 +319,17 @@ class PipelineLogStreamer:
 
     def __iter__(self):
         url = self._wait_for_entry()
-        buffer = ''
+        buffer = ""
         bytes_read = 0
         while True:
             self.request.assert_alive()
 
             try:
-                headers = {'Range': f'bytes={bytes_read}-'}
+                headers = {"Range": f"bytes={bytes_read}-"}
                 # load all returned data via .decode() rather than streaming it
                 # in chunks, because we don't want to leave the connection open
                 # (blocking others) while the user code runs between __next__ calls
-                reply = _http.request('GET', url, headers=headers)
+                reply = _http.request("GET", url, headers=headers)
 
                 # 416=Range Not Satisfiable, typically meaning "no new data to send"
                 if reply.status == 416:
@@ -313,9 +339,9 @@ class PipelineLogStreamer:
                     raise BadHTTPError(f"got {reply.status} when trying to GET {url}", reply)
 
                 bytes_read += len(reply.data)
-                buffer += reply.data.decode(errors='ignore')
+                buffer += reply.data.decode(errors="ignore")
 
-                while (index := buffer.find('\n')) != -1:
+                while (index := buffer.find("\n")) != -1:
                     yield buffer[:index]
                     buffer = buffer[index+1:]
 
@@ -333,14 +359,17 @@ class Reserve:
     When used in a context manager, it produces a ReservedMachine tuple with
     connection details for an ssh client:
 
-        with Reserve(compose='CentOS-Stream-9', timeout=720) as m:
-            subprocess.run(['ssh', '-i', m.ssh_key, f'{m.user}@{m.host}', 'ls /'])
+        with Reserve(compose="CentOS-Stream-9", timeout=720) as m:
+            subprocess.run(["ssh", "-i", m.ssh_key, f"{m.user}@{m.host}", "ls /"])
     """
 
-    Reserved = collections.namedtuple('ReservedMachine', ['user', 'host', 'ssh_key', 'request'])
+    Reserved = collections.namedtuple(
+        "ReservedMachine",
+        ("host", "port", "user", "ssh_key", "request"),
+    )
 
     def __init__(
-        self, compose=None, arch='x86_64', pool=None, hardware=None, kickstart=None,
+        self, *, compose, arch="x86_64", pool=None, hardware=None, kickstart=None,
         timeout=60, ssh_key=None, source_host=None, api=None,
     ):
         """
@@ -377,81 +406,77 @@ class Reserve:
         'api' is a TestingFarmAPI instance - if unspecified, a sensible default
         will be used.
         """
+        util.info(f"Will reserve compose:{compose} on arch:{arch} for {timeout}min")
         spec = {
-            'test': RESERVE_TASK,
-            'environments': [{
-                'arch': arch,
-                'os': {
-                    'compose': compose,
+            "test": RESERVE_TASK,
+            "environments": [{
+                "arch": arch,
+                "os": {
+                    "compose": compose,
                 },
-                'pool': pool,
-                'settings': {
-                    'pipeline': {
-                        'skip_guest_setup': True,
+                "pool": pool,
+                "settings": {
+                    "pipeline": {
+                        "skip_guest_setup": True,
                     },
-                    'provisioning': {
-                        'tags': {
-                            'ArtemisUseSpot': 'false',
+                    "provisioning": {
+                        "tags": {
+                            "ArtemisUseSpot": "false",
                         },
-                        'security_group_rules_ingress': [],
+                        "security_group_rules_ingress": [],
                     },
                 },
-                'secrets': {},
+                "secrets": {},
             }],
-            'settings': {
-                'pipeline': {
-                    'timeout': timeout,
+            "settings": {
+                "pipeline": {
+                    "timeout": timeout,
                 },
             },
         }
         if hardware:
-            spec['environments'][0]['hardware'] = hardware
+            spec["environments"][0]["hardware"] = hardware
         if kickstart:
-            spec['environments'][0]['kickstart'] = kickstart
+            spec["environments"][0]["kickstart"] = kickstart
 
         self._spec = spec
         self._ssh_key = Path(ssh_key) if ssh_key else None
         self._source_host = source_host
         self.api = api or TestingFarmAPI()
 
+        self.lock = threading.RLock()
         self.request = None
         self._tmpdir = None
 
     @staticmethod
     def _guess_host_ipv4():
-        curl_agent = {'User-Agent': 'curl/1.2.3'}
+        curl_agent = {"User-Agent": "curl/1.2.3"}
         try:
-            r = _http.request('GET', 'https://ifconfig.me', headers=curl_agent)
+            r = _http.request("GET", "https://ifconfig.me", headers=curl_agent)
             if r.status != 200:
                 raise ConnectionError()
         except (ConnectionError, urllib3.exceptions.RequestError):
-            r = _http.request('GET', 'https://ifconfig.co', headers=curl_agent)
+            r = _http.request("GET", "https://ifconfig.co", headers=curl_agent)
         return r.data.decode().strip()
 
-    @staticmethod
-    def _gen_ssh_keypair(tmpdir):
-        tmpdir = Path(tmpdir)
-        subprocess.run(
-            ['ssh-keygen', '-t', 'rsa', '-N', '', '-f', tmpdir / 'key_rsa'],
-            stdout=subprocess.DEVNULL,
-            check=True,
-        )
-        return (tmpdir / 'key_rsa', tmpdir / 'key_rsa.pub')
+    def reserve(self):
+        with self.lock:
+            if self.request:
+                raise RuntimeError("reservation already in progress")
 
-    def __enter__(self):
         spec = self._spec.copy()
 
         try:
             # add source_host firewall filter
-            source_host = self._source_host or f'{self._guess_host_ipv4()}/32'
+            source_host = self._source_host or f"{self._guess_host_ipv4()}/32"
             ingress = \
-                spec['environments'][0]['settings']['provisioning']['security_group_rules_ingress']
+                spec["environments"][0]["settings"]["provisioning"]["security_group_rules_ingress"]
             ingress.append({
-                'type': 'ingress',
-                'protocol': '-1',
-                'cidr': source_host,
-                'port_min': 0,
-                'port_max': 65535,
+                "type": "ingress",
+                "protocol": "-1",
+                "cidr": source_host,
+                "port_min": 0,
+                "port_max": 65535,
             })
 
             # read user-provided ssh key, or generate one
@@ -459,42 +484,46 @@ class Reserve:
             if ssh_key:
                 if not ssh_key.exists():
                     raise FileNotFoundError(f"{ssh_key} specified, but does not exist")
-                ssh_pubkey = Path(f'{ssh_key}.pub')
+                ssh_pubkey = Path(f"{ssh_key}.pub")
             else:
-                self._tmpdir = tempfile.TemporaryDirectory()
-                ssh_key, ssh_pubkey = self._gen_ssh_keypair(self._tmpdir.name)
+                with self.lock:
+                    self._tmpdir = tempfile.TemporaryDirectory()
+                    ssh_key, ssh_pubkey = util.ssh_keygen(self._tmpdir.name)
 
             pubkey_contents = ssh_pubkey.read_text().strip()
-            secrets = spec['environments'][0]['secrets']
-            secrets['RESERVE_SSH_PUBKEY'] = pubkey_contents
+            secrets = spec["environments"][0]["secrets"]
+            secrets["RESERVE_SSH_PUBKEY"] = pubkey_contents
 
-            self.request = Request(api=self.api)
-            self.request.submit(spec)
+            with self.lock:
+                self.request = Request(api=self.api)
+                self.request.submit(spec)
             util.debug(f"submitted request:\n{textwrap.indent(str(self.request), '    ')}")
 
             # wait for user/host to ssh to
             ssh_user = ssh_host = None
             for line in PipelineLogStreamer(self.request):
-                util.debug(f"pipeline: {line}")
+                # the '\033[0m' is to reset colors sometimes left in a bad
+                # state by pipeline.log
+                util.debug(f"pipeline: {line}\033[0m")
                 # find hidden login details
-                m = re.search(r'\] Guest is ready: ArtemisGuest\([^,]+, (\w+)@([0-9\.]+), ', line)
+                m = re.search(r"\] Guest is ready: ArtemisGuest\([^,]+, (\w+)@([0-9\.]+), ", line)
                 if m:
                     ssh_user, ssh_host = m.groups()
                     continue
                 # but wait until much later despite having login, at least until
                 # the test starts running (and we get closer to it inserting our
                 # ~/.ssh/authorized_keys entry)
-                if ssh_user and re.search(r'\] starting tests execution', line):
+                if ssh_user and re.search(r"\] starting tests execution", line):
                     break
 
             # wait for a successful connection over ssh
             # (it will be failing to login for a while, until the reserve test
             #  installs our ssh pubkey into authorized_keys)
-            ssh_attempt_cmd = [
-                'ssh', '-q', '-i', ssh_key, f'-oConnectionAttempts={API_QUERY_DELAY}',
-               '-oStrictHostKeyChecking=no', '-oUserKnownHostsFile=/dev/null',
-                f'{ssh_user}@{ssh_host}', 'exit 123',
-            ]
+            ssh_attempt_cmd = (
+                "ssh", "-q", "-i", ssh_key, f"-oConnectionAttempts={API_QUERY_DELAY}",
+               "-oStrictHostKeyChecking=no", "-oUserKnownHostsFile=/dev/null",
+                f"{ssh_user}@{ssh_host}", "exit 123",
+            )
             while True:
                 # wait for API_QUERY_DELAY between ssh retries, seems like GEFN sleep time
                 time.sleep(API_QUERY_DELAY)
@@ -508,25 +537,34 @@ class Reserve:
                 if proc.returncode == 123:
                     break
 
-            return self.Reserved(ssh_user, ssh_host, ssh_key, self.request)
+            return self.Reserved(
+                host=ssh_host,
+                port=22,
+                user=ssh_user,
+                ssh_key=ssh_key,
+                request=self.request,
+            )
 
         except:
-            self.__exit__(*sys.exc_info())
+            self.release()
             raise
 
-    def __exit__(self, exc_type, exc_value, traceback):
-        if self.request:
-            try:
-                self.request.cancel()
-            except APIError:
-                pass
-            finally:
-                self.request = None
+    def release(self):
+        with self.lock:
+            if self.request:
+                try:
+                    self.request.cancel()
+                except APIError:
+                    pass
+                finally:
+                    self.request = None
+
+            if self._tmpdir:
+                self._tmpdir.cleanup()
+                self._tmpdir = None
 
-        if self._tmpdir:
-            self._tmpdir.cleanup()
-            self._tmpdir = None
+    def __enter__(self):
+        return self.reserve()
 
-        # cancel request
-        # clear out stored self.request
-        pass
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.release()