atex 0.13__py3-none-any.whl → 0.15__py3-none-any.whl

atex/provisioner/testingfarm/api.py

@@ -1,6 +1,7 @@
 import os
 import re
 import time
+import logging
 import tempfile
 import datetime
 import textwrap
@@ -15,11 +16,13 @@ from ... import util
 import json
 import urllib3
 
+logger = logging.getLogger("atex.provisioner.testingfarm")
+
 DEFAULT_API_URL = "https://api.testing-farm.io"
 
 DEFAULT_RESERVE_TEST = {
     "url": "https://github.com/RHSecurityCompliance/atex-reserve",
-    "ref": "main",
+    "ref": "0.12",
     "path": ".",
     "name": "/plans/reserve",
 }
@@ -42,6 +45,8 @@ _http = urllib3.PoolManager(
         # retry on API server errors too, not just connection issues
         status=10,
         status_forcelist={403,404,408,429,500,502,503,504},
+        # retry POST as well, even if risky
+        allowed_methods=urllib3.Retry.DEFAULT_ALLOWED_METHODS | {"POST"},
     ),
 )
 
@@ -75,11 +80,11 @@ class TestingFarmAPI:
 
     def __init__(self, url=DEFAULT_API_URL, token=None):
         """
-        'url' is Testing Farm API URL, a sensible default is used
-        if unspecified.
+        - `url` is Testing Farm API URL, a sensible default is used
+          if unspecified.
 
-        'token' is a secret API token generated by Testing Farm admins,
-        if empty, the TESTING_FARM_API_TOKEN env var is read instead.
+        - `token` is a secret API token generated by Testing Farm admins,
+          if empty, the `TESTING_FARM_API_TOKEN` env var is read instead.
 
         Note that token-less operation is supported, with limited functionality.
         """
@@ -132,7 +137,7 @@ class TestingFarmAPI:
 
     def composes(self, ranch=None):
         """
-        'ranch' is 'public' or 'redhat', autodetected if token was given.
+        - `ranch` is `public` or `redhat`, autodetected if token was given.
         """
         if not ranch:
             if not self.api_token:
@@ -146,19 +151,21 @@ class TestingFarmAPI:
         created_before=None, created_after=None,
     ):
         """
-        'state' is one of 'running', 'queued', etc., and is required by the API.
+        - `state` is one of `running`, `queued`, etc., and is required by the
+          API.
 
-        'ranch' is 'public' or 'redhat', or (probably?) all if left empty.
+        - `ranch` is `public` or `redhat`, or (probably?) all if left empty.
 
-        If 'mine' is True and a token was given, return only requests for that
-        token (user), otherwise return *all* requests (use extra filters pls).
+        - If `mine` is `True` and a token was given, return only requests for
+          that token (user), otherwise return *all* requests (use extra filters
+          pls).
 
-        'user_id' and 'token_id' are search API parameters - if not given and
-        'mine' is True, these are extracted from a user-provided token.
+        - `user_id` and `token_id` are search API parameters - if not given and
+          `mine` is `True`, these are extracted from a user-provided token.
 
-        'created_*' take ISO 8601 formatted strings, as returned by the API
-        elsewhere, ie. 'YYYY-MM-DD' or 'YYYY-MM-DDTHH:MM:SS' (or with '.MS'),
-        without timezone.
+        - `created_*` take ISO 8601 formatted strings, as returned by the API
+          elsewhere, ie. `YYYY-MM-DD` or `YYYY-MM-DDTHH:MM:SS` (or with `.MS`),
+          without timezone (UTC is used always).
         """
         fields = {"state": state}
         if ranch:
@@ -186,11 +193,11 @@ class TestingFarmAPI:
         An unofficial wrapper for search_requests() that can search a large
         interval incrementally (in "pages") and yield batches of results.
 
-        Needs 'created_after', with 'created_before' defaulting to now().
+        Needs `created_after`, with `created_before` defaulting to `now()`.
 
-        'page' specifies the time interval of one page, in seconds.
+        - `page` specifies the time interval of one page, in seconds.
 
-        'args' and 'kwargs' are passed to search_requests().
+        - `args` and `kwargs` are passed to `search_requests()`.
         """
         assert "created_after" in kwargs, "at least 'created_after' is needed for paging"
 
@@ -229,13 +236,13 @@ class TestingFarmAPI:
 
     def get_request(self, request_id):
         """
-        'request_id' is the UUID (string) of the request.
+        - `request_id` is the UUID (string) of the request.
         """
         return self._query("GET", f"/requests/{request_id}")
 
     def submit_request(self, spec):
         """
-        'spec' is a big dictionary with 'test', 'environment', 'settings', etc.
+        - `spec` is a big dictionary with 'test', 'environment', 'settings', etc.
         keys that specify what should be run and where.
         """
         if not self.api_token:
@@ -244,7 +251,7 @@ class TestingFarmAPI:
 
     def cancel_request(self, request_id):
         """
-        'request_id' is the UUID (string) of the request.
+        - `request_id` is the UUID (string) of the request.
         """
         return self._query("DELETE", f"/requests/{request_id}")
 
@@ -261,11 +268,13 @@ class Request:
 
     def __init__(self, id=None, api=None, initial_data=None):
         """
-        'id' is a Testing Farm request UUID
+        - `id` is a Testing Farm request UUID.
 
-        'api' is a TestingFarmAPI instance - if unspecified, a sensible default
+        - `api` is a TestingFarmAPI instance - if unspecified, a new one
+          is instantiated.
 
-        'initial_data' (dict) can be used to pre-fill an initial Request state.
+        - `initial_data` (dict) can be used to pre-fill an initial Request
+          state.
         """
         self.id = id
         self.api = api or TestingFarmAPI()
@@ -274,8 +283,8 @@ class Request:
 
     def submit(self, spec):
         """
-        'spec' is a big dictionary with 'test', 'environment', 'settings', etc.
-        keys that specify what should be run and where.
+        - `spec` is a big dictionary with 'test', 'environment', 'settings',
+          etc. keys that specify what should be run and where.
         """
         if self.id:
             raise ValueError("this Request instance already has 'id', refusing submit")
@@ -310,7 +319,7 @@ class Request:
 
     def wait_for_state(self, state):
         """
-        'state' is a str or a tuple of states to wait for.
+        - `state` is a string or a tuple of states to wait for.
         """
         watched = (state,) if isinstance(state, str) else state
         while True:
@@ -371,12 +380,12 @@ class PipelineLogStreamer:
                 # 403: happens on internal OSCI artifacts server, probably
                 #      due to similar reasons (folder exists without log)
                 if reply.status in (404,403):
-                    util.debug(f"got {reply.status} for {log}, retrying")
+                    logger.info(f"got {reply.status} for {log}, retrying")
                     continue
                 elif reply.status != 200:
                     raise APIError(f"got HTTP {reply.status} on HEAD {log}", reply)
 
-                util.info(f"artifacts: {artifacts}")
+                logger.info(f"artifacts: {artifacts}")
 
                 return log
 
@@ -440,54 +449,59 @@ class Reserve:
         api=None,
     ):
         """
-        'compose' (str) is the OS to install, chosen from the composes supported
-        by the Testing Farm ranch of the authenticated user.
-
-        'arch' (str) is one of 'x86_64', 's390x', etc.
-
-        'pool' (str) is a name of a Testing Farm infrastructure pool.
-
-        'hardware' (dict) is a complex specification of hardware properties
-        the reserved system should have, see:
-        https://docs.testing-farm.io/Testing%20Farm/0.1/test-request.html#hardware
-
-        'kickstart' (dict) is a Beaker-style specification of Anaconda Kickstart
-        hacks, passed directly to Testing Farm POST /requests API.
-
-        'timeout' (int) is the maximum time IN MINUTES a Testing Farm request
-        is alive, which includes initial creation, waiting in queue, preparing
-        an OS, and the entire reservation period.
-        Make sure to set it high enough (not just the pure reservation time).
-
-        'ssh_key' (str) is a path to an OpenSSH private key file (with an
-        associated public key file in .pub), to be added to the reserved OS.
-        If unspecified, an attempt to read ~/.ssh/id_rsa will be made and if
-        that is also unsuccessful, a temporary keypair will be generated.
-
-        'source_host' (str) is an IPv4 network specified as ie. '1.2.3.4/32'
-        to be allowed incoming traffic to the reserved system (such as ssh).
-        If unspecified, an Internet service will be queried to get an outside-
-        facing address of the current system.
-        Ignored on the 'redhat' ranch.
-
-        'reserve_test' is a dict with a fmf test specification to be run on the
-        target system to reserve it, ie.:
-            {
-                "url": "https://some-host/path/to/repo",
-                "ref": "main",
-                "name": "/plans/reserve",
-            }
+        - `compose` (str) is the OS to install, chosen from the composes
+          supported by the Testing Farm ranch of the authenticated user.
+
+        - `arch` (str) is one of 'x86_64', 's390x', etc.
+
+        - `pool` (str) is a name of a Testing Farm infrastructure pool.
+
+        - `hardware` (dict) is a complex specification of hardware properties
+          the reserved system should have, see:
+          https://docs.testing-farm.io/Testing%20Farm/0.1/test-request.html#hardware
+
+        - `kickstart` (dict) is a Beaker-style specification of Anaconda
+          Kickstart hacks, passed directly to Testing Farm POST /requests API.
+
+        - `timeout` (int) is the maximum time **in minutes** a Testing Farm
+          request is alive, which includes initial creation, waiting in queue,
+          preparing an OS, and the entire reservation period.
+
+          Make sure to set it high enough (not just the pure reservation time).
+
+        - `ssh_key` (str) is a path to an OpenSSH private key file (with an
+          associated public key file in .pub), to be added to the reserved OS.
+
+          If unspecified, an attempt to read `~/.ssh/id_rsa` will be made, and
+          if that is also unsuccessful, a temporary keypair will be generated.
+
+        - `source_host` (str) is an IPv4 network specified as ie. `1.2.3.4/32`
+          to be allowed incoming traffic to the reserved system (such as ssh).
+
+          If unspecified, an Internet service will be queried to get an outside-
+          facing address of the current system.
+
+          Ignored on the `redhat` ranch.
+
+        - `reserve_test` is a dict with a fmf test specification to be run on
+          the target system to reserve it, ie.:
+              {
+                  "url": "https://some-host/path/to/repo",
+                  "ref": "main",
+                  "name": "/plans/reserve",
+              }
 
-        'variables' and 'secrets' are dicts with environment variable key/values
-        exported for the reserve test - variables are visible via TF API,
-        secrets are not (but can still be extracted from pipeline log).
+        - `variables` and `secrets` are dicts with environment variable
+          key/values exported for the reserve test - variables are visible via
+          TF API, secrets are not (but can still be extracted from pipeline
+          log).
 
-        'tags' is a dict of custom key/values to be submitted in TF Request as
-        environments->settings->provisioning->tags, useful for storing custom
-        metadata to be queried later.
+        - `tags` is a dict of custom key/values to be submitted in TF Request as
+          `environments->settings->provisioning->tags`, useful for storing
+          custom metadata to be queried later.
 
-        'api' is a TestingFarmAPI instance - if unspecified, a sensible default
-        will be used.
+        - `api` is a TestingFarmAPI instance - if unspecified, a new one
+          is instantiated.
         """
         spec = {
             "test": {
@@ -592,8 +606,8 @@ class Reserve:
             with self.lock:
                 self.request = Request(api=self.api)
                 self.request.submit(spec)
-            util.debug(f"submitted request {self.request.id}")
-            util.extradebug(
+            logger.info(f"submitted request {self.request.id}")
+            logger.debug(
                 f"request {self.request.id}:\n{textwrap.indent(str(self.request), '    ')}",
             )
 
@@ -602,7 +616,7 @@ class Reserve:
             for line in PipelineLogStreamer(self.request):
                 # the '\033[0m' is to reset colors sometimes left in a bad
                 # state by pipeline.log
-                util.extradebug(f"{line}\033[0m")
+                logger.debug(f"{line}\033[0m")
                 # find hidden login details
                 m = re.search(
                     # host address can be an IP address or a hostname

atex/provisioner/testingfarm/testingfarm.py

@@ -1,4 +1,5 @@
 import time
+import logging
 import tempfile
 import threading
 import concurrent.futures
@@ -8,6 +9,8 @@ from .. import Provisioner, Remote
 
 from . import api
 
+logger = logging.getLogger("atex.provisioner.testingfarm")
+
 
 class TestingFarmRemote(Remote, connection.ssh.ManagedSSHConnection):
     """
@@ -17,12 +20,13 @@ class TestingFarmRemote(Remote, connection.ssh.ManagedSSHConnection):
 
     def __init__(self, request_id, ssh_options, *, release_hook):
         """
-        'request_id' is a string with Testing Farm request UUID (for printouts).
+        - `request_id` is a string with Testing Farm request UUID
+          (for printouts).
 
-        'ssh_options' are a dict, passed to ManagedSSHConnection __init__().
+        - `ssh_options` are a dict, passed to ManagedSSHConnection `__init__()`.
 
-        'release_hook' is a callable called on .release() in addition
-        to disconnecting the connection.
+        - `release_hook` is a callable called on `.release()` in addition
+          to disconnecting the connection.
         """
         # NOTE: self.lock inherited from ManagedSSHConnection
         super().__init__(options=ssh_options)
@@ -59,12 +63,12 @@ class TestingFarmProvisioner(Provisioner):
 
     def __init__(self, compose, arch="x86_64", *, max_retries=10, **reserve_kwargs):
         """
-        'compose' is a Testing Farm compose to prepare.
+        - `compose` is a Testing Farm compose to prepare.
 
-        'arch' is an architecture associated with the compose.
+        - `arch`' is an architecture associated with the compose.
 
-        'max_retries' is a maximum number of provisioning (Testing Farm) errors
-        that will be reprovisioned before giving up.
+        - `max_retries` is a maximum number of provisioning (Testing Farm) errors
+          that will be reprovisioned before giving up.
         """
         self.lock = threading.RLock()
         self.compose = compose
@@ -90,7 +94,7 @@ class TestingFarmProvisioner(Provisioner):
         # distribute load on TF servers
         # (we can sleep here as this code is running in a separate thread)
         if initial_delay:
-            util.debug(f"delaying for {initial_delay}s to distribute load")
+            logger.info(f"delaying for {initial_delay}s to distribute load")
             time.sleep(initial_delay)
 
         # 'machine' is api.Reserve.ReservedMachine namedtuple
@@ -135,7 +139,7 @@ class TestingFarmProvisioner(Provisioner):
         # instantiate a class Reserve from the Testing Farm api module
         # (which typically provides context manager, but we use its .reserve()
         #  and .release() functions directly)
-        util.info(f"{repr(self)}: reserving new remote")
+        logger.info(f"{repr(self)}: reserving new remote")
         tf_reserve = api.Reserve(
             compose=self.compose,
             arch=self.arch,
@@ -204,7 +208,7 @@ class TestingFarmProvisioner(Provisioner):
                 exc_str = f"{type(e).__name__}({e})"
                 with self.lock:
                     if self.retries > 0:
-                        util.warning(
+                        logger.warning(
                             f"caught while reserving a TF system: {exc_str}, "
                             f"retrying ({self.retries} left)",
                         )
@@ -215,7 +219,7 @@ class TestingFarmProvisioner(Provisioner):
                         else:
                             return None
                     else:
-                        util.warning(
+                        logger.warning(
                             f"caught while reserving a TF system: {exc_str}, "
                             "exhausted all retries, giving up",
                         )

atex/util/__init__.py

@@ -1,3 +1,26 @@
+"""
+The point of this directory is to have miscellaneous utilities (for text
+formatting, subprocess wrappers, whatever) accessible from the `util.*`
+namespace, while being able to break them down into multiple `*.py` files
+for readability.
+
+These multiple `*.py` files then get automatically imported into one `globals()`
+of the entire `util` module (package), appearing as a singular `util`.
+
+Since the individual submodules cannot easily `from .* import *` themselves,
+and since the intention is to give the impression of a single big `util.py`,
+any local/relative imports between files should extract the necessary
+identifiers via ie.
+
+    # in wrappers.py
+    from .custom_dedent import dedent
+
+    dedent(...)
+
+rather than trying to preserve `custom_dedent.dedent()` or reaching beyond
+parent with `from .. import util` (creating an infinite recursion).
+"""
+
 import importlib as _importlib
 import pkgutil as _pkgutil
 import inspect as _inspect

atex/util/dedent.py

@@ -8,10 +8,10 @@ This allows raw blocks like
         ''')
 
 without the leading or trailing newlines and any common leading whitespaces.
-You might think using '''\ would eliminate the first newline, but the string
+You might think using `'''\` would eliminate the first newline, but the string
 is 'raw', it doesn't have escapes.
 
-textwrap.dedent() does only the common leading whitespaces.
+`textwrap.dedent()` does only the common leading whitespaces.
 """
 
 import textwrap

atex/util/named_mapping.py

@@ -27,7 +27,7 @@ as a dict, that are used if omitted from the constructor:
 
     m = MyMap()  # will have m.key == 678
 
-A class instance can unpack via ** with the entirety of its mapping contents:
+A class instance can unpack via `**` with the entirety of its mapping contents:
 
     m = MyMap(key2=456)
     both = {'key1': 123, **m}  # contains both keys
@@ -59,7 +59,7 @@ another dict-like object (does not have to be a parent of the class):
     s = SmallMap._from(b, extra=555)  # can pass extra **kwargs to __init__
     s = SmallMap(**b)                 # will copy all keys
 
-Note that this is a fairly basic implementation without __hash__, etc.
+Note that this is a fairly basic implementation without `__hash__`, etc.
 """
 
 import abc
@@ -153,6 +153,6 @@ class NamedMapping(collections.abc.Mapping, metaclass=_NamedMappingMeta):
     def __repr__(self):
         return (
             f"{self.__class__.__name__}("
-            + ", ".join((f"{k}={repr(v)}" for k,v in self._data.items()))
+            + ", ".join(f"{k}={repr(v)}" for k,v in self._data.items())
             + ")"
         )

atex/util/path.py

@@ -3,7 +3,7 @@ import os
 
 def normalize_path(path):
     """
-    Transform a potentially dangerous path (leading slash, relative ../../../
+    Transform a potentially dangerous path (leading slash, relative `../../../`
     leading beyond parent, etc.) to a safe one.
 
     Always returns a relative path.

atex/util/subprocess.py

@@ -1,52 +1,58 @@
+import logging
 import subprocess
 
-from .log import extradebug
+logger = logging.getLogger("atex.util.subprocess")
 
 
 def subprocess_run(cmd, **kwargs):
     """
-    A simple wrapper for the real subprocess.run() that logs the command used.
+    A simple wrapper for the real `subprocess.run()` that logs the command used.
     """
     # when logging, skip current stack frame - report the place we were called
     # from, not util.subprocess_run itself
-    extradebug(f"running: '{cmd}' with {kwargs=}")
+    logger.info(f"running: '{cmd}' with {kwargs=}")
     return subprocess.run(cmd, **kwargs)
 
 
 def subprocess_output(cmd, *, check=True, text=True, **kwargs):
     """
-    A wrapper simulating subprocess.check_output() via a modern .run() API.
+    A wrapper simulating `subprocess.check_output()` via a modern `.run()` API.
     """
-    extradebug(f"running: '{cmd}' with {check=}, {text=} and {kwargs=}")
+    logger.info(f"running: '{cmd}' with {check=}, {text=} and {kwargs=}")
     proc = subprocess.run(cmd, check=check, text=text, stdout=subprocess.PIPE, **kwargs)
     return proc.stdout.rstrip("\n") if text else proc.stdout
 
 
 def subprocess_Popen(cmd, **kwargs):  # noqa: N802
     """
-    A simple wrapper for the real subprocess.Popen() that logs the command used.
+    A simple wrapper for the real `subprocess.Popen()` that logs the command used.
     """
-    extradebug(f"running: '{cmd}' with {kwargs=}")
+    logger.info(f"running: '{cmd}' with {kwargs=}")
     return subprocess.Popen(cmd, **kwargs)
 
 
 def subprocess_stream(cmd, *, stream="stdout", check=False, input=None, **kwargs):
     """
-    Run 'cmd' via subprocess.Popen() and return an iterator over any lines
+    Run `cmd` via `subprocess.Popen()` and return an iterator over any lines
     the command outputs on stdout, in text mode.
 
-    The 'stream' is a subprocess.Popen attribute (either 'stdout' or 'stderr')
-    to read from.
-    To capture both stdout and stderr as yielded lines, use 'stream="stdout"'
-    and pass an additional 'stderr=subprocess.STDOUT'.
+    - The `stream` is a subprocess.Popen attribute (either `stdout` or `stderr`)
+      to read from.
 
-    With 'check' set to True, raise a CalledProcessError if the 'cmd' failed.
+      To capture both stdout and stderr as yielded lines, use `stream="stdout"`
+      and pass an additional `stderr=subprocess.STDOUT`.
 
-    Similarly, 'input' simulates the 'input' arg of subprocess.run().
-    Note that the input is written to stdin of the process *before* any outputs
-    are streamed, so it should be sufficiently small and/or not cause a deadlock
-    with the process waiting for outputs to be read before consuming more input.
-    Use 'stdin=subprocess.PIPE' and write to it manually if you need more.
+    - With `check` set to `True`, raise a CalledProcessError if the `cmd`
+      failed.
+
+    - Similarly, `input` simulates the `input` arg of `subprocess.run()`.
+
+      Note that the input is written to stdin of the process *before* any
+      outputs are streamed, so it should be sufficiently small and/or not cause
+      a deadlock with the process waiting for outputs to be read before
+      consuming more input.
+
+      Use `stdin=subprocess.PIPE` and write to it manually if you need more.
     """
     all_kwargs = {
         "text": True,
@@ -56,7 +62,7 @@ def subprocess_stream(cmd, *, stream="stdout", check=False, input=None, **kwargs
         all_kwargs["stdin"] = subprocess.PIPE
     all_kwargs |= kwargs
 
-    extradebug(f"running: '{cmd}' with {all_kwargs=}")
+    logger.info(f"running: '{cmd}' with {all_kwargs=}")
     proc = subprocess.Popen(cmd, **all_kwargs)
 
     def generate_lines():
@@ -78,9 +84,9 @@ def subprocess_log(cmd, **kwargs):
     A wrapper to stream every (text) line output from the process to the
     logging module.
 
-    Uses subprocess_stream() to gather the lines.
+    Uses `subprocess_stream()` to gather the lines.
     """
-    extradebug(f"running: '{cmd}' with {kwargs=}")
+    logger.info(f"running: '{cmd}' with {kwargs=}")
     _, lines = subprocess_stream(cmd, **kwargs)
     for line in lines:
-        extradebug(line)
+        logger.info(line)

atex/util/threads.py

@@ -39,12 +39,12 @@ class ThreadQueue:
 
     def start_thread(self, target, target_args=None, target_kwargs=None, **user_kwargs):
         """
-        Start a new thread and call 'target' as a callable inside it, passing it
-        'target_args' as arguments and 'target_kwargs' as keyword arguments.
+        Start a new thread and call `target` as a callable inside it, passing it
+        `target_args` as arguments and `target_kwargs` as keyword arguments.
 
-        Any additional 'user_kwargs' specified are NOT passed to the callable,
+        Any additional `user_kwargs` specified are NOT passed to the callable,
         but instead become part of the ThreadReturn namespace returned by the
-        .get_raw() method.
+        `.get_raw()` method.
         """
         t = threading.Thread(
             target=self._wrapper,
@@ -59,8 +59,8 @@ class ThreadQueue:
     def get_raw(self, block=True, timeout=None):
         """
         Wait for and return the next available ThreadReturn instance on the
-        queue, as enqueued by a finished callable started by the .start_thread()
-        method.
+        queue, as enqueued by a finished callable started by the
+        `.start_thread()` method.
         """
         with self.lock:
             if block and timeout is None and not self.threads:
@@ -75,7 +75,7 @@ class ThreadQueue:
     def get(self, block=True, timeout=None):
         """
         Wait for and return the next available return value of a callable
-        enqueued via the .start_thread() method.
+        enqueued via the `.start_thread()` method.
 
         If the callable raised an exception, the exception is re-raised here.
         """
@@ -100,7 +100,7 @@ class ThreadQueue:
 
     def qsize(self):
         """
-        Return the amount of elements .get() can retrieve before it raises
-        queue.Empty.
+        Return the amount of elements `.get()` can retrieve before it raises
+        `queue.Empty`.
         """
         return self.queue.qsize()

{atex-0.13.dist-info → atex-0.15.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: atex
-Version: 0.13
+Version: 0.15
 Summary: Ad-hoc Test EXecutor
 Project-URL: Homepage, https://github.com/RHSecurityCompliance/atex
 License-Expression: GPL-3.0-or-later