toil 7.0.0__py3-none-any.whl → 8.0.0__py3-none-any.whl

toil/lib/misc.py

@@ -7,9 +7,9 @@ import socket
 import subprocess
 import sys
 import time
-import typing
+from collections.abc import Iterator
 from contextlib import closing
-from typing import Iterator, List, Optional
+from typing import Optional
 
 logger = logging.getLogger(__name__)
 
@@ -21,19 +21,20 @@ def get_public_ip() -> str:
     try:
         # Try to get the internet-facing IP by attempting a connection
         # to a non-existent server and reading what IP was used.
-        ip = '127.0.0.1'
+        ip = "127.0.0.1"
         with closing(socket.socket(socket.AF_INET, socket.SOCK_DGRAM)) as sock:
             # 203.0.113.0/24 is reserved as TEST-NET-3 by RFC 5737, so
             # there is guaranteed to be no one listening on the other
             # end (and we won't accidentally DOS anyone).
-            sock.connect(('203.0.113.1', 1))
+            sock.connect(("203.0.113.1", 1))
             ip = sock.getsockname()[0]
         return ip
     except:
         # Something went terribly wrong. Just give loopback rather
         # than killing everything, because this is often called just
         # to provide a default argument
-        return '127.0.0.1'
+        return "127.0.0.1"
+
 
 def get_user_name() -> str:
     """
@@ -46,20 +47,23 @@ def get_user_name() -> str:
         except KeyError:
             # This is expected if the user isn't in /etc/passwd, such as in a
             # Docker container when running as a weird UID. Make something up.
-            return 'UnknownUser' + str(os.getuid())
+            return "UnknownUser" + str(os.getuid())
     except Exception as e:
         # We can't get the UID, or something weird has gone wrong.
-        logger.error('Unexpected error getting user name: %s', e)
-        return 'UnknownUser'
+        logger.error("Unexpected error getting user name: %s", e)
+        return "UnknownUser"
+
 
 def utc_now() -> datetime.datetime:
     """Return a datetime in the UTC timezone corresponding to right now."""
     return datetime.datetime.utcnow().replace(tzinfo=datetime.timezone.utc)
 
+
 def unix_now_ms() -> float:
     """Return the current time in milliseconds since the Unix epoch."""
     return time.time() * 1000
 
+
 def slow_down(seconds: float) -> float:
     """
     Toil jobs that have completed are not allowed to have taken 0 seconds, but
@@ -77,9 +81,25 @@ def slow_down(seconds: float) -> float:
 
     return max(seconds, sys.float_info.epsilon)
 
-def printq(msg: str, quiet: bool) -> None:
+
+def printq(msg: str, quiet: bool, log: bool = False) -> None:
+    """
+    This is for functions used simultaneously in Toil proper and in the admin scripts.
+
+    Our admin scripts "print" to stdout, while Toil proper uses logging.  For a script that,
+    for example, cleans up IAM, EC2, etc. cruft leftover after failed CI runs, we can call
+    an AWS delete IAM role function, and this prints or logs progress (unless quiet is True),
+    depending on whether the function is called in, say, the jobstore or a script.
+
+    :param msg: The string to print or log to stdout.
+    :param quiet: Silent output to stdout.
+    :param log: Use logging (else "print" to the screen).
+    """
     if not quiet:
-        print(msg)
+        if not log:
+            print(msg)
+        else:
+            logger.debug(msg)
 
 
 def truncExpBackoff() -> Iterator[float]:
@@ -102,12 +122,23 @@ class CalledProcessErrorStderr(subprocess.CalledProcessError):
         if (self.returncode < 0) or (self.stderr is None):
             return str(super())
         else:
-            err = self.stderr if isinstance(self.stderr, str) else self.stderr.decode("ascii", errors="replace")
+            err = (
+                self.stderr
+                if isinstance(self.stderr, str)
+                else self.stderr.decode("ascii", errors="replace")
+            )
             return "Command '%s' exit status %d: %s" % (self.cmd, self.returncode, err)
 
 
-def call_command(cmd: List[str], *args: str, input: Optional[str] = None, timeout: Optional[float] = None,
-                useCLocale: bool = True, env: Optional[typing.Dict[str, str]] = None, quiet: Optional[bool] = False) -> str:
+def call_command(
+    cmd: list[str],
+    *args: str,
+    input: Optional[str] = None,
+    timeout: Optional[float] = None,
+    useCLocale: bool = True,
+    env: Optional[dict[str, str]] = None,
+    quiet: Optional[bool] = False
+) -> str:
     """
     Simplified calling of external commands.
 
@@ -138,14 +169,30 @@ def call_command(cmd: List[str], *args: str, input: Optional[str] = None, timeou
 
     logger.debug("run command: {}".format(" ".join(cmd)))
     start_time = datetime.datetime.now()
-    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
-                            encoding='utf-8', errors="replace", env=env)
+    proc = subprocess.Popen(
+        cmd,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        encoding="utf-8",
+        errors="replace",
+        env=env,
+    )
     stdout, stderr = proc.communicate(input=input, timeout=timeout)
     end_time = datetime.datetime.now()
     runtime = (end_time - start_time).total_seconds()
     sys.stderr.write(stderr)
     if proc.returncode != 0:
-        logger.debug("command failed in {}s: {}: {}".format(runtime, " ".join(cmd), stderr.rstrip()))
-        raise CalledProcessErrorStderr(proc.returncode, cmd, output=stdout, stderr=stderr)
-    logger.debug("command succeeded in {}s: {}{}".format(runtime, " ".join(cmd), (': ' + stdout.rstrip()) if not quiet else ''))
+        logger.debug(
+            "command failed in {}s: {}: {}".format(
+                runtime, " ".join(cmd), stderr.rstrip()
+            )
+        )
+        raise CalledProcessErrorStderr(
+            proc.returncode, cmd, output=stdout, stderr=stderr
+        )
+    logger.debug(
+        "command succeeded in {}s: {}{}".format(
+            runtime, " ".join(cmd), (": " + stdout.rstrip()) if not quiet else ""
+        )
+    )
     return stdout

toil/lib/objects.py

@@ -126,10 +126,10 @@ class InnerClass:
         if instance is None:
             return self.inner_class
         else:
-            return self._bind( instance )
+            return self._bind(instance)
 
     @sync_memoize
-    def _bind( self, _outer):
+    def _bind(self, _outer):
         class BoundInner(self.inner_class):
             outer = _outer
 

toil/lib/resources.py

@@ -12,11 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import fnmatch
-import os
 import math
-import sys
+import os
 import resource
-from typing import List, Tuple
+import sys
+
 
 class ResourceMonitor:
     """
@@ -52,14 +52,20 @@ class ResourceMonitor:
         cls._extra_cpu_seconds += seconds
 
     @classmethod
-    def get_total_cpu_time_and_memory_usage(cls) -> Tuple[float, int]:
+    def get_total_cpu_time_and_memory_usage(cls) -> tuple[float, int]:
         """
         Gives the total cpu time of itself and all its children, and the maximum RSS memory usage of
         itself and its single largest child (in kibibytes).
         """
         me = resource.getrusage(resource.RUSAGE_SELF)
         children = resource.getrusage(resource.RUSAGE_CHILDREN)
-        total_cpu_time = me.ru_utime + me.ru_stime + children.ru_utime + children.ru_stime + cls._extra_cpu_seconds
+        total_cpu_time = (
+            me.ru_utime
+            + me.ru_stime
+            + children.ru_utime
+            + children.ru_stime
+            + cls._extra_cpu_seconds
+        )
         total_memory_usage = me.ru_maxrss + children.ru_maxrss
         if sys.platform == "darwin":
             # On Linux, getrusage works in "kilobytes" (really kibibytes), but on
@@ -74,10 +80,16 @@ class ResourceMonitor:
         """Gives the total cpu time, including the children."""
         me = resource.getrusage(resource.RUSAGE_SELF)
         childs = resource.getrusage(resource.RUSAGE_CHILDREN)
-        return me.ru_utime + me.ru_stime + childs.ru_utime + childs.ru_stime + cls._extra_cpu_seconds
+        return (
+            me.ru_utime
+            + me.ru_stime
+            + childs.ru_utime
+            + childs.ru_stime
+            + cls._extra_cpu_seconds
+        )
 
 
-def glob(glob_pattern: str, directoryname: str) -> List[str]:
+def glob(glob_pattern: str, directoryname: str) -> list[str]:
     """
     Walks through a directory and its subdirectories looking for files matching
     the glob_pattern and returns a list=[].

toil/lib/retry.py

@@ -131,35 +131,30 @@ import sqlite3
 import time
 import traceback
 import urllib.error
+from collections.abc import Generator, Iterable, Sequence
 from contextlib import contextmanager
-from typing import (Any,
-                    Callable,
-                    ContextManager,
-                    Generator,
-                    Iterable,
-                    List,
-                    Optional,
-                    Sequence,
-                    Tuple,
-                    Type,
-                    Union, TypeVar)
+from typing import Any, Callable, ContextManager, Optional, TypeVar, Union
 
 import requests.exceptions
 import urllib3.exceptions
 
-SUPPORTED_HTTP_ERRORS = [http.client.HTTPException,
-                         urllib.error.HTTPError,
-                         urllib3.exceptions.HTTPError,
-                         requests.exceptions.HTTPError]
+SUPPORTED_HTTP_ERRORS = [
+    http.client.HTTPException,
+    urllib.error.HTTPError,
+    urllib3.exceptions.HTTPError,
+    requests.exceptions.HTTPError,
+]
 
 try:
     import kubernetes.client.rest
+
     SUPPORTED_HTTP_ERRORS.append(kubernetes.client.rest.ApiException)
 except ModuleNotFoundError:
     kubernetes = None
 
 try:
     import botocore.exceptions
+
     SUPPORTED_HTTP_ERRORS.append(botocore.exceptions.ClientError)
 except ModuleNotFoundError:
     botocore = None
@@ -175,12 +170,14 @@ class ErrorCondition:
     whether to retry.
     """
 
-    def __init__(self,
-                 error: Optional[Any] = None,
-                 error_codes: List[int] = None,
-                 boto_error_codes: List[str] = None,
-                 error_message_must_include: str = None,
-                 retry_on_this_condition: bool = True):
+    def __init__(
+        self,
+        error: Optional[Any] = None,
+        error_codes: list[int] = None,
+        boto_error_codes: list[str] = None,
+        error_message_must_include: str = None,
+        retry_on_this_condition: bool = True,
+    ):
         """
         Initialize this ErrorCondition.
 
@@ -227,12 +224,14 @@ class ErrorCondition:
 # There is a better way to type hint this with python 3.10
 # https://stackoverflow.com/a/68290080
 RT = TypeVar("RT")
+
+
 def retry(
-    intervals: Optional[List] = None,
+    intervals: Optional[list] = None,
     infinite_retries: bool = False,
-    errors: Optional[Sequence[Union[ErrorCondition, Type[Exception]]]] = None,
-    log_message: Optional[Tuple[Callable, str]] = None,
-    prepare: Optional[List[Callable]] = None,
+    errors: Optional[Sequence[Union[ErrorCondition, type[Exception]]]] = None,
+    log_message: Optional[tuple[Callable, str]] = None,
+    prepare: Optional[list[Callable]] = None,
 ) -> Callable[[Callable[..., RT]], Callable[..., RT]]:
     """
     Retry a function if it fails with any Exception defined in "errors".
@@ -266,7 +265,9 @@ def retry(
     errors = errors if errors else [Exception]
 
     error_conditions = {error for error in errors if isinstance(error, ErrorCondition)}
-    retriable_errors = {error for error in errors if not isinstance(error, ErrorCondition)}
+    retriable_errors = {
+        error for error in errors if not isinstance(error, ErrorCondition)
+    }
 
     if log_message:
         post_message_function = log_message[0]
@@ -275,7 +276,10 @@ def retry(
     # if a generic error exists (with no restrictions),
     # delete more specific error_condition instances of it
     for error_condition in error_conditions:
-        if error_condition.retry_on_this_condition and error_condition.error in retriable_errors:
+        if (
+            error_condition.retry_on_this_condition
+            and error_condition.error in retriable_errors
+        ):
             error_conditions.remove(error_condition)
 
     # if a more specific error exists that isn't in the general set,
@@ -306,13 +310,17 @@ def retry(
                             raise
 
                     interval = intervals_remaining.pop(0)
-                    logger.warning(f"Error in {func}: {e}. Retrying after {interval} s...")
+                    logger.warning(
+                        f"Error in {func}: {e}. Retrying after {interval} s..."
+                    )
                     time.sleep(interval)
                     if prepare is not None:
                         for prep_function in prepare:
                             # Reset state for next attempt
                             prep_function(*args, **kwargs)
+
         return call
+
     return decorate
 
 
@@ -323,17 +331,18 @@ def return_status_code(e):
 
     if botocore:
         if isinstance(e, botocore.exceptions.ClientError):
-            return e.response.get('ResponseMetadata', {}).get('HTTPStatusCode')
+            return e.response.get("ResponseMetadata", {}).get("HTTPStatusCode")
 
     if isinstance(e, requests.exceptions.HTTPError):
         return e.response.status_code
-    elif isinstance(e, http.client.HTTPException) or \
-            isinstance(e, urllib3.exceptions.HTTPError):
+    elif isinstance(e, http.client.HTTPException) or isinstance(
+        e, urllib3.exceptions.HTTPError
+    ):
         return e.status
     elif isinstance(e, urllib.error.HTTPError):
         return e.code
     else:
-        raise ValueError(f'Unsupported error type; cannot grok status code: {e}.')
+        raise ValueError(f"Unsupported error type; cannot grok status code: {e}.")
 
 
 def get_error_code(e: Exception) -> str:
@@ -342,21 +351,21 @@ def get_error_code(e: Exception) -> str:
 
     Returns empty string for other errors.
     """
-    if hasattr(e, 'error_code') and isinstance(e.error_code, str):
+    if hasattr(e, "error_code") and isinstance(e.error_code, str):
         # A Boto 2 error
         return e.error_code
-    if hasattr(e, 'code') and isinstance(e.code, str):
+    if hasattr(e, "code") and isinstance(e.code, str):
         # A (different?) Boto 2 error
         return e.code
-    elif hasattr(e, 'response') and hasattr(e.response, 'get'):
+    elif hasattr(e, "response") and hasattr(e.response, "get"):
         # A Boto 3 error
-        code = e.response.get('Error', {}).get('Code')
+        code = e.response.get("Error", {}).get("Code")
         if isinstance(code, str):
             return code
         else:
-            return ''
+            return ""
     else:
-        return ''
+        return ""
 
 
 def get_error_message(e: Exception) -> str:
@@ -366,18 +375,18 @@ def get_error_message(e: Exception) -> str:
     Note that error message conditions also check more than this; this function
     does not fall back to the traceback for incompatible types.
     """
-    if hasattr(e, 'error_message') and isinstance(e.error_message, str):
+    if hasattr(e, "error_message") and isinstance(e.error_message, str):
         # A Boto 2 error
         return e.error_message
-    elif hasattr(e, 'response') and hasattr(e.response, 'get'):
+    elif hasattr(e, "response") and hasattr(e.response, "get"):
         # A Boto 3 error
-        message = e.response.get('Error', {}).get('Message')
+        message = e.response.get("Error", {}).get("Message")
         if isinstance(message, str):
             return message
         else:
-            return ''
+            return ""
     else:
-        return ''
+        return ""
 
 
 def get_error_status(e: Exception) -> int:
@@ -391,22 +400,23 @@ def get_error_status(e: Exception) -> int:
 
     Returns 0 from other errors.
     """
+
     def numify(x):
         """Make sure a value is an integer."""
         return int(str(x).strip())
 
-    if hasattr(e, 'status'):
+    if hasattr(e, "status"):
         # A Boto 2 error, kubernetes.client.rest.ApiException,
         # http.client.HTTPException, or urllib3.exceptions.HTTPError
         return numify(e.status)
-    elif hasattr(e, 'response'):
-        if hasattr(e.response, 'status_code'):
+    elif hasattr(e, "response"):
+        if hasattr(e.response, "status_code"):
             # A requests.exceptions.HTTPError
             return numify(e.response.status_code)
-        elif hasattr(e.response, 'get'):
+        elif hasattr(e.response, "get"):
             # A Boto 3 error
-            return numify(e.response.get('ResponseMetadata', {}).get('HTTPStatusCode'))
-    elif hasattr(e, 'code'):
+            return numify(e.response.get("ResponseMetadata", {}).get("HTTPStatusCode"))
+    elif hasattr(e, "code"):
         # A urllib.error.HTTPError
         return numify(e.code)
     else:
@@ -419,16 +429,17 @@ def get_error_body(e: Exception) -> str:
 
     Returns the code and message if the error does not have a body.
     """
-    if hasattr(e, 'body'):
+    if hasattr(e, "body"):
         # A Boto 2 error
         if isinstance(e.body, bytes):
             # Decode the body first
-            return e.body.decode('utf-8')
+            return e.body.decode("utf-8")
         elif isinstance(e.body, str):
             return e.body
 
     # Anything else
-    return f'{get_error_code(e)}: {get_error_message(e)}'
+    return f"{get_error_code(e)}: {get_error_message(e)}"
+
 
 def meets_error_message_condition(e: Exception, error_message: Optional[str]):
     if error_message:
@@ -440,7 +451,9 @@ def meets_error_message_condition(e: Exception, error_message: Optional[str]):
             if isinstance(e, botocore.exceptions.ClientError):
                 return error_message in str(e)
 
-        if isinstance(e, http.client.HTTPException) or isinstance(e, urllib3.exceptions.HTTPError):
+        if isinstance(e, http.client.HTTPException) or isinstance(
+            e, urllib3.exceptions.HTTPError
+        ):
             return error_message in e.reason
         elif isinstance(e, sqlite3.OperationalError):
             return error_message in str(e)
@@ -448,7 +461,7 @@ def meets_error_message_condition(e: Exception, error_message: Optional[str]):
             return error_message in e.msg
         elif isinstance(e, requests.exceptions.HTTPError):
             return error_message in e.raw
-        elif hasattr(e, 'msg'):
+        elif hasattr(e, "msg"):
             return error_message in e.msg
         else:
             return error_message in traceback.format_exc()
@@ -456,7 +469,7 @@ def meets_error_message_condition(e: Exception, error_message: Optional[str]):
         return True
 
 
-def meets_error_code_condition(e: Exception, error_codes: Optional[List[int]]):
+def meets_error_code_condition(e: Exception, error_codes: Optional[list[int]]):
     """These are expected to be normal HTTP error codes, like 404 or 500."""
     if error_codes:
         status_code = get_error_status(e)
@@ -465,7 +478,9 @@ def meets_error_code_condition(e: Exception, error_codes: Optional[List[int]]):
         return True
 
 
-def meets_boto_error_code_condition(e: Exception, boto_error_codes: Optional[List[str]]):
+def meets_boto_error_code_condition(
+    e: Exception, boto_error_codes: Optional[list[str]]
+):
     """These are expected to be AWS's custom error aliases, like 'BucketNotFound' or 'AccessDenied'."""
     if boto_error_codes:
         status_code = get_error_code(e)
@@ -478,21 +493,37 @@ def error_meets_conditions(e, error_conditions):
     condition_met = False
     for error in error_conditions:
         if isinstance(e, error.error):
-            if error.error_codes or error.boto_error_codes or error.error_message_must_include:
-                error_message_condition_met = meets_error_message_condition(e, error.error_message_must_include)
-                error_code_condition_met = meets_error_code_condition(e, error.error_codes)
-                boto_error_code_condition_met = meets_boto_error_code_condition(e, error.boto_error_codes)
-                if error_message_condition_met and error_code_condition_met and boto_error_code_condition_met:
+            if (
+                error.error_codes
+                or error.boto_error_codes
+                or error.error_message_must_include
+            ):
+                error_message_condition_met = meets_error_message_condition(
+                    e, error.error_message_must_include
+                )
+                error_code_condition_met = meets_error_code_condition(
+                    e, error.error_codes
+                )
+                boto_error_code_condition_met = meets_boto_error_code_condition(
+                    e, error.boto_error_codes
+                )
+                if (
+                    error_message_condition_met
+                    and error_code_condition_met
+                    and boto_error_code_condition_met
+                ):
                     if not error.retry_on_this_condition:
                         return False
                     condition_met = True
     return condition_met
 
+
 DEFAULT_DELAYS = (0, 1, 1, 4, 16, 64)
 DEFAULT_TIMEOUT = 300
 
 E = TypeVar("E", bound=Exception)  # so mypy understands passed through types
 
+
 # TODO: Replace the use of this with retry()
 #  The aws provisioner and jobstore need a large refactoring to be boto3 compliant, so this is
 #  still used there to avoid the duplication of future work
@@ -575,38 +606,45 @@ def old_retry(
     if timeout is None:
         timeout = DEFAULT_TIMEOUT
     if timeout > 0:
-        go = [ None ]
+        go = [None]
 
         @contextmanager
-        def repeated_attempt( delay ):
+        def repeated_attempt(delay):
             try:
                 yield
             except Exception as e:
-                if time.time( ) + delay < expiration:
-                    if predicate( e ):
-                        logger.info('Got %s, trying again in %is.', e, delay)
-                        time.sleep( delay )
+                if time.time() + delay < expiration:
+                    if predicate(e):
+                        logger.info("Got %s, trying again in %is.", e, delay)
+                        time.sleep(delay)
                     else:
-                        logger.error('Got a %s: %s which is not retriable according to %s', type(e), e, predicate)
+                        logger.error(
+                            "Got a %s: %s which is not retriable according to %s",
+                            type(e),
+                            e,
+                            predicate,
+                        )
                         raise
                 else:
-                    logger.error('Got %s and no time is left to retry', e)
+                    logger.error("Got %s and no time is left to retry", e)
                     raise
             else:
-                go.pop( )
+                go.pop()
 
-        delays = iter( delays )
-        expiration = time.time( ) + timeout
-        delay = next( delays )
+        delays = iter(delays)
+        expiration = time.time() + timeout
+        delay = next(delays)
         while go:
-            yield repeated_attempt( delay )
-            delay = next( delays, delay )
+            yield repeated_attempt(delay)
+            delay = next(delays, delay)
     else:
+
         @contextmanager
-        def single_attempt( ):
+        def single_attempt():
             yield
 
-        yield single_attempt( )
+        yield single_attempt()
+
 
 # Decorator to retry tests that fail. Needs to be called with
 # prepare=[tearDown, setUp] if the test class has tear down and set up that