toil 7.0.0__py3-none-any.whl → 8.1.0b1__py3-none-any.whl

toil/lib/humanize.py

@@ -25,7 +25,9 @@ def bytes2human(n: SupportsInt) -> str:
     """
     Convert n bytes into a human readable string.
     """
-    logger.warning('Deprecated toil method.  Please use "toil.lib.conversions.bytes2human()" instead."')
+    logger.warning(
+        'Deprecated toil method.  Please use "toil.lib.conversions.bytes2human()" instead."'
+    )
     return b2h(n)
 
 
@@ -36,5 +38,7 @@ def human2bytes(s: str) -> int:
 
     When unable to recognize the format ValueError is raised.
     """
-    logger.warning('Deprecated toil method.  Please use "toil.lib.conversions.human2bytes()" instead."')
+    logger.warning(
+        'Deprecated toil method.  Please use "toil.lib.conversions.human2bytes()" instead."'
+    )
     return h2b(s)

toil/lib/io.py

@@ -1,16 +1,86 @@
+import hashlib
 import logging
 import os
 import shutil
 import stat
+import sys
 import tempfile
 import uuid
+from collections.abc import Iterator
 from contextlib import contextmanager
 from io import BytesIO
-from typing import IO, Any, Callable, Iterator, Optional, Union
+from typing import IO, Any, Callable, Optional, Protocol, Union
+
+from toil.lib.memoize import memoize
 
 logger = logging.getLogger(__name__)
 
-def mkdtemp(suffix: Optional[str] = None, prefix: Optional[str] = None, dir: Optional[str] = None) -> str:
+@memoize
+def get_toil_home() -> str:
+    """
+    Get the Toil home directory for storing configuration and global state.
+
+    Raises an error if it does not exist and cannot be created. Safe to run
+    simultaneously in multiple processes.
+    """
+   
+    # TODO: should this use an XDG config directory or ~/.config to not clutter the
+    # base home directory?
+    toil_home_dir = os.path.join(os.path.expanduser("~"), ".toil")
+
+    dir_path = try_path(toil_home_dir)
+    if dir_path is None:
+        raise RuntimeError(
+            f"Cannot create or access Toil configuration directory {toil_home_dir}"
+        )
+    return dir_path
+
+TOIL_URI_SCHEME = "toilfile:"
+
+
+STANDARD_SCHEMES = ["http:", "https:", "s3:", "gs:", "ftp:"]
+REMOTE_SCHEMES = STANDARD_SCHEMES + [TOIL_URI_SCHEME]
+ALL_SCHEMES = REMOTE_SCHEMES + ["file:"]
+
+def is_standard_url(filename: str) -> bool:
+    return is_url_with_scheme(filename, STANDARD_SCHEMES)
+
+def is_remote_url(filename: str) -> bool:
+    """
+    Decide if a filename is a known, non-file kind of URL
+    """
+    return is_url_with_scheme(filename, REMOTE_SCHEMES)
+
+def is_any_url(filename: str) -> bool:
+    """
+    Decide if a string is a URI like http:// or file://.
+
+    Otherwise it might be a bare path.
+    """
+    return is_url_with_scheme(filename, ALL_SCHEMES)
+
+def is_url_with_scheme(filename: str, schemes: list[str]) -> bool:
+    """
+    Return True if filename is a URL with any of the given schemes and False otherwise.
+    """
+    # TODO: "http:myfile.dat" is a valid filename and *not* a valid URL
+    for scheme in schemes:
+        if filename.startswith(scheme):
+            return True
+    return False
+
+def is_toil_url(filename: str) -> bool:
+    return is_url_with_scheme(filename, [TOIL_URI_SCHEME])
+
+def is_file_url(filename: str) -> bool:
+    return is_url_with_scheme(filename, ["file:"])
+
+
+def mkdtemp(
+    suffix: Optional[str] = None,
+    prefix: Optional[str] = None,
+    dir: Optional[str] = None,
+) -> str:
     """
     Make a temporary directory like tempfile.mkdtemp, but with relaxed permissions.
 
@@ -27,10 +97,13 @@ def mkdtemp(suffix: Optional[str] = None, prefix: Optional[str] = None, dir: Opt
     # Make the directory
     result = tempfile.mkdtemp(suffix=suffix, prefix=prefix, dir=dir)
     # Grant all the permissions: full control for user, and execute for group and other
-    os.chmod(result, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
+    os.chmod(
+        result, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH
+    )
     # Return the path created
     return result
 
+
 def robust_rmtree(path: Union[str, bytes]) -> None:
     """
     Robustly tries to delete paths.
@@ -45,7 +118,7 @@ def robust_rmtree(path: Union[str, bytes]) -> None:
     if not isinstance(path, bytes):
         # Internally we must work in bytes, in case we find an undecodeable
         # filename.
-        path = path.encode('utf-8')
+        path = path.encode("utf-8")
 
     if not os.path.exists(path):
         # Nothing to do!
@@ -107,7 +180,7 @@ def atomic_tmp_file(final_path: str) -> str:
     as finalPath.  It the final path is in /dev (/dev/null, /dev/stdout), it is
     returned unchanged and atomic_tmp_install will do nothing."""
     final_dir = os.path.dirname(os.path.normpath(final_path))  # can be empty
-    if final_dir == '/dev':
+    if final_dir == "/dev":
         return final_path
     final_basename = os.path.basename(final_path)
     final_ext = os.path.splitext(final_path)[1]
@@ -117,14 +190,15 @@ def atomic_tmp_file(final_path: str) -> str:
 
 def atomic_install(tmp_path, final_path) -> None:
     """atomic install of tmp_path as final_path"""
-    if os.path.dirname(os.path.normpath(final_path)) != '/dev':
+    if os.path.dirname(os.path.normpath(final_path)) != "/dev":
         os.rename(tmp_path, final_path)
 
+
 @contextmanager
 def AtomicFileCreate(final_path: str, keep: bool = False) -> Iterator[str]:
     """Context manager to create a temporary file.  Entering returns path to
     the temporary file in the same directory as finalPath.  If the code in
-    context succeeds, the file renamed to its actually name.  If an error
+    context succeeds, the file renamed to its actual name.  If an error
     occurs, the file is not installed and is removed unless keep is specified.
     """
     tmp_path = atomic_tmp_file(final_path)
@@ -140,7 +214,9 @@ def AtomicFileCreate(final_path: str, keep: bool = False) -> Iterator[str]:
         raise
 
 
-def atomic_copy(src_path: str, dest_path: str, executable: Optional[bool] = None) -> None:
+def atomic_copy(
+    src_path: str, dest_path: str, executable: Optional[bool] = None
+) -> None:
     """Copy a file using posix atomic creations semantics."""
     if executable is None:
         executable = os.stat(src_path).st_mode & stat.S_IXUSR != 0
@@ -150,10 +226,12 @@ def atomic_copy(src_path: str, dest_path: str, executable: Optional[bool] = None
             os.chmod(dest_path_tmp, os.stat(dest_path_tmp).st_mode | stat.S_IXUSR)
 
 
-def atomic_copyobj(src_fh: BytesIO, dest_path: str, length: int = 16384, executable: bool = False) -> None:
+def atomic_copyobj(
+    src_fh: BytesIO, dest_path: str, length: int = 16384, executable: bool = False
+) -> None:
     """Copy an open file using posix atomic creations semantics."""
     with AtomicFileCreate(dest_path) as dest_path_tmp:
-        with open(dest_path_tmp, 'wb') as dest_path_fh:
+        with open(dest_path_tmp, "wb") as dest_path_fh:
             shutil.copyfileobj(src_fh, dest_path_fh, length=length)
         if executable:
             os.chmod(dest_path_tmp, os.stat(dest_path_tmp).st_mode | stat.S_IXUSR)
@@ -179,9 +257,11 @@ def make_public_dir(in_directory: str, suggested_name: Optional[str] = None) ->
             return generated_dir_path
         except FileExistsError:
             pass
-    for i in range(4, 32 + 1):  # make random uuids and truncate to lengths starting at 4 and working up to max 32
+    for i in range(
+        4, 32 + 1
+    ):  # make random uuids and truncate to lengths starting at 4 and working up to max 32
         for _ in range(10):  # make 10 attempts for each length
-            truncated_uuid: str = str(uuid.uuid4()).replace('-', '')[:i]
+            truncated_uuid: str = str(uuid.uuid4()).replace("-", "")[:i]
             generated_dir_path: str = os.path.join(in_directory, truncated_uuid)
             try:
                 os.mkdir(generated_dir_path)
@@ -194,6 +274,7 @@ def make_public_dir(in_directory: str, suggested_name: Optional[str] = None) ->
     os.chmod(this_should_never_happen, 0o777)
     return this_should_never_happen
 
+
 def try_path(path: str, min_size: int = 100 * 1024 * 1024) -> Optional[str]:
     """
     Try to use the given path. Return it if it exists or can be made,
@@ -291,3 +372,31 @@ class WriteWatchingStream:
         """
 
         self.backingStream.close()
+
+class ReadableFileObj(Protocol):
+    """
+    Protocol that is more specific than what file_digest takes as an argument.
+    Also guarantees a read() method.
+    Would extend the protocol from Typeshed for hashlib but those are only
+    declared for 3.11+.
+    """
+    def readinto(self, buf: bytearray, /) -> int: ...
+    def readable(self) -> bool: ...
+    def read(self, number: int) -> bytes: ...
+
+# hashlib._Hash seems to not appear at runtime
+def file_digest(f: ReadableFileObj, alg_name: str) -> "hashlib._Hash":
+    """
+    Polyfilled hashlib.file_digest that works on Python <3.11.
+    """
+    if sys.version_info >= (3, 11):
+        return hashlib.file_digest(f, alg_name)
+    BUFFER_SIZE = 1024 * 1024
+    hasher = hashlib.new(alg_name)
+    buffer = f.read(BUFFER_SIZE)
+    while buffer:
+        hasher.update(buffer)
+        buffer = f.read(BUFFER_SIZE)
+    return hasher
+
+

toil/lib/iterables.py

@@ -12,8 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from collections.abc import Iterable, Iterator
+
 # 5.14.2018: copied into Toil from https://github.com/BD2KGenomics/bd2k-python-lib
-from typing import Any, Iterable, Iterator, TypeVar
+from typing import Any, TypeVar
 
 IT = TypeVar("IT")
 
@@ -102,7 +104,7 @@ class concat:
                 try:
                     i = x.__iter__()
                 except AttributeError:
-                    i = x,
+                    i = (x,)
             else:
                 i = x
             return i

toil/lib/memoize.py

@@ -17,7 +17,7 @@ import datetime
 import re
 from functools import lru_cache, wraps
 from threading import Lock
-from typing import Any, Callable, Dict, Tuple, TypeVar
+from typing import Any, Callable, TypeVar
 
 memoize = lru_cache(maxsize=None)
 """
@@ -31,13 +31,14 @@ more than once with the same arguments.
 MAT = TypeVar("MAT")
 MRT = TypeVar("MRT")
 
+
 def sync_memoize(f: Callable[[MAT], MRT]) -> Callable[[MAT], MRT]:
     """
     Like memoize, but guarantees that decorated function is only called once, even when multiple
     threads are calling the decorating function with multiple parameters.
     """
     # TODO: Think about an f that is recursive
-    memory: Dict[Tuple[Any, ...], Any] = {}
+    memory: dict[tuple[Any, ...], Any] = {}
     lock = Lock()
 
     @wraps(f)
@@ -53,13 +54,14 @@ def sync_memoize(f: Callable[[MAT], MRT]) -> Callable[[MAT], MRT]:
                     r = f(*args)
                     memory[args] = r
                     return r
+
     return new_f
 
 
 def parse_iso_utc(s: str) -> datetime.datetime:
     """
     Parses an ISO time with a hard-coded Z for zulu-time (UTC) at the end. Other timezones are
-    not supported. Returns a timezone-naive datetime object. 
+    not supported. Returns a timezone-naive datetime object.
 
     :param s: The ISO-formatted time
 
@@ -74,20 +76,22 @@ def parse_iso_utc(s: str) -> datetime.datetime:
     ...
     ValueError: Not a valid ISO datetime in UTC: 2016-04-27T00:28:04X
     """
-    rfc3339_datetime = re.compile(r'^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(?:\.(\d+))?(Z|[+-]\d{2}:\d{2})$')
+    rfc3339_datetime = re.compile(
+        r"^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(?:\.(\d+))?(Z|[+-]\d{2}:\d{2})$"
+    )
     m = rfc3339_datetime.match(s)
     if not m:
-        raise ValueError(f'Not a valid ISO datetime in UTC: {s}')
+        raise ValueError(f"Not a valid ISO datetime in UTC: {s}")
     else:
-        fmt = '%Y-%m-%dT%H:%M:%S' + ('.%f' if m.group(7) else '') + 'Z'
+        fmt = "%Y-%m-%dT%H:%M:%S" + (".%f" if m.group(7) else "") + "Z"
         return datetime.datetime.strptime(s, fmt)
 
 
 def strict_bool(s: str) -> bool:
     """Variant of bool() that only accepts two possible string values."""
-    if s == 'True':
+    if s == "True":
         return True
-    elif s == 'False':
+    elif s == "False":
         return False
     else:
         raise ValueError(s)

toil/lib/misc.py

@@ -7,9 +7,9 @@ import socket
 import subprocess
 import sys
 import time
-import typing
+from collections.abc import Iterator
 from contextlib import closing
-from typing import Iterator, List, Optional
+from typing import Optional
 
 logger = logging.getLogger(__name__)
 
@@ -21,19 +21,20 @@ def get_public_ip() -> str:
     try:
         # Try to get the internet-facing IP by attempting a connection
         # to a non-existent server and reading what IP was used.
-        ip = '127.0.0.1'
+        ip = "127.0.0.1"
         with closing(socket.socket(socket.AF_INET, socket.SOCK_DGRAM)) as sock:
             # 203.0.113.0/24 is reserved as TEST-NET-3 by RFC 5737, so
             # there is guaranteed to be no one listening on the other
             # end (and we won't accidentally DOS anyone).
-            sock.connect(('203.0.113.1', 1))
+            sock.connect(("203.0.113.1", 1))
             ip = sock.getsockname()[0]
         return ip
     except:
         # Something went terribly wrong. Just give loopback rather
         # than killing everything, because this is often called just
         # to provide a default argument
-        return '127.0.0.1'
+        return "127.0.0.1"
+
 
 def get_user_name() -> str:
     """
@@ -46,20 +47,41 @@ def get_user_name() -> str:
         except KeyError:
             # This is expected if the user isn't in /etc/passwd, such as in a
             # Docker container when running as a weird UID. Make something up.
-            return 'UnknownUser' + str(os.getuid())
+            return "UnknownUser" + str(os.getuid())
     except Exception as e:
         # We can't get the UID, or something weird has gone wrong.
-        logger.error('Unexpected error getting user name: %s', e)
-        return 'UnknownUser'
+        logger.error("Unexpected error getting user name: %s", e)
+        return "UnknownUser"
+
 
 def utc_now() -> datetime.datetime:
     """Return a datetime in the UTC timezone corresponding to right now."""
     return datetime.datetime.utcnow().replace(tzinfo=datetime.timezone.utc)
 
+
 def unix_now_ms() -> float:
     """Return the current time in milliseconds since the Unix epoch."""
     return time.time() * 1000
 
+def unix_seconds_to_timestamp(timestamp: float) -> str:
+    """
+    Convert a time in seconds since the Unix epoch to an ISO 8601 string.
+    """
+    return datetime.datetime.fromtimestamp(timestamp, tz=datetime.timezone.utc).isoformat()
+
+def unix_seconds_to_local_time(timestamp: float) -> datetime.datetime:
+    """
+    Returns a local time corresponding to the given number of seconds since the Unix epoch.
+    """
+    return datetime.datetime.fromtimestamp(timestamp, tz=datetime.timezone.utc).astimezone()
+
+def seconds_to_duration(time_difference: float) -> str:
+    """
+    Convert a time difference in seconds to an ISO 8601 duration string.
+    """
+    return f"PT{time_difference:.3f}S"
+
+
 def slow_down(seconds: float) -> float:
     """
     Toil jobs that have completed are not allowed to have taken 0 seconds, but
@@ -77,9 +99,25 @@ def slow_down(seconds: float) -> float:
 
     return max(seconds, sys.float_info.epsilon)
 
-def printq(msg: str, quiet: bool) -> None:
+
+def printq(msg: str, quiet: bool, log: bool = False) -> None:
+    """
+    This is for functions used simultaneously in Toil proper and in the admin scripts.
+
+    Our admin scripts "print" to stdout, while Toil proper uses logging.  For a script that,
+    for example, cleans up IAM, EC2, etc. cruft leftover after failed CI runs, we can call
+    an AWS delete IAM role function, and this prints or logs progress (unless quiet is True),
+    depending on whether the function is called in, say, the jobstore or a script.
+
+    :param msg: The string to print or log to stdout.
+    :param quiet: Silent output to stdout.
+    :param log: Use logging (else "print" to the screen).
+    """
     if not quiet:
-        print(msg)
+        if not log:
+            print(msg)
+        else:
+            logger.debug(msg)
 
 
 def truncExpBackoff() -> Iterator[float]:
@@ -102,12 +140,23 @@ class CalledProcessErrorStderr(subprocess.CalledProcessError):
         if (self.returncode < 0) or (self.stderr is None):
             return str(super())
         else:
-            err = self.stderr if isinstance(self.stderr, str) else self.stderr.decode("ascii", errors="replace")
+            err = (
+                self.stderr
+                if isinstance(self.stderr, str)
+                else self.stderr.decode("ascii", errors="replace")
+            )
             return "Command '%s' exit status %d: %s" % (self.cmd, self.returncode, err)
 
 
-def call_command(cmd: List[str], *args: str, input: Optional[str] = None, timeout: Optional[float] = None,
-                useCLocale: bool = True, env: Optional[typing.Dict[str, str]] = None, quiet: Optional[bool] = False) -> str:
+def call_command(
+    cmd: list[str],
+    *args: str,
+    input: Optional[str] = None,
+    timeout: Optional[float] = None,
+    useCLocale: bool = True,
+    env: Optional[dict[str, str]] = None,
+    quiet: Optional[bool] = False
+) -> str:
     """
     Simplified calling of external commands.
 
@@ -138,14 +187,30 @@ def call_command(cmd: List[str], *args: str, input: Optional[str] = None, timeou
 
     logger.debug("run command: {}".format(" ".join(cmd)))
     start_time = datetime.datetime.now()
-    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
-                            encoding='utf-8', errors="replace", env=env)
+    proc = subprocess.Popen(
+        cmd,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        encoding="utf-8",
+        errors="replace",
+        env=env,
+    )
     stdout, stderr = proc.communicate(input=input, timeout=timeout)
     end_time = datetime.datetime.now()
     runtime = (end_time - start_time).total_seconds()
     sys.stderr.write(stderr)
     if proc.returncode != 0:
-        logger.debug("command failed in {}s: {}: {}".format(runtime, " ".join(cmd), stderr.rstrip()))
-        raise CalledProcessErrorStderr(proc.returncode, cmd, output=stdout, stderr=stderr)
-    logger.debug("command succeeded in {}s: {}{}".format(runtime, " ".join(cmd), (': ' + stdout.rstrip()) if not quiet else ''))
+        logger.debug(
+            "command failed in {}s: {}: {}".format(
+                runtime, " ".join(cmd), stderr.rstrip()
+            )
+        )
+        raise CalledProcessErrorStderr(
+            proc.returncode, cmd, output=stdout, stderr=stderr
+        )
+    logger.debug(
+        "command succeeded in {}s: {}{}".format(
+            runtime, " ".join(cmd), (": " + stdout.rstrip()) if not quiet else ""
+        )
+    )
     return stdout

toil/lib/objects.py

@@ -126,10 +126,10 @@ class InnerClass:
         if instance is None:
             return self.inner_class
         else:
-            return self._bind( instance )
+            return self._bind(instance)
 
     @sync_memoize
-    def _bind( self, _outer):
+    def _bind(self, _outer):
         class BoundInner(self.inner_class):
             outer = _outer
 

toil/lib/resources.py

@@ -12,11 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import fnmatch
-import os
 import math
-import sys
+import os
 import resource
-from typing import List, Tuple
+import sys
+
 
 class ResourceMonitor:
     """
@@ -52,14 +52,20 @@ class ResourceMonitor:
         cls._extra_cpu_seconds += seconds
 
     @classmethod
-    def get_total_cpu_time_and_memory_usage(cls) -> Tuple[float, int]:
+    def get_total_cpu_time_and_memory_usage(cls) -> tuple[float, int]:
         """
         Gives the total cpu time of itself and all its children, and the maximum RSS memory usage of
         itself and its single largest child (in kibibytes).
         """
         me = resource.getrusage(resource.RUSAGE_SELF)
         children = resource.getrusage(resource.RUSAGE_CHILDREN)
-        total_cpu_time = me.ru_utime + me.ru_stime + children.ru_utime + children.ru_stime + cls._extra_cpu_seconds
+        total_cpu_time = (
+            me.ru_utime
+            + me.ru_stime
+            + children.ru_utime
+            + children.ru_stime
+            + cls._extra_cpu_seconds
+        )
         total_memory_usage = me.ru_maxrss + children.ru_maxrss
         if sys.platform == "darwin":
             # On Linux, getrusage works in "kilobytes" (really kibibytes), but on
@@ -74,10 +80,16 @@ class ResourceMonitor:
         """Gives the total cpu time, including the children."""
         me = resource.getrusage(resource.RUSAGE_SELF)
         childs = resource.getrusage(resource.RUSAGE_CHILDREN)
-        return me.ru_utime + me.ru_stime + childs.ru_utime + childs.ru_stime + cls._extra_cpu_seconds
+        return (
+            me.ru_utime
+            + me.ru_stime
+            + childs.ru_utime
+            + childs.ru_stime
+            + cls._extra_cpu_seconds
+        )
 
 
-def glob(glob_pattern: str, directoryname: str) -> List[str]:
+def glob(glob_pattern: str, directoryname: str) -> list[str]:
     """
     Walks through a directory and its subdirectories looking for files matching
     the glob_pattern and returns a list=[].