toil 6.1.0a1__py3-none-any.whl → 8.0.0__py3-none-any.whl

toil/lib/memoize.py

@@ -17,7 +17,7 @@ import datetime
 import re
 from functools import lru_cache, wraps
 from threading import Lock
-from typing import Any, Callable, Dict, Tuple, TypeVar
+from typing import Any, Callable, TypeVar
 
 memoize = lru_cache(maxsize=None)
 """
@@ -31,13 +31,14 @@ more than once with the same arguments.
 MAT = TypeVar("MAT")
 MRT = TypeVar("MRT")
 
+
 def sync_memoize(f: Callable[[MAT], MRT]) -> Callable[[MAT], MRT]:
     """
     Like memoize, but guarantees that decorated function is only called once, even when multiple
     threads are calling the decorating function with multiple parameters.
     """
     # TODO: Think about an f that is recursive
-    memory: Dict[Tuple[Any, ...], Any] = {}
+    memory: dict[tuple[Any, ...], Any] = {}
     lock = Lock()
 
     @wraps(f)
@@ -53,13 +54,14 @@ def sync_memoize(f: Callable[[MAT], MRT]) -> Callable[[MAT], MRT]:
                     r = f(*args)
                     memory[args] = r
                     return r
+
     return new_f
 
 
 def parse_iso_utc(s: str) -> datetime.datetime:
     """
     Parses an ISO time with a hard-coded Z for zulu-time (UTC) at the end. Other timezones are
-    not supported. Returns a timezone-naive datetime object. 
+    not supported. Returns a timezone-naive datetime object.
 
     :param s: The ISO-formatted time
 
@@ -74,20 +76,22 @@ def parse_iso_utc(s: str) -> datetime.datetime:
     ...
     ValueError: Not a valid ISO datetime in UTC: 2016-04-27T00:28:04X
     """
-    rfc3339_datetime = re.compile(r'^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(?:\.(\d+))?(Z|[+-]\d{2}:\d{2})$')
+    rfc3339_datetime = re.compile(
+        r"^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(?:\.(\d+))?(Z|[+-]\d{2}:\d{2})$"
+    )
     m = rfc3339_datetime.match(s)
     if not m:
-        raise ValueError(f'Not a valid ISO datetime in UTC: {s}')
+        raise ValueError(f"Not a valid ISO datetime in UTC: {s}")
     else:
-        fmt = '%Y-%m-%dT%H:%M:%S' + ('.%f' if m.group(7) else '') + 'Z'
+        fmt = "%Y-%m-%dT%H:%M:%S" + (".%f" if m.group(7) else "") + "Z"
         return datetime.datetime.strptime(s, fmt)
 
 
 def strict_bool(s: str) -> bool:
     """Variant of bool() that only accepts two possible string values."""
-    if s == 'True':
+    if s == "True":
         return True
-    elif s == 'False':
+    elif s == "False":
         return False
     else:
         raise ValueError(s)

toil/lib/misc.py

@@ -7,11 +7,9 @@ import socket
 import subprocess
 import sys
 import time
-import typing
+from collections.abc import Iterator
 from contextlib import closing
-from typing import Iterator, List, Optional
-
-import pytz
+from typing import Optional
 
 logger = logging.getLogger(__name__)
 
@@ -23,19 +21,20 @@ def get_public_ip() -> str:
     try:
         # Try to get the internet-facing IP by attempting a connection
         # to a non-existent server and reading what IP was used.
-        ip = '127.0.0.1'
+        ip = "127.0.0.1"
         with closing(socket.socket(socket.AF_INET, socket.SOCK_DGRAM)) as sock:
             # 203.0.113.0/24 is reserved as TEST-NET-3 by RFC 5737, so
             # there is guaranteed to be no one listening on the other
             # end (and we won't accidentally DOS anyone).
-            sock.connect(('203.0.113.1', 1))
+            sock.connect(("203.0.113.1", 1))
             ip = sock.getsockname()[0]
         return ip
     except:
         # Something went terribly wrong. Just give loopback rather
         # than killing everything, because this is often called just
         # to provide a default argument
-        return '127.0.0.1'
+        return "127.0.0.1"
+
 
 def get_user_name() -> str:
     """
@@ -48,20 +47,23 @@ def get_user_name() -> str:
         except KeyError:
             # This is expected if the user isn't in /etc/passwd, such as in a
             # Docker container when running as a weird UID. Make something up.
-            return 'UnknownUser' + str(os.getuid())
+            return "UnknownUser" + str(os.getuid())
     except Exception as e:
         # We can't get the UID, or something weird has gone wrong.
-        logger.error('Unexpected error getting user name: %s', e)
-        return 'UnknownUser'
+        logger.error("Unexpected error getting user name: %s", e)
+        return "UnknownUser"
+
 
 def utc_now() -> datetime.datetime:
     """Return a datetime in the UTC timezone corresponding to right now."""
-    return datetime.datetime.utcnow().replace(tzinfo=pytz.UTC)
+    return datetime.datetime.utcnow().replace(tzinfo=datetime.timezone.utc)
+
 
 def unix_now_ms() -> float:
     """Return the current time in milliseconds since the Unix epoch."""
     return time.time() * 1000
 
+
 def slow_down(seconds: float) -> float:
     """
     Toil jobs that have completed are not allowed to have taken 0 seconds, but
@@ -79,9 +81,25 @@ def slow_down(seconds: float) -> float:
 
     return max(seconds, sys.float_info.epsilon)
 
-def printq(msg: str, quiet: bool) -> None:
+
+def printq(msg: str, quiet: bool, log: bool = False) -> None:
+    """
+    This is for functions used simultaneously in Toil proper and in the admin scripts.
+
+    Our admin scripts "print" to stdout, while Toil proper uses logging.  For a script that,
+    for example, cleans up IAM, EC2, etc. cruft leftover after failed CI runs, we can call
+    an AWS delete IAM role function, and this prints or logs progress (unless quiet is True),
+    depending on whether the function is called in, say, the jobstore or a script.
+
+    :param msg: The string to print or log to stdout.
+    :param quiet: Silent output to stdout.
+    :param log: Use logging (else "print" to the screen).
+    """
     if not quiet:
-        print(msg)
+        if not log:
+            print(msg)
+        else:
+            logger.debug(msg)
 
 
 def truncExpBackoff() -> Iterator[float]:
@@ -104,12 +122,23 @@ class CalledProcessErrorStderr(subprocess.CalledProcessError):
         if (self.returncode < 0) or (self.stderr is None):
             return str(super())
         else:
-            err = self.stderr if isinstance(self.stderr, str) else self.stderr.decode("ascii", errors="replace")
+            err = (
+                self.stderr
+                if isinstance(self.stderr, str)
+                else self.stderr.decode("ascii", errors="replace")
+            )
             return "Command '%s' exit status %d: %s" % (self.cmd, self.returncode, err)
 
 
-def call_command(cmd: List[str], *args: str, input: Optional[str] = None, timeout: Optional[float] = None,
-                useCLocale: bool = True, env: Optional[typing.Dict[str, str]] = None, quiet: Optional[bool] = False) -> str:
+def call_command(
+    cmd: list[str],
+    *args: str,
+    input: Optional[str] = None,
+    timeout: Optional[float] = None,
+    useCLocale: bool = True,
+    env: Optional[dict[str, str]] = None,
+    quiet: Optional[bool] = False
+) -> str:
     """
     Simplified calling of external commands.
 
@@ -140,14 +169,30 @@ def call_command(cmd: List[str], *args: str, input: Optional[str] = None, timeou
 
     logger.debug("run command: {}".format(" ".join(cmd)))
     start_time = datetime.datetime.now()
-    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
-                            encoding='utf-8', errors="replace", env=env)
+    proc = subprocess.Popen(
+        cmd,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        encoding="utf-8",
+        errors="replace",
+        env=env,
+    )
     stdout, stderr = proc.communicate(input=input, timeout=timeout)
     end_time = datetime.datetime.now()
     runtime = (end_time - start_time).total_seconds()
     sys.stderr.write(stderr)
     if proc.returncode != 0:
-        logger.debug("command failed in {}s: {}: {}".format(runtime, " ".join(cmd), stderr.rstrip()))
-        raise CalledProcessErrorStderr(proc.returncode, cmd, output=stdout, stderr=stderr)
-    logger.debug("command succeeded in {}s: {}{}".format(runtime, " ".join(cmd), (': ' + stdout.rstrip()) if not quiet else ''))
+        logger.debug(
+            "command failed in {}s: {}: {}".format(
+                runtime, " ".join(cmd), stderr.rstrip()
+            )
+        )
+        raise CalledProcessErrorStderr(
+            proc.returncode, cmd, output=stdout, stderr=stderr
+        )
+    logger.debug(
+        "command succeeded in {}s: {}{}".format(
+            runtime, " ".join(cmd), (": " + stdout.rstrip()) if not quiet else ""
+        )
+    )
     return stdout

toil/lib/objects.py

@@ -126,10 +126,10 @@ class InnerClass:
         if instance is None:
             return self.inner_class
         else:
-            return self._bind( instance )
+            return self._bind(instance)
 
     @sync_memoize
-    def _bind( self, _outer):
+    def _bind(self, _outer):
         class BoundInner(self.inner_class):
             outer = _outer
 

toil/lib/resources.py

@@ -12,31 +12,84 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import fnmatch
+import math
 import os
 import resource
-from typing import List, Tuple
+import sys
 
 
-def get_total_cpu_time_and_memory_usage() -> Tuple[float, int]:
+class ResourceMonitor:
     """
-    Gives the total cpu time of itself and all its children, and the maximum RSS memory usage of
-    itself and its single largest child.
+    Global resource monitoring widget.
+
+    Presents class methods to get the resource usage of this process and child
+    processes, and other class methods to adjust the statistics so they can
+    account for e.g. resources used inside containers, or other resource usage
+    that *should* be billable to the current process.
     """
-    me = resource.getrusage(resource.RUSAGE_SELF)
-    children = resource.getrusage(resource.RUSAGE_CHILDREN)
-    total_cpu_time = me.ru_utime + me.ru_stime + children.ru_utime + children.ru_stime
-    total_memory_usage = me.ru_maxrss + children.ru_maxrss
-    return total_cpu_time, total_memory_usage
 
+    # Store some extra usage to tack onto the stats as module-level globals
+    _extra_cpu_seconds: float = 0
+    _extra_memory_ki: int = 0
+
+    @classmethod
+    def record_extra_memory(cls, peak_ki: int) -> None:
+        """
+        Become responsible for the given peak memory usage, in kibibytes.
+
+        The memory will be treated as if it was used by a child process at the time
+        our real child processes were also using their peak memory.
+        """
+        cls._extra_memory_ki = max(cls._extra_memory_ki, peak_ki)
+
+    @classmethod
+    def record_extra_cpu(cls, seconds: float) -> None:
+        """
+        Become responsible for the given CPU time.
+
+        The CPU time will be treated as if it had been used by a child process.
+        """
+        cls._extra_cpu_seconds += seconds
+
+    @classmethod
+    def get_total_cpu_time_and_memory_usage(cls) -> tuple[float, int]:
+        """
+        Gives the total cpu time of itself and all its children, and the maximum RSS memory usage of
+        itself and its single largest child (in kibibytes).
+        """
+        me = resource.getrusage(resource.RUSAGE_SELF)
+        children = resource.getrusage(resource.RUSAGE_CHILDREN)
+        total_cpu_time = (
+            me.ru_utime
+            + me.ru_stime
+            + children.ru_utime
+            + children.ru_stime
+            + cls._extra_cpu_seconds
+        )
+        total_memory_usage = me.ru_maxrss + children.ru_maxrss
+        if sys.platform == "darwin":
+            # On Linux, getrusage works in "kilobytes" (really kibibytes), but on
+            # Mac it works in bytes. See
+            # <https://github.com/python/cpython/issues/74698>
+            total_memory_usage = int(math.ceil(total_memory_usage / 1024))
+        total_memory_usage += cls._extra_memory_ki
+        return total_cpu_time, total_memory_usage
 
-def get_total_cpu_time() -> float:
-    """Gives the total cpu time, including the children."""
-    me = resource.getrusage(resource.RUSAGE_SELF)
-    childs = resource.getrusage(resource.RUSAGE_CHILDREN)
-    return me.ru_utime + me.ru_stime + childs.ru_utime + childs.ru_stime
+    @classmethod
+    def get_total_cpu_time(cls) -> float:
+        """Gives the total cpu time, including the children."""
+        me = resource.getrusage(resource.RUSAGE_SELF)
+        childs = resource.getrusage(resource.RUSAGE_CHILDREN)
+        return (
+            me.ru_utime
+            + me.ru_stime
+            + childs.ru_utime
+            + childs.ru_stime
+            + cls._extra_cpu_seconds
+        )
 
 
-def glob(glob_pattern: str, directoryname: str) -> List[str]:
+def glob(glob_pattern: str, directoryname: str) -> list[str]:
     """
     Walks through a directory and its subdirectories looking for files matching
     the glob_pattern and returns a list=[].